diff --git a/.env.template b/.env.template index 0f7ea3ea2..b76bfd3c8 100644 --- a/.env.template +++ b/.env.template @@ -38,13 +38,34 @@ CLAUDE_CODE_ATTREBUTION_HEADER=0 # api = the raw-API keys above # cursor = Cursor subscription via the cursor-agent CLI (no API key; # the CLI must be installed and logged in) -# claude-code / codex = declared, ship in M2/M3 +# claude-code = Claude subscription/API via the claude CLI (M2, shipped) +# codex = ChatGPT subscription via the codex CLI (M3; needs a login) STRICT_BACKEND=api # STRICT_BACKEND_MODEL= # model id inside the harness (optional) # STRICT_BACKEND_CONCURRENCY=2 # concurrent harness sessions # STRICT_BACKEND_CLI= # binary path override (else PATH lookup) # STRICT_BACKEND_TIMEOUT_S=1800 # per-session wall-clock ceiling +# --- Review model routing (measured on the wave-4/5 gates; see +# --- eval/dataset/results/model_comparison.md before changing these) ------- +# Cheap seats: the planner's gray-zone call and the tool-less coverage/ +# residue promotion do not carry the precision result, so a cheap model is +# the right default. NOTE a reasoning model needs headroom — the planner +# silently failed for two campaigns at a 400-token ceiling because thinking +# consumed it before any JSON appeared. +# REVIEW_PLANNER_MODEL=deepseek-v4-flash +# REVIEW_PROMOTION_MODEL=deepseek-v4-flash +# +# Per-pass backends: map a review pass to "provider:model" to run that seat +# on a different engine. Measured configurations (same gate, same judge): +# (unset) — all-DS "v15": the precision-leaning config, ~$1/item. +# adversary+round2 on claude-code — "v16": the recall-leaning config; the +# only one that reproduced the GOLD latent-gap catch inside the pipeline. +# Costs 2 harness sessions/item on top of the DS core. +# investigator/docs/round2 on cursor — subscription-priced breadth; it +# measured WORSE precision and no recall gain. Not recommended. +# REVIEW_LENS_BACKENDS={"adversary": "claude-code:claude-fable-5", "round2": "claude-code:claude-fable-5"} + # Repos: JSON map of repo name -> local path (machine-specific; .env is gitignored, # which is why absolute paths belong HERE and never in a committed file). REPO_PATHS={"vllm-omni": "/path/to/vllm-omni", "afd-plugin": "/path/to/afd-plugin"} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54372444e..e5c2977f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: tests # The suite is ~485 offline cases with no network and no paid LLM calls, so it can # run on every push. Until this existed it ran only when someone remembered to type -# `pytest` (doc/CODE_TOUR.md documents it as a manual step) — tests that nothing runs +# `pytest` (doc/architecture/CODE_TOUR.md documents it as a manual step) — tests that nothing runs # protect nothing. # # NOTE: this workflow is a SIGNAL, not a merge gate. Making it blocking needs a branch @@ -21,7 +21,16 @@ jobs: suite: runs-on: ubuntu-latest steps: + # Full history, not the default depth-1 shallow clone. check_spec_freshness + # dates each source file with `git log -1 -- `, and in a shallow clone + # the single commit has no parent, so git reports EVERY tracked file as + # created in it. All 62 SPEC pages then read as modified today and the + # strict check fails wholesale. It only ever passed because the tip commit + # happened to predate the pages' verified-against: dates; the first commit + # made on a later day broke all 62 at once. - uses: actions/checkout@v4 + with: + fetch-depth: 0 - uses: actions/setup-python@v5 with: @@ -74,6 +83,22 @@ jobs: ", ".join(_RESOURCE_MARKERS)) PY + # Doc integrity. These exist because the doc/ refactor measured the damage + # they prevent: 29 in-code citations of doc/ paths that no linter checked, + # and 44 of 45 SPEC pages silently behind their own source. + - name: Doc linters + run: | + python tools/check_doc_links.py + python tools/check_doc_citations.py + + # BLOCKING as of the P5 rewrite: all 92 modules have a SPEC page and all + # 62 pages carry a `verified-against:` date. --strict additionally fails + # on a page whose source changed after that date, or one that dropped its + # marker — which is the whole point: 44 of 45 pages had silently fallen + # behind before this check existed. + - name: SPEC coverage and freshness + run: python tools/check_spec_freshness.py --strict + # Written, tested, and until now invoked only from prose (AGENTS.md, DEVELOPMENT.md). - name: Knowledge linters run: | diff --git a/.github/workflows/vllm-omni-release-audit.yml b/.github/workflows/vllm-omni-release-audit.yml index 2c9891e63..46cd48346 100644 --- a/.github/workflows/vllm-omni-release-audit.yml +++ b/.github/workflows/vllm-omni-release-audit.yml @@ -6,7 +6,7 @@ on: - .github/workflows/vllm-omni-release-audit.yml - adapters/vllm_omni/manifest.yaml - adapters/vllm_omni/release_baseline.yaml - - doc/VLLM_OMNI_RELEASE_MAINTENANCE.md + - doc/contributing/release-maintenance.md - plugins/infermatrix-copilot/skills/imupdate/** - knowledge/repos/vllm-omni/** - tools/audit_vllm_omni_release.py diff --git a/AGENTS.md b/AGENTS.md index 57080ea1f..bc0bead0a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,28 +2,32 @@ ## Knowledge edits -`knowledge/` is a vendored adaptation of -`zuiho-kai/claude-workflow-starter`. Before adding, moving, or deleting any -knowledge page: +Before adding, moving, or deleting any knowledge page: -1. Read `doc/PLAN-knowledge-reorg.md`, `knowledge/CONTRIBUTING.md`, then exactly - the one linked contribution topic that matches the change. The plan's - owner-scoped inventory and detail-retention rules remain authoritative for - this vendored tree. +1. Read `doc/knowledge/CONTRIBUTING.md`, then exactly the one linked contribution + topic that matches the change. The current contract is that entry page plus + `doc/knowledge/contributing/`, `doc/knowledge/SCHEMA.md` and the two validators; + `doc/archive/PLAN-knowledge-reorg.md` is archived history, not the directory + contract. Keep working owner-scoped and merge synonymous conclusions union-first. 2. Route each conclusion by both purpose and code owner. PR-learning and review- experience intake may produce only executable rules in the nearest owner's `rules.md`; model-specific rules stay with that model owner. Raw PR material is temporary input outside the repository and must be deleted after the owner-scoped rule batch passes validation. Never add PR case/history/result/ incident pages or collapse heterogeneous owners into one catch-all page. -3. Treat `knowledge/SCHEMA.md` as an additive metadata overlay only. Existing +3. Treat `doc/knowledge/SCHEMA.md` as an additive metadata overlay only. Existing `incidents/`, `history/`, and `results/` files are legacy material, not an allowed destination for new PR learning. Generated evaluation cases, hidden labels, predictions, judgments, and run reports belong under `eval/`, not in the product knowledge tree. 4. Never persist raw evidence pages or replay outputs. Update the nearest `_index.md` for every current rules page. -5. Run both validators after the complete owner-scoped batch: +5. Editing `knowledge/repos/vllm-omni/` also touches a machine contract the two + validators cannot see: `owner_documents` entry pages, in-body SHA pins and + upstream `sources:` are reconciled by `tools/audit_vllm_omni_release.py`, and + any PR touching that slice runs it in `enforce` mode + (`doc/knowledge/contributing/validation.md`). +6. Run both validators after the complete owner-scoped batch: ```powershell python knowledge/tools/check_knowledge_tree.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 729691a58..661fd7d98 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -17,7 +17,7 @@ | Git、PR、rebase | [general/git](knowledge/general/git/_index.md) | [Git](knowledge/repos/vllm-omni/git/_index.md) / [rebase](knowledge/repos/vllm-omni/rebase/_index.md) | | Benchmark | [general/benchmark](knowledge/general/benchmark/_index.md) | [vLLM-Omni benchmark](knowledge/repos/vllm-omni/benchmark/_index.md) | | 远端验证 | [general/remote](knowledge/general/remote/_index.md) | [vLLM-Omni remote](knowledge/repos/vllm-omni/remote/_index.md) | -| 新增或整理知识 | [knowledge/CONTRIBUTING](knowledge/CONTRIBUTING.md) | 先确定最近的 owner | +| 新增或整理知识 | [knowledge/CONTRIBUTING](doc/knowledge/CONTRIBUTING.md) | 先确定最近的 owner | vLLM-Omni 的代码 owner： @@ -31,7 +31,7 @@ vLLM-Omni 的代码 owner： | queue、token budget、prefix cache、调度 | [scheduler](knowledge/repos/vllm-omni/components/scheduler/rules.md) | | 明确的模型或 registry key | [models](knowledge/repos/vllm-omni/models/_index.md) | -完整知识入口仍在 [knowledge/README](knowledge/README.md)。只有 owner 不明确时才看 +完整知识入口仍在 [知识树地图](doc/knowledge/tree-map.md)。只有 owner 不明确时才看 [组件职责表](knowledge/repos/vllm-omni/components/_index.md)；不要默认打开事故记录、 history、results 或所有 `_index.md`。 @@ -49,12 +49,12 @@ src/infermatrix_copilot/ playbooks/ Strict/Autonomous 工作流定义 adapters/ 发布包中的仓库配置 knowledge/ 通用、仓库、组件和模型知识 -doc/SPEC/ 与 src 文件对应的约束和职责 +doc/architecture/SPEC/ 与 src 文件对应的约束和职责 test/ 离线测试 ``` -整体架构先看 [CODE_TOUR](doc/CODE_TOUR.md)，修改具体源码前再看 -[file-level SPEC](doc/SPEC/README.md) 中对应文件。 +整体架构先看 [CODE_TOUR](doc/architecture/CODE_TOUR.md)，修改具体源码前再看 +[file-level SPEC](doc/architecture/SPEC/README.md) 中对应文件。 ## 常见改动从哪里开始 @@ -65,7 +65,7 @@ test/ 离线测试 | 模型、Key、Base URL | `config.py`、`llm.py` | `test_llm_providers.py`、`test_tier_split.py` | | 工作流步骤 | `playbooks/*.yaml`、`engine/steps/` | 对应 step 测试和 playbook 加载测试 | | 新仓库支持 | `adapters//`、`knowledge/repos//` | adapter 测试和知识检查 | -| 更新 vLLM-Omni 知识 | `/imupdate` 维护流程 | [release maintenance](doc/VLLM_OMNI_RELEASE_MAINTENANCE.md) | +| 更新 vLLM-Omni 知识 | `/imupdate` 维护流程 | [release maintenance](doc/contributing/release-maintenance.md) | | 安装或发布包 | `scripts/install_mcp.py`、`pyproject.toml` | installer 测试和 wheel 内容检查 | ## 本地开发 @@ -117,9 +117,11 @@ required status check —— 这是仓库设置，任何一次提交都做不到 ## 进一步阅读 -- [设计](doc/DESIGN.md) -- [知识来源和同步边界](doc/KNOWLEDGE.md) -- [扩展知识库](doc/EXTENDING-KNOWLEDGE.md) -- [MCP 安装与接口](doc/MCP.md) -- [Autonomous workflow](docs/autonomous-workflow.md) +- [指南](doc/GUIDE.md) —— 操作者使用 + 维护者开发，外加 playbook / step / tool + 清单和性能对比（`CODE_TOUR` 的结构视角对照版） +- [设计](doc/architecture/DESIGN.md) +- [知识来源和同步边界](doc/architecture/KNOWLEDGE.md) +- [扩展知识库](doc/knowledge/writing.md) +- [MCP 安装与接口](doc/guide/mcp.md) +- [Autonomous workflow](doc/guide/autonomous-workflow.md) - [评测](eval/README.md) diff --git a/QUICKSTART.md b/QUICKSTART.md index 0044698a5..592c0f163 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -1,207 +1,214 @@ -# Quickstart — InferMatrixCopilot +# 快速上手 —— InferMatrixCopilot -A playbook-driven repo-maintenance copilot for vLLM-Omni. One natural-language -interface reviews PRs, debugs CI, rebases branches, answers/triages issues, and -profiles repos — with a hard safety model: **everything is dry-run by default**, -and natural language can never widen permissions. +面向 vLLM-Omni 的 playbook 驱动仓库维护 copilot。一个自然语言入口即可审 PR、 +调 CI、rebase 分支、回答/分流 issue、建立仓库 profile —— 并带一套硬安全模型： +**默认一切 dry-run**，且自然语言永远无法扩大权限。 -New to the code? Read [`doc/CODE_TOUR.md`](doc/CODE_TOUR.md) (data-flow -walkthrough) and [`doc/DESIGN.md`](doc/DESIGN.md) (the why). This page just gets -you running. +本页讲的是**独立 CLI**。如果你想在 Codex / Claude Code / Cursor **里面**用 +copilot，那是 Direct 模式，见 [`doc/guide/hosts/`](doc/guide/hosts/README.md)。 + +第一次接触代码？[`doc/GUIDE.md`](doc/GUIDE.md) 是唯一总览； +[`doc/architecture/CODE_TOUR.md`](doc/architecture/CODE_TOUR.md) 按数据流走一遍代码， +[`doc/architecture/DESIGN.md`](doc/architecture/DESIGN.md) 讲为什么这么设计。 --- -## 1. Install (one command) +## 1. 安装（一条命令） ```bash bash install.sh ``` -Creates/reuses a venv, installs the package editable, seeds `.env` from the -template (never overwrites), writes the repo-local `./infermatrix-copilot` wrapper (no -PATH mutation), and runs `doctor`. `bash install.sh --uninstall` removes only -what it created (never touches `.env`). +创建或复用 venv、以 editable 方式安装包、从模板生成 `.env`（**永不覆盖**已有的）、 +写出仓库内的 `./infermatrix-copilot` 包装器（不改 PATH），最后跑一次 `doctor`。 +`bash install.sh --uninstall` 只删除它自己创建的东西（永不动 `.env`）。 -> If it's already installed in your workspace venv, `./infermatrix-copilot` works as-is — -> skip to step 3 unless you changed `.env`. +> 如果工作区 venv 里已经装好了，`./infermatrix-copilot` 直接可用 —— 除非你改过 +> `.env`，否则跳到第 3 步。 -## 2. Configure `.env` (git-ignored — NEVER commit it) +## 2. 配置 `.env`（已被 git 忽略 —— **绝对不要提交**） -Minimum to be operational: +能跑起来的最小集： ```bash -ANTHROPIC_API_KEY=sk-... # or a DeepSeek /anthropic key -ANTHROPIC_BASE_URL= # set if using a compatible endpoint -AGENT_MODEL=claude-sonnet-5 # default reasoning model -REPO_PATHS={"vllm-omni": "/path/to/vllm-omni"} # JSON map: repo name -> path +ANTHROPIC_API_KEY=sk-... # 或一个 DeepSeek 的 /anthropic key +ANTHROPIC_BASE_URL= # 使用兼容端点时填写 +AGENT_MODEL=claude-sonnet-5 # 默认推理模型 +REPO_PATHS={"vllm-omni": "/path/to/vllm-omni"} # JSON 映射：仓库名 -> 路径 DEFAULT_REPO=vllm-omni ``` -Leave the write guards **off** until you deliberately want outward writes: +在你**确实想要**对外写入之前，两个写闸保持关闭： ```bash -ALLOW_PUSH=0 # 1 = allow git push (still --force-with-lease only, never to main) -ALLOW_POST=0 # 1 = allow posting PR comments / issue replies +ALLOW_PUSH=0 # 1 = 允许 git push（仍然只有 --force-with-lease，永不推 main） +ALLOW_POST=0 # 1 = 允许发布 PR 评论 / issue 回复 ``` -Optional: `REVIEWER_MODEL`, `INTENT_MODEL`, the dual-path tier backends -(`ECO_MODEL`/`PERFORMANCE_MODEL`, each optionally with its own -`*_BASE_URL`+`*_API_KEY` — set all three or none; an unconfigured performance -tier makes performance requests fail upfront rather than silently running the -eco model), `NOTIFY_EMAIL`+`RESEND_API_KEY`/SMTP (escalation email), -`LLM_MIXTURE` (MoA). See `.env.template` for all knobs. +可选项：`REVIEWER_MODEL`、`INTENT_MODEL`、双路径两档后端 +（`ECO_MODEL`/`PERFORMANCE_MODEL`，各自可带自己的 `*_BASE_URL`+`*_API_KEY` —— +三个要么都填要么都不填；未配置 performance 档时，performance 请求会**当场失败** +而不是悄悄跑 eco 模型）、`NOTIFY_EMAIL`+`RESEND_API_KEY`/SMTP（升级邮件）、 +`LLM_MIXTURE`（MoA）。全部开关见 `.env.template`。 -Every response's served model is verified against the request (fail-closed — -endpoints that silently substitute models, e.g. Claude names on DeepSeek's -`/anthropic`, are caught, not mislabeled). `doctor` checks tier/host -plausibility for free; `doctor --probe` makes one 1-token call per tier and -prints `requested → served`. +每次响应实际服务的模型都会与请求核对（fail-closed —— 那些悄悄替换模型的端点， +例如在 DeepSeek 的 `/anthropic` 上返回 Claude 名字，会被抓出来而不是被误标）。 +`doctor` 免费检查档位/主机的合理性；`doctor --probe` 每档发一次 1-token 调用并 +打印 `requested → served`。 -## 3. Preflight +## 3. 预检 ```bash -./infermatrix-copilot doctor # every ✗ prints the exact fix +./infermatrix-copilot doctor # 每个 ✗ 都会打印确切的修复命令 ``` -Checks: package installed, `ANTHROPIC_API_KEY` set (name only, value never -printed), `gh` installed **and authenticated** (`gh auth login` once), -`REPO_PATHS` exist, `.env` parses, MoA config valid if set. +检查项：包是否装好、`ANTHROPIC_API_KEY` 是否设置（只打印名字，**永不打印值**）、 +`gh` 是否安装**且已认证**（`gh auth login` 一次即可）、`REPO_PATHS` 是否存在、 +`.env` 能否解析、MoA 配置（如果设了）是否合法。 --- -## 4. First commands +## 4. 最初几条命令 -You never need to memorize task kinds or trigger phrases — plain English routes -itself; a full GitHub URL routes to the right repo/workflow. +你不需要背任务 kind 或触发短语 —— 平实的中文/英文会自己路由；完整的 GitHub URL +会路由到正确的仓库和工作流。 ```bash -# Review a PR (read-only; produces a review, does not post) +# 审一个 PR（只读；产出评审意见，不发布） ./infermatrix-copilot -p "review pr 5134" ./infermatrix-copilot -p "do a full-depth review of pr 5134" ./infermatrix-copilot -p "review https://github.com/vllm-project/vllm-omni/pull/5134" -# Debug failing CI on a PR (report-only = read-only triage) +# 调 PR 上失败的 CI（report only = 只读分诊） ./infermatrix-copilot -p "debug pr 5134, report only" -# Answer / triage issues +# 回答 / 分流 issue ./infermatrix-copilot -p "answer issue 4842, do not post" ./infermatrix-copilot -p "triage recent open issues" -# Rebase a PR onto its base +# 把 PR rebase 到它的 base 上 ./infermatrix-copilot -p "rebase pr 5134" -# Compound request -> ordered queue, target carries over +# 复合请求 -> 有序队列，目标自动延续 ./infermatrix-copilot -p "rebase pr 5134, then review it" ``` -Useful flags: +常用 flag： -| Flag | Effect | +| Flag | 作用 | |---|---| -| `--yes` | skip the `[y/N]` confirmation (headless/scripting) | -| `--plan-only` | resolve and print the plan, execute nothing | -| `--performance` | use the high-capability model tier for this run (default: eco) | -| `--resume` | re-enter the last run at its first incomplete step | -| `--playbook ` | run a specific playbook (incl. candidates) | -| `--report-only` | with `--playbook`: read-only variant | -| `--task-param k=v` | with `--playbook`: pass a param (repeatable) | -| `--no-chat` | plain command REPL instead of chat | +| `--yes` | 跳过 `[y/N]` 确认（headless / 脚本） | +| `--plan-only` | 只解析并打印计划，不执行任何步骤 | +| `--performance` | 本次 run 使用高能力模型档（默认 eco） | +| `--resume` | 从上一次 run 的首个未完成 step 重新进入 | +| `--playbook ` | 运行指定 playbook（含 candidate） | +| `--report-only` | 配合 `--playbook`：只读变体 | +| `--task-param k=v` | 配合 `--playbook`：传参（可重复） | +| `--no-chat` | 用朴素命令 REPL 代替对话模式 | -In `-p --yes` mode an ambiguous request exits **nonzero** with a clarifying -question instead of guessing — safe for eval harnesses and CI. +在 `-p --yes` 模式下，含糊的请求会带着**澄清问题以非零码退出**，而不是猜测执行 +—— 这对评测 harness 和 CI 是安全的。 -## 5. Chat mode (default, no `-p`) +## 5. 对话模式（默认，不加 `-p`） ```bash ./infermatrix-copilot ``` -A Claude-Code-style conversation: ask about the repo or past runs, and run work -through tools — same TaskSpec, planner, and `[y/N]` gates as the flag CLI (chat -can never widen permissions). Built-ins: `/status`, `/logs [n]`, `/playbooks`, -`/resume`, `/quit`. Repo reads are jailed to configured repos; `.env*` refused. +Claude-Code 风格的对话：问仓库的事、问过往 run，也可以直接通过工具执行工作 —— +走的是与 flag CLI **完全相同**的 TaskSpec、planner 和 `[y/N]` 确认门（对话模式 +永远无法扩大权限）。内置命令：`/status`、`/logs [n]`、`/playbooks`、`/resume`、 +`/quit`。仓库读取被囚禁在已配置的仓库内；`.env*` 一律拒绝。 --- -## 6. Safety model (read this before you post or push) +## 6. 安全模型（发布或推送之前请先读这一节） -Outward effects are **double-gated** — both must be true: +对外生效的动作是**双闸**，两者必须同时成立： -- **Post a PR comment / issue reply** → the request must carry explicit `post` - intent (e.g. `"review pr 5134 and post it"`) **AND** `ALLOW_POST=1`. -- **git push** → an allowing push policy **AND** `ALLOW_PUSH=1`. Force is - `--force-with-lease` only; **`main` is never pushed to**, policy or not. +- **发 PR 评论 / issue 回复** → 请求必须带明确的 `post` 意图 + （例如 `"review pr 5134 and post it"`）**并且** `ALLOW_POST=1`。 +- **git push** → 推送策略允许**并且** `ALLOW_PUSH=1`。force 只有 + `--force-with-lease`；**`main` 永远不会被推送**，无论策略怎么写。 -Anything short of both is a **dry run** that shows exactly what it would do. -Adapter zero (`adapters/vllm_omni/manifest.yaml`) declares `push.allowed: false` -— vLLM-Omni changes ship via PR, never a direct push. Blocked runs write -`ESCALATION.md` and exit 3 (notify, never guess). +任何一个条件不满足，都是一次 **dry run**，它会准确显示"本来会做什么"。 +Adapter zero（`adapters/vllm_omni/manifest.yaml`）声明了 `push.allowed: false` +—— vLLM-Omni 的改动一律走 PR，绝不直接推送。被阻塞的 run 会写出 +`ESCALATION.md` 并以退出码 3 结束（通知，绝不猜测）。 -## 7. Where output goes +## 7. 产物在哪 ``` ~/.infermatrix-copilot/runs/run--/ - RUN_REPORT.md the deliverable (review / answer / debug summary) - DIAGNOSTICS.md per-step diagnostics - run_trace.jsonl append-only fact log - progress.json step checkpoints (what --resume reads) - metrics.json CATQ run metrics - ESCALATION.md only when blocked -~/.infermatrix-copilot/worktrees/-pr/ PR-time checkout for reviews -~/.infermatrix-copilot/sessions/ chat transcripts + RUN_REPORT.md 交付物（评审 / 回答 / 调试摘要） + DIAGNOSTICS.md 逐 step 诊断 + run_trace.jsonl 仅追加的事实日志 + progress.json step 检查点（--resume 读的就是它） + metrics.json CATQ run 指标 + ESCALATION.md 仅在被阻塞时出现 +~/.infermatrix-copilot/worktrees/-pr/ 评审用的 PR-time checkout +~/.infermatrix-copilot/sessions/ 对话记录 ``` -## 8. Playbooks +## 8. Playbook ```bash -./infermatrix-copilot -p "…" # planner picks the vetted playbook -./infermatrix-copilot --playbook pr-review --plan-only # run one by name +./infermatrix-copilot -p "…" # 由 planner 挑选已审核的 playbook +./infermatrix-copilot --playbook pr-review --plan-only # 按名字点名运行 ``` -Registered: `pr-review`, `pr-debug`, `pr-rebase`, `issue-assist`, -`issue-triage`, `repo-rebase` (**locked** — delegates to the proven 5-phase -nightly orchestrator), `repo-rebase-native` (candidate), `repo-profile`, -`profile-consolidate` (candidate; Stage-4 maintenance). Candidates are invisible -to the planner — run them explicitly with `--playbook`. +已注册：`pr-review`、`pr-debug`、`pr-rebase`、`issue-assist`、`issue-triage`、 +`repo-rebase`（**locked** —— 委托给已验证的 5 阶段夜跑编排器）、 +`repo-rebase-native`（candidate）、`repo-profile`、`profile-consolidate` +（candidate；Stage-4 维护）。candidate 对 planner **不可见**，只能用 +`--playbook` 显式运行。 -## 9. Onboard a new repo (profile) +## 9. 接入一个新仓库（profile） ```bash -./infermatrix-copilot -p "profile the repo" --yes # fingerprint -> draft profile -./infermatrix-copilot --playbook profile-consolidate --yes # Stage-4 dedupe/refresh +./infermatrix-copilot -p "profile the repo" --yes # 指纹 -> draft profile +./infermatrix-copilot --playbook profile-consolidate --yes # Stage-4 去重/刷新 ``` -Repo knowledge lives at the edge in `adapters//` (human-gated -`manifest.yaml` + agent-established, evidence-gated `profile/`) — never in -`src/`. `PROFILE_BRIEFING_ENABLED=0` runs the {no-profile} eval ablation arm. +仓库知识住在边缘的 `adapters//`（人工门禁的 `manifest.yaml` + agent 建立、 +证据门禁的 `profile/`）—— **永远不在 `src/` 里**。`PROFILE_BRIEFING_ENABLED=0` +用于跑 {无 profile} 的评测对照臂。 -## 10. Advanced: MCP server (Claude Code / Codex / Cursor) +## 10. 进阶：MCP 服务端（Claude Code / Codex / Cursor） ```bash -pip install -e '.[mcp]' # optional extra, kept out of the base install +pip install -e '.[mcp]' # 可选 extra，不进基础安装 ``` -Exposes read-only tools — `start_review` / `start_issue_answer` / -`start_issue_triage` (return a `run_id`), `get_status` / `get_result` (poll), -`list_playbooks`, `doc_read` / `doc_search`. Reviews take 5–12 min, so the API -is start-then-poll. Only the read-only kinds are reachable and `post` is -force-disabled at both the server boundary and the run subprocess. Setup: -[`doc/MCP.md`](doc/MCP.md). +暴露只读工具 —— `start_review` / `start_issue_answer` / `start_issue_triage` +（返回 `run_id`）、`get_status` / `get_result`（轮询）、`list_playbooks`、 +`doc_read` / `doc_search`。评审要跑 5–12 分钟，所以接口是"先启动再轮询"。 +只有只读 kind 可达，且 `post` 在**服务端边界和 run 子进程两处**都被强制关闭。 +配置方法见 [`doc/guide/mcp.md`](doc/guide/mcp.md)。 --- -## Troubleshooting - -- **"capability gap … run repo_profile"** — the task needs repo knowledge the - profile doesn't provide; profile the repo first (step 9). -- **Clarifying question in `-p --yes`** — the request was ambiguous; it exits - nonzero by design. Add the PR/issue number or the `post`/`report only` intent. -- **Review/answer looks blocked but has content** — check `RUN_REPORT.md`; a - substantive draft is salvaged and delivered even when a step escalates. -- **Anything red in `doctor`** — the message *is* the fix. `gh` auth is the most - common one: `gh auth login`. - -Deeper reading: [`doc/DESIGN.md`](doc/DESIGN.md) · -[`doc/CODE_TOUR.md`](doc/CODE_TOUR.md) · -[`doc/IMPLEMENTATION_STATUS.md`](doc/IMPLEMENTATION_STATUS.md) · -[`doc/SPEC/`](doc/SPEC/README.md) · [`doc/MCP.md`](doc/MCP.md) +## 排查 + +- **"capability gap … run repo_profile"** —— 任务需要 profile 尚未提供的仓库知识； + 先执行第 9 步建立 profile。 +- **`-p --yes` 下抛出澄清问题** —— 请求含糊，按设计以非零码退出。补上 PR/issue + 编号，或补上 `post` / `report only` 意图。 +- **评审/回答看起来被阻塞但有内容** —— 打开 `RUN_REPORT.md`；即使某个 step 升级， + 实质性草稿也会被抢救并交付。 +- **`doctor` 里任何一条红色** —— 那条消息**本身就是修复方法**。最常见的是 `gh` + 认证：`gh auth login`。 + +深入阅读 —— 从文档地图 [`doc/README.md`](doc/README.md) 开始： + +- [`doc/GUIDE.md`](doc/GUIDE.md) —— 唯一总览（功能、使用、开发、playbook、step、 + tool、实测性能） +- [`doc/guide/backends.md`](doc/guide/backends.md) —— 怎么选和配 Strict 后端 +- [`doc/architecture/CODE_TOUR.md`](doc/architecture/CODE_TOUR.md) —— 按数据流讲代码 · + [`doc/architecture/DESIGN.md`](doc/architecture/DESIGN.md) —— 为什么 +- [`doc/architecture/SPEC/`](doc/architecture/SPEC/README.md) —— 逐文件契约： + 改一个文件之前先读它对应的那页 + +注意 `IMPLEMENTATION_STATUS.md` 已移到 +[`doc/archive/`](doc/archive/IMPLEMENTATION_STATUS.md) 并冻结 —— 它描述的是 7 月的 +构建，不是现状。现状以 `doc/GUIDE.md` 为准。 diff --git a/README.md b/README.md index 42b482259..b47b0e12b 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ release_baseline.yaml 的 audited_sha `imupdate` 不会根据一次 diff 自动编造 owner 规则，也不会自动 commit、push 或开 PR。 详细的审计项和底层命令见 -[`doc/VLLM_OMNI_RELEASE_MAINTENANCE.md`](doc/VLLM_OMNI_RELEASE_MAINTENANCE.md)。 +[`doc/contributing/release-maintenance.md`](doc/contributing/release-maintenance.md)。 ## Skill 命令和 MCP 接口不是一回事 @@ -202,7 +202,7 @@ release_baseline.yaml 的 audited_sha - `get_review_result(run_id, offset=0)`：轮询 Strict 结果；报告较长时按 `next_offset` 继续读取。 - `update_knowledge(repo="vllm-omni")`：只返回知识贡献入口 - `knowledge/CONTRIBUTING.md`。它不是 `imupdate` 的发版审计器。 + `doc/knowledge/CONTRIBUTING.md`。它不是 `imupdate` 的发版审计器。 - `doc_search(query, repo="vllm-omni", limit=20)`：搜索模型或组件知识。 - `doc_read(path, repo="vllm-omni", offset=0)`：读取搜索到的知识页面。 @@ -233,14 +233,25 @@ Strict 会返回 `run_id`，Agent 再用 `get_review_status` 查看进度，并 Autonomous workflow 不是 `imreview` 的第三种模式，而是同一工作流引擎的独立 CLI/MCP 入口。默认安装器不会注册它；只有需要独立执行 issue、CI、rebase 等任务时 才需要阅读 -[`docs/autonomous-workflow.md`](docs/autonomous-workflow.md)。 +[`doc/guide/autonomous-workflow.md`](doc/guide/autonomous-workflow.md)。 -## 维护者文档 +## 更多文档 -- [开发者入口和文档地图](DEVELOPMENT.md) -- [安装和通用 MCP 配置](doc/MCP.md) -- [vLLM-Omni 发版漂移审计](doc/VLLM_OMNI_RELEASE_MAINTENANCE.md) -- [知识库贡献规范](knowledge/CONTRIBUTING.md) -- [知识维护示例](docs/knowledge-maintainer.zh-CN.md) -- [项目设计与实现](doc/) -- [评测说明](eval/README.md) +**[`doc/README.md`](doc/README.md) 是文档地图**——一行一条，按你要做的事挑一篇。 +下面只列最常用的四条路： + +| 你要做什么 | 看这篇 | +|---|---| +| 完整了解它是什么、能做什么 | [指南](doc/GUIDE.md)（概览 / 功能 / 使用 / 开发 / playbook / step / tool / 性能） | +| 在 Codex / Claude Code / Cursor 里装上用 | [宿主端](doc/guide/hosts/README.md) | +| 让 Strict 跑在订阅或别的模型上 | [后端](doc/guide/backends.md) | +| 维护这个仓库本身 | [开发者入口](DEVELOPMENT.md) | + +> **宿主 ≠ 后端**：`claude-code` / `codex` / `cursor` 这三个名字两边都出现， +> 方向相反。宿主提供模型给你用（Direct），后端是 copilot 拿去用的模型（Strict）。 + +其余：[MCP 接口](doc/guide/mcp.md) · +[发版漂移审计](doc/contributing/release-maintenance.md) · +[知识库贡献规范](doc/knowledge/CONTRIBUTING.md) · +[知识维护示例](doc/knowledge/maintainer-walkthrough.md) · +[评测结论](doc/evaluation/README.md) diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index ee417d5c9..b7395720d 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -25,8 +25,8 @@ upstream: # Repo-specific slice of the SHARED, human-curated knowledge base (the vendored # community docs at the repo-root `knowledge/` = settings.knowledge_dir; # cross-repo general/ knowledge is shared, NOT nested here). Replaces the -# retired AI profile/. Provenance + the author's copy authorization are in -# doc/KNOWLEDGE.md. Paths below are relative to settings.knowledge_dir. +# retired AI profile/. Layout and consumption rules are in +# doc/architecture/KNOWLEDGE.md. Paths below are relative to settings.knowledge_dir. knowledge: source: zuiho-kai/claude-workflow-starter repo_subdir: repos/vllm-omni # this repo's slice within the shared base @@ -38,6 +38,11 @@ knowledge: # on-demand through doc_read/doc_search. briefing_docs_extra: - repos/vllm-omni/review/guides/maintainer-pattern-routing.md + # repo-specific review checklist injected (first 4k chars) into the Strict + # reviewer's system prompt — triggered checks distilled from maintainer + # review patterns; the page lives in the shared knowledge tree so Direct + # and knowledge maintenance see the same source + review_checklist: repos/vllm-omni/review/guides/strict-review-checklist.md modules: worker_runner: @@ -61,11 +66,54 @@ modules: model_config: local_paths: [vllm_omni/config/] wave: 1 + # risk high: platform code sets process-wide defaults (op priorities, + # capability gates) inherited by EVERY model on that platform — a 3-file + # 61-line default flip here is a repo-wide behavior change (PR 4964/5009 + # class), so small diffs must still get the full review ensemble platform: local_paths: [vllm_omni/platforms/] wave: 1 + risk: high + # shared diffusion pipeline: denoise loop, attention backends, DiT/VAE — + # multi-model shared code where one-model-validated changes have cross-model + # blast radius (kernel-provider and hub-loading changes live here) + diffusion: + local_paths: [vllm_omni/diffusion/] + wave: 1 + risk: high + # deploy YAMLs are the serving registry: defaults and per-platform blocks + # (cudagraph modes, capacity) that decide runtime behavior for CLI and yaml + # launches alike. NOT risk-tagged: a 2-line deploy deletion measured as a + # precision WIN at light depth (pr4970), so tiny deploy diffs keep the + # cheap tier; deploy changes bundled with code still escalate via the + # code-side signals + deploy: + local_paths: [vllm_omni/deploy/] + wave: 1 + distributed: + local_paths: [vllm_omni/distributed/] + wave: 1 + risk: high + engine: + local_paths: [vllm_omni/engine/] + wave: 1 + attention: + local_paths: [vllm_omni/attention/] + wave: 1 + quantization: + local_paths: [vllm_omni/quantization/] + wave: 1 + sampling: + local_paths: [vllm_omni/sample/] + wave: 1 + tokenizers: + local_paths: [vllm_omni/tokenizers/] + wave: 1 + metrics: + local_paths: [vllm_omni/metrics/] + wave: 2 benchmarks: - local_paths: [benchmarks/] + local_paths: [benchmarks/, vllm_omni/benchmarks/] wave: 2 review_routes: diff --git a/adapters/vllm_omni/release_baseline.yaml b/adapters/vllm_omni/release_baseline.yaml index 01267ee32..d7e0dc30f 100644 --- a/adapters/vllm_omni/release_baseline.yaml +++ b/adapters/vllm_omni/release_baseline.yaml @@ -134,7 +134,7 @@ ignored_paths: reason: package dependencies pin_documents: -- doc/KNOWLEDGE.md +- doc/architecture/KNOWLEDGE.md - knowledge/repos/vllm-omni/models/catalog.md - knowledge/repos/vllm-omni/components/configuration/_index.md - knowledge/repos/vllm-omni/components/diffusion/_index.md diff --git a/adapters/vllm_omni/skills/_candidates.json b/adapters/vllm_omni/skills/_candidates.json index 9e26dfeeb..792ba1d42 100644 --- a/adapters/vllm_omni/skills/_candidates.json +++ b/adapters/vllm_omni/skills/_candidates.json @@ -1 +1,30 @@ -{} \ No newline at end of file +{ + "behavior-lens-device-rng-review": { + "name": "behavior-lens-device-rng-review", + "description": "Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist", + "body": "When a diff changes `device.type == \"cuda\"` to `device.type != \"cpu\"` (or similar) in shared model code, treat the PR's stated allowlist as a contract. Grep the platform plugin registry and recipes to enumerate every device type that reaches the path; if the PR validates only a subset, the finding is \"implement the explicit allowlist or validate the rest\". Also check whether a device module is captured once with no argument (`torch.get_device_module()`) instead of bound to the actual tensor device.", + "modules": [], + "proposed_at": 1786614914.8996305 + }, + "verification-lens-pytestmark-deselection": { + "name": "verification-lens-pytestmark-deselection", + "description": "PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers.", + "body": "When reviewing a PR that adds test files, grep each added file for `pytestmark`. In vllm-omni, CI selects tests with `-m 'core_model and cpu'` (`.buildkite/cuda/test-ready.yml:13/23/28`, `.buildkite/amd/test-amd-ready.yml:19`) and the XPU lane with `-m 'core_model and xpu and B60'`. There is no `pytest_collection_modifyitems` hook that auto-adds `core_model`, so a new test file without `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` is silently deselected (green CI, zero execution). Test integrity outranks test existence: report it as major and ask for the module-level marker matching every sibling file in the directory.", + "modules": [], + "proposed_at": 1786692376.3324893 + }, + "vllm-omni-docs-image-reference-convention": { + "name": "vllm-omni-docs-image-reference-convention", + "description": "When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs.", + "body": "# vLLM-Omni docs figure-reference convention\n\n- docs/design/feature/*.md conventionally embeds figures with absolute\n `https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/...`\n URLs (see docs/design/feature/async_chunk.md), which render in both the\n MkDocs site and GitHub's rendered PR/file preview.\n- A repo-relative `../../../source/...` from `docs/design/feature/.md`\n resolves to `/source/...` on GitHub (broken: assets live under\n `docs/source/...`) while resolving correctly in the MkDocs site only because\n MkDocs default `use_directory_urls: true` adds the page's directory level.\n- Review action: for any new `` in a docs diff, check the sibling\n convention with `grep -n 'src=' docs/design/feature` and flag inconsistent or\n GitHub-broken relative paths.", + "modules": [], + "proposed_at": 1786716286.719784 + }, + "vllm-omni-async-omni-output-review-contract": { + "name": "vllm-omni-async-omni-output-review-contract", + "description": "Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files.", + "body": "# Async Omni output materialization review contract\n\nWhen a PR documents or changes async Omni output materialization, verify these\nclaims against the PR-time tree rather than trusting the prose:\n\n- Guard: `GPUARModelRunner._should_use_async_omni_output`\n (vllm_omni/worker/gpu_ar_model_runner.py) requires, in order:\n `use_async_scheduling`, `omni_prefix_cache is None`, `speculative_config is\n None`, `model_config.async_chunk`, `not enable_return_routed_experts`,\n model `use_async_omni_output`, and `not has_postprocess or\n eager_omni_postprocess_before_async_output`.\n- Opt-ins: Qwen3-Omni thinker/talker in qwen3_omni.py, Qwen3-TTS talker in\n qwen3_tts_talker.py (talkers set `eager_omni_postprocess_before_async_output`\n and `omni_pooler_payload_include_hidden = False`); moss_tts opts out.\n- Deploy: qwen3_omni_moe.yaml and qwen3_tts.yaml ship `async_chunk: true` and\n prefix-cache off on the AR stages; `async_scheduling` defaults True.\n- NPU: NPUARModelRunner builds `OmniModelRunnerOutput` synchronously and only\n then wraps in `AsyncGPUModelRunnerOutput`.\n- Full-payload accumulation branch is unreachable under async_chunk because\n `should_accumulate_full_payload_output` returns False when async_chunk is on.", + "modules": [], + "proposed_at": 1786716300.3188832 + } +} \ No newline at end of file diff --git a/doc/DESIGN.md b/doc/DESIGN.md deleted file mode 100644 index f270fe776..000000000 --- a/doc/DESIGN.md +++ /dev/null @@ -1,523 +0,0 @@ -# Design - -This repo implements the **InferMatrixCopilot** design — originally specified as -"vLLM-Omni Copilot" in `vllm-omni-rebase-agent/docs/copilot/copilot_design`, plus -its `implementation/` milestone plan. This document maps the design onto the code -here; the rationale lives in those source docs. - -Part I below describes the v1 architecture as built. **Part II (Design v2, -2026-07-10)** extends it with three goals: fix the known v1 problems, make the -copilot **repo-invariant** (stable task performance across repositories), and -give it the ability to **establish a repo profile** (code format, skills, -constraints, module analysis, and supporting artifacts) for any repository it -is pointed at. - ---- - -# Part I — v1 architecture (as built) - -## Architecture - -``` - user NL ──► intent.py ──► TaskSpec (task_spec.py) §3.Y CLI layer - │ tier fixed by kind (L0/L1/L2), confirm gates - ▼ - engine/planner.py §3.2 reuse>adapt>generate - reuse ─────────┼───────── generate (read-only kinds only) - ▼ - playbooks/store.py registry §3.2 candidate/active/locked - ▼ - engine/executor.py §3.X engine substrate - per-step checkpoint/resume · retries · typed failures · escalation - ▼ - engine/steps/* (Step library) §3.X.1 Steps - tools.py + scopes.py (ToolScope/PathScope choke point) §3.3(2) - review/ (diff summary → triggers → read-only reviewer) §3.3(3) - memory/ (RunTrace, DebugMemory FTS5, gated skills) §3.3(4) - adapters/ (repo knowledge, draft bootstrap) adapter layer - push.py (PushPolicy + guard_push) push authorization - notify.py (ESCALATION.md + email, exit 3) goal #4 -``` - -## Key decisions - -1. **Wrap, don't rewrite the rebase.** The locked `repo-rebase` playbook - delegates to the existing 5-phase orchestrator via `rebase.run_external` - (`REBASE_ORCHESTRATOR_CMD`). Zero regression: this repo adds orchestration - *around* the proven pipeline, it does not fork it. Native step-level - decomposition of the rebase (waves, module agents) migrates later per the - milestone plan. -2. **Own executor instead of LangGraph.** The design (§3.X.5) is explicit that - LangGraph is a possible backend, not the domain abstraction. The executor - here is ~150 lines, async, with per-step checkpoint/resume — dependency-free - and fully testable. A LangGraph backend can be swapped in behind the same - `Playbook`/`StepSpec` contracts if/when this merges with the parent agent. -3. **Tier is derived, never parsed.** `TaskSpec` has no tier field an LLM or - user could set; `tier` is a property of the task kind. Natural language can - therefore never widen permissions (§3.Y.4). -4. **Generation is structurally read-only.** The planner's generate path - raises if any composed step has `risk` ∈ {write_workspace, push}; write-capable - kinds without a vetted playbook escalate instead of improvising. -5. **Fail-closed reviews.** `run_patch_review` returns `unavailable` without a - reviewer LLM; unparseable reviewer output degrades to `revise`. Push gates - treat anything but `lgtm` as not-passing. -6. **Untrusted data is fenced.** GitHub-fetched text enters agent prompts inside - `` markers with an explicit "not instructions" preamble, and - only terminal input reaches the intent parser (channel separation, §3.Y.4). - Intent classification is **LLM-only** (no deterministic keyword parser); a - low-confidence or suspicious command clarifies rather than runs, and no - configured LLM → clarify (never a guessed execution). Injection resistance - comes from channel separation, not from the parser. - -## Module map - -| Path | Design concept | -|---|---| -| `src/infermatrix_copilot/task_spec.py`, `intent.py`, `cli/` | §3.Y conversational CLI, TaskSpec, clarify-not-guess (intent classification is LLM-only) | -| `src/infermatrix_copilot/engine/step.py`, `registry.py` | §3.X Step abstraction, vetted step library | -| `src/infermatrix_copilot/engine/executor.py` | engine substrate: checkpoint/resume, typed failure routing | -| `src/infermatrix_copilot/engine/planner.py` | §3.2 reuse > adapt > generate, L0/L1/L2 | -| `src/infermatrix_copilot/engine/steps/` | vetted step library (guard, rebase, review gate, push, gh reads, agent steps) — self-registering | -| `src/infermatrix_copilot/engine/agent_runtime/` | unified Agent-Step runtime (修正方案; package: dispatch/knowledge/utils substrate + runner/ensemble entries): AgentDispatchContext, evidence pack (cap+archive), skill/memory retrieval + gated candidates, enforced scopes, structured output contract, full RunTrace; `run_agent_step_ensemble` — perspective-diverse fan-out + verify-and-merge for run-to-run robustness (eval-informed; any list-output agent step) | -| `src/infermatrix_copilot/playbooks/store.py`, `playbooks/*.yaml` | Playbook registry: versioned, provenance, candidate/active/locked/retired | -| `src/infermatrix_copilot/scopes.py`, `tools.py`, `agent_loop.py` | 框架层改进 (2): ToolScope/PathScope at one choke point | -| `src/infermatrix_copilot/review/` | 框架层改进 (3): diff summary → trigger rules → read-only patch reviewer | -| `src/infermatrix_copilot/run_trace.py`, `memory/` | 框架层改进 (4): RunTrace / DebugMemory / gated skills | -| `src/infermatrix_copilot/adapters/` | RepoAdapter: adapter zero, registry, deterministic bootstrap → draft | -| `src/infermatrix_copilot/push.py` | Push authorization: `PushPolicy` + unified `guard_push` (no separate Target layer) | -| `src/infermatrix_copilot/notify.py` | goal #4: escalation channel, blocked exit code 3 | - -## Data & artifacts per run - -`~/.infermatrix-copilot/runs/run-/`: `run_trace.jsonl` (facts), `progress.json` -(step checkpoints — resume re-enters the first incomplete step), -`RUN_REPORT.md`, `ESCALATION.md` (only when blocked). - ---- - -# Part II — Design v2: repo-invariant copilot (2026-07-10) - -v1 already states the multi-repo success criterion ("onboarding a second repo -must not touch the core engine — only a new adapter"). v2 strengthens it from -*runs at all* to **runs with stable, measured performance**, and closes the -gaps that currently make that impossible. Three workstreams: - -1. **§V2.1 Fix the existing problems** — correctness bugs, repo-knowledge - leaks into the core, and v1 boundaries left open. -2. **§V2.2 Repo invariance** — an enforceable contract that the engine is - repo-neutral and that quality is measured per repo, not assumed. -3. **§V2.3 Repo profile** — the copilot can establish, maintain, and consume - a full profile of any repo: code format, skills, constraints, module - analysis, and the other artifacts tasks need. - -## V2.0 Prior art and evidence (survey, 2026-07-10) - -What the field has learned about repo context for agents — each point below -changed a v2 decision. Sources at the end of the section. - -1. **Auto-generated repo context files HURT.** The ETH Zurich study - ([arXiv:2602.11988](https://arxiv.org/html/2602.11988v1), 438 tasks, - 4 agents) found LLM-generated AGENTS.md-style files *reduce* resolve rates - (−0.5 to −2%) while adding 20–23% inference cost and 2–4 extra steps; - agents dutifully follow the extra instructions (more greps, more tests) - without accuracy gains. Codebase/directory overviews — present in ~100% of - generated files — showed **zero** effect on how fast agents reach the - relevant code. Human-written files help only mildly (+4%) and only when - they carry *non-redundant, repo-specific tooling facts* (which package - manager, exact build/test commands); when the same info already exists in - README/docs, the context file is pure cost. GitHub's own guidance for - `copilot-instructions.md` agrees: start with 10–20 short imperative - directives, iterate against observed behavior. - → **v2 consequence**: the profile is NOT a prompt dump. Most of it must be - machine-consumed or retrieved-on-demand; the always-on prompt slice is - minimal, curated, and must prove itself on the eval (§V2.3.4, §V2.3.5). -2. **The useful "map" is computed per task, not written once.** Aider's repo - map (tree-sitter symbol graph + personalized PageRank, sized to a token - budget per conversation) and the RepoGraph line - ([arXiv:2410.14684](https://arxiv.org/html/2410.14684v1), +32.8% relative - on SWE-bench; CodexGraph [arXiv:2408.03910](https://arxiv.org/abs/2408.03910)) - both show repo *structure* pays off as a queryable graph ranked against - the current goal — not as a static overview document. - → **v2 consequence**: `modules.yaml` is a machine artifact (feeds - fan-out, patch triggers, wave scheduling) plus an on-demand `repo_map` - tool for agent steps; it is never injected wholesale into prompts. -3. **Trigger-gated knowledge beats always-on knowledge.** OpenHands - microagents/skills (keyword-triggered .md files), Devin's Knowledge Base - (every item = content + a *trigger description*, recalled semantically - when relevant), and Agent Workflow Memory - ([arXiv:2409.07429](https://arxiv.org/html/2409.07429v1), workflows - induced from successful trajectories, +24.6–51.1% relative on web-agent - benchmarks and robust cross-domain) converge on the same shape: small - knowledge units, each carrying its own recall condition, injected only - when triggered. Our skills store already has this shape — v2 extends it - with per-repo namespacing and induction from successful RunTraces. -4. **Per-repo performance variance is the norm, not the exception.** - SWE-bench-family analyses show resolve rates swinging from <10% to >50% - across repositories for the same model, systematic language effects, and - sizable drops on private/unseen codebases (e.g. Opus 4.1 22.7%→17.8% on - SWE-bench Pro's private set). Repo properties — architectural complexity, - adapter/hook systems, API surface clarity — predict difficulty. - → **v2 consequence**: invariance must be *measured* per repo (§V2.2.5); - an agent that is only ever evaluated on vllm-omni has unknown performance - everywhere else, and profile quality is a first-order lever on the gap. -5. **Curated-top-layer-over-evidence is the converging memory architecture.** - The 2026 agent-memory literature and shipping systems (Zep/Graphiti, Mem0, - Letta sleep-time agents; continuous LLM rewriting *corrupts* memory — - arXiv:2605.12978) converge on: an immutable evidence store under a small, - curated, human-readable profile; typed patch operations as the only write - surface; scheduled (not per-interaction) consolidation; provenance and - stability scores gating rewrites. Our own personal-agent implements - exactly this and it works in production (§V2.3.2 borrows its skeleton). - -Sources: [ETH AGENTS.md study](https://arxiv.org/html/2602.11988v1) · -[Aider repo map](https://aider.chat/docs/repomap.html) · -[RepoGraph](https://arxiv.org/html/2410.14684v1) · -[CodexGraph](https://arxiv.org/abs/2408.03910) · -[OpenHands repo skills](https://docs.openhands.dev/overview/skills/repo) · -[Devin Knowledge/DeepWiki](https://docs.devin.ai/work-with-devin/deepwiki) · -[Agent Workflow Memory](https://arxiv.org/html/2409.07429v1) · -[SWE-bench Pro](https://scale.com/blog/swe-bench-pro) · -[Cline memory bank](https://docs.cline.bot/best-practices/memory-bank) · -[Copilot custom instructions](https://docs.github.com/copilot/customizing-copilot/adding-custom-instructions-for-github-copilot) - -## V2.1 Existing problems and fixes - -### (a) Correctness bugs (P0 — fix before anything else) - -| # | Problem | Evidence | Fix | -|---|---|---|---| -| 1 | **Resume loses in-memory state.** Steps hand results to later steps by writing `ctx.state` directly (`pr.fetch_diff` → `diff_text`, `issue.fetch` → `issue_text`, `pr.gate_check` → `gate_report`, `pr.checkout_branch` → `push_policy`/branch refs, `agent.review_diff` → `review_text`), but the executor's resume path restores only `outputs.state_updates`. `--resume` past a completed fetch re-enters `agent.review_diff` with no `diff_text` → spurious BLOCKED; resuming pr-rebase at the push step sees the default (deny-all) `PushPolicy` → spurious FORBIDDEN. | `executor.py` resume branch vs. `steps/pr/`/`steps/review/` state writes | Contract: **every state key a later step consumes must be published via `outputs.state_updates`** (JSON-simple; `PushPolicy` serialized). Enforced by a resume-integrity test per playbook: run to step N, drop the process, resume, assert no BLOCKED-on-missing-state at any N. | -| 2 | **`foreach` fan-out drops `state_updates`.** `_merge` re-keys item outputs by index, so `state_updates` published by fan-out items never reach `state.update(...)`. | `executor.py::_merge` | Merge `state_updates` across items explicitly (last-writer-wins per key is acceptable; conflicts traced). Covered by the same resume-integrity test. | -| 3 | **`when:` silently evaluates against TaskSpec only.** A condition on a computed state key (e.g. `has_conflicts`) evaluates false with no error. | `executor.py::_eval_when` | Evaluate against `state` with TaskSpec fallback; unknown key → planning-time error, not a silent skip. | -| 4 | **Per-repo knowledge is designed but not wired.** `RepoAdapter.skills_dir` and `RepoAdapter.debug_memory_db` exist, but `agent_runtime` reads the *global* `settings.skills_dir` / `settings.memory_db` — every repo shares one skill store and one debug memory. | `adapters/base.py` vs `engine/agent_runtime/knowledge.py::_retrieve_skills/_retrieve_memories` | Resolve knowledge stores through the active adapter/profile first, global shared pool second (§V2.2.6). | -| 5 | **High-risk-module list is hardcoded vLLM-Omni names in core settings**, so the patch-review `high_risk_modules` trigger silently never fires on any other repo. | `config.py::high_risk_modules` | Move to the profile (`modules.yaml` risk tiers, §V2.3); the setting becomes a fallback only. | - -### (b) Repo-knowledge leaks in the core (violate "knowledge at the edge"; block invariance) - -These are not bugs on vllm-omni, but each one is a place where a second repo -gets silently degraded behavior. All are resolved by **profile injection** -(§V2.3) — the core keeps only repo-neutral scaffolding: - -- `_REVIEW_SYSTEM` and `_REVIEW_LENSES` hardcode "vLLM-Omni" and a - Python/ML-repo review checklist inside `engine/steps/review/` (the - prompt/checklist data now lives in `engine/steps/review/prompts.py` after - the module was split into a package). → Split into a repo-neutral core - prompt + profile-supplied domain sections (`review.md`: domain checklist, - lens extensions, severity norms). -- `_sweep_targets` assumes Python surface syntax (`xs[0]`, `elif`, - `tests/` layout). → Language-keyed extractor selected by - `profile.language`, with a generic diff-heuristic fallback. -- `TaskSpec.repo` / `Settings.default_repo` default to `"vllm-omni"`; the - intent parser cannot extract a repo at all ("review pr 12 in repo-b" is - unparseable). → Intent gains repo extraction; default comes from the single - configured profile when only one exists. -- Playbooks declare `repos: [vllm-omni]`, so a second repo needs copy-pasted - playbooks that will drift. → Repo-neutral playbooks (§V2.2.3). -- `gh` is assumed as the only forge and Buildkite handling is scattered/stub. - → Forge/CI adapters declared by the profile (§V2.2.4). - -### (c) v1 boundaries to close - -- **Buildkite log download** (stubbed; PR-debug reads `gh pr checks` only) → - a `CIProvider` adapter interface (fetch failing jobs, download logs, poll a - build), implementations for GitHub Actions and Buildkite, selected by - `profile.ci.provider`. The parent rebase-agent's known baseline-signature - weakness (exact-string compare misclassifying flaky/known failures) must - not be inherited: signature matching in the adapter is normalized - (timestamps/hashes/paths stripped) before comparison. -- **Metrics feedback collectors** (per METRICS_RESEARCH roadmap): gh - collector filling useful/accepted/conflict online, post-push CI snapshot. -- **Native-rebase prelude env export** mutates the copilot's own process env - → run the prelude/phases in a subprocess with an explicit env, keeping the - traced `env_exported` delta. -- **Ensemble coverage**: `run_agent_step_ensemble` is wired only for review. - Triage and debug-hypothesis steps adopt lens ensembles once their run-to-run - variance is measured (same replicate-mean discipline as the review - campaign; do not ship unmeasured "improvements"). - -## V2.2 Repo invariance - -**Definition.** For every task kind the copilot supports, quality on a new -repo with an established profile stays within a declared band of the -reference repo (vllm-omni), and onboarding a repo requires zero core-code -changes. Invariance is a *contract plus a measurement*, not an aspiration — -the benchmark literature (§V2.0.4) shows same-model resolve rates swinging -<10%→>50% across repos, so unmeasured cross-repo claims are worthless. - -1. **Repo-neutral core, enforced.** No repo-specific literal (repo names, - module names, domain prompts, absolute paths) anywhere in - `src/infermatrix_copilot/` — such knowledge lives only under `adapters//`. - Pinned by a `test_repo_neutral_core` test that scans the source for the - known-leak patterns and fails on new ones. Prompts in the core may state - *how to review/debug/triage*; only the profile states *what this repo is*. -2. **Single injection path.** Everything repo-varying reaches steps through - the DispatchContext `profile` section (§V2.3.4): domain prompt sections, - module map, risk tiers, test/format commands, CI/forge adapters, - conventions. A step that needs repo knowledge and finds none in the - profile must degrade explicitly (see 4), never guess. -3. **Repo-neutral playbooks.** Playbooks declare `repos: ["*"]` plus a - `requires:` list of profile capabilities (e.g. `ci.provider`, - `upstream.fork_tracking` for repo-rebase). `PlaybookStore.find()` matches - task kind + capability satisfaction instead of repo name; a repo-specific - playbook override (exact repo match) still wins when present. The locked - `repo-rebase` playbook stays vllm-omni-specific by declaration — it - `requires` the external orchestrator capability that only the vllm-omni - profile provides; zero-regression is untouched. -4. **Graceful capability degradation.** Missing capability → a *declared* - downgrade recorded as a `capability_gap` RunTrace event and surfaced in - the run report: no CI provider → pr-debug runs report-only triage; no - upstream declared → repo_rebase refuses at planning time with the reason; - unknown language → generic sweep extractor, flagged in the review output. - Degraded ≠ silently worse: every gap is visible. -5. **Measured invariance.** Extend the eval harness to a cross-repo - benchmark: the same arms and RQS3/RQS3e/CATQ metrics per repo, scored on - replicate means (the campaign's hard lesson: single rolls are ±0.1 noise). - Report an **invariance index** = min(repo score) / mean(repo score) per - task kind; target ≥ 0.8. A new repo's profile is promoted - (draft → active) only after its eval run lands within the band — the - profile promotion gate *is* the invariance gate. -6. **Knowledge namespacing.** Skills and debug memories resolve per-repo - first (`adapters//skills/`, `adapters//store/debug_memory.db` — - the already-designed fields from §(a)4), then a shared general pool. - Writes always land in the active repo's namespace; promotion of a repo - skill to the shared pool is a curator decision (it must be genuinely - repo-agnostic, e.g. "how to bisect a flaky test" vs. "HunyuanImage SSIM - threshold"). - -## V2.3 Repo profile - -The profile generalizes adapter zero + Phase-0 bootstrap into the copilot's -full picture of a repository. Two evidence-driven design rules govern -everything below (§V2.0.1–2): - -- **Rule 1 — a profile is not a prompt.** Most profile content is consumed - by *code* (step logic, adapters, triggers) or retrieved *on demand*; the - always-on prompt slice is a curated minimum of non-obvious, non-redundant - directives. Auto-generated overviews injected wholesale are measured to be - worse than nothing. -- **Rule 2 — no fact without provenance, no rewrite without stability.** - Every fact cites how it was derived and when it was last confirmed; - curated summaries are rewritten only by a scheduled consolidation tier, - never continuously. This is the personal-agent profile architecture, which - is running in production and whose failure modes (fragmentation, - evidence dumps, continuous-rewrite corruption) are already mapped. - -### V2.3.0 Two tiers of repo knowledge - -Repo knowledge splits into two tiers by **who may write it and how much trust it -carries** — the axis that matters for safety, orthogonal to the machine / -briefing / retrieved *consumption* split of §V2.3.4: - -- **Tier 1 — the manifest** (`manifest.yaml`): *human-authored, human-gated* config - the agent would be unsafe to guess — repo/upstream paths, push policy - (protected branches, `allowed`), CI provider, capabilities. Plain declarative - YAML; the high-risk sections (`repo`/`upstream`/`push`) are human-only - (`HIGH_RISK_SECTIONS`, enforced in `update_manifest`). **Required** (a repo - can't be operated safely without it), low-churn — the safety gate (**D2**). -- **Tier 2 — the profile** (`profile/`, plus `skills/`, `store/`, `repo_map/`): - *agent-established, evidence-gated* knowledge the agent earns — per-fact - provenance, typed patch ops as the only write surface, stability + staleness - gates (§V2.3.2). **Optional** enrichment (retrieval falls back to the shared - pool when absent), high-churn, self-correcting, auditable — the quality lever - (**D1**). - -The manifest is the repo's *constitution*; the profile is the knowledge that -accumulates under it. They are deliberately **not merged**: folding Tier 1 into -the evidence-gated Tier-2 write path would make the push policy agent-writable (a -safety regression), and the alternative — a human-only region inside the profile -— just re-draws the same boundary inside one file. The two data models are also -incompatible (a manifest fact like `repo.path` has no "evidence" or "staleness"; -a profile fact without evidence is *rejected*). The container `adapters//` -already unifies them on disk — **one repo edge, two trust tiers**. The agent -*may draft* Tier-1 structure (`profile.structure_scan` seeds the module list), -but Tier 1's risky sections always require human activation, so anything -safety-bearing is *agent-proposes / human-confirms*. - -### V2.3.1 Layout - -``` -adapters// - manifest.yaml identity, repo/upstream paths, push policy (human-gated) - profile/ - profile.yaml the curated core (small, human-readable): per-fact - provenance fields — see §V2.3.2 - briefing.md RENDERED always-on prompt slice: 10-30 short - imperative directives (tooling commands, hard - constraints, known traps). Budgeted (<400 words), - non-redundant with repo docs — validated, see §V2.3.5 - format.yaml formatters/linters + exact commands, naming and - comment conventions (machine: validation cmds) - constraints.md protected branches, review/merge norms, required - checks, licensing/DCO, never-touch paths (machine: - push guard + PathScope; plus briefing lines) - modules.yaml module map (paths -> module), import-graph waves, - module -> test commands, ownership, risk tiers - (machine: fan-out, triggers, scheduling; never - injected as prose) - review.md domain review checklist + lens extensions + severity - norms (retrieved by pr_review) - ci.yaml provider(s), pipelines, log access, required checks, - normalized flaky-signature baseline (machine) - PROFILE_REPORT.md rendered provenance view: how each fact was derived - (deterministic | agent | human), evidence, confidence - skills/ per-repo skills, each with a TRIGGER description - (Devin-style: content + when to recall it) - store/debug_memory.db per-repo debug memory - repo_map/ cached symbol graph for the on-demand repo_map tool - (Aider/RepoGraph-style, ranked per query, token- - budgeted; rebuilt on drift) -``` - -`manifest.yaml` keeps its v1 role and high-risk human-only sections -(`repo`/`upstream`/`push`); `profile/` holds the establishable knowledge. - -> **Naming (renamed 2026-07-11, was "plugin").** The old name "plugin" was a -> misnomer — this bundle is declarative repo knowledge, not executable extension -> code, which is what "plugin" implies everywhere else in software. The design's -> headline property is *repo-invariance*: a generic engine specialized per repo — -> i.e. the **adapter pattern** — so the concepts are now self-documenting: -> - container `adapters//` (was `plugins//`) — -> `RepoAdapter`/`AdapterRegistry` (were `RepoPlugin`/`PluginRegistry`) -> - Tier-1 file `manifest.yaml` (was `plugin.yaml`) — already the `.manifest` -> attribute internally, and "manifest" names the config tier honestly -> -> Read as: the **adapter** (container) holds a **manifest** (Tier-1 config) and a -> **profile** (Tier-2 knowledge). The `ADAPTERS_DIR` env var (was `PLUGINS_DIR`) -> points at the adapters root. The rename was mechanical (git-mv + symbol sweep, -> tests green throughout); on-disk `adapters//manifest.yaml` replaces the -> old `plugins//plugin.yaml`. - -### V2.3.2 Profile memory architecture (borrowed from the personal agent) - -The personal agent (a sibling project — `profile_store.py` + -`RESEARCH_AGENT_MEMORY_2026.md`) maintains a person-profile with exactly the -lifecycle a repo profile needs. We adopt its skeleton wholesale, translated -to repo terms: - -| Personal-agent mechanism | Repo-profile translation | -|---|---| -| Immutable evidence store (`events.db`) under a small curated `profile.yaml` | RunTraces + Stage-1 evidence archives are the immutable layer; `profile.yaml` is the curated layer. Facts summarize evidence, never replace it | -| **Typed patch ops as the only write surface** (`add_evidence`, `bump_last_seen`, `merge_*`, `mark_dormant`; malformed ops rejected) | Profile mutations are typed ops validated by the store (`add_fact`, `add_evidence`, `bump_confirmed`, `merge_facts`, `mark_stale`, `update_command`); agents never free-edit profile files | -| **Two write tiers**: daily additive (no rewrite) vs. weekly consolidation (only tier allowed `rewrite_entry`) — continuous LLM rewriting measurably corrupts memory | Per-run tier: runs may only *add* evidence/candidates. Consolidation tier (scheduled or post-eval): dedupe, merge, rewrite `briefing.md` — the only place prose is regenerated | -| **Join key against fragmentation**: owner-editable `aliases.yaml`; every op must name its initiative | The **module map is the join key**: every profile op names the module (or `repo-wide`) it concerns, so knowledge converges per module instead of fragmenting | -| Provenance per entry: `evidence[]`, `first_seen`, `last_seen`, `confirmations: n` | Same fields per fact, plus `source: deterministic|agent|human` and source commit | -| **Stability gate**: entries with ≥3 confirmations can't be rewritten to cite less evidence | Same rule: a consolidation rewrite may not drop cited evidence from a stable fact; superseded text goes to `history[]`, never deleted | -| **Dormancy decay, never delete** (30/60-day windows) | Facts unconfirmed past their window flip to `stale` (excluded from injection/consumption, kept for audit); refresh re-confirms or retires them | -| Protected sections the LLM cannot touch (`identity`, `preferences`) | `manifest.yaml` high-risk sections (already enforced in `update_manifest`) + human-authored constraint lines marked `source: human` | -| **Read-only LLM judge** (weekly): reports contradictions/stale claims, never auto-fixes | Profile judge audits profile vs. last N RunTraces (commands that failed, modules that moved, checklist items that never fire) — findings become refresh proposals | -| Git repo per profile: every save a commit, weekly diff emailed, `git revert` rollback | `adapters//` is committed; every consolidation is one reviewed commit with the diff in the run report — rollback is `git revert` | - -### V2.3.3 Establishment pipeline (extends Phase-0 bootstrap) - -- **Stage 0 — deterministic fingerprint** (exists: `fingerprint_repo`): - language mix, layout, CI systems, remotes, default branch. Extended with - the deterministic parts of `modules.yaml` (directory clustering, - import-graph waves via static analysis) and `repo_map/` construction. No - LLM, no writes to the target repo. -- **Stage 1 — profiling agents** (new): read-only governed agent steps, one - per profile artifact, through the unified agent runtime: *structure* - (module semantics + test mapping on top of the Stage-0 graph), *format* - (detect config files, infer unwritten conventions, emit exact check - commands), *constraints* (CONTRIBUTING/CODEOWNERS/PR templates/branch - protection via forge API), *ci* (pipelines, log access), *review* (domain - checklist from docs + recent human review threads). Every emitted fact - must cite its evidence — uncited facts are rejected at the contract layer. -- **Stage 1.5 — redundancy filter** (new, the ETH-study lesson): a - deterministic pass drops any candidate briefing/checklist line whose - content is already stated in the repo's README/docs/AGENTS.md (the agent - reads those anyway) and any codebase-overview prose. What remains is the - non-obvious residue — tooling commands, traps, invariants. If an - `AGENTS.md`/`CLAUDE.md`/`.github/copilot-instructions.md` exists, it is - ingested as *human-authored* briefing input (highest-trust source) rather - than re-derived. -- **Stage 2 — human review gate** (exists, extended): profile lands as - `status: draft` + `PROFILE_REPORT.md`; high-risk sections human-only. - Human review flips draft → candidate. -- **Stage 3 — live calibration**: first N runs treat low-confidence facts as - hypotheses; corrections (module map fixes, commands that actually pass, - flaky baseline entries) are proposed as candidates through the curator - gate. Successful-run RunTraces feed **skill induction** (AWM-style): - recurring resolution patterns become per-repo skill candidates with - trigger descriptions. Passing the eval band (§V2.2.5 + §V2.3.5) promotes - candidate → active. -- **Stage 4 — consolidation & refresh** (the personal agent's weekly pass): - scheduled consolidation dedupes/merges facts per module, regenerates - `briefing.md` under its word budget, applies dormancy decay; drift - detectors (module path gone, format command fails, CI pipeline renamed, - N profile-contradicting runs, judge findings) trigger targeted re-runs of - the affected Stage-1 agent. One reviewed commit per cycle. - -### V2.3.4 Consumption: three channels - -The ETH result forces the discipline; every artifact declares its channel: - -1. **Machine-consumed (never prose-injected)**: `modules.yaml` → fan-out, - wave scheduling, patch-review `high_risk_modules`, retrieval queries; - `format.yaml` → post-edit validation commands run before the patch gate; - `ci.yaml` → adapter selection, required-checks list, flaky filtering; - `constraints.md` machine lines → push guard + PathScope. -2. **Always-on prompt slice**: only `briefing.md` (word-budgeted, curated, - redundancy-filtered) enters every DispatchContext for the repo, plus the - handful of constraint lines relevant to the step's risk level. -3. **Triggered/on-demand**: per-repo skills recalled by trigger match; - `review.md` retrieved only by pr_review steps; the `repo_map` tool - answers structure queries ranked against the step's goal under a token - budget — agents pull structure when they need it instead of being fed an - overview. - -### V2.3.5 Profile efficacy is measured, not assumed - -The same study showed even human-written context files can be net-negative -per repo. So the profile itself is an eval arm: on each profiled repo, the -task-kind eval runs {no profile} vs {profile} (replicate means, as always). -A profile (or a briefing revision) is promoted only if it is non-negative on -quality **and** does not blow the cost budget (CATQ's C term already prices -the +2–4 extra steps context files induce). Consolidation-tier briefing -rewrites re-run the ablation. This closes the loop that AGENTS.md authors -skip: we never assume injected context helps. - -### V2.3.6 Governance - -No new machinery — profiles reuse the existing gates: profile edits are -knowledge edits (already a patch-review trigger); agents propose typed-op -candidates only (`update_manifest` already rejects agent writes to high-risk -sections); draft/candidate/active/retired mirrors the playbook lifecycle; -every accepted fact is traceable via `PROFILE_REPORT.md`; every -consolidation is one reviewed, revertable commit. - -## V2.4 Milestones and acceptance - -- **P0 — correctness**: §V2.1(a) fixes + resume-integrity test per playbook + - `test_repo_neutral_core` (initially xfail-listing the known leaks so the - list can only shrink). -- **P1 — profile substrate**: profile store with typed patch ops + - provenance fields (§V2.3.2), `profile/` schema + loader, Stage-0/1 - establishment + Stage-1.5 redundancy filter + PROFILE_REPORT, - DispatchContext `briefing` injection, review/sweep/intent - parameterization off the profile (de-leaking §V2.1(b)). -- **P2 — repo-neutral execution**: playbook `requires:` capability matching, - CI/forge adapter interface (GitHub Actions + Buildkite), capability-gap - degradation paths, per-repo knowledge namespacing wired, `repo_map` - on-demand tool. -- **P3 — measured invariance**: cross-repo eval harness (reference repo + at - least one structurally different second repo), invariance index reporting, - **profile ablation arm** (§V2.3.5), promotion gates bound to the eval - band; Stage-3 live calibration + AWM-style skill induction; Stage-4 - consolidation/judge/refresh loops on a schedule. - -**Acceptance criteria** - -1. A second repository is onboarded end-to-end (fingerprint → profile → - active) with **zero commits to `src/infermatrix_copilot/`**. -2. All six task kinds execute on the second repo — at full capability where - the profile provides it, and with *recorded* `capability_gap` degradation - where it does not; no silent quality loss. -3. pr-review RQS3 (replicate mean) on the second repo within the declared - band of the vllm-omni reference; invariance index ≥ 0.8. -4. Resume-integrity tests green for every playbook (no BLOCKED/FORBIDDEN - caused by resume itself). -5. The locked `repo-rebase` nightly remains byte-identical in behavior - (zero-regression constraint carries through v2 unchanged). -6. The profile earns its keep: on every profiled repo the {profile} eval arm - is non-negative vs {no profile} on quality at acceptable cost (§V2.3.5), - and every `briefing.md` stays within budget with zero doc-redundant lines. diff --git a/doc/DOCSTRING_STYLE.md b/doc/DOCSTRING_STYLE.md deleted file mode 100644 index 61a5365e5..000000000 --- a/doc/DOCSTRING_STYLE.md +++ /dev/null @@ -1,82 +0,0 @@ -# Docstring style — infermatrix-copilot - -The house convention for docstrings, grounded in [PEP 257](https://peps.python.org/pep-0257/) -(structure) and the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html) -(what to document), adapted to this codebase's terse, rationale-first voice. - -**Every module, class, function, and method carries a docstring.** It explains -the *logic* (what it does and the non-obvious *why*) and the *inputs/outputs* — -not a restatement of the signature. - -## Form (PEP 257) - -- Triple double-quotes `"""..."""`. The module docstring is the file's very first - statement (before `from __future__ import ...` and all imports). -- **Summary line**: one imperative-mood sentence, ≤ ~88 cols, ending in a period. - For obvious cases the summary line alone is the whole docstring (one-liner). -- Non-trivial cases: summary line, blank line, then the body. -- Closing `"""` on its own line for multi-line docstrings. - -## What each kind documents - -- **Module** — the one thing the file owns + a one-line note of what it exports. - For a re-export `__init__.py`, say what the package groups and that the surface - is re-exported. -- **Class** — its behavior/role in one line; note the key attributes or the - invariant it upholds when that isn't obvious. `@dataclass` field meaning goes - in the class docstring (or an inline `#` comment), not a separate `Attributes` - block unless there are many. -- **Function / method** — summary of behavior, then the inputs → output and the - non-obvious logic, **in prose**. Name the parameters that carry meaning and say - what the return represents. Reserve explicit `Args:`/`Returns:` sections for - functions with many parameters where prose would be harder to read. - -## Voice (match the existing code) - -- Rationale-first: prefer explaining *why* over narrating *what* the code line-by-line - already shows. Cite the design/eval when it explains a choice (e.g. "single - untooled reduction call: a tool-looped reducer over-dropped in live runs"). -- Keep the safety/contract framing that the specs use: mention when a function is - a fail-closed gate, publishes state for a later step (**B2**), degrades to a - typed BLOCKED result, or records a `capability_gap`. -- Do not invent behavior. If unsure what a branch does, describe it at the level - the code supports — never guess specifics (line numbers, thresholds) not present. - -## Examples (from this repo) - -Function — inputs → output + the why: - -```python -def guard_push(policy: PushPolicy, protected_branches: list[str]) -> PushDecision: - """Authorize a push: allow only when `policy.allowed` AND the target branch - is not in `protected_branches`. Returns a PushDecision carrying the concrete - git command (never run here) or a deny reason — the single C4 choke point, so - every push path is authorized in one place.""" -``` - -Class — role + key attribute: - -```python -class StepResult: - """The outcome of one step: `ok` plus, on failure, a typed `failure_kind` - (drives the engine's retry/replan/escalate branch) and the `state_updates` - a later step consumes. Never raises across the step boundary — failures are - values, not exceptions.""" -``` - -Re-export `__init__.py` module: - -```python -"""Review subsystem: diff summarization, trigger evaluation, and the patch/plan -reviewers. Re-exports the public surface (`build_diff_summary`, `ReviewVerdict`, -`run_patch_review`) so callers import from the package, not its modules.""" -``` - -## Don't - -- Don't add `Args:`/`Returns:`/`Raises:` scaffolding to trivial accessors — a - one-liner is correct and PEP-257-preferred. -- Don't change code, signatures, or existing (correct) docstrings while adding - new ones. Docstrings are additive. -- Don't duplicate the per-file SPEC — the docstring is the local, in-code view; - the spec is the normative contract. diff --git a/doc/EXTENDING-KNOWLEDGE.md b/doc/EXTENDING-KNOWLEDGE.md deleted file mode 100644 index fef41cd0f..000000000 --- a/doc/EXTENDING-KNOWLEDGE.md +++ /dev/null @@ -1,392 +0,0 @@ -# Extending the knowledge base (`knowledge/`) - -A practical guideline for users and developers who want to **add or change a page** -in the curated Markdown wiki at repo-root `knowledge/`. - -Chinese maintainer guide: -[`docs/knowledge-maintainer.zh-CN.md`](../docs/knowledge-maintainer.zh-CN.md). - -`knowledge/` is a **vendored** copy of `zuiho-kai/claude-workflow-starter` -(provenance and copy authorization in [KNOWLEDGE.md](KNOWLEDGE.md)); it is -maintained in-repo as ordinary tracked files. The wiki's own short entry point is -[`knowledge/CONTRIBUTING.md`](../knowledge/CONTRIBUTING.md), which routes to one -topic page in [`knowledge/contributing/`](../knowledge/contributing/) per action — -read it when you need the authoritative rule. **This guide adds copy-paste -templates and a decision tree on top of that**, plus how the copilot actually -*consumes* the wiki so you know what to keep tight. - -> The same workflow is distilled into a retrievable skill, -> [`skills/knowledge-base-contribution/SKILL.md`](../skills/knowledge-base-contribution/SKILL.md), -> so the copilot's own agents follow it during retrospectives. - ---- - -## 0. TL;DR — the four-step loop - -1. **Pick the owner directory** (§1). Route by the *verified root cause*, never by - where the symptom first showed. -2. **Write/edit one page** of the right type (rules / architecture / guide / - incident). Keep exactly **one canonical copy** of each fact; everything else - links to it. -3. **Register it in the same change**: add a `遇到什么 → 查看哪里` row (or a - child-dir link) in the **nearest `_index.md`**. An unregistered page fails the gate. -4. **Validate** (both gates must print 0 errors): - - ```bash - python knowledge/tools/check_knowledge_tree.py # structure / index / links / incidents - python knowledge/tools/check_wiki_lint.py # synthesis-layer frontmatter + tag taxonomy - ``` - -Deliver via PR (the wiki gate runs in review); never direct-push. See §5. - ---- - -## 1. Where does it go? — the owner decision tree - -### Layer 1 — general vs repo-specific - -| The lesson is… | Put it under | -|---|---| -| True on **any** repo (review, ci, debug, git, planning, remote, docs, benchmark, agents, environment) | `knowledge/general//` | -| Specific to **one repo** | `knowledge/repos//` | -| A **current-machine** fact (host, path, account, cache, venv, token) | `knowledge/local/` — git-ignored, never tracked | - -### Layer 2 — inside a repo slice - -| The lesson is about… | Put it under | -|---|---| -| A repo-wide workflow topic (review, ci, git, benchmark, remote, rebase…) | `repos///` | -| **Shared source code** used by several models (diffusion, scheduler, serving…) | `repos//components//` | -| **One model's** own implementation / config / checkpoint | `repos//models//` | - -Route by the **verified** root cause. "Frontend saw a 404" does not mean the root -cause lives in frontend — sink the page where the cause was proven, not where the -symptom appeared. - -### Layer 3 — which page type - -| You have… | Page type | Notes | -|---|---|---| -| A rule that must change the **next** run (trigger → do → don't → how to verify) | `rules.md` in the nearest owner dir | The default product of a retrospective. Always-on (see §2) — keep it tight. | -| A **stable** data-flow / responsibility / boundary description | `architecture.md` | No title-only stubs. | -| A longer method that isn't a hard gate | a feature page in the owner root; work-topic collections may use `guides/` | Component/model owners stay flat; all deeper pages are pulled on demand. | -| **Complex reproducible history** a rule can't carry | `incidents/YYYY-MM-DD-short-name.md` | Optional. Only when the repro chain / evidence still has independent lookup value. | - -**Retrospective rule of thumb:** the default output of "复盘 / record a lesson" is a -**rule**, not an incident. Add an incident only when the evidence chain is complex -enough that the rule alone can't carry it. - ---- - -## 2. How the copilot consumes it (so you know what to optimize) - -Wiring lives in `src/infermatrix_copilot/engine/agent_runtime/knowledge.py` + -`src/infermatrix_copilot/config.py` + each `adapters//manifest.yaml`. - -- **Always-on briefing** — injected into *every* run, each capped: - - the general slice `settings.knowledge_general_docs` (`general/_index.md`); - - the adapter's `briefing_docs` — for vllm-omni: `repos/vllm-omni/rules.md` + - `repos/vllm-omni/_index.md`; - - `performance_briefing_docs` for strong models (compact review patterns). -- **On demand** — the `doc_search` / `doc_read` tools recursively reach every - deeper Markdown page in `general/` **plus the active - adapter's `repo_subdir` only** (other repos' slices are refused; path traversal - is blocked; `doc_read` windows 24k chars and pages with an offset). -- **MCP** — the same repo-scoped `doc_search` / `doc_read` are exposed read-only - over MCP, so a host model can query the wiki without starting a run. - -**Practical implication for authors:** `rules.md` and `_index.md` are *budget* — -they load on every task. Keep them to triggers + gates + navigation. Push -narrative, long repros, and step-by-step method into on-demand feature pages or `incidents/`, -which are pulled only when a run actually needs them. - -To register a **new repo's** slice, point its adapter manifest at it -(`knowledge.repo_subdir`, `briefing_docs`, `performance_briefing_docs`) — see §3.8. - ---- - -## 3. Templates (copy-paste) - -**Ready-to-copy files live in [`knowledge-templates/`](knowledge-templates/)** (see -its [README](knowledge-templates/README.md) for the copy → register → validate -recipe) — `cp` one into place instead of retyping the blocks below. The blocks -here are the same skeletons, inline for reading in context. - -All templates are gate-valid once you (a) replace placeholders and (b) register -the page in the nearest `_index.md`. **Frontmatter is required on synthesis-layer -pages** — `rule` / `guide` / `architecture` / `index` under `general/` and -`repos/` — and enforced by `knowledge/tools/check_wiki_lint.py`: it needs `title`, -`created` + `updated` (both `YYYY-MM-DD`), `type` (one of those four), and a -non-empty `tags` list drawn from the taxonomy in -[`knowledge/SCHEMA.md`](../knowledge/SCHEMA.md) (`## 标签分类法`); `confidence`, if -present, must be `high|medium|low`. **Evidence-layer pages** (`incidents/`, -`history/`, `results/`) take **no** frontmatter — the incident template uses plain -`- 编号/…` fields instead (checked by `check_knowledge_tree.py`). Directory names -use lowercase -`a-z0-9-`; `_index`, `local`, `components`, `models`, `incidents`, `guides` are -reserved role names — don't reuse them as custom topics. - -### 3.1 `_index.md` — any topic / directory (the required routing table) - -```markdown ---- -title: "" -created: 2026-07-20 -updated: 2026-07-20 -type: index -tags: [] -sources: [] ---- - -# - -## 什么时候查这里 - -- - -## 不放什么 - -- → `` - -## 目录内容 - -| 遇到什么 | 查看哪里 | 说明 | -|---|---|---| -| | [](.md) | | -| | [incidents](incidents/_index.md) | optional | -``` - -Every non-index page and every child directory in this folder **must** appear -exactly once as a link in this table (that link is what the gate checks). - -### 3.2 `_index.md` — a **repo** entry (`repos//_index.md`) - -Same as 3.1, plus an identity header before the tables: - -```markdown -# - -- 上游仓库：`/` -- 常用分支：默认分支 `
`； -- 适用范围： -- 组件源码映射已按 ` main @ ` 校验 - -## 什么时候查这里 -- 当前 Git 仓库或用户明确目标是 。 - -## 不放什么 -- 跨仓库通用方法（放 `general/`）；其他仓库的规则。 - -## 当前入口 -| 遇到什么 | 查看哪里 | 说明 | -|---|---|---| -| 开始任何 修改、测试、发布任务 | [硬门禁](rules.md) | 仓库硬规则 | -| 查看共享代码模块 | [components](components/_index.md) | 模块职责地图 | -| 查看某个模型 | [models](models/_index.md) | 模型入口 | -``` - -### 3.3 `_index.md` — a **component** or **model** (extra required fields) - -Component `components//_index.md` must additionally list: the source -paths it owns, its responsibility / IO boundary, its test entry, and which -models/features it affects. Model `models//_index.md` must additionally -list: canonical name + aliases, source paths, which shared components it depends -on, and how checkpoints/sizes/quantizations differ. - -### 3.4 `rules.md` — the always-on gate page - -```markdown ---- -title: " 硬门禁" -created: 2026-07-20 -updated: 2026-07-20 -type: rule -tags: [] -sources: [] ---- - -# 硬门禁 - -只在当前任务明确属于 时应用本页。先遵守根 `CLAUDE.md` 的通用 P0。 - -## 场景触发器 - -| 用户提到 | 必读 | 硬约束 | -|---|---|---| -| | [](.md) | | - -## 规则（每条给稳定 ID） - -### -- 触发： -- 必须： -- 禁止： -- 验收： -``` - -Give every independent, auditable constraint a **stable ID** (e.g. `HY3-2c`). -One ID = one behavioral invariant. Keep rules readable without knowing any -incident number. Don't pre-create an empty `rules.md`; create it when the first -rule exists and link it from the sibling `_index.md`. - -### 3.5 `architecture.md` — stable boundaries (no title-only stubs) - -Synthesis-layer, so it needs frontmatter (`type: architecture`). Component variant: - -```markdown ---- -title: " 架构" -created: 2026-07-21 -updated: 2026-07-21 -type: architecture -tags: [] -sources: [] ---- - -# 架构 - -## 职责和边界 -## 主要源码和调用入口 -## 数据怎样流动 -## 怎样验证 -``` - -Model variant (same frontmatter, `type: architecture`): - -```markdown -# 架构 - -## 模型专有部分与共享模块的边界 -## 配置、checkpoint 和兼容范围 -## 从输入到输出的主要流程 -## 怎样验证功能、精度和性能 -``` - -### 3.6 An ordinary method page - -```markdown ---- -title: "" -created: 2026-07-20 -updated: 2026-07-20 -type: guide -tags: [] -sources: [] ---- - -# - -## 什么时候用 -- - -## 步骤 / 方法 -1. - -## 怎样验证 -- -``` - -For a component/model, put the page directly in the owner root and link it from -the owner's `_index.md`. Work-topic collections may place it in `guides/` and -register it in `guides/_index.md`. - -### 3.7 An `incidents/` page (validator-checked fields) - -File name **must** match `YYYY-MM-DD-short-name.md`. The body must contain these -exact field labels, and the state must be one of -`待归类 / 处理中 / 已验证 / 已提炼 / 仅历史`. The `编号` must be unique across the tree. - -```markdown -# <人能读懂的现象标题> - -- 编号：`inc-2026-07-20-short-name` -- 归属：`repos//` -- 状态：处理中 -- 搜索词：、、 -- 影响范围： - -## 现象 -## 根因（live 证据） -## 修复 -## 验收 -## 已提炼的规则 -- 见 [ rules](../rules.md#) -``` - -One incident = one canonical write-up; other places link to it. Normal work still -starts from rules, never from an incident path. - -### 3.8 Onboarding a **new repo** slice - -1. Create the wiki skeleton and register each level up the chain: - - ```text - knowledge/repos//_index.md # from template 3.2 - knowledge/repos//rules.md # only if a real per-repo gate exists - # add a row for in knowledge/repos/_index.md - ``` - -2. Point the adapter at the slice (`adapters//manifest.yaml`): - - ```yaml - knowledge: - repo_subdir: repos/ # its slice under knowledge/ - briefing_docs: # always-on: keep tiny - - repos//rules.md - - repos//_index.md - performance_briefing_docs: # optional, strong-model only - - repos//review/guides/.md - ``` - -3. If you forked the wiki from upstream, delete the `repos/` slices that aren't - yours and update `repos/_index.md` accordingly. - ---- - -## 4. Gate cheat-sheet — what the two gates enforce - -`check_knowledge_tree.py` (structure) and `check_wiki_lint.py` (frontmatter/schema) -both run in review; both must be clean. - -| Check | Gate | Rule | -|---|---|---| -| Index present | tree | Every directory containing Markdown has an `_index.md`. | -| Registration | tree | Every non-index page **and** every child dir is linked **exactly once** from the nearest `_index.md`. | -| Links | tree | Relative links must resolve; absolute paths are rejected. | -| Size | tree | Warn at ≥300 non-empty lines or 16 KiB; **must split** at ≥500 lines or 32 KiB (or note `暂不拆分` + a review date in the index). | -| Directory fan-out | tree | Non-group dir warns above 7 ordinary pages; group dirs (`guides` `incidents` `history` `references` `results` `rfcs`) hard-fail above 20. | -| Incident format | tree | `YYYY-MM-DD-short-name.md` + the five `- 编号/归属/状态/搜索词/影响范围` fields + a valid state + a unique 编号. | -| Privacy | tree | No real IPv4, Windows `C:\Users\…`, remote user home, or private-key blocks in tracked pages. | -| Safety | tree | No `StrictHostKeyChecking=no`, global `safe.directory *`, `--gpus all`, `pkill`, `rm -rf`, or `find … -exec rm`. | -| `local/` | tree | Must stay git-ignored (untracked). | -| Root entry | tree | `knowledge/CONTRIBUTING.md` stays ≤100 non-empty lines / ≤8 KiB (detail sinks into `contributing/`). | -| Frontmatter | lint | Synthesis pages (`rule/guide/architecture/index` in `general/`+`repos/`) need `title`, `created`+`updated` (`YYYY-MM-DD`), `type` (those four), non-empty `tags`; optional `confidence: high\|medium\|low`. | -| Tag taxonomy | lint | Every `tags` value must appear in `SCHEMA.md`'s `## 标签分类法`. | -| Evidence layer | lint | `incidents/` `history/` `results/` pages are **not** frontmatter-checked (they use the plain incident fields). | -| Adapter briefings | lint | `manifest.yaml` `knowledge:` allows only `source`/`repo_subdir`/`briefing_docs`/`performance_briefing_docs`, and briefing docs must not point at evidence-layer pages. | - -`rules.md`, `architecture.md`, `_index.md` are "special" pages and don't count -toward the 7-page fan-out warning. - ---- - -## 5. Delivery - -- Edit `knowledge/` in place like any tracked content; ship via **PR**. The wiki - gate runs in review — don't direct-push to a protected branch. -- The tree is **vendored** — never edit upstream. To pull a future upstream page, - diff against `zuiho-kai/claude-workflow-starter` and import the specific pages - deliberately (there is no submodule link). -- Machine facts (host, path, account, token, cache, venv) go **only** in the - git-ignored `knowledge/local/`; keep tracked pages free of them (the gate - enforces this, but check before you commit anyway). - ---- - -## 6. The retrieval skill - -[`skills/knowledge-base-contribution/SKILL.md`](../skills/knowledge-base-contribution/SKILL.md) -encodes this workflow for the copilot's own agents (owner routing, page-type -choice, `_index.md` registration, always-on tightness, and the validator). It is -retrieved via `skill_search` and surfaces during retrospectives when an agent is -told to record a lesson. Update it in the same PR whenever this guideline changes. diff --git a/doc/GUIDE.md b/doc/GUIDE.md new file mode 100644 index 000000000..8a7a71c8d --- /dev/null +++ b/doc/GUIDE.md @@ -0,0 +1,768 @@ +# InferMatrixCopilot 指南 + +一份文档覆盖两类读者：**操作者**（用它审 PR、修 CI、答 issue，不读 `src/`） +和**维护者**（改这个仓库本身）。按结构组织——概览、功能、使用、开发、 +playbook、step、tool、性能——与 [`CODE_TOUR`](architecture/CODE_TOUR.md) 的数据流视角互补： +那份文档回答"数据怎么流"，这份回答"有哪些东西、各自在哪、怎么用"。 + +> 配套：[`DESIGN.md`](architecture/DESIGN.md) 为什么这么设计 · +> [`SPEC/`](architecture/SPEC/README.md) 每个源文件不能破坏什么 · +> [`README`](../README.md) 安装与 MCP 入口 · [`DEVELOPMENT`](../DEVELOPMENT.md) 文档地图。 + +--- + +## 目录 + +1. [概览](#1-概览) +2. [功能](#2-功能) +3. [使用指南（操作者）](#3-使用指南操作者) +4. [开发指南（维护者）](#4-开发指南维护者) +5. [Playbooks](#5-playbooks) +6. [Steps](#6-steps) +7. [Tools](#7-tools) +8. [性能对比](#8-性能对比) + +--- + +## 1. 概览 + +### 1.1 一个仓库，两个产品 + +最容易混淆的一点先说清楚：这个仓库里住着两个形态不同的产品，共用一套知识和 +治理，但执行路径几乎不重叠。 + +| | **Direct / MCP** | **Strict / CLI** | +|---|---|---| +| 谁读代码 | 宿主 Agent 自己的模型 | InferMatrixCopilot 配置的模型 | +| 入口 | `/imreview` `/imdesign` `/imcifix` `/imupdate` | `infermatrix-copilot -p "…"` | +| 用到 playbook / step | **完全不用** | 全用 | +| 服务端跑模型 | 否（零模型、零状态） | 是（隔离子进程） | +| 需要 API Key | 否 | 是（或订阅制 harness 后端） | +| 典型耗时 | 同步一次调用 | 5–12 分钟 | + +Direct 是默认形态：MCP 服务端把"该看哪些知识、该守哪些门"作为**结构化数据** +返回给宿主，宿主用自己的模型完成阅读和判断。Strict 才走下面 §5–§7 描述的 +执行主脊。**看到 playbook、step、tool 这些词时，说的都是 Strict 侧。** + +### 1.2 三条数据流 + +``` +第一条 · 执行主脊（Strict / CLI，一个任务从左到右） + 终端串 → intent → TaskSpec → planner → Playbook → executor → steps → 产物 + (reuse>adapt>generate) (共享 state 字典) + +第二条 · 知识平面（正交叠加，每次 run 读入 / 写出） + profile briefing · skills · debug memory · repo_map + 读宽写窄：agent 只能提案 candidate，人类晋升 + +第三条 · MCP 服务端（宿主入口，非交互） + 宿主 → thin_mcp_server ─┬→ Direct：同步返回知识路由 + 治理契约（主脊不参与） + └→ Strict：预约 run_id → 隔离子进程 → 轮询取结果 +``` + +### 1.3 三条边界 + +贯穿全部三条流，不是流程的某一段： + +- **信任边界**——指令只来自终端输入（只有 `intent.py` 这一个入口）；抓取回来的 + diff / issue / CI 文本一律包在 `` 里，永远不会成为指令。 +- **权限边界**——`tier`（爆炸半径）由 `kind` 推导，不是字段。用户措辞和模型输出 + 都无法扩权。 +- **知识边界**——`knowledge/` 和 `adapters/` 是数据面；仓库、组件、模型的具体规则 + 不允许硬编码进 `src/`（由 `test_repo_neutral_core` 强制）。 + +### 1.4 规模 + +92 个 Python 文件 / ~19.5k 行（`src/`）· 9 个 playbook · 38 个注册 step · +457 页知识 · 59 个测试文件。目标仓库是 vLLM-Omni（adapter zero）。 + +--- + +## 2. 功能 + +### 2.1 四个宿主 Skill（Direct） + +| Skill | 做什么 | 边界 | +|---|---|---| +| `imreview` | 审 PR 或本地改动：固定 PR 快照 → 取知识路由 → 读源码 + 跑验证 → 带文件行号的结论 | 默认只在当前对话输出，不发 GitHub | +| `imdesign` | 写代码前的协同设计：问题、现状、方案、接口变化、实施步骤、验证计划、风险 | 不自动改代码 | +| `imcifix` | 从 issue 出发，本地复现 → 最小修复 → 针对性验证 | 不 commit / push / 开 PR / 发评论 | +| `imupdate` | 上游发版后，对比新旧版本，更新模型清单、registry、deploy、路径路由、source pin | 只改知识，不动目标仓库 | + +Codex 用 `$imreview`，Claude Code / Cursor 用 `/imreview`。 + +### 2.2 七种任务 kind（Strict / CLI） + +| kind | tier | 只读 | 做什么 | +|---|---|---|---| +| `pr_review` | L2 | ✅ | PR 评审 | +| `issue_answer` | L2 | ✅ | 起草 issue 回复 | +| `issue_filter` | L2 | ✅ | issue 批量分类路由 | +| `repo_profile` | L2 | ✍️ 知识 | 建立仓库 profile | +| `pr_debug` | L1 | ✍️ | CI 失败分组 → 修复 → 增量 push | +| `pr_rebase` | L1 | ✍️ | fork-aware checkout → rebase → 验证 → with-lease push | +| `repo_rebase` | L0 | ✍️ | 夜跑全量 rebase（委托给已有 5 阶段编排器） | + +tier 语义：**L0** 只能原样复用 locked playbook；**L1** 可用已审核 playbook 改编； +**L2** 允许现场生成计划——而 generate 路径**结构上**不可能包含 write/push step。 + +### 2.3 质量与成本机制 + +- **多 lens ensemble**——评审扇出成多个视角（logic / behavior / contracts…）， + 逐条 keep/drop/dup 裁决后确定性合并；零产出的 lens 单独重问一次而不是整组重跑。 +- **审查深度自适应**——确定性规则先裁明确案例（小而低风险 → light 单遍；大或 + 高风险路径 → full 全 ensemble），只有灰区花一次小 LLM 调用，且该调用**只能选 + standard 或 full**：PR 内容说服不了 planner 降级。 +- **双路径 mode**——`eco`（默认，便宜模型）/ `performance`（显式要求才升级）， + 与 `tier` 正交：便宜模型永不放大任务权限。 +- **五种后端**——`api`（Anthropic/OpenAI 兼容）+ 四个 harness：`cursor`、 + `claude-code`、`codex`（订阅制）、`deepseek`（dsh，唯一 API-keyed 的 harness）。 + harness 会话通过 MCP tool bridge 拿回 copilot 自己的工具，每次调用仍过 + `tools.dispatch`。 +- **MoA**（Mixture-of-Agents）——lens/draft 提案可跑在异构成员上，预算在代码里 + 硬封顶（先原子 `reserve()` 保守上界，结算时替换实际用量）。 + +--- + +## 3. 使用指南（操作者） + +### 3.1 安装 + +**作为宿主 Skill（Direct，最常用）：** + +```bash +git clone git@github.com:JiusiServe/InferMatrixCopilot.git +cd InferMatrixCopilot + +./install-mcp.sh # macOS / Linux +./install-mcp.sh --repo-path /path/to/vllm-omni +install.cmd --repo-path D:\path\to\vllm-omni # Windows +``` + +安装器识别本机的 Codex / Claude Code / Cursor，注册 MCP 与四个 Skill，并创建 +`~/.infermatrix-copilot/.env`。未识别到已知 Agent 时生成标准 +`infermatrix-copilot.mcp.json` 供其他 MCP 客户端导入。装完**重启 Agent**。 + +**作为独立 CLI（Strict）：** + +```bash +bash install.sh # 建 venv + editable 安装 + ./infermatrix-copilot 包装器 + 跑 doctor +bash install.sh --uninstall # 只删自己创建的东西，不碰 .env +``` + +### 3.2 配置 + +密钥、机器路径、本地 checkout 只放 `~/.infermatrix-copilot/.env`，**不要提交**。 + +```bash +# 模型后端（Direct 不需要；Strict 必需其一） +ANTHROPIC_API_KEY=... # 或 OPENAI_API_KEY +ANTHROPIC_BASE_URL= # 可选，代理或兼容网关 +LLM_PROVIDER=anthropic # 两种 Key 都填时才需要指定 +STRICT_BACKEND= # 空=api；或 cursor / claude-code / codex / deepseek + +# 模型分层 +AGENT_MODEL=claude-sonnet-5 # 默认推理模型 +ECO_MODEL= / PERFORMANCE_MODEL= # 双路径两档（各有独立 BASE_URL / API_KEY） +REVIEWER_MODEL= / INTENT_MODEL= # patch reviewer / 意图分类 + +# 目标仓库 +REPO_PATHS={"vllm-omni": "/path/to/vllm-omni"} # JSON：仓库名 → 本地路径 +DEFAULT_REPO=vllm-omni +MCP_REPO_ALLOWLIST= # MCP 侧允许的仓库集 + +# 写出闸门——默认全关，想要外部写入时才手动打开 +ALLOW_PUSH=0 # 1 = 允许 git push（仍只 --force-with-lease，永不推保护分支） +ALLOW_POST=0 # 1 = 允许发 PR 评论 / issue 回复 +``` + +配置不完整时，`review` 会直接返回缺少的项目，**不会**启动一个注定失败的后台任务。 + +### 3.3 确认装好了 + +宿主的 MCP 页面里应能看到 `infermatrix-copilot` 已连接；Codex 也可以 +`codex mcp list`。CLI 侧： + +```bash +infermatrix-copilot doctor # 逐项 ✓/✗，失败即给出唯一修复命令 +infermatrix-copilot doctor --json # 供 CI +infermatrix-copilot doctor --probe # 每档模型 1-token 实探（唯一付费检查） +``` + +doctor 只读、只打印密钥名不打印值。 + +### 3.4 Direct 怎么用 + +```text +$imreview https://github.com/vllm-project/vllm-omni/pull/5172 # Codex +/imreview # 当前 PR / 本地改动 +/imupdate vllm-omni # 也接受路径、别名、URL +``` + +宿主没主动调用时，直接说："Use InferMatrixCopilot in Direct mode to review this PR." + +一次 `imreview` 的过程：固定 PR 快照并**在 60 秒内**先报 head SHA / CI / +mergeability → 调用 `review(mode="direct")` → 拿到 ≤3 条知识路由（含内嵌 +`quick_map`，3.5k 封顶）+ `execution_budget`（硬顶）+ checklist → 并行读源码、 +跑验证 → `validate_direct_review` 完成门 → 输出**单条**合并结论。 + +要跑 Strict 就明说："Use InferMatrixCopilot in Strict mode to review …"， +拿到 `run_id` 后用 `get_review_status` / `get_review_result` 轮询。 + +### 3.5 CLI 怎么用 + +```bash +infermatrix-copilot # 默认进对话式 REPL +infermatrix-copilot -p "review pr 4830" # 一次性命令 +infermatrix-copilot -p "review pr 4830" --plan-only # 只看计划不执行 +infermatrix-copilot -p "profile the repo" --yes # 跳过确认（headless） +infermatrix-copilot --resume # 从首个未完成 step 重入 +infermatrix-copilot --playbook profile-consolidate --yes # 直接点名 playbook（含 candidate） +infermatrix-copilot --playbook pr-debug --report-only --task-param limit=5 +infermatrix-copilot -p "review pr 4830" --performance # 升到高性能模型档 +infermatrix-copilot --no-chat # 用朴素命令 REPL 代替对话界面 +``` + +REPL 内置 `/status`、`/logs`、`/playbooks`。复合命令（"… then review it"）会被切分成 +有序队列并延续 PR/issue 引用。 + +**写操作一律需要确认**：`confirm_required` 由 kind 推导，只读 kind 之外都要过 +确认门；`--yes` 才跳过。 + +### 3.6 产物在哪 + +``` +~/.infermatrix-copilot/runs/run--/ + RUN_REPORT.md 每个交付物恰好一次 + checkout 注记 + 待裁决 skill candidate + DIAGNOSTICS.md 逐 step 诊断 + run_trace.jsonl 仅追加的事实记录 + progress.json step 检查点（--resume 的依据） + metrics.json CATQ 成本/质量指标 + ESCALATION.md 仅在被阻塞时出现 +~/.infermatrix-copilot/worktrees/-pr/ PR 评审用的 PR-time checkout +adapters//profile/ 仓库 profile +``` + +### 3.7 安全默认值一览 + +| 行为 | 默认 | 打开它需要 | +|---|---|---| +| git push | dry-run | `ALLOW_PUSH=1` **且** PushPolicy 允许 **且** 非保护分支 | +| 发 PR 评论 / issue 回复 | 关 | `ALLOW_POST=1` **且** 调用显式传 `post=true` | +| force push | 仅 `--force-with-lease` | 无法放宽 | +| 脏工作树 | 拒绝启动 | 先清理 | +| 未知仓库 | 生成 DRAFT adapter 后停下 | 人工审核后激活 | +| 高风险 manifest 段（`repo`/`upstream`/`push`） | agent 写入被拒 | 只能人工改 | + +--- + +## 4. 开发指南（维护者） + +### 4.1 代码地图 + +```text +src/infermatrix_copilot/ + intent.py task_spec.py 终端串 → 受治理的 TaskSpec + cli/ entry / copilot（编排核）/ doctor / utils + chat.py ui.py 对话式前门与终端 chrome + engine/ + planner.py reuse > adapt > generate + 能力匹配 + executor.py ★ 共享 state、检查点/resume、扇出、typed failure + step.py registry.py step 类型词汇与"名字→handler"解析 + steps/ 38 个 step 的实现（@step 自注册） + agent_runtime/ runner / dispatch / ensemble / moa / knowledge / utils + providers/ ★ 五种后端的统一注册表（api + 四个 harness） + tool_bridge.py harness 会话拿回 copilot 工具的 MCP 桥 + tools.py scopes.py 原子能力与权限（每次调用的门） + push.py ★ 全库唯一 push choke point + review/ 条件式 Patch Review + 审查深度 planner + profiles/ memory/ 知识平面：profile store / skills / debug memory + thin_mcp_server.py ★ 默认 MCP：Direct 路由 + Strict 入口 + mcp_server.py run_status.py Strict 后台任务、状态、结果 + mcp_policy.py MCP 安全闸（边界 + 子进程各跑一次） + llm.py config.py tracing.py 后端封装 / Settings / span 树 +playbooks/*.yaml 工作流定义（数据，不是代码） +adapters// manifest（人工门禁）+ profile（证据门禁） +knowledge/ 通用 / 仓库 / 组件 / 模型知识（457 页） +doc/architecture/SPEC/ 与 src 文件一一对应的"不能破坏什么" +test/ 59 个离线测试文件 +``` + +### 4.2 会静默损坏的四条不变量 + +改代码前先记住这四条——它们坏掉时不会报错，只会让行为悄悄退化。 + +1. **`state_updates` 契约**（`engine/executor.py`）。step 之间只通过一个共享 + `state` 字典交接。**任何被后续 step 消费的键，必须经 `outputs.state_updates` + 发布**，否则 `--resume` 会在恢复时丢掉它——表现为莫名其妙的 BLOCKED（缺 + `diff_text`）或 FORBIDDEN（PushPolicy 退回默认的 deny-all）。扇出的 updates 由 + `_merge` 提升到顶层；`when:` 条件遇未知键**大声阻塞**而不是静默跳过。 +2. **`tier` 是推导出来的**（`task_spec.py`）。它是 `kind` 的 property，不是字段。 + 不要为了方便加一个可写的 tier/权限字段——那等于把提权入口交给自然语言。 +3. **`guard_push` 是唯一出口**（`push.py`）。所有对外写入必须经过它， + `engine/steps/pr/publish.py` 是它唯一的调用点。新增任何 push 能力都要走这里。 +4. **核心必须仓库中立**。`src/` 里不允许出现仓库名、模块名、领域 prompt、绝对 + 路径——这些只能住在 `adapters//` 或 `knowledge/`。核心 prompt 可以说 + *怎么审查*，只有 profile 能说*这个仓库是什么*。`test_repo_neutral_core` 用一份 + 只能变短的泄漏清单封顶。 + +### 4.3 常见改动从哪里开始 + +| 要改什么 | 主要位置 | 最少验证 | +|---|---|---| +| Direct 返回内容或路由 | `thin_mcp_server.py`、Direct Skill | `test_thin_mcp_server.py`、`test_imreview_output_contract.py` | +| Strict 启动 / 轮询 / 发布门禁 | `mcp_server.py`、`mcp_policy.py` | `test_mcp.py` | +| 模型、Key、Base URL | `config.py`、`llm.py` | `test_llm_providers.py`、`test_tier_split.py` | +| 新增后端 | `providers/`（注册表 + transport） | `test_providers.py` + 对应 `test_provider_*.py` | +| 工作流步骤 | `playbooks/*.yaml`、`engine/steps/` | 对应 step 测试 + playbook 加载测试 | +| 新仓库支持 | `adapters//`、`knowledge/repos//` | adapter 测试 + 知识检查 | +| 评审质量 | `engine/steps/review/`、`review/planner.py` | `test_review_*.py`、`test_agent_ensemble.py` | +| 安装或发布包 | `scripts/install_mcp.py`、`pyproject.toml` | installer 测试 + wheel 内容检查 | + +**加一个 step**：在 `engine/steps/*.py` 写 handler，用 +`@step(name, kind, risk, description)` 装饰（自注册，不用改中央注册表）， +把被消费的键经 `state_updates` 发布，补一个护栏测试。 + +**加一个 task kind**：`task_spec.py`（kind + tier）→ 写 playbook yaml → +intent hint → chat enum。planner / executor 不动。 + +**加仓库知识**：用类型化 patch op 写 profile fact，或加一个 skill。三条注入面 +都有硬上限（briefing 350 词 / 每个 SKILL.md body 1,500 字符 / `review.md` +4,000 字符），超出**静默丢弃**——改完先量字符数。 + +### 4.4 本地开发与验证 + +```bash +python -m pip install -e ".[dev,mcp]" ruff build +PYTHONPATH=src python -m pytest # 59 个测试文件 +python -m ruff check . +git diff --check +``` + +改了知识文件额外跑： + +```bash +python knowledge/tools/check_knowledge_tree.py +python knowledge/tools/check_wiki_lint.py +``` + +改了发布包：`python -m build --wheel`，并确认 wheel 里含 `knowledge/`、 +`_runtime/playbooks/`、`_runtime/adapters/`、`_runtime/skills/`——只验证源码 +checkout 证明不了这件事（`test_packaged_runtime.py` 只断言 pyproject 的**文本**）。 + +CI（`.github/workflows/test.yml`）跑上面全部检查，外加把 wheel 装进干净 venv 验证 +四棵数据树确实随包发布。**注意：CI 目前只是信号，不是合并门禁**——要让它真正挡住 +合并需要在仓库 Settings 里把 `suite` job 设为 required status check。 + +### 4.5 护栏测试：你不能破坏的行为 + +| 测试 | 固定的不变量 | +|---|---| +| `test_v2_p0.py::test_repo_neutral_core` | `src/` 仓库字面量被泄漏清单封顶 | +| `test_v2_p0.py::test_resume_restores_state_handoffs` | `state_updates` 契约 | +| `test_capabilities.py` | 能力匹配；locked rebase 不泄漏 | +| `test_profile_store.py::test_stability_gate_and_history` | stable fact 永不丢证据 | +| `test_profile_steps.py::test_profile_agent_applies_gated_facts` | 无证据的 fact 被拒 | +| `test_p3_machinery.py::test_judge_reports_but_never_mutates` | profile judge 只读 | +| `test_agent_runtime.py`（参数化 dispatch） | `kind=="agent"` ⇒ 统一运行时治理 | +| `test_agent_runtime.py::test_unparseable_after_repair_is_salvaged_as_escalation` | 非空失败答复被抢救 | +| `test_agent_loop.py::test_final_round_nudge_follows_tool_results` | 收尾提示排在 tool_result 之后 | +| `test_agent_ensemble.py::test_render_verdict_calibration` | 仅已验证的 blocker/major 阻塞 | +| `test_agent_ensemble.py::test_zero_yield_lens_gets_one_retry` | 零产出 lens 单独重问 | +| `test_moa.py` | MoA 预算台账封顶；密钥永不落 trace | +| `test_review_step.py::test_review_salvaged_when_agent_escalates_with_comments` | 带 comments 的 escalate 被抢救 | +| `test_pr_steps.py::test_fetch_diff_pins_pr_time_checkout` | 评审树 pin 到 PR head | +| `test_memory.py::test_skill_touch_increments_usage` | 用量先验累计 / debug memory 写入 | +| `test_push_and_steps.py` | `guard_push` 语义 | +| `test_mcp.py` | 篡改防御、单写者对账、分页、只读工具集 | + +### 4.6 知识树的编辑规矩 + +增删移任何知识页之前，先读 `doc/knowledge/CONTRIBUTING.md`，再读**恰好一个**匹配的 +贡献主题（`doc/archive/PLAN-knowledge-reorg.md` 是已归档历史，不是现行契约）。要点： + +- 每条结论按**用途 + 代码 owner** 双重路由；PR 学习只能产出可执行规则，写进最近 + owner 的 `rules.md`。 +- 原始 PR 材料是仓库外的临时输入，规则批次验证通过后**必须删除**。 +- 不要新增 PR case / history / result / incident 页，也不要把异构 owner 合并成一个 + 大杂烩页面。现存的 `incidents/`、`history/`、`results/` 是遗留物，不是新知识的 + 合法去处。 +- 评测生成的用例、隐藏标签、预测、判分、run 报告属于 `eval/`，不属于产品知识树。 +- 每个现行 rules 页都要更新最近的 `_index.md`。 +- 整批改完后跑两个 validator（见 §4.4）。 + +--- + +## 5. Playbooks + +Playbook 是**数据**（`playbooks/*.yaml`），不是代码。`PlaybookStore.find(kind, repo, +capabilities)` 负责召回：精确 repo 优先；`repos: []` 的仓库无关 playbook 仅当 +`requires ⊆ capabilities` 才匹配；**candidate 永不被自动召回**（只能 +`--playbook ` 点名）。 + +### 5.1 全部 9 个 + +| playbook | ver | status | kind | repos | requires | +|---|---|---|---|---|---| +| `pr-review` | 6 | active | `pr_review` | 任意 | `repo.path` | +| `pr-debug` | 2 | active | `pr_debug` | 任意 | `repo.path` | +| `pr-rebase` | 2 | active | `pr_rebase` | 任意 | `repo.path` | +| `issue-answer` | 2 | active | `issue_answer` | 任意 | `repo.path` | +| `issue-triage` | 2 | active | `issue_filter` | 任意 | `repo.path` | +| `repo-profile` | 1 | active | `repo_profile` | 任意 | — | +| `repo-rebase` | 2 | **locked** | `repo_rebase` | vllm-omni | `orchestrator.external` | +| `repo-rebase-native` | 1 | candidate | `repo_rebase` | vllm-omni | `orchestrator.external` | +| `profile-consolidate` | 1 | candidate | `repo_profile` | 任意 | `repo.path` | + +### 5.2 各自的 step 链 + +**`pr-review`** — 评审主线。`agent.review_diff` 内部才是多 lens ensemble。 + +``` +pr.fetch_diff → pr.gate_check → agent.review_diff → [post] pr.post_review → report.final_summary +``` + +**`pr-debug`** — `report_only` 时停在分组，成为独立可用的只读 triage。 + +``` +workspace.guard_clean → pr.checkout_branch → pr.fetch_ci_failures → pr.group_failures + → [not report_only] agent.debug_group (foreach failure_groups) + → [not report_only] review.patch_gate → [not report_only] ci.push → report.final_summary +``` + +**`pr-rebase`** — push 只针对 PR head 分支，force 仅 with-lease。 + +``` +workspace.guard_clean → pr.checkout_branch → pr.rebase_onto_base → pr.analyze_diff + → agent.verify_module (foreach affected_modules) + → [not report_only] review.patch_gate → [not report_only] ci.push → report.final_summary +``` + +**`issue-answer`** / **`issue-triage`** + +``` +issue.fetch → agent.draft_issue_answer → [post] issue.post_answer → report.final_summary +issue.fetch → agent.triage_issues → report.final_summary +``` + +**`repo-profile`** — Stage 0–1.5，对目标仓库只读，只写 `adapters//`，产出是 DRAFT。 + +``` +profile.fingerprint → profile.structure_scan → profile.ingest_docs + → [not report_only] agent.profile_repo → report.final_summary +``` + +**`profile-consolidate`** — Stage 4，**设计上就是 candidate**：对 planner 不可见， +只能显式或定时触发，让"重写"留在有节奏的独立层里（个人 agent 的教训：连续重写会 +腐蚀记忆）。 + +``` +profile.fingerprint → profile.detect_drift → profile.decay_stale + → [not report_only] agent.profile_consolidate → profile.judge → report.final_summary +``` + +**`repo-rebase`**（locked）— 夜跑全量 rebase，L0 原样复用，永不改编或重新生成。 +它委托给已有的 5 阶段编排器，**零回归**：这个仓库只在成熟流水线*外面*加编排。 + +``` +workspace.guard_clean → rebase.run_external → report.final_summary +``` + +**`repo-rebase-native`**（candidate）— 上面那条的原生分解版：包装父仓库自己的函数 +而不是重写，用于并排验证后再晋升。 + +``` +workspace.guard_clean → rebase.prelude → rebase.phase1 → rebase.phase2_prepare + → rebase.module_rebase (foreach wave1_modules) → rebase.module_rebase (foreach wave2_modules) + → rebase.phase2_finalize → review.patch_gate → rebase.phase3 → rebase.phase4 + → rebase.phase5 → rebase.compare_with_locked → report.final_summary +``` + +### 5.3 生命周期 + +``` +candidate ──人工晋升──▶ active ──▶ locked 回滚 = git revert + ▲ │ + └──────────── retired ◀────────────┘ +``` + +- **locked** 拒绝改编（planner 抛错而不是悄悄改）。 +- **candidate** 对 `find()` 不可见，只能点名运行——晋升前的验证期就靠这个。 +- 每个 playbook 带 `provenance`（谁创建、来源），改编产生的新版本要经 plan review 门。 + +--- + +## 6. Steps + +38 个 step 在 `engine/steps/` 就地 `@step` 自注册（`rebase_native.py` / `issue.py` / +`pr/publish.py` 里有几个用 `register_step` 直接注册）。每个 step 声明两个属性： + +- **kind**——`deterministic`（17）· `agent`（8）· `script`（7）· `validation`（3）· + `report`（3）。`kind == "agent"` 是有语义的：它意味着这个 step 走统一的 + `run_agent_step` 运行时治理（dispatch context、证据围栏、scope、结构化输出契约、 + 完整 trace），由一个参数化测试钉死。 +- **risk**——`read` · `knowledge` · `write_workspace` · `push` · `report`。 + +失败是**值不是异常**：`StepResult.failure` 是六种 `FailureKind` 之一， +BLOCKED / ESCALATE / FORBIDDEN 会通知并停机，只有 RETRYABLE 会有界重试。 + +### 6.1 deterministic（17）— 不花模型调用 + +| step | risk | 做什么 | +|---|---|---| +| `workspace.guard_clean` | read | 脏工作树拒绝启动 | +| `analysis.diff_summary` | read | 廉价 diffstat + 越界/全量写标志 | +| `pr.fetch_diff` | read | 取 PR diff；内含 **PR-time checkout**（把评审树 pin 到 PR head） | +| `pr.gate_check` | read | draft / merge-state / failing-checks 门禁报告 | +| `pr.checkout_branch` | read | fork-aware 检出 PR head，推导 PushPolicy | +| `pr.analyze_diff` | read | 改动文件 → 受影响模块（adapter 映射） | +| `pr.fetch_ci_failures` | read | 收集失败的 check（可注入，离线可测） | +| `pr.group_failures` | read | 按根因签名分组；超硬上限则升级 | +| `issue.fetch` | read | 取 issue | +| `profile.fingerprint` | knowledge | Stage 0 指纹；未知仓库产出 DRAFT adapter 后停下 | +| `profile.structure_scan` | knowledge | Stage 0 模块草稿（**永不覆盖**已声明模块） | +| `profile.ingest_docs` | knowledge | 摄入 AGENTS.md/CLAUDE.md 式人工指令（文档冗余行丢弃） | +| `profile.detect_drift` | read | Stage 4 漂移报告——刷新材料，绝不自动修 | +| `profile.decay_stale` | knowledge | Stage 4 休眠衰减：未确认的 fact 转 stale（排除但**不删**） | +| `rebase.prelude` | read | 父运行时设置 + wave 列表 | +| `rebase.phase2_prepare` | read | 预检 curator + phase-2 进度初始化 | +| `rebase.phase2_finalize` | read | 两个 wave 完成后推进父标记 | + +### 6.2 agent（8）— 走统一运行时治理 + +| step | risk | 做什么 | +|---|---|---| +| `agent.review_diff` | read | 证据锚定的两阶段评审：工具循环调查草稿 → verify-and-rewrite 编辑轮 | +| `agent.draft_issue_answer` | read | 起草 issue 回复（含 close/keep-open/duplicate-of-#N 处置槽） | +| `agent.triage_issues` | read | 批量分类路由 | +| `agent.debug_group` | write_workspace | 修一个失败组的根因并提交；成功后写回 debug memory | +| `pr.rebase_onto_base` | write_workspace | rebase 到最新 base；冲突交给受治理 agent 或 abort+升级 | +| `agent.profile_repo` | knowledge | Stage 1：推导非显然、**必须引证据**的 profile fact | +| `agent.profile_consolidate` | knowledge | Stage 4：**唯一**允许 rewrite/merge 的层，强制稳定性门禁 | +| `profile.judge` | read | Stage 4 只读审计 → JUDGE_REPORT.md，findings 只呈现不自动应用 | + +### 6.3 script（7）— 委托或对外写 + +| step | risk | 做什么 | +|---|---|---| +| `ci.push` | **push** | 受门禁的 push（PushPolicy ∧ 保护分支双闸，默认 dry-run） | +| `pr.post_review` | **push** | 单条 GitHub review + inline threads（显式 post 标志 + `ALLOW_POST`） | +| `issue.post_answer` | **push** | 发布 issue 回复（同样双闸） | +| `rebase.phase4` | **push** | 父 Phase 4（push + Buildkite CI），走 copilot 的 push guard | +| `rebase.run_external` | write_workspace | 委托给已有 5 阶段编排器（locked 流水线） | +| `rebase.phase1` | write_workspace | 父 Phase 1（init） | +| `rebase.module_rebase` | write_workspace | 单模块 rebase，委托父仓库自己的 `node_rebase_module` | + +> 全库只有这 4 个 `risk == "push"` 的 step，全部经 `push.py::guard_push`。 + +### 6.4 validation（3）与 report（3） + +| step | kind | risk | 做什么 | +|---|---|---|---| +| `review.patch_gate` | validation | read | 条件式 patch review，push 前 fail-closed | +| `agent.verify_module` | validation | read | 逐模块 rebase 损伤检查——纯 LLM 建议，**不是**受治理 agent step | +| `rebase.phase3` | validation | write_workspace | 父 Phase 3（本地流水线测试 + SDK debug 循环） | +| `report.final_summary` | report | report | 写 `RUN_REPORT.md` + `DIAGNOSTICS.md` | +| `rebase.phase5` | report | report | 父 Phase 5 最终摘要 + 运行后 curator | +| `rebase.compare_with_locked` | report | report | `COMPARISON.md`：原生 run vs locked 基线（晋升证据） | + +--- + +## 7. Tools + +"工具"在这个仓库里指**三个不同的面**，混为一谈正是 Direct/Strict 混淆的起点。 + +### 7.1 内置 agent 工具（6 个，沙箱内） + +给 Strict 侧 agent step 用，定义在 `tools.py::TOOLS`。 + +| 工具 | 说明 | +|---|---| +| `read_file` | **窗口化**读取（每次 48k 字符，用 `offset` 翻页） | +| `write_file` | 写/覆盖文件 | +| `edit_file` | 精确一次匹配的文本替换 | +| `list_dir` | 列目录 | +| `grep` | 递归搜索。pattern **默认按字面量匹配**，要正则得传 `regex:true` | +| `run_shell` | 执行命令，stdout/stderr 各截断返回 | + +**一条铁律：每一次工具调用都过 `tools.dispatch`**，检查 `ToolScope`（允许哪些工具） +和 `PathScope`（允许写哪些路径）并记 trace，三种结果——允许 / 拒绝 / 越界但记录。 +agent 永远看不到 scope 不允许的工具（`tool_definitions_for` 按 scope 过滤）。 + +> `read_file` 为什么必须窗口化：整文件读会吹爆会话历史、倍增未缓存 token、把对话 +> 推出可靠缓存长度。这不是保守，是实测过的成本项。 + +### 7.2 Step 提供的工具（按需注入） + +由具体 step 在 dispatch 时追加，已被该 step 审核过： + +- **只读 gh**——`gh_pr_view`、`gh_issue_view`、`gh_issue_timeline`、`gh_ci_read` +- **知识**——`skill_search`、`memory_search`、`skill_update_candidate` + （**只能提案 candidate**，人类晋升） +- **结构**——`repo_map`（按查询排序、有字符预算的符号索引；由 agent 主动拉取， + 绝不散文注入） +- **考古**（v14 新增，只读）——`diff_stat`、`file_at_base`、`show_commit`、 + `search_history`、`calc` + +harness 后端通过 `tool_bridge.py` 以 MCP 形式拿到这些工具，每次调用仍然回到 +`tools.dispatch`——所以换后端不会绕过权限闸。跨进程的 candidate 写入是**故意 +没有开放**的。 + +### 7.3 MCP 工具（7 个，面向宿主） + +`thin_mcp_server.py` 暴露给 Claude Code / Codex / Cursor： + +| 工具 | 说明 | +|---|---| +| `review(target, repo, mode, post, review_depth, title, body, changed_files, repo_path)` | Direct 返回 ≤3 条知识路由 + 治理契约；Strict 返回 `run_id` | +| `validate_direct_review(subtraction_signal, subtraction?, minimality_proof?, final_comment_count, evidence_head_sha)` | Direct 最终输出前的**完成门** | +| `get_review_status(run_id)` | Strict 后台任务的步骤与进度 | +| `get_review_result(run_id, offset)` | 轮询 Strict 结果，按 `next_offset` 分页 | +| `update_knowledge(repo)` | 只返回知识贡献入口，**不是** `imupdate` 的发版审计器 | +| `doc_search(query, repo, limit)` | 搜索模型/组件知识 | +| `doc_read(path, repo, offset)` | 读取搜索到的知识页（24k/页） | + +两点值得注意： + +- `validate_direct_review` 是**机械的结构检查**——它验证形状（`none` 不得附带证据； + `triggered` 需要减法项或最小性证明；恰好 1 条最终评论；`evidence_head_sha` 必须是 + 本次固定的 head），它**不验证证据真伪**。server 管不住宿主的模型，所以把"该怎么 + 审"编码成结构化字段随返回值下发，再用完成门机械验收。 +- `mcp_policy.py` 在**边界**（server）和**子进程**（权威）各跑一次：kind 必须 ∈ + `READ_ONLY_KINDS`，`post` 硬置 False，repo 必须在 allowlist，未知 params 剥除。 + 允许集直接引用 `task_spec.py` 的 `READ_ONLY_KINDS`，永不与代码漂移。 + +--- + +## 8. 性能对比 + +> **数据日期 2026-08-17。** 后端矩阵是本仓库变动最快的数据（过去一周重写三次）， +> 引用前请以 `eval/dataset/results/model_comparison.md` 为准；本节的方法论和结论 +> 结构比具体数字更耐久。 + +### 8.1 一句话结论 + +**在 recall 上落后 CC + Opus 5 基线约五个 rubric 点，precision 打平，成本约为其 +三分之一。** 这个结论由预注册的 20 项测量给出，此前一次"优于基线"的宣称已被撤回。 + +### 8.2 主结论（v14→v16，预注册） + +判官 claude-sonnet-5，盲配对，3 replicate，对同一批冻结快照上的 pinned Opus 5 基线。 +Δ = arm − baseline，在**每条 verdict 内部配对**、按 item 聚类。 + +| 统计量 | 值 | 触发的决策规则 | +|---|---|---| +| **Δrecall** | **−.049 [−.097, −.001]** | CI 整体低于 0 → 缺口**真实存在**，停止宣称 recall 打平 | +| **Δprecision** | **−.000 [−.041, +.040]** | CI 含 0 → 打平 | + +成本 $0.97/item vs 基线 $3.09/item。 + +这个 recall 缺口是**边缘且集中**的：上界 −.001；20 个 item 里 6 个偏向 arm、4 个在 +±.02 以内，而单个 item（pr5978，−.37）贡献了约三分之一的缺口。按预注册的规则， +**没有跑第三个 wave**。 + +### 8.3 测量纪律（本节最值得保留的部分） + +这份报告的中期版本本来会宣称"在新 holdout 上 precision 高于基线"。该结论**被撤回**， +因为原始均值支撑不了它：三组 wave-4 judgment 对**同一批**基线评审打分，基线 recall +均值分别是 .335 / .338 / .416——±.08 的纯判官漂移，比所有被研究的效应都大。 + +三条推论，都超出这次战役本身： + +1. **绝不拿一组 judgment 里 arm 的均值去比另一组里 baseline 的均值。** 要在 verdict + 内部配对（同一次判官调用同时给两个候选打分，它的宽严自然抵消）。 +2. **同一个 item 的多个 replicate 不是独立观测。** 10 items × 3 replicates 对 CI 来说 + 是 n=10 而不是 n=30；当成 30 会把标准误低估约一倍。 +3. **花掉 holdout 之前先算功效。** 在实测的 item 级 sd（.103–.127）下，分辨 .05 的 + 差异需要约 32–38 个 item。wave 4 的 10-item 门根本无法回答它被用来回答的问题—— + 三个配置是对着一个测不出来的标准提交的。 + +配套落地的三道流程闸（2026-08-17，每道对应一次真实事故）：manifest 记录**解析后** +的设置而非 env 字符串（一个默认值在底下变了，15 项探针实际跑成三厂商混合而 artifact +里看不出来）；余额预检**拒绝启动**而不是跑到一半死（三天两次 402，空账户变成 rc=3 +桩再被判成零分）；混合门覆盖 MoA 路径（此前只检查 `REVIEW_LENS_BACKENDS`）。 + +### 8.4 两个受支持的配置 + +| 配置 | 特点 | 什么时候用 | +|---|---|---| +| **v15**（默认，全 DS） | 偏 precision，~$1/item | 走量的日常评审 | +| **v16**（`REVIEW_LENS_BACKENDS` 把 Fable 路由进 adversary + 第二轮） | 偏 recall，每 item 多两次 harness 会话 | 高风险 PR——漏掉缺陷的代价高于多一条噪声评论 | + +配置效应**在符号上可复现**，即使单个 delta 不显著：v15 在两次独立生成里都偏 +precision（+.029 / +.022）；v16 拿到新 split 上最好的 recall（.336），并且是唯一在 +流水线内部复现 GOLD 缺口捕获的配置。 + +### 8.5 后端矩阵（2026-08-17） + +五个后端，一张注册表（`providers/registry.py`）。`api` 是原有路径被收编进同一注册表 +的结果——用 `api` 时行为逐字节不变。 + +| 后端 | kind | 凭据 | 能力 | 状态 | +|---|---|---|---|---| +| `api` | api | `.env` 里的 Key | 完整（我们自己的工具循环） | 基线路径，behavior 平价棘轮 | +| `cursor` | harness | 订阅（cursor-agent CLI） | `mcp_tools` `usage_reporting` | 已实测 | +| `claude-code` | harness | 订阅（claude CLI） | 五项能力最全（含 `builtin_tools_off` `max_turns` `cost_reporting`） | 订阅认证下实测 | +| `codex` | harness | 订阅（codex CLI） | `mcp_tools` `sandbox_read_only` `usage_reporting` | 仅离线测试（开发机无 ChatGPT 登录） | +| `deepseek`（dsh） | harness | **API Key**（唯一 api-keyed 的 harness，走 SDK 无 PATH 二进制） | `mcp_tools` `sandbox_read_only` `system_prompt` | 最新加入 | + +**同一批 10 个 train item 上的后端冠军是 api/DeepSeek 核心，不是 Composer：** + +| 配置 | Δrecall [95% CI] | Δprecision [95% CI] | win share | +|---|---|---|---| +| v17ds train（api/DeepSeek，10 项） | **+.024 [−.030, +.077]** | **+.016 [−.061, +.093]** | .47 | +| v17ds pooled（15 项） | −.004 [−.056, +.049] | −.008 [−.078, +.062] | .38 | +| v17cb train（Composer 2.5 / cursor） | −.052 [−.165, +.061] | −.097 [−.215, +.021] | .20 | +| v17cb val（Composer 2.5 / cursor，5 项） | −.003 [−.222, +.215] | −.040 [−.322, +.242] | .40 | + +val 一度把 Composer 排在前面，那是小样本幻觉：n=5、区间 ±.22，而 train 的 item 方差 +紧四倍（sd .075 api vs .158 cursor）。Composer 的分散来自大摆动（pr4970 +.317 对 +pr4923 −.233），api arm 则在十项里有七项落在 ±.06 内。 + +**读这张表时必须知道的三件事：** + +- 这两个 split 已经被整个战役迭代过，是 probe 不是 gate，**不能支撑关于新数据的 + 结论**；每个 CI 都跨零，结果是"n=15 下测量精度内的打平"，不是"更优"。 +- val（−.059）与 train（+.024）**符号相反**，pooled 的 −.004 是在平均一个真实的 + split 差异；val 的缺口几乎由 `pr4893` 一项承担（DeepSeek −.250，Composer −.267）。 + 同一个 item 在两个不同模型下几乎同样失败，说明那是**流水线缺陷而非模型缺陷**。 +- 2026-08-16 曾报告"转换修复没能缩小 recall 缺口"（pooled −.050），该测量被一次 + 非预期的厂商混合污染；同样 10 项从 −.052（污染）变成 +.024（干净），摆动 +.076， + **原结论已撤回**。 + +### 8.6 成本与时延（wave-2 holdout，每 item） + +| arm | $/item | 墙钟 | +|---|---|---| +| CC + Opus 5 基线 | $3.58 | 547 s | +| DS v4-pro（api） | $0.78–0.79 | 1139–1322 s | +| MiMo-v2.5（api） | $0.43 | 1728 s | +| Composer-2.5（harness） | 订阅 | 177–397 s | +| Grok-4.5 / 4.6（harness） | 订阅 | 284–1111 s | +| MoA r1（混合） | $0.33 API + 订阅 | 1195 s | + +harness 后端快 2–4 倍且吃订阅，但每项都记到 `tok_out=0`——harness 会话不向 span +记账暴露 token，所以它们的成本优势是**假定的，不是测出来的**。这是刻意不伪造： +metrics 记 source 为 `subscription`，绝不编造 USD。 + +### 8.7 一个污染教训 + +grok 家族（4.5 和 4.6，Composer 从未出现）会在评审会话中**主动搜索并读取 copilot +自己的 `imreview` 评审方法论 skill**。相关 arm 已进污染台账并作废。跑 cursor 系 +评测前，三份 skill 副本必须移出 `$HOME` 并在事后恢复。 + +### 8.8 继任者从哪里开始 + +1. **缺口是一类 item，不是一个水平差。** 三分之一的 pooled 缺口来自单个 item； + wave-3 forensics 已命名了复发类别（test/gate 认知论、文档信息架构、CI 经济性）。 + 修这一类比再来一轮通用 duties 划算。 +2. **先给 gate 编预算，再改流水线。** 10 个 item 上测出来的任何东西在这个量级都 + 不可解释。 +3. **唯一没试过的杠杆是全程用强生成器。** Fable 占两个 seat 时产出了本战役最好的 + recall 和唯一一次流水线内 GOLD 缺口捕获；Fable 占满所有 seat 估算 ~$10–20/item + （成本故事反转），已被明确 scope 并**刻意没跑**。 + +--- + +*本文档覆盖到 2026-08-17 的 `rfc/strict-review-deep-engine` 分支状态。 +数字类内容以 `eval/dataset/results/` 与 `doc/evaluation/EVAL-*.md` 为准。* diff --git a/doc/MCP.md b/doc/MCP.md deleted file mode 100644 index c48b1de26..000000000 --- a/doc/MCP.md +++ /dev/null @@ -1,93 +0,0 @@ -# 安装 InferMatrixCopilot - -InferMatrixCopilot 的核心发布物保持一份： - -- `plugins/infermatrix-copilot/.mcp.json`：标准 stdio MCP 描述； -- `plugins/infermatrix-copilot/skills/`：开放 Agent Skills； -- Claude、Codex、Cursor 的市场文件只负责把同一个插件展示出来，不包含各自的安装逻辑。 - -## 私有仓库过渡安装 - -```text -git clone git@github.com:JiusiServe/InferMatrixCopilot.git -cd InferMatrixCopilot - -# Windows cmd、PowerShell 或双击 -install.cmd -install.cmd --repo-path D:\path\to\vllm-omni - -# macOS / Linux -./install-mcp.sh -./install-mcp.sh --repo-path /path/to/vllm-omni -``` - -两个入口共用 `scripts/install_mcp.py`，但用户不需要安装或直接运行 Python： -入口会自动安装 `uv`，再由 `uv` 提供隔离运行环境。安装器会： - -- 自动识别 Codex、Claude Code 和 Cursor，检测到几个就安装几个； -- 保留 Cursor 已有配置并先备份； -- 没识别到已知 Agent 时生成 `infermatrix-copilot.mcp.json`，不会直接失败。 -- 创建 `~/.infermatrix-copilot/.env` 作为 Strict 的稳定配置入口；安装时传入 - `--repo-path` 会写入本地 vLLM-Omni checkout。模型密钥不会自动复制，使用 - Strict 前在该文件填写 `ANTHROPIC_API_KEY` 或 `OPENAI_API_KEY`。两者分别支持 - 可选的 `ANTHROPIC_BASE_URL` 和 `OPENAI_BASE_URL`；只填一种 Key 时自动选择， - 两种都填时用 `LLM_PROVIDER` 指定。 - -## 公共市场上线后 - -在 Agent 的插件市场搜索 `infermatrix-copilot` 并点击安装，然后直接运行： - -```text -# Codex -$imreview -$imupdate - -# Claude Code / Cursor -/imreview -/imupdate -``` - -市场只安装同一份 MCP 与 Skill；不会把逐 Agent 逻辑带回核心包。 - -## 其他 MCP Agent - -安装器生成的 `infermatrix-copilot.mcp.json` 与 -[`plugins/infermatrix-copilot/.mcp.json`](../plugins/infermatrix-copilot/.mcp.json) -内容相同，可直接导入支持 stdio MCP 的客户端。开放 Skill 位于： - -```text -plugins/infermatrix-copilot/skills/imreview/SKILL.md -plugins/infermatrix-copilot/skills/imupdate/SKILL.md -``` - -核心 MCP 配置： - -```json -{ - "mcpServers": { - "infermatrix-copilot": { - "type": "stdio", - "command": "uvx", - "args": [ - "--from", - "infermatrix-copilot[mcp] @ git+https://github.com/JiusiServe/InferMatrixCopilot.git@main", - "infermatrix-copilot-mcp" - ] - } - } -} -``` - -这里用 `uvx` 自动拉取并缓存隔离运行环境。宿主如果能消费 MCP Registry，则应优先 -使用 Registry 条目；项目不为每个新 Agent 增加安装分支。 - -## 使用 - -```text -# Codex -$imreview https://github.com/vllm-project/vllm-omni/pull/5172 -$imupdate D:\path\to\vllm-omni -$imupdate vllmomni -``` - -默认 Direct 模式只提供知识库，由 Agent 当前模型完成审查；不会自动评论或推送代码。 diff --git a/doc/PLAN-doc-refactor.md b/doc/PLAN-doc-refactor.md new file mode 100644 index 000000000..4e36fb7ab --- /dev/null +++ b/doc/PLAN-doc-refactor.md @@ -0,0 +1,412 @@ +# Plan —— `doc/` 全量重构 + +> 状态：**已完成（P0–P6，2026-08-17）**。执行中发现并修正的偏差记录在各阶段下。 +> 决策已定（2026-08-17 grilling）：RFC 归入 `doc/features/`；使用指南新增 +> 后端介绍；**现有文件只能作为参考材料，不得直接沿用**——以代码为准重写； +> SPEC 采用**逐文件全量重写**。 + +--- + +## 1. 目标与非目标 + +**目标** + +1. 一个新人打开 `doc/` 能在 10 秒内知道该看哪一篇。 +2. 每条事实**只有一个归属**，其他文档只链接不复述。 +3. 文档内容与当前代码一致——不是"曾经正确"，是**核对过**。 +4. 让"再次腐烂"变成可检测的：加闸门，而不是靠自觉。 + +**非目标** + +- 不做 mkdocs 站点化（本轮不引入构建依赖）。 +- 不改 `knowledge/`（那是数据面，有自己的治理协议；本轮只修它对 `doc/` 的 + 两处路径引用）。 +- 不改 `eval/` 目录本身（评测产物留在原地，`doc/evaluation/` 只放叙述性报告）。 + +--- + +## 2. 现状诊断（实测，2026-08-17） + +`doc/` **204 个文件**、`docs/` **8 个文件**： + +| 分组 | 文件数 | 状态 | +|---|---|---| +| `doc/archive/reorg-audit/` | **127（62%）** | 7 月一次性迁移取证；**但被引用**——`knowledge/SCHEMA.md`、`knowledge/repos/vllm-omni/rebase/workflow.md`、`doc/architecture/KNOWLEDGE.md` 都把它当作溯源基线 | +| `doc/architecture/SPEC/` | 49 | **42 个内容页里 41 个落后于其源码，仅 1 个同步**；另有约 17 个模块**完全没有页面** | +| `doc/` 顶层 `.md` | 20 | 六种体裁混放，无任何分组signpost | +| `doc/contributing/knowledge-templates/` | 8 | 活跃，被 EXTENDING-KNOWLEDGE 与 knowledge-maintainer 引用 | +| `docs/`（小写） | 8 | 与 `doc/` 只差一个字母；内含三类东西：使用指南、示例、**配置模板** | + +**四个具体缺陷** + +1. **`doc/` 与 `docs/` 一字之差**，且 `docs/` 里混着 `codex/config.toml`、 + `cursor/mcp.json` 这类**根本不是文档**的配置模板。 +2. **文件名已经编码了体裁，目录树没有反映**：`RFC-`×4、`PLAN-`×2、`EVAL-`×4、 + `RESEARCH-`×1 与常驻参考文档平铺在一起。 +3. **SPEC 的空洞正好落在开发最活跃的地方**：整个 `providers/`（8 个文件）、 + 整个 MCP 面（`thin_mcp_server`/`mcp_server`/`mcp_policy`/`run_status`/ + `knowledge_docs`）、`tool_bridge`、`tracing`、`review/planner` 全无 SPEC 页。 + SPEC 描述的是 7 月的架构。 +4. **过期材料以现行姿态陈列**：`IMPLEMENTATION_STATUS.md` 写着 pr-review v4 / + 64 tests / 20 steps，实际是 **v6 / 430+ 用例 / 38 steps**； + `PLAN-mcp-plugin-and-community-merge.md` 自称 "PROPOSED rev 6，尚未执行"， + 停在 07-22。 + +**两个降低风险的事实** + +- `doc/` 和 `docs/` **都不进 wheel**（`pyproject.toml` 只 force-include + `knowledge` / `playbooks` / `adapters` / `skills`）——迁移不影响发布包。 +- `integrations/config-templates/codex/config.toml`、`integrations/config-templates/cursor/mcp.json` 只被散文引用，没有代码依赖。 + +**一个必须先处理的风险** + +代码里一共有 **29 处指向 `doc/` 的引用**，全部是散文形式、**没有任何 linter +检查**，改路径会静默失效： + +| 引用方 | 次数 | 指向 | +|---|---|---| +| `src/` | 20 | `RFC-provider-registry.md`×14、`DESIGN.md`×4、`CODE_TOUR.md`×1、`KNOWLEDGE.md`×1 | +| `test/` | 7 | `DESIGN.md`×4、`RFC-provider-registry.md`×3 | +| `skills/` | 2 | `DOCSTRING_STYLE.md`、`EXTENDING-KNOWLEDGE.md` | + +其中 `RFC-provider-registry.md` 一篇就占 17 次——而它恰好是本轮要改名并迁入 +`features/` 的文件。 + +--- + +## 3. 目标结构 + +```text +doc/ + README.md NEW —— 文档地图（一行一条，唯一索引） + GUIDE.md 总入口：概览/功能/使用/开发/playbook/step/tool/性能 + + guide/ 面向使用者 + hosts/ **宿主端**：copilot 跑在谁里面（Direct） + README.md NEW —— 宿主 vs 后端的消歧表（见下方警告） + codex.md ← doc/guide/hosts/codex.md（重写） + claude-code.md NEW —— 从 README 安装章节抽出 + cursor.md NEW —— 同上 + backends.md NEW —— **后端**：谁跑在 copilot 里面（Strict） + mcp.md ← doc/guide/mcp.md（重写） + autonomous-workflow.md ← doc/guide/autonomous-workflow.md（重写） + knowledge-maintainer.md ← doc/guide/knowledge-maintainer.md（重写） + samples/ ← doc/guide/samples/ 三篇（核对） + + features/ 面向"这个特性怎么回事" + provider-registry.md ← RFC-provider-registry（已实现） + strict-review-deep-engine.md ← RFC-strict-review-deep-engine（已实现） + review-recall.md ← RFC-review-recall-v14（已实现） + auto-run.md ← RFC-auto-run（**未实现**，状态头标 draft） + + architecture/ + CODE_TOUR.md 核对 + 补 providers/ 与 MCP 段 + DESIGN.md 重构：v1 / v2 / **v3（provider registry + MCP 面）** + KNOWLEDGE.md 核对 + SPEC/ **62 页逐文件全量重写** + + contributing/ + DOCSTRING_STYLE.md 核对 + EXTENDING-KNOWLEDGE.md 核对 + release-maintenance.md ← VLLM_OMNI_RELEASE_MAINTENANCE.md + knowledge-templates/ 原样保留（活跃引用） + + evaluation/ + README.md NEW —— 评测索引 + 当前结论一句话 + EVAL-goal-strict-vs-opus5.md 冻结记录（加日期头） + EVAL-v14-v16-recall-campaign.md + EVAL-goal-report.md + EVAL-PR20-report.md + RESEARCH-reference-agents.md + + archive/ 冻结，不再维护，全部加 superseded 头 + IMPLEMENTATION_STATUS.md + PLAN-knowledge-reorg.md + PLAN-mcp-plugin-and-community-merge.md + PLAN-doc-refactor.md （本文件，完成后归档） + reorg-audit/ ← 连同 2 处 knowledge 引用一并更新 + +integrations/config-templates/ ← 从 docs/ 迁出（不是文档，是模板） + codex/config.toml + cursor/mcp.json + +docs/ ← 目录取消 +``` + +--- + +### 3.1 必须消歧：宿主 ≠ 后端 + +`claude-code`、`codex`、`cursor` 这三个名字**同时是宿主和后端**，方向相反。 +不把它讲清楚，`guide/backends.md` 和 `guide/hosts/codex.md` 在读者眼里就是同 +一篇的两个标题。 + +| 名字 | 作为**宿主**（Direct：copilot 跑在它里面，用它的模型） | 作为**后端**（Strict：它被 copilot 拉起，跑一个 agent step） | +|---|---|---| +| Claude Code | ✅ `/imreview` | ✅ `STRICT_BACKEND=claude-code` | +| Codex | ✅ `$imreview` | ✅ `STRICT_BACKEND=codex` | +| Cursor | ✅ `/imreview` | ✅ `STRICT_BACKEND=cursor` | +| DeepSeek（dsh） | ✗ | ✅ `STRICT_BACKEND=deepseek` | +| api（Anthropic/OpenAI） | ✗ | ✅ 默认 | + +判据一句话：**宿主提供模型给你用，后端是 copilot 拿去用的模型。** +宿主端不需要 API Key（`doc/guide/hosts/codex.md` 原话："there is no API key or +model configuration"）；后端端要么要 Key，要么要订阅登录。 + +两篇文档各自的开头**必须**放这张表加一句 "你要找的可能是另一篇"，并互相链接。 + +## 4. 内容归属矩阵 + +重构最容易失败的方式，是把同一件事写进三个文档然后各自腐烂。本轮为每类事实 +指定**唯一归属**，其他文档只允许链接： + +| 事实类型 | 唯一归属 | 其他文档的义务 | +|---|---|---| +| 装什么、配什么、怎么跑 | `GUIDE §3` + `guide/*` | 只链接 | +| **宿主端**安装与用法（Direct） | `guide/hosts/.md` | `README` 只留一条最短路径 + 链接 | +| **后端**选择与配置（Strict） | `guide/backends.md` | `GUIDE §2.3` 只给一句话 + 链接 | +| kind / playbook / step / tool **清单** | `GUIDE §2/§5/§6/§7` | `CODE_TOUR` 只讲数据流，**不再列清单** | +| 数据如何流动、模块为何在此处 | `architecture/CODE_TOUR.md` | — | +| 为什么这样设计（含被否决的选项） | `architecture/DESIGN.md` | — | +| 单个源文件"不能破坏什么" | `architecture/SPEC/.md` | `GUIDE §4.2` 只留跨文件那四条 | +| 某特性怎么用/怎么配/实测如何 | `features/.md` | — | +| 测量数字与方法论 | `evaluation/*` | `GUIDE §8` 给结论摘要 + 链接 | +| 怎么写/维护知识库 | `knowledge/`（doc 侧，2026-08-18 合并） | `knowledge/` 树本身只留数据；规范由 doc 侧承载 | + +**判定规则**：写一段话之前问"这条事实的归属在哪"。如果不在你正在写的文档， +写一个链接。 + +--- + +## 5. 逐组处置 + +> 注：本表的"原路径"一列在 P1 执行时被路径扫描一并改写成了新位置——扫描器不区分 +> "引用某个文件"和"记录它搬家前的位置"。留着不改，因为它本身就是一条教训： +> 机械替换会顺手改掉历史记录，迁移计划这类文档应当写在被扫描的范围之外。 + +| 原路径（见上注） | 去向 | 动作 | +|---|---|---| +| `CODE_TOUR.md` | `architecture/` | 核对更新（08-12，主体仍准；补 providers/、MCP、清单去重） | +| `DESIGN.md` | `architecture/` | 重构（07-22，缺 v3 期全部内容） | +| `KNOWLEDGE.md` | `architecture/` | 核对 + 改 reorg-audit 新路径 | +| `MCP.md` | `guide/mcp.md` | 重写 | +| `GUIDE.md` | 原地（`doc/`） | 修链接以适配新树 | +| `DOCSTRING_STYLE.md` `EXTENDING-KNOWLEDGE.md` | `contributing/` | 核对 | +| `VLLM_OMNI_RELEASE_MAINTENANCE.md` | `contributing/release-maintenance.md` | 核对 | +| `knowledge-templates/` | `contributing/` | 原样移动 | +| `RFC-*.md` ×4 | `features/` | **重写为特性文档**（RFC 作为素材） | +| `EVAL-*.md` ×4、`RESEARCH-*` | `evaluation/` | 加日期/冻结头，不改内容 | +| `IMPLEMENTATION_STATUS.md` | `archive/` | 冻结（其"现状"职能由 GUIDE 承担） | +| `PLAN-*.md` ×2 | `archive/` | 冻结 | +| `SPEC/` | `architecture/SPEC/` | **62 页全量重写** | +| `reorg-audit/` | `archive/reorg-audit/` | 移动 + 修 2 处 knowledge 引用 | +| `doc/guide/autonomous-workflow.md` | `guide/` | 重写 | +| `doc/guide/knowledge-maintainer.md` | `guide/knowledge-maintainer.md` | 重写 | +| `doc/guide/samples/*` ×3 | `guide/samples/` | 核对 | +| `doc/guide/hosts/codex.md` | `guide/hosts/codex.md` | 重写（并加 §3.1 消歧表） | +| （无对应文件） | `guide/hosts/claude-code.md`、`cursor.md` | 新写：从 `README` 安装章节抽出 | +| `integrations/config-templates/codex/config.toml` `integrations/config-templates/cursor/mcp.json` | `integrations/config-templates/` | 移动（非文档） | + +--- + +## 6. 分阶段执行 + +每个阶段有**退出标准**；未达标不进入下一阶段。全程 `git mv` 保留历史。 + +### P0 —— 先造闸门，再动文件 ✅ 完成 + +没有工具就搬 200 个文件，等于把断链埋进去。 + +1. `tools/check_doc_links.py`：校验所有 `.md` 内部相对链接可解析。跳过围栏代码块 + 与 `<占位符>`；排除 `eval/dataset/`（生成的评测产物**引用的是目标仓库**的文档 + 路径，检查它们产生 250+ 个假阳性）和 `contributing/knowledge-templates/` + （模板的相对链接按"复制到 `knowledge/` 之后"书写，原地就该是断的）。 +2. `tools/check_doc_citations.py`：扫描代码中形如 `doc/...` 的引用，断言存在。 + 只匹配单数 `doc/`——复数 `docs/` 指的是**目标仓库**的文档树，匹配它全是假阳性。 + 另外解析 `ROOT / "docs" / "codex" / "README.md"` 这种**按段拼接**的路径：本次 + 迁移中恰好有一处这样的引用逃过了字符串替换并弄坏了一个测试。 + 合法的例外用行内 `doc-citation-exempt` 标注。 +3. `tools/check_spec_freshness.py`：**以页内 `verified-against:` 声明为准**， + 而不是 git 日期。发现于执行途中的关键缺陷：`git mv` 会把页面的最后提交日期 + 重置为迁移那一次，于是 45 个 7 月的页面在 P1 之后会**永远显示为最新**—— + 迁移会亲手废掉这个闸门。声明式标记不受改名、重排版和错字修复影响。 +4. 建目录骨架 + `doc/README.md` 索引。 + +**退出标准**（已达成）：三个工具在当前树上跑通并如实报出既有问题——links 与 +citations 现为绿；spec 报出 **45 页无声明 + 17 个模块无页面 = 62 页待写**。 +CI 已接入：links/citations 阻塞，SPEC 覆盖率**先只报告**（17 个空缺是已知在办 +事项，让 CI 常红没有意义），P5 收尾时改为阻塞并加 `--strict`。 + +### P1 —— 纯迁移（不改内容） ✅ 完成 + +`git mv` 全部文件到目标位置；修代码内引用；修 `knowledge/` 的 `reorg-audit` +路径；`docs/` 目录取消。 + +实际战果：**移动 ~210 个文件**，重写 **55 个文件**里的路径引用（含 `src/` 20 处、 +`test/` 7 处、`skills/` 2 处、CI workflow、`knowledge/` 4 处），再逐篇修正 5 个 +被移动文档内部的 **17 条相对链接**（`../` 深度变了，仓库根路径替换救不了它们）。 + +**退出标准**（已达成）：三个 doc 工具零断链零悬空引用；`docs/` 目录消失； +知识两个 validator 绿（0 错误）；`pytest` 只剩 1 个**先于本次改动即存在**的失败 +（`test_mcp.py::test_subprocess_tamper_defense`，已在 stash 后的干净树上复现， +与本重构无关）。 + +### P2 —— 使用者文档 ✅ 完成 + +分两条互不重叠的线，边界见 §3.1： + +- **后端线**：`guide/backends.md`（**新写**：五个 provider 的凭据模型、能力 + 矩阵、`STRICT_BACKEND` 怎么配、harness 与 api 的差别、工具桥如何保持权限 + 闸、每个后端的实测特点）。 +- **宿主线**：`guide/hosts/README.md`（新写，消歧表）+ `codex.md`（重写）+ + `claude-code.md`、`cursor.md`（新写，从 `README` 抽出）。 + +其余：`guide/mcp.md`、`autonomous-workflow.md`、`knowledge-maintainer.md` +重写，`samples/` 核对。 + +**退出标准**：(a) 每篇的命令/配置项在当前 `.env.template` 与 `cli/entry.py` +里逐条核对过；(b) `backends.md` 与每篇 `hosts/*.md` 都带 §3.1 消歧表并互链—— +**这条单独验收**，它是本组最容易塌掉的地方。 + +### P3 —— `features/` ✅ 完成 + +四篇特性文档，从 RFC + 代码重写。每篇固定小节：**做什么 / 怎么开关 / 怎么配 / +实测结果 / 已知边界**。`auto-run.md` 状态头必须明确标注**未实现**。 + +**退出标准**：每个提到的 setting 在 `config.py` 里存在；每个实测数字在 +`evaluation/` 或 `eval/dataset/results/` 里有出处。 + +### P4 —— `architecture/` 三件 ✅ 完成 + +`CODE_TOUR` 核对（并按归属矩阵**删掉清单类内容**，改为链接 GUIDE）； +`DESIGN` 补 v3 期（provider registry、MCP 双入口、双路径 mode）； +`KNOWLEDGE` 核对。 + +**退出标准**：`CODE_TOUR` 的文件清单与 `src/` 实际文件数一致（当前它自称 +覆盖 80 个 `.py`，实际 92 个）。 + +### P5 —— SPEC 全量重写（工作量主体）✅ 完成 + +62 页，按包分 13 批，每批一个提交： +`engine/` · `engine/agent_runtime/` · `engine/steps/` · `providers/` ★ · +MCP 面 ★ · `profiles/` · `memory/` · `review/` · `ci/` · `cli/` · +`adapters/` · `playbooks/` · 顶层散文件。（★ = 全新，零基础） + +每页强制格式：**职责 / 不变量（逐条指向源码符号）/ 失败模式 / 相关测试**。 +"不变量"一栏必须能指到具体符号或测试，写不出来的条目不许留。 + +**退出标准**：`check_spec_freshness.py` 报零落后；每页至少引用一个存在的测试。 + +### P6 —— 根目录收口 ✅ 完成 + +`README.md`、`DEVELOPMENT.md`、`QUICKSTART.md`、`AGENTS.md` 重新指路。 +处理 `QUICKSTART.md` 与 `GUIDE §3` 的重叠——按归属矩阵，**QUICKSTART 收缩为 +指向 GUIDE 的短页**，或直接并入 `guide/`（待定，见 §8 未决项）。 + +**退出标准**：三个 doc 工具全绿；从 `README` 出发三跳内可达任意文档。 + +--- + +## 7. 风险与缓解 + +| 风险 | 影响 | 缓解 | +|---|---|---| +| 29 处代码内 doc 引用静默失效 | 注释指向不存在的文件，`RFC-provider-registry` 一篇占 17 处 | P0 的 `check_doc_citations.py` **先于**迁移落地，并进 CI | +| `knowledge/` 两处 reorg-audit 引用 | 溯源链断裂 | P1 内一并修改，跑两个知识 validator | +| 重写 59 页 SPEC 引入**错误**的契约 | 比过期更糟——错的约束会误导修改 | 每条不变量必须指向源码符号或测试；写不出来就不写 | +| 重复内容卷土重来 | 三处描述同一事实，各自腐烂 | §4 归属矩阵 + `CODE_TOUR` 清单类内容在 P4 主动删除 | +| 再次腐烂（41/42 复发） | 半年后回到原点 | `check_spec_freshness.py` 进 CI；源码改了而 SPEC 未改即失败 | +| 阶段做一半停工 | 半新半旧比全旧更难导航 | P1 是**纯迁移**，任何时刻中断，树都是自洽的；P2–P5 每篇独立可交付 | + +--- + +## 8. 未决项（需要你拍板，不阻塞 P0–P1） + +1. **`QUICKSTART.md` 的归宿**：收缩为指路短页，还是并入 `guide/` 后删除？ + （它是英文，`GUIDE` 是中文——并入涉及语言统一。） +2. **`archive/` 的存续**：`reorg-audit/` 127 个文件是否值得留在主仓库？ + 备选是打成 tarball 或迁到独立分支，但会削弱 `knowledge/SCHEMA.md` 的溯源 + 链条。**建议：留在仓库内**（它是被引用的证据基线，不是垃圾）。 +3. **`evaluation/` 与顶层 `eval/` 的边界**：本计划让 `doc/evaluation/` 只放 + 叙述性报告，数据与脚本留在 `eval/`。若你希望合并，P0 前告诉我。 +4. **语言**：新写与重写的文档默认中文（与 `GUIDE.md` 一致），标识符/路径/ + flag 保留英文。`DESIGN.md` 现为英文——重构时是否统一为中文？ + +--- + +## 9. 工作量估算 + +| 阶段 | 产出 | 相对规模 | +|---|---|---| +| P0 | 3 个校验工具 + 索引 + 目录骨架 | 小 | +| P1 | ~210 次移动 + ~24 处引用修正 | 小（机械） | +| P2 | 9 篇（`backends.md` + `hosts/` 四篇全新） | 中 | +| P3 | 4 篇特性文档 | 中 | +| P4 | 3 篇（`DESIGN` 需补一整个时期） | 中 | +| **P5** | **62 页 SPEC** | **大——约等于其余全部之和** | +| P6 | 4 篇根文档收口 | 小 | + +P0–P1 可立即执行且低风险；P5 建议按批推进，每批独立提交、独立可回滚。 + + +--- + +## 10. 完成记录（2026-08-17） + +### 最终状态 + +``` +check_doc_links OK（279 个文件，零断链） +check_doc_citations OK（39 处代码内引用全部解析） +check_spec_freshness 62 页 —— 62 declared-verified / 0 stale / 0 undeclared + / 0 orphan；**0 个模块无覆盖**（起点：45 无声明 + 17 无页面） +knowledge validators 0 错误 +pytest 1 处失败，**先于本次改动即存在** + （test_mcp.py::test_subprocess_tamper_defense，已在 stash + 后的干净树上复现） +``` + +CI 已接入三个校验器；SPEC 一项在 P5 收尾后按计划从"只报告"改为 +**阻塞 + `--strict`**。 + +### 执行中改掉的计划本身 + +1. **SPEC 的新鲜度不能靠 git 日期。** 原计划用"页面提交日期 vs 源码提交日期"。 + 实测发现 `git mv` 会把页面日期重置为迁移那一次——**迁移会亲手废掉这个闸门**， + 45 个 7 月的页面在 P1 之后会永远显示为最新。改为页内声明 + ``，不受改名/重排版影响。 +2. **62 页，不是 59 页。** 初测把 `engine/steps/_common.md` 这类嵌套下划线页误判 + 为跨切页；`__main__.md` 也因 dunder 名被同一条规则吞掉。工具修正后真实数字是 + 45 + 17 = 62。 +3. **45 页里只有 15 页真的需要改写。** 其余 30 页的唯一改动来自同一个提交 + （`91be10a` 重命名扫描），不改契约。对这 30 页做了**程序化的公共契约核对** + （规格里点名的每个符号是否仍存在于源码），29 页干净，1 页有真实漂移 + （`profiles/establish.md` 仍列着 `LANGUAGE_SUFFIXES`，它已被并入 + `profiles/languages.py` 的 `suffixes()`）——这正是那页自己的 refactor note + 当初预告的 K2 合并。盲目重写会把这条漂移一起抄下去。 +4. **字符串扫描抓不到按段拼接的路径。** `ROOT / "docs" / "codex" / "README.md"` + 逃过了路径替换并弄坏一个测试。`check_doc_citations.py` 因此增加了段拼接解析， + 合法例外用行内 `doc-citation-exempt` 标注。 + +### 顺带发现的代码缺陷（已修） + +- **`providers/registry.py` 给 `deepseek` 声明了 `mcp_tools`，而该 transport + 测得根本用不了工具桥**（其运行时未编译进 `dsh-mcp-client`，会话跑在原生 bash + 上并记 `capability_gap`）。`base.ProviderSpec` 把 capabilities 定义为"集成方 + 可以依赖的标志"，所以这是一条假声明。没有任何代码分支读它——**正因如此这个 + 错误才活了下来：它只会误导读者**（本次就误导了 `guide/backends.md` 初稿）。已移除。 +- **`tool_bridge.py` 的模块 docstring 与自身代码矛盾**：说 `repo_map` 未被桥接， + 而第 270–282 行就在桥接它。已改正，并补上"skill/memory 检索刻意不开放"的真实边界。 +- `doc/architecture/CODE_TOUR.md` 自称覆盖"全部 80 个 `.py`"，实际 92 个，缺的 + 9 个正好是 `providers/` 与 `tool_bridge`、`review/anchor`、`review/repo_tools`。 + 已补齐并核对为 92。 + +### 仍未做（明确留白） + +- `doc/guide/mcp.md`、`autonomous-workflow.md`、`knowledge-maintainer.md`、 + `samples/` 只做了链接修正与位置迁移，**未逐句重写**。它们内容当前无已知错误， + 但没有像 SPEC 那样被逐条核对。 +- `QUICKSTART.md` 仍是英文，与中文文档树并存（§8 未决项 1 按"收缩为指路页"处理， + 未合并删除）。 +- `doc/architecture/DESIGN.md` 仍是英文（§8 未决项 4 未统一）。 +- 顶层 `eval/` 与 `doc/evaluation/` 的边界按计划划定（数据 vs 叙述），未合并。 diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 000000000..fcf6d4c55 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,92 @@ +# 文档地图 + +一行一条，按你要做的事挑一篇。**这份索引是唯一的目录**——每篇文档只负责自己那 +一类事实，别处只链接不复述（归属规则见 +[`PLAN-doc-refactor.md`](PLAN-doc-refactor.md) §4）。 + +## 从这里开始 + +| 你要做什么 | 看这篇 | +|---|---| +| 先搞清楚这东西是什么、有哪些能力 | [`GUIDE.md`](GUIDE.md) —— 总入口：概览 / 功能 / 使用 / 开发 / playbook / step / tool / 性能 | +| 装上并开始用 | [`GUIDE.md §3`](GUIDE.md#3-使用指南操作者) | +| 改这个仓库的代码 | [`GUIDE.md §4`](GUIDE.md#4-开发指南维护者) → 再看下面的 architecture | + +## guide/ —— 使用者 + +| 文档 | 内容 | +|---|---| +| [`guide/backends.md`](guide/backends.md) | **后端**：五个 provider 怎么选、怎么配、权限如何保持（Strict） | +| [`guide/hosts/`](guide/hosts/README.md) | **宿主端**：在 Codex / Claude Code / Cursor 里使用 copilot（Direct） | +| [`guide/mcp.md`](guide/mcp.md) | MCP 安装与七个工具接口 | +| [`guide/autonomous-workflow.md`](guide/autonomous-workflow.md) | 独立执行器（默认不注册） | +| [`guide/knowledge-maintainer.md`](knowledge/maintainer-walkthrough.md) | 知识维护实操 | +| [`guide/samples/`](knowledge/samples) | 三个知识贡献示例 | + +> **宿主 ≠ 后端。** `claude-code` / `codex` / `cursor` 三个名字两边都出现，方向 +> 相反：**宿主**提供模型给你用（Direct），**后端**是 copilot 拿去用的模型 +> （Strict）。对照表见 [`guide/hosts/README.md`](guide/hosts/README.md)。 + +## features/ —— 特性（做什么 / 怎么开关 / 实测如何） + +| 文档 | 状态 | +|---|---| +| [`features/provider-registry.md`](features/provider-registry.md) | 已实现——五种后端的统一注册表 | +| [`features/strict-review-deep-engine.md`](features/strict-review-deep-engine.md) | 已实现（默认开） | +| [`features/review-recall.md`](features/review-recall.md) | 已实现（默认开）——v14/v15 召回攻坚 | +| [`features/auto-run.md`](features/auto-run.md) | **draft，未实现**——GitHub 事件触发 | + +## architecture/ —— 维护者 + +| 文档 | 内容 | +|---|---| +| [`architecture/CODE_TOUR.md`](architecture/CODE_TOUR.md) | 按**数据流**讲每个源文件在哪一环 | +| [`architecture/DESIGN.md`](architecture/DESIGN.md) | **为什么**这么设计（含被否决的选项） | +| [`architecture/KNOWLEDGE.md`](architecture/KNOWLEDGE.md) | 知识来源与同步边界 | +| [`architecture/SPEC/`](architecture/SPEC/README.md) | 逐文件契约：这个文件**不能破坏什么** | + +## contributing/ —— 参与进来 + +| 文档 | 内容 | +|---|---| +| [`contributing/DOCSTRING_STYLE.md`](contributing/DOCSTRING_STYLE.md) | docstring 约定 | +| [`knowledge/writing.md`](knowledge/writing.md) | **怎么写和更新知识库**——落盘位置、页面类型、copilot 怎么消费、两道门禁 | +| [`contributing/release-maintenance.md`](contributing/release-maintenance.md) | vLLM-Omni 发版漂移审计（`imupdate` 底层） | +| [`knowledge/templates/`](knowledge/templates/README.md) | 可直接复制的知识页模板（7 种页型，各自对照一篇 vLLM-Omni 实页） | + +## evaluation/ —— 测量（冻结记录） + +索引与当前结论：[`evaluation/README.md`](evaluation/README.md)。 +数据与脚本在顶层 [`eval/`](../eval/README.md)；这里只放叙述性报告。 + +| 文档 | 结论 | +|---|---| +| [`evaluation/EVAL-v14-v16-recall-campaign.md`](evaluation/EVAL-v14-v16-recall-campaign.md) | **最新**——Δrecall −.049，precision 打平，成本约 1/3 | +| [`evaluation/EVAL-goal-strict-vs-opus5.md`](evaluation/EVAL-goal-strict-vs-opus5.md) | v7→v13：combined 15—15 | +| [`evaluation/EVAL-goal-report.md`](evaluation/EVAL-goal-report.md) | 早期一键 NL copilot 对比 | +| [`evaluation/EVAL-PR20-report.md`](evaluation/EVAL-PR20-report.md) | 20 例 PR 评审全量重跑 | +| [`evaluation/RESEARCH-reference-agents.md`](evaluation/RESEARCH-reference-agents.md) | 参考 agent 机制调研 | + +## archive/ —— 冻结，不再维护 + +读它们要知道：**内容停在各自的日期上，不代表现状**。 + +| 文档 | 为什么留着 | +|---|---| +| [`archive/IMPLEMENTATION_STATUS.md`](archive/IMPLEMENTATION_STATUS.md) | 设计 15 项任务的历史交付记录；"现状"职能已由 `GUIDE.md` 承担 | +| [`archive/PLAN-knowledge-reorg.md`](archive/PLAN-knowledge-reorg.md) | 知识重组的历史迁移计划，仍是溯源规则的出处 | +| [`archive/PLAN-mcp-plugin-and-community-merge.md`](archive/PLAN-mcp-plugin-and-community-merge.md) | PROPOSED rev 6，从未执行 | +| [`archive/reorg-audit/`](archive/reorg-audit/) | 知识重组的**证据基线**：`knowledge/SCHEMA.md` 与 rebase workflow 页引用它做溯源 | + +## 文档自身的闸门 + +三个校验器，都在 CI 里跑： + +```bash +python tools/check_doc_links.py # 所有 .md 相对链接可解析 +python tools/check_doc_citations.py # 代码里引用的 doc/ 路径都存在 +python tools/check_spec_freshness.py # SPEC 页有 verified-against 且不落后于源码 +``` + +进行中的重构：[`PLAN-doc-refactor.md`](PLAN-doc-refactor.md) +（P0–P2 完成；P3 features / P4 architecture / P5 SPEC 62 页 / P6 根目录收口待做）。 diff --git a/doc/RFC-auto-run.md b/doc/RFC-auto-run.md deleted file mode 100644 index f0784387f..000000000 --- a/doc/RFC-auto-run.md +++ /dev/null @@ -1,105 +0,0 @@ -# RFC: Auto-run InferMatrixCopilot on GitHub events (PR review first) - -- **Status**: draft v3, for team discussion (two gpt-5.6 review rounds folded) -- **Author**: tzhouam (+ copilot analysis) -- **Date**: 2026-07-23 -- **Discussion**: https://github.com/JiusiServe/InferMatrixCopilot/issues/3 - -## Summary - -Give the copilot a trigger layer so it runs unattended against our GitHub repos — starting with automatic PR review, later issue answering/triage. Today every run is manually invoked. The headless building blocks exist; this RFC covers the trigger, the idempotency/trigger-ownership rules, the security hardening that must land first, and the rollout gates for letting it post. - -## Motivation - -- The copilot is now used beyond vllm-omni (AgentInfer, afd-plugin); issues #1/#2 came from that adoption and are fixed. Manual invocation is the remaining friction. -- Measured envelope (20-PR campaign, 3 generation replicates over the same 20 PRs — not 60 independent items): ~$0.19 and 6–13 min per review on the eco path; precision 0.777 ± 0.026, recall 0.430 ± 0.018, with documented context asymmetry vs the recorded baseline. Read: comments are usually right but coverage is partial — an *additional* reviewer, cheap enough to run on every PR; never a replacement for human review. -- Reviews that arrive minutes after a PR opens change author behavior; reviews that must be requested mostly don't happen. - -## What already exists - -- Headless one-shot: `-p "review and post the review" --yes`; URL/bare-ref routing is deterministic (no LLM intent call) and explicit post intent survives it (issue #1 fix). -- Double-gated outward writes: command must carry post intent AND `ALLOW_POST=1`; anything less is a dry run. Push safety (force-with-lease only, protected `main`) is unchanged and out of scope here. -- `pr.gate_check` reports draft/merge-state/failing CI — as review *context*, deliberately non-blocking. The trigger layer must do its own eligibility filtering (draft/closed/merged); it cannot lean on this step. -- Unattended failure path: `ESCALATION.md`, exit 3, escalation email. - -## Prerequisite hardening (blockers before ANY auto-run) - -Auto-run turns hostile PR/issue content from a theoretical input into a routine one, and the current posture has real gaps. These land first: - -- **H1 — repo-bounded reads.** Today the agent's read tools resolve relative paths against the run's repo root but pass absolute paths through untouched, and `PathScope` bounds only writes. A prompt-injected review agent could read arbitrary host files (keys, `.env`) and quote them into its output. Fix: refuse absolute paths outside the run's root/worktree for `read_file`/`list_dir`/`grep`/`run_shell` cwd, and refuse `.env*`/key-material patterns (the jail chat's `repo_read` already has, applied to the step runtime). -- **H2 — outbound scrubber.** Before `pr.post_review` publishes, scan the body for credential-shaped strings (key prefixes, PEM headers, high-entropy tokens) and refuse/redact with an escalation. Defense in depth for exfiltration-via-comment; note `` fencing is prompt guidance, not an enforcement boundary — H1/H2 are the boundary. -- **H3 — least-privilege identity & isolation.** The poller must not use ambient `gh` auth (which may hold broad rights). Dedicated machine account + fine-grained PAT: selected repos only, PR read/write, nothing else — and the poller runs as a dedicated OS user whose environment holds only that PAT and its own LLM key, not the host's other credentials. This also puts a bot name on every posted comment. -- **H4 — no agent-controlled persistent writes in auto-run.** Review agents normally may propose skill candidates (`skill_update_candidate`); under auto-run that is an injection-to-persistent-state path, so knowledge-write tools are disabled for auto-triggered runs. Auto-runs are pure read + report; knowledge accrual stays a human-invoked flow. - -## Design options - -### A. GitHub Actions workflow in each target repo - -`pull_request` (opened / ready_for_review) → install copilot → one-shot → post via repo token. - -- Pros: event-driven, visible/re-runnable by the team, per-repo opt-in. -- Cons: fork PRs get no secrets and a read-only token on `pull_request`; `pull_request_target` would hand secrets to untrusted-PR-triggered runs — not acceptable before H1/H2 have soaked. Ephemeral runners reset the knowledge plane (debug memory, skill priors); only the committed profile briefing persists. LLM keys spread into each repo's Actions secrets. - -### B. Poller on the copilot host (recommended start) - -A `copilot-watch` script on a systemd timer on the machine already running the nightly rebase: per managed repo, list open PRs → eligibility filter → dedupe (below) → run one-shot → record + post (Phase 1+). - -- Pros: inherits the full setup (checkouts, worktree cache) and the persistent knowledge plane; no LLM secrets exported to GitHub; ~100–200 lines; same operational pattern as the nightly rebase. -- Cons: trigger is invisible to the team (results aren't); single host; polling latency (below). - -### C. GitHub App / webhook service — explicit non-goal - -The productized form; real hosting + auth surface. Not at our scale. - -## Proposal - -**Phase 0 — report-only shadow (≥1 week).** Option B on team-nominated repos, H1–H4 landed. Shadow runs carry **no post intent at all** (a genuinely read-only TaskSpec — `ALLOW_POST=0` is the belt, not the mechanism). Reviews accumulate in run dirs; escalations email the operator. Humans adjudicate outputs (gate below). - -**Phase 1 — posting.** Flip `ALLOW_POST=1` per repo that passes the gate. Posted via `gh pr comment` under the bot identity with a signature footer (`copilot-review: # @ / `). Known limitation, deliberate: a *comment*, not a GitHub review submission — no APPROVE/REQUEST_CHANGES state, no inline threads. Upgrading to `gh pr review` (and whether a bot may ever set blocking state) is a separate later proposal. - -**Phase 2 — optional Actions front-end (sketch only).** For repos wanting team-visible triggers: same-repo PRs only (no `pull_request_target`), `/copilot review` comment command for re-runs with actor-permission validation before any secret use. **One trigger owner per repo**: a repo is served by the poller OR Actions, never both; migration moves ownership. Phase 2 is deliberately underspecified here — it gets its own RFC (trigger security, permission checks, workflow provenance) before any implementation; nothing in Phase 0/1 depends on it. - -### Trigger & idempotency spec (Phase 0/1) - -- **Eligibility**: open, non-draft, not merged/closed, opened after the repo's enablement timestamp (no historical backlog sweep). -- **Dedupe key**: `(repo, pr_number)` — a PR is auto-reviewed **once**, at first eligibility. New pushes (`synchronize`, force-push) do NOT re-trigger; re-review is on-demand only (`/copilot review`, keyed to the new head SHA, commenters with write permission only). This keeps cost and comment noise linear in PRs, not pushes. -- **Ledger is a cache; GitHub is the source of truth.** Transactional order: record `started` → run → re-resolve head SHA and PR state immediately before posting (advanced/closed ⇒ mark `stale`, don't post) → post → record `posted` + comment URL. On startup and on any crash-uncertainty, reconcile by querying the bot's signature comments — a crash between post and record therefore cannot double-post, and a `started`-but-dead entry retries instead of being suppressed. Record the reviewed SHA in the footer so humans can see review-vs-head drift. -- **Run states & limits**: ledger entries move `started → reviewed → posted | stale | failed | post_unknown`. `failed` retries at most twice with backoff, then escalates — a crashing PR cannot burn budget every cycle; `post_unknown` (posted but unrecorded) is resolved only by the signature-comment query. Hard per-run timeout (kill at 30 min), daily cost circuit breaker (configurable, e.g. $10/day across repos), run-dir retention cap. -- **Queueing honesty**: reviews take 6–13 min; one worker, ≤4 new reviews per cycle, FIFO across repos. Latency is "~15 min under light load" and grows linearly in bursts — a queue-age alert (not just email) is part of the health story, alongside the existing `/status` and a weekly human glance at the ledger. API/rate-limit failures skip the cycle; ≥3 consecutive failures escalate by email. -- **Loop prevention**: the trigger ignores PRs and comments authored by the bot account or any `[bot]` actor — the copilot never triggers on its own (or another bot's) output. - -### Phase-1 promotion gate (per repo) - -Report-only output is adjudicated by the repo owner + one other reviewer (disagreements resolve toward the stricter label): each finding labeled useful / neutral / wrong. Promote when, over ≥15 PRs AND ≥30 findings: useful-rate ≥ 0.75, ≥50% of PRs got at least one useful finding (a mostly silent reviewer must not pass), zero critical-wrong findings (a "wrong blocker" that would have misdirected a merge), and no unresolved injection/exfiltration incident. Recall is tracked but not gated — the copilot is additive. Sign-off: repo owner. This is a judgment gate, not a significance test — the floors exist to make the rate meaningful, and the owner can hold a repo in shadow longer on any doubt. - -### Policy defaults (debatable) - -- Review depth `auto`; eco model path. -- Issue auto-answering: off in Phase 0/1; separate proposal with its own gate — wrong PR comments waste reviewer minutes, wrong issue answers mislead users. -- Write-capable kinds (pr_debug fixes, rebase pushes): permanently out of scope for auto-run. - -## Security considerations - -- Posting stays double-gated; auto-run changes *when* reviews run, not what they may do. -- Injection worst case (post-H1/H2): a wrong or hostile *comment* — which can itself trigger other repos' comment-bots; the bot identity (H3) lets other automation filter it, and the scrubber bounds exfiltration. -- Fork PRs: poller-only (H3 identity holds comment rights only); `pull_request_target` stays banned until explicitly revisited. -- Secrets: Phase 0/1 keep all LLM keys on the copilot host; Phase 2 adds them only to participating repos' Actions secrets. - -## Rollback - -- `systemctl stop` the timer **and** the service unit (terminating any active worker/run), then revoke the bot PAT ⇒ no new runs and no in-flight posts. -- Posted comments are NOT auto-retracted: rollback procedure includes bulk-minimizing the bot's comments (GraphQL `minimizeComment`) when the reason is quality/incident, and disabling `/copilot review` handling in config. -- Phase 2: delete the workflow file; trigger ownership reverts per repo. - -## Open questions for the team - -1. Which repos are in scope for Phase 0? -2. Is the promotion gate (useful ≥0.75 over ≥10 PRs / ≥20 findings, zero critical-wrong) right, and who besides the repo owner adjudicates? -3. Once-per-PR + on-command re-review: acceptable, or does anyone want re-review on every push despite the cost/noise? -4. Should `/copilot review` land in Phase 1 (poller reads comments) rather than waiting for Phase 2? -5. Who operates the poller host and owns the escalation inbox? -6. Comment vs GitHub-review submission: is the no-status limitation fine for Phase 1, or is inline/review-state support a requirement before anyone relies on it? - -## Review record - -Drafted with the copilot's analysis, then run twice through our external plan-review hook (gpt-5.6). Round 1 (verdict REVISE) surfaced, verified against source, and got folded: the absolute-path read gap (→ H1), the gate_check non-blocking misstatement, comment-vs-review-submission, the synchronize/ledger contradiction (→ once-per-PR keying), eval-caveat honesty, and the transactional posting order. Round 2 (verdict REVISE) added: skill-candidate writes as an injection path (→ H4), post-intent-free shadow runs, retry/timeout/cost limits, loop prevention, gate floors, and kill-active-workers rollback — all folded. Not adopted (logged for discussion, judged disproportionate at our scale): full container isolation with restricted egress, and blocking Phase 1 on GitHub-review- state posting. The reviewer's position on both is available on request. diff --git a/doc/RFC-provider-registry.md b/doc/RFC-provider-registry.md deleted file mode 100644 index 69d88367d..000000000 --- a/doc/RFC-provider-registry.md +++ /dev/null @@ -1,343 +0,0 @@ -# RFC — Provider registry: subscription-auth harness backends for Strict - -- Status: accepted (grilling session 2026-08-14); M1 — registry, api - parity, tool bridge, cursor transport — implemented on this branch; - claude-code (M2) and codex (M3) declared but unshipped -- Owner: LLM/backend layer (`llm.py`, `config.py`), agent runtime - (`engine/agent_runtime/runner.py`), new `src/infermatrix_copilot/providers/` -- Prior art studied: Hermes Agent `api_mode` transports - (hermes-agent.nousresearch.com/docs/developer-guide/adding-providers) · - opencode provider registry (opencode.ai/v2/docs/providers) - -## Motivation - -Strict mode runs the full execution spine, and every model call goes through -`llm.py::LLM` — a raw Anthropic/OpenAI-compatible completions client that -requires `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`. Users whose only model -access is a coding-agent subscription (Claude Code Pro/Max, ChatGPT plan for -Codex CLI, Cursor) cannot run Strict at all. - -Claude Code, Codex CLI and cursor-agent expose no raw completions API on -subscription auth — they are *agent harnesses* that own their tool loop. So -the plug-in point cannot be `LLM.create()` (stateless -`system+messages+tools → tool_use` round trips); it has to sit one level up, -where a whole agent step is delegated. - -Hermes solves the adjacent problem with an `api_mode` transport abstraction -(canonical internal message shape; per-mode adapters for request building, -response normalization, usage extraction; one standardized runtime-resolution -record). This codebase is already half-way there: the Anthropic block -protocol is our canonical shape, `_openai_messages()` is a transport adapter, -and `Settings.tier_target()` → `ResolvedTarget` is runtime resolution. This -RFC formalizes that into a provider registry and adds the axis Hermes only -needed for its out-of-process Codex path: **transport kind** (`api` vs -`harness`). - -## Decisions (locked with the maintainer, 2026-08-14) - -1. **Purpose: product reach.** Strict on subscription auth for users without - raw API keys. Fidelity is best-effort per harness and labeled per run. - Eval/campaign tables are unaffected — a harness backend measures - loop+model and is *never* pooled into generator-ablation tables (standing - rule: generator arms ride our pipeline with only the model swapped). -2. **Explicit selection, hard error.** A `.env` key selects the backend. For - a Strict run with no selected backend the server errors upfront with the - exact missing item (same philosophy as `TierNotConfiguredError` / - `strict_readiness`). No auto-detection, no silent fallback. -3. **Tools bridged, preventive-first.** Harness sessions get the copilot's - own tools over MCP, every call passing `tools.dispatch` - (ToolScope/PathScope choke point preserved). Vendor built-ins are - disabled where the harness supports it; cursor-agent additionally gets - the productized post-run audit (defense in depth), with the control class - disclosed in RUN_REPORT. -4. **Unified registry.** The existing API path becomes provider `api` under - the same registry — one resolution path, one trace vocabulary. Parity - ratchet: with provider `api`, behavior is byte-identical and the existing - test suite is the gate. -5. **Phasing: cursor-agent first**, then Claude Code, then Codex. - -## Design - -### Vocabulary - -- **provider id** — registry key and the value of the selection config: - `api`, `cursor`, `claude-code`, `codex`. -- **kind** — `api` (stateless completions; implements `complete`) or - `harness` (owns a tool loop; implements `run_session` + one-shot - `complete`). -- **api_mode** — the wire protocol *within* the `api` provider: - `anthropic_messages` | `chat_completions`. This is exactly today's - `Settings.resolved_llm_provider`; the term "provider" in existing code - (`llm_provider`, `ResolvedTarget.provider`) keeps meaning the API vendor - protocol and maps onto api_mode unchanged. - -### Package layout - -``` -src/infermatrix_copilot/providers/ - __init__.py re-exports; register_builtin_providers() - base.py ProviderSpec + Transport protocol + session dataclasses - api.py provider "api": wraps the existing LLM class (both - api_modes). Wrapper, not rewrite — LLM internals, - tier_target, served-model guard, tracing all unchanged. - cursor.py M1 harness transport (cursor-agent CLI) - claude_code.py M2 harness transport (claude -p) - codex.py M3 harness transport (codex exec) - audit.py post-run session audit (productized from - eval/dataset/run_cursor_arm.py) - registry.py PROVIDER_REGISTRY + resolve_provider(settings) -src/infermatrix_copilot/tool_bridge.py - stdio MCP server exposing the run's scoped tools - (entry: python -m infermatrix_copilot.tool_bridge) -``` - -### base.py — the contracts - -```python -@dataclass(frozen=True) -class ProviderSpec: - id: str # "api" | "cursor" | "claude-code" | "codex" - kind: Literal["api", "harness"] - display: str - cli_names: tuple[str, ...] = () # binaries to probe (harness) - capabilities: frozenset[str] = frozenset() - # capability flags: "mcp_tools", "builtin_tools_off", "max_turns", - # "system_prompt", "usage_reporting", "cost_reporting" - -class Transport(Protocol): - # api kind + tool-less harness calls; signature mirrors LLM.create - def complete(self, *, system, messages, tools=None, model=None, - max_tokens=None, on_text=None, role="") -> Reply: ... - # harness kind only: one whole agent step - def run_session(self, req: AgentSessionRequest) -> AgentOutcome: ... - -@dataclass -class AgentSessionRequest: - system: str # contract preamble (harnesses without a system- - prompt: str # prompt channel prepend it to the prompt) - scope: ToolScope - bridge_spec_path: Path # written by the runner; see tool bridge - model: str - max_iters: int # mapped to --max-turns where supported, else - timeout_s: float # wall-clock timeout + budget-discipline prompt - run_dir: Path -``` - -`run_session` returns the existing `agent_loop.AgentOutcome` (text, -iterations, tool_calls, truncated, refusals, token usage, tools_used) so -everything downstream of the runner branch — `_coerce_output`, the -`agent_output` trace event, `_to_step_result` — is untouched. Fields a -harness cannot know (iterations) are best-effort; `tools_used` comes from -the bridge trace. - -### Resolution — one place, extended - -`ResolvedTarget` (config.py) gains two fields with defaults that keep every -existing construction site valid: - -```python -provider_id: str = "api" -kind: Literal["api", "harness"] = "api" -``` - -`Settings.tier_target()` stays the only place model×endpoint×credential pair -up; when `strict_backend` selects a harness provider it returns a target -with that provider id/kind and the harness model (`STRICT_BACKEND_MODEL` or -the provider default) — base_url/api_key empty (subscription auth lives in -the harness CLI, never in our config). `LLM.for_target()` is only consulted -for `kind == "api"` targets. - -Selection scope: `STRICT_BACKEND` is **required for Strict runs** — checked -in `CopilotMCP.strict_readiness()` (mcp_server.py:175, alongside the current -`shared_api_key` check, which becomes the `api`-provider readiness item) and -re-checked at `--execute-strict-reserved` startup in the child. The CLI path -(`run_task`) treats empty as `api` — no breaking change for maintainers — -and may grow a `--backend` flag later. - -### Runner branch — the single integration point - -In `run_agent_step` (engine/agent_runtime/runner.py:170), the current call - -```python -outcome = await asyncio.to_thread(run_agent, step_llm, system=..., ...) -``` - -becomes a two-way branch on the resolved target's kind. `harness` targets -write the bridge spec (below) into the run dir and call -`provider.run_session(...)` in the same worker thread. Everything before the -branch (dispatch context, evidence pack, briefing, scope binding to the -PR-time worktree root, `agent_dispatch` trace) and everything after -(`_coerce_output` with its one repair round and escalation salvage, -`agent_output` trace, skill touch) is shared — the harness rides the exact -v13 prompt bundle and output contract. - -Tool-less roles (review planner gray-zone call, ensemble reducer/merge, -coverage promotion, `_coerce_output` repair) go through a `HarnessLLM` -adapter implementing the `LLM.create()` signature: it raises if `tools` is -non-empty, runs a one-shot CLI invocation, and returns a normalized `Reply` -with usage from the CLI's JSON. Its `available` property mirrors doctor's -CLI probe so `run_agent_step`'s `ctx.llm.available` gate keeps working. For -cursor, one-shot completions run in an **empty scratch cwd** so native tools -have nothing to read. - -MoA (`for_member`) rejects harness providers as mixture members in v1 — the -reservation ledger requires per-request pricing that subscription CLIs -cannot provide. - -### The tool bridge — choke-point preservation - -`python -m infermatrix_copilot.tool_bridge --spec /bridge/.json` -is a stdio MCP server (FastMCP, already in the `[mcp]` extra) exposing -exactly the tools the step's scope allows. The spec file serializes: - -- the `ToolScope` (frozen dataclass → JSON: name, allowed_tools, read_only, - root, path_scope patterns) — trivially round-trippable; -- repo name + repo_path (the PR-time worktree), run_dir; -- which extra tool families to reconstruct (knowledge/doc/repo_map) — these - are rebuilt in the bridge process from `Settings()` + the adapter, the - same factories the runner uses (`_knowledge_tools`, `_repo_map_tool`, - `_repo_docs_tool`). - -Every MCP tool handler is a thin shim over `tools.dispatch(name, args, -scope=scope, trace=bridge_trace, extra=extra)` — refusals, out-of-scope -records, path resolution against the worktree root, and result bounding all -behave identically to the in-process loop. The bridge appends to -`bridge_trace.jsonl` in the run dir (its own file: append-only jsonl from a -second process must not interleave with the parent's `run_trace.jsonl`); -after the session the runner folds a summary (tool_calls, tools_used, -refusals) into the normal trace events. - -Guard test (the exfiltration case): a bridged `read_file` of -`~/.infermatrix-copilot/.env` — outside the scope root — is refused and -traced. - -### Environment sanitization — load-bearing - -Harness subprocesses get a constructed allowlist env (PATH, HOME, TERM, -LANG/LC_*, PYTHON* as needed, plus provider-specific vars), **never** -`os.environ` passthrough. Concrete hazards on this very machine: - -- our `.env` points `ANTHROPIC_BASE_URL` at a DeepSeek-compatible gateway — - leaked into `claude`, subscription calls would route to the wrong endpoint - and bill the wrong account; -- `ANTHROPIC_API_KEY`/`OPENAI_API_KEY` present in env would silently switch - the CLIs from subscription auth to API billing; -- `CLAUDECODE`/host markers confuse nested-CC when the Strict host *is* - Claude Code. - -The Strict child itself already receives a curated env from the server -worker (mcp_server.py:100–115); the same discipline extends one level down. - -### Guards and accounting - -- **Served-model guard**: harness CLIs report the model used in their JSON - output; recorded via the existing `llm.served` event with the same - verdict logic (mismatch → fail per `MODEL_MISMATCH_POLICY`; absent → - `unverified`, warn). -- **Usage/cost**: token counts parsed from CLI JSON (claude `-p - --output-format json` includes usage + cost; codex `--json` emits token - events; cursor stream-json includes usage). `metrics.py` records cost - source `subscription` and does not consult `MODEL_PRICES` for harness - providers — CATQ's C term must not fabricate USD. -- **Ensemble stagger** (`ensemble_stagger_seconds`) is disabled for harness - targets: there is no shared prompt-cache prefix across CLI sessions to - warm. Each delegated pass is one harness session, so context caching - within the 32-iter deep pass is the harness's own concern. -- **Concurrency**: `STRICT_BACKEND_CONCURRENCY` (default 2) is a semaphore - over concurrent sessions (deep passes + verify fan-out) — subscription - rate windows are real; auth/limit errors surface as BLOCKED, never - retried into a lockout. - -### Config schema (.env) - -``` -STRICT_BACKEND= # REQUIRED for Strict: api | cursor | claude-code | codex -STRICT_BACKEND_MODEL= # model id inside the harness (optional) -STRICT_BACKEND_CONCURRENCY=2 # concurrent harness sessions -STRICT_BACKEND_CLI= # binary path override (else PATH lookup) -STRICT_BACKEND_TIMEOUT_S=1800 # per-session wall-clock ceiling -``` - -Installer and `.env.template` write `STRICT_BACKEND=api` so existing setups -migrate with the file refresh; `strict_readiness` names the exact fix line -when it is missing. `doctor` gains per-provider checks: binary found + -version, auth status, bridge self-test (spawn, list tools, one scoped read); -`doctor --probe` performs one cheap round trip through the selected -provider. - -### Cursor transport (M1 specifics) - -- Invocation: `cursor-agent --print --output-format stream-json --force`, - prompt on stdin, `--model` from config, MCP config file pointing at the - tool bridge, cwd = the PR-time worktree. -- Parsing, session accounting, and the boundary prompt line reuse what the - Composer eval arm already learned (`eval/dataset/run_cursor_arm.py`) — - including the incident where a smoke run read `~/.claude/skills/`. -- **Open question to spike first**: whether cursor-agent can disable its - built-in tools when MCP tools are configured. If yes → preventive; if no → - the bridge is additive and `providers/audit.py` is the enforcement layer: - realpath-normalized worktree bound on every file access, write ban, - discussion-access regex; verdict + flags traced and rendered in - RUN_REPORT ("backend: cursor — native tools possible, audit: clean/N - flags"). -- No `--max-turns` equivalent → wall-clock timeout + the existing - budget-discipline prompt lines (reserve final rounds for the contract). - -### Claude Code (M2) and Codex (M3) - -- `claude -p --output-format json --mcp-config --allowedTools - --max-turns --model `; cleanest harness - citizen (built-ins fully off, native max-turns, system prompt supported, - usage+cost in output). -- `codex exec --json --sandbox read-only` + MCP servers in config; no - system-prompt channel (contract prepended to prompt); budget via timeout. - -## Milestones and acceptance - -**M1 — provider layer + cursor-agent** -1. `providers/` + registry + `ResolvedTarget` extension + `api` provider - wrapping `LLM`. Acceptance: full existing suite green with no config - change; `STRICT_BACKEND=api` byte-identical (parity ratchet). -2. Tool bridge + bridge tests. Acceptance: dispatch parity (same refusals/ - bounds as in-process), `.env` exfiltration read refused + traced. -3. Cursor transport + audit + doctor checks + readiness wiring. Acceptance: - fake-CLI offline tests green; one live Strict smoke on a small PR with - backend label, usage, and audit verdict in RUN_REPORT; unset - `STRICT_BACKEND` on a Strict start yields the exact fix-line error. - -**M2 — claude-code** · **M3 — codex**: same acceptance shape (offline -fake-CLI tests + one live smoke each); M2 additionally proves nested-CC -(Strict host = Claude Code) works under the sanitized env. - -## Test plan (offline-first, house discipline) - -- `test_providers.py` — registry resolution; strict-vs-CLI empty-selection - behavior; `HarnessLLM` raising on tools; MoA member rejection; tier - interplay (`tier_target` returning harness targets). -- `test_tool_bridge.py` — spec round-trip; scope enforcement through the - bridge; bridge trace contents. -- `test_provider_cursor.py` — fixture script emitting canned stream-json: - final-text extraction, usage parsing, timeout → truncated outcome, audit - flag surfacing. -- Doctor: per-provider check rendering + fix lines. -- Ratchets untouched: `test_repo_neutral_core`, `test_llm_providers.py`, - `test_tier_split.py`, `test_thin_mcp_server.py`, `test_mcp.py`. -- New SPEC pages under `doc/SPEC/` for `providers/` and `tool_bridge.py` - once implemented (house rule: file-level constraints live there). - -## Risks and open questions - -1. cursor-agent built-in tool restriction unknown → M1 spike; governance - fallback already decided (audit as enforcement). -2. Contract compliance without a system-prompt channel (cursor/codex) — - absorbed by the existing repair + escalation-salvage path; watched in - smoke before trusting. -3. Harness inner reasoning is invisible to our trace; only bridged tool - calls are audited. Disclosed per run — never presented as full-fidelity - Strict tracing. -4. Subscription ToS/rate limits: concurrency capped, limit errors BLOCK - loudly. Users choose their own account exposure by selecting the - backend explicitly. -5. Model naming inside harnesses (composer ids, codex model slugs) — - `doctor --probe` reports what the CLI actually serves; served-model - guard records it per call. diff --git a/doc/RFC-strict-review-deep-engine.md b/doc/RFC-strict-review-deep-engine.md new file mode 100644 index 000000000..c40d85bf0 --- /dev/null +++ b/doc/RFC-strict-review-deep-engine.md @@ -0,0 +1,127 @@ +# RFC — Strict review for a strong generator: deep passes, per-comment verification, self-proving evidence + +- Status: implemented behind flags (all default ON); measured on the 20-PR + campaign splits (val + one-shot frozen test) +- Owner: review pipeline (`engine/steps/review/`, `review/planner.py`) +- Evidence: `doc/EVAL-goal-strict-vs-opus5.md`, + `eval/dataset/judgments/goal_v13_val/`, `eval/dataset/judgments/goal_v13_test/` + +## Motivation + +The Strict review pipeline was engineered around a weak generator: four +narrow lenses forcing enumeration inside templates, tight tool budgets, a +tool-less reducer, and a 5-comment severity cap. With the official +`deepseek-v4-pro` (a large capability jump over the `[1m]` preview), that +scaffolding became the bottleneck — measured symptoms across judged sweeps: + +1. **Recall losses were mechanical, not analytical**: the budget cap deleted + reducer-kept maintainer concerns; depth planning misrouted small + high-blast-radius diffs to the no-ensemble light tier; a 170k-char diff + lost 30% of its hunks to the 120k evidence cap; whole question classes + (blast radius, dependency floors, test integrity, lifecycle-on-abort) had + no owning lens. +2. **Precision was stuck at 0.54–0.56 across five different configurations** + while recall swung 0.55–0.86 — because the blind judge is TOOL-LESS: it + sees only the diff and the human threads, so any repo-side finding + (consumers elsewhere, CI lane rules, version floors) scores as + speculation unless the comment itself carries checkable proof. + +## Design + +Three mechanisms, composed with the existing ensemble/reducer machinery: + +### 1. Hybrid pass set (`review_deep_engine`, default on) + +Full depth runs two **deep passes** — `investigator` (central-change-first +free investigation; verify-before-assert; budget discipline: reserve the +final rounds for filing) and `adversary` (independent hunt over the +systematically-missed classes) — **plus** the two highest-value breadth +lenses (`behavior`, `verification`). Standard depth runs +investigator+behavior. Light is unchanged. Deep passes get +`review_deep_max_iters` (32) tool rounds; measured alone they ground +claims well (precision .82 on the hardest train items) but under-cover +(val recall .55); measured with the breadth lenses they hold recall .71–.86. + +### 2. Per-comment agentic verification (`review_verify_comments`, default on) + +Every merged draft comment gets a small tool-loop (4 rounds, concurrency 6, +shared diff cache prefix) that must re-derive the claim on the PR-time +tree: `refuted` drops, `unverifiable` demotes one severity step, +`confirmed` may tighten wording/position — and a verification failure +keeps the comment (the pass can only raise precision, never silently +delete recall). Skip/failure reasons are traced +(`review_comments_verified` / `review_coverage_skipped`). + +### 3. Self-proving evidence (drafting + verify + render) + +The single biggest measured lever (4—11 → 8—7 on the val gate): every +comment's `evidence` field must QUOTE the decisive code line(s) verbatim +with file:line so a reader holding only the review and the diff can check +the claim. Enforced in the drafting contract, re-emitted by the verifier +from the code it actually read, rendered as-is. + +Supporting changes shipped with this RFC: `value_flips` diff signal (a +default flip can never rule light), verification lens in the planner +fallback set, evidence caps 120k→260k / reducer 60k→280k, commit timeline ++ deterministic changed-symbol consumer sweep (`git grep -nw`) + per-file +hunk-location paging targets in the evidence pack, cross-lens +corroboration tags with protected-class reducer rules, comment budget +8 + rich-only overflow, coverage-promotion pass (mines the run's own +findings/blockers for uncovered protected-class concerns), knowledge page +`repos/vllm-omni/review/guides/strict-review-checklist.md` injected via +the new adapter manifest key `knowledge.review_checklist`, and the +vllm-omni module map extended to 17 modules with 7 risk tiers. + +## Measured results (blind gpt-5.6 judge, 3 replicates, vs recorded CC+Opus 5) + +| gate | config | head-to-head (arm—opus) | arm r/p | opus r/p | +|---|---|---|---|---| +| pre-campaign | preview model, 4 lenses | 1—4 | .42/.51 | .82/.54 | +| v8 | official model, lenses | 3—12 | .86/.56 | .79/.66 | +| v12 | hybrid, narrative evidence | 4—11 | .71/.55 | .81/.66 | +| **v13 val** | hybrid + self-proving | **8—7** | .857/.555 | .889/.589 | +| **v13 frozen test** (one shot) | same | 7—8 | .689/.687 | .762/.719 | + +Combined fresh verdicts: **15—15** at ~1/3 the baseline's cost +(~$0.75–1.2/item vs ~$1.5–3). GOLD latent-gap items: pr4810 2/3, pr4834 +swept 3/3 clear. Rubric-mean gaps (.03–.07) are at the magnitude of +measured judge drift on identical baseline reviews (±.05–.10). + +## Alternatives considered + +- **MoA re-test with stronger members** — excluded by owner direction. +- **Deep passes only** (no breadth lenses): precision transferred but val + recall collapsed to .55; rejected (gate v11, 2—13). +- **Budget/reducer tuning alone**: five configurations moved precision by + ≤.02; the deficit was structural (tool-less judge), not calibrational. + +## Rollout / compatibility + +Two of the three mechanisms sit behind kill switches +(`review_deep_engine`, `review_verify_comments`; legacy lens set retained +verbatim and used when the deep engine is off). The third — self-proving +evidence rendering — has no setting of its own: `config.py` carries no +boolean for it, so it cannot be rolled back independently of the deep +engine. Open Question 2 is the same gap seen from the other side; treat +evidence rendering as fixed behavior, not as something an operator can +switch off. Offline tests cover pass +selection, verify verdict handling (drop/demote/tighten/fail-open), +budget/overflow, corroboration ordering, promotion, and the planner +signal; the full suite and both knowledge validators are green. Cost is +~2–3× the old lens pipeline per item and remains ~3× cheaper than the +baseline. `ARM_JOBS` in `run_copilot_arm.py` enables item-level parallel +sweeps (endpoint allows 500 concurrent requests on v4-pro). + +## Open questions + +1. Both tuning splits are spent (8 val submissions, 1 frozen-test shot). + Further iteration needs wave-2 items (`gt/pr5957*`, `gt/pr5976*`, + `build_wave2.py`) judged under the same protocol. +2. The judge protocol's tool-less-ness now shapes the reviews (verbose + quoted evidence). If the product's real audience is maintainers rather + than a repo-less judge, the evidence verbosity should become a render + option rather than a fixed behavior. +3. `pr4762`/`pr4954`-class losses (broad multi-concern PRs) remain: the + baseline still out-reads the arm per finding. The next structural lever + is a coverage-driven second investigation round seeded by the reducer's + uncovered-GT-class checklist rather than a fixed pass count. diff --git a/doc/SPEC/README.md b/doc/SPEC/README.md deleted file mode 100644 index 4b3ab4910..000000000 --- a/doc/SPEC/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# InferMatrixCopilot — Specification (file-level) - -A **normative, file-level** specification. The `doc/SPEC/` tree mirrors -`src/infermatrix_copilot/`: **one spec file per source file**, at the same relative -path (`engine/steps/pr/` → `doc/SPEC/engine/steps/pr.md`). Each spec covers a -fixed set of lenses. Two cross-cutting docs sit at the root and are referenced -(not repeated) by every per-file spec. - -This SPEC is written to drive a **codebase refactor**: for any source file you -touch, its spec tells you what the file must keep guaranteeing, what does not -belong in it, which global constraints bind it, and where it is currently messy. - -## Cross-cutting (read first) - -- **[_ARCHITECTURE.md](_ARCHITECTURE.md)** — layers, dependency direction rules, - functionality (the 7 task kinds), scope, data & artifacts, safety model, - repo-invariance contract. The "big picture" every per-file spec assumes. -- **[_CONSTRAINTS.md](_CONSTRAINTS.md)** — global programming constraints - (A structural, B contract, C safety, D knowledge, E observability) + the - invariant catalog. Per-file specs cite these by id (e.g. `A2`, `C4`) instead - of restating them. -- **[_CONCISION.md](_CONCISION.md)** — the plan for making the codebase smaller: - a prioritized, grep-backed catalog of dead code, duplication, and boilerplate - to remove (K1–K7), each with the shared helper to introduce and the invariant - to preserve. This is what the "make it concise" refactor follows; per-file - specs carry a matching **Concision** note where an opportunity applies. - -## Per-file spec template (the lenses) - -Every `*.md` under the mirrored tree uses these headings, in this order: - -1. **Header line** — `LOC ~N · role · refactor-status`. - `refactor-status ∈ {ok, oversized, split-candidate, shim-to-retire, - trivial}`. -2. **Responsibility** — the single thing the file owns. -3. **Functionality** — what it actually does (behavior), briefly. -4. **Public contract** — exported symbols other code may use + their guarantees. -5. **Invariants** — properties that must hold on every path (the "don't break" - list), each tagged with the constraint id it realizes. -6. **Scope — not here** — what explicitly does NOT belong in this file. -7. **Dependencies (allowed)** — the imports this file may have (enforces the - §_ARCHITECTURE dependency rules). Anything else is a layering violation. -8. **Extension points** — the sanctioned way to add capability here. -9. **Tests** — the guard tests that pin the invariants. -10. **Refactor notes** — size/cohesion/coupling smells and concrete suggested - moves/splits (a *cohesion* lens — improves readability, adds files). -11. **Concision** (only where an opportunity applies) — what in this file is - dead, duplicated, or boilerplate, and which `_CONCISION.md` item (K1–K7) - removes it. This is the lens the *make-it-concise* refactor consumes; - where it conflicts with a cohesion split, concision wins. - -## How to use this during refactor - -- **Before editing a file**: read its spec. If your change adds a responsibility - the spec doesn't list, the change belongs in a different file (or a new one) — - add/adjust the spec first. -- **When splitting a file**: create the new spec files, move the relevant - lenses, and keep the "Dependencies (allowed)" honest — a split that creates a - cross-import the rules forbid is not done. -- **When deleting a shim** (`refactor-status: shim-to-retire`): migrate importers - to the real module first (the spec lists who imports it), then delete both the - code and its spec. -- **Invariants are the contract**: a refactor may move code freely but must - preserve every invariant and every guard test named in the spec. If a test - must change, the invariant changed — call it out. - -## File index (mirrors `src/infermatrix_copilot/`) - -| Layer | Spec files | -|---|---| -| Interface / task | `task_spec` `intent` `cli` `chat` `ui` `config` | -| Engine substrate | `engine/step` `engine/registry` `engine/executor` `engine/planner` `engine/agent_runtime` `agent_loop` `tools` `scopes` `llm` | -| Step library | `engine/steps/__init__` `engine/steps/_common` `engine/steps/{workspace,rebase_ext,review,report,pr,issue,profile,rebase_native}` | -| Planning data | `playbooks/store` `playbooks/PLAYBOOKS` (the yaml) | -| Edge — languages | `profiles/languages` (per-language rules, shared) | -| Edge | `adapters/base` `ci/normalize` `ci/providers` `rebase/monitor` | -| Safety primitives | `scopes` `push` | -| Profiles | `profiles/store` `profiles/establish` `profiles/repo_map` `profiles/consolidate` | -| Cross-cutting | `review/{diff_summary,triggers,reviewer}` `memory/{debug_memory,skills}` `run_trace` `notify` `metrics` | - -Trivial `__init__.py` re-export files are not specified individually -(`engine/steps/__init__` is, because it defines `register_builtin_steps`). diff --git a/doc/SPEC/_ARCHITECTURE.md b/doc/SPEC/_ARCHITECTURE.md deleted file mode 100644 index 772687117..000000000 --- a/doc/SPEC/_ARCHITECTURE.md +++ /dev/null @@ -1,138 +0,0 @@ -# 00 — Architecture, Scope, Functionality - -## 1. What the system is - -A playbook-driven repo-maintenance copilot for vLLM-Omni (and, by design, any -repo with an established profile). It turns a natural-language or flag command -into a governed, resumable, auditable pipeline of vetted steps, and escalates -rather than guesses when blocked. - -## 2. Functionality (the task kinds) - -Exactly seven task kinds exist; each has a fixed blast-radius **tier** that no -input can change (see `task_spec.py`): - -| Kind | Tier | Read-only | What it does | -|---|---|---|---| -| `repo_rebase` | L0 | no | rebase the fork onto upstream (delegates to the locked 5-phase orchestrator) | -| `pr_rebase` | L1 | no | replay a PR branch onto its latest base | -| `pr_debug` | L1 | no | diagnose & fix a PR's failing CI | -| `pr_review` | L2 | yes* | evidence-grounded inline review | -| `issue_answer` | L2 | yes* | draft an answer to an issue | -| `issue_filter` | L2 | yes* | triage/label/route issues | -| `repo_profile` | L2 | no** | establish/refresh a repo's profile | - -\* read-only unless the explicit `post` intent is set. \*\* reads the target -repo but writes knowledge under `adapters//`. - -Tiers (`§3.2` of DESIGN): **L0** reuse a locked playbook verbatim; **L1** adapt -a vetted playbook (plan-review gated); **L2** may fall back to generation, but -only for read-only kinds. - -## 3. The three layers + the engine substrate - -``` - Interface cli/ / chat.py / ui.py NL & flags -> one dispatcher - │ - Task task_spec.py / intent.py NL -> TaskSpec (tier from kind) - │ - Planning engine/planner.py / playbooks/ reuse > adapt > generate - │ - Engine engine/{step,registry,executor} run steps: checkpoint, foreach, - │ engine/agent_runtime.py typed failures, governed agents - │ agent_loop.py, tools.py, scopes.py - │ - Steps engine/steps/* the vetted step library - │ - Edge adapters/, ci/, profiles/ repo knowledge & capabilities - review/, memory/, run_trace, notify, metrics cross-cutting - scopes.py, push.py pure safety primitives -``` - -There are three code layers (Interface / Task+Planning / Engine+Steps) over an -edge of repo knowledge and cross-cutting services. The design's "Target layer" -is **not** a code layer: its task-definition role is carried by `TaskSpec` + -`Playbook`, and its only surviving artifact — push authorization — is the -`push.py` safety primitive (sibling of `scopes.py`). - -- **Engine is a substrate, not a pipeline.** It provides a vetted step library, - task-agnostic execution guarantees (checkpoint/resume, typed failure routing, - scope enforcement, escalation, RunTrace), and a plan→execute loop. It does not - hold repo knowledge and does not let the planner compose raw tools. -- **Knowledge lives at the edge.** Repo-specific facts (modules, CI, prompts, - conventions, push policy) live under `adapters//` (manifest + profile), - never in the core. Adding a repo is an edge addition, not a core change. - -## 4. Dependency rules (enforced layering) - -Imports may only point **down and outward**. Verified constraints a change may -not violate: - -1. Leaf edge packages (`profiles/`, `ci/`, `adapters/`, `review/`, `memory/`) - and safety primitives (`scopes.py`, `push.py`) MUST NOT import `engine/`. - The engine depends on them. -2. Interface (`cli/`, `chat.py`) MUST NOT be imported by any lower layer. - `chat.py` may reference `cli.Copilot` under `TYPE_CHECKING` only. -3. `engine/step.py` is the base vocabulary: it may depend only on `run_trace`. - Nothing task- or repo-specific. -4. `engine/steps/*` compose tools and edge packages; they reach the agent - runtime via `engine.agent_runtime`. A step MUST NOT be imported by the - engine substrate (steps sit above it). -5. Cross-step shared helpers live in `engine/steps/_common.py` — step modules - MUST NOT import each other (the old `pr_steps -> builtin_steps` late import - is gone and may not return). - -## 5. Scope - -**In scope:** the seven task kinds above; a single orchestration engine; -repo-profile establishment/maintenance; conversational + flag CLIs; multi-repo -support via profiles; safety guardrails (plan/patch review, scope enforcement, -push guard, escalation); per-run metrics. - -**Out of scope (by design):** reimplementing the nightly rebase (it is -delegated, wrapped-not-rewritten); a general agent framework; running paid -evals from inside the app (eval harness is offline machinery in `eval/`); -being a forge other than what a CI/forge adapter declares; storing secrets -(only `.env`, git-ignored, never committed). - -## 6. Data & artifacts (the ground truth) - -- **Per run** — `~/.infermatrix-copilot/runs/run-/`: `run_trace.jsonl` (append-only - facts), `progress.json` (step checkpoints), `RUN_REPORT.md`, `metrics.json`, - `ESCALATION.md` (only when blocked), plus step-specific artifacts - (`rebase_status.json`, `COMPARISON.md`, evidence archives). -- **Per repo (knowledge)** — `adapters//`, **two trust tiers** (DESIGN - §V2.3.0): **Tier 1 — the manifest** `manifest.yaml` (human-authored, human-gated: - identity/paths/push/ci — the safety config the agent may not self-edit), and - **Tier 2 — the profile** `profile/` (agent-established, evidence-gated: - `profile.yaml`, `PROFILE_REPORT.md`, `JUDGE_REPORT.md`, `ops_log.jsonl`, - `format.yaml`/`review.md`/… as established) + `skills/`, - `store/debug_memory.db`, `repo_map/` cache. (The container is a repo *adapter* - holding a *manifest* + a *profile* — renamed from "plugin" 2026-07-11.) -- **Governance rule:** RunTrace/evidence are the immutable layer; the profile is - the curated layer over it. Facts summarize evidence, never replace it. - -## 7. Safety model (defense in depth) - -Five independent gates, ANDed — a change may strengthen but not remove any: - -1. **Tier from kind** — NL/user text can never widen permissions - (`_CONSTRAINTS.md` C1). -2. **Plan review** — adapted/generated plans are reviewed before execution; - generation is structurally barred from write/push steps. -3. **ToolScope/PathScope** — every tool call passes one dispatcher; out-of-scope - writes execute but are recorded, never silent. -4. **Patch review** — conditional on 7 risk triggers, fail-closed before pushes. -5. **Push guard** — one choke point; protected branches never pushed to; force - is with-lease only; dry-run unless `ALLOW_PUSH=1`. - -Blocked runs write `ESCALATION.md`, notify, and exit 3 — notify, never guess. - -## 8. Repo invariance (the multi-repo contract) [partly planned] - -Quality on a profiled repo must stay within a declared band of the reference -repo, and onboarding a repo requires **zero** edits under `src/infermatrix_copilot/`. -Enforced by: repo-neutral core (`test_repo_neutral_core`), capability-matched -playbooks, explicit `capability_gap` degradation, per-repo knowledge -namespacing. Measured by the cross-repo eval + invariance index -(`eval/invariance.py`) — the eval campaign itself is `[planned]`. diff --git a/doc/SPEC/_CONCISION.md b/doc/SPEC/_CONCISION.md deleted file mode 100644 index f47ed06c7..000000000 --- a/doc/SPEC/_CONCISION.md +++ /dev/null @@ -1,133 +0,0 @@ -# Concision plan — making the codebase smaller without losing a guarantee - -This is the cross-cutting plan the refactor follows to make the codebase -**concise**: remove duplication, delete dead code, and collapse repeated -boilerplate into shared helpers. Every item names the exact sites (grep-backed), -the estimated line saving, the abstraction to introduce, and the invariant that -must survive. Per-file specs carry a matching **Concision** subsection. - -## Status — K1–K7 applied on `main` (2026-07-10) - -All seven items are **done** (211 tests green throughout; each shipped as its -own commit). Outcome: −2 net files (3 shims + 4 dead dataclasses removed, 1 -shared `profiles/languages.py` added), and every duplication below collapsed to -a single source of truth. Raw line count is roughly flat because K2 trades -three divergent copies for one shared module (DRY, not fewer lines); the real -win is maintainability — a new step now reuses `_common` guards, and language -rules live in one place. Keep this doc as the record of *why* those helpers -exist so they are not re-inlined. - -## Two axes — do not confuse them - -- **Concision** (this doc): fewer lines, less duplication, less dead code. - Preferred for this refactor. -- **Cohesion split** (per-file "Refactor notes"): break an oversized file into - focused files — improves readability but adds files and does NOT reduce total - lines. - -Where they conflict, **prefer concision**. Example: `agent_runtime.py` (685 LOC) -had a cohesion-split note, but the concision win there was smaller (dedupe the -repair-round and evidence-cap code), not the split — so the dedup came first. -The cohesion split followed once the file was still hard to navigate. - -## Cohesion splits applied (2026-07-10, after K1–K7) - -Two oversized files were split into packages once the concision passes above -left them still dense (219 tests green throughout; public import surfaces -preserved via re-exporting `__init__`s, so no importer changed): -- `engine/agent_runtime.py` (685 LOC) → `engine/agent_runtime/` — - `dispatch`/`knowledge`/`utils` (substrate) + `runner`/`ensemble` (entries). -- `engine/steps/review.py` (341 LOC) → `engine/steps/review/` — - `prompts` (eval-tuned text) + `utils` (deterministic helpers) + `steps` - (handlers). -- `engine/steps/pr.py` (484 LOC) → `engine/steps/pr/` — split by concern: - `fetch` (read-only) + `rebase` + `debug` + `publish` (both risk=push steps) - + `utils` (`extract_signature`). -- `cli.py` (406 LOC) → `cli/` — `copilot` (the orchestrator class, kept whole) - + `entry` (argparse/REPL) + `utils` (pure formatters); `__init__`/`__main__` - preserve the `infermatrix_copilot.cli:main` entry point and `python -m` parity. -In every case the stateless/pure helpers moved to a `utils.py` so the class/ -handler files carry control flow, not plumbing. Eval-citation comments moved -with their code, and public import surfaces were preserved via re-exporting -`__init__`s (one test's white-box monkeypatch target was updated to the new -submodule — `pr.debug._gh` — the only test change across all four splits). - -## Rules the concision refactor must not break - -- Every invariant tagged in a per-file spec survives verbatim; every guard test - still passes (a helper may change *how* a guarantee is met, never *whether*). -- New helpers live where the dependency rules allow (`engine/steps/_common.py` - for step helpers; a leaf data module for shared data). A helper must not - create a forbidden cross-import (`_ARCHITECTURE.md` §4). -- A helper that would be used **once** is not worth it — only extract at ≥2 - real call sites (the numbers below all qualify). - -## Prioritized catalog (highest value first) - -### K1 — Delete dead target dataclasses · ~35 LOC · risk: none -`ModuleTask`, `ModuleSchedule`, `ValidationPlan`, `RebaseRunSpec` in -`targets/base.py` (now `push.py`) were used nowhere but their re-exports -(verified: 0 other importers). Delete the four dataclasses + their re-exports. -**Keep** `PushPolicy`/`PushDecision`/`guard_push` (the live push guard). -*Invariant to preserve:* C4 (push guard) untouched. - -### K2 — Centralize per-language rules · ~30–40 LOC + prevents drift · risk: low -The "what is a source file / a symbol / a branch, per language" data is -triplicated: `review._sweep_targets` (`line_rules`), `establish` -(`LANGUAGE_SUFFIXES` + `scan_modules`), `repo_map` (`_SYMBOL_RES` + `_SUFFIXES`). -Introduce one leaf data module `profiles/languages.py` (pure data + tiny -accessors: suffixes, symbol regex, branch/index regex per language). The three -consumers import it. *Invariant to preserve:* unknown language degrades honestly -in each consumer (file-level sweep / empty module scan / "use grep"). - -### K3 — Step-handler guard helpers · ~40–60 LOC · risk: low -Add to `engine/steps/_common.py` and replace the repeated blocks: -- `require_repo(ctx) -> Path | StepResult` — the **8** `repo is None → BLOCKED` - guards across step files become one line each. -- `adapter_or_result(ctx) -> RepoAdapter | StepResult` (or a `@needs_adapter` - decorator) — the **7** `_adapter_from_state` + `isinstance(..., StepResult)` - guards in `profile.py`. -- `no_llm_gap(ctx, step, effect) -> StepResult` — the **4** identical - "no LLM → record `capability_gap` → return ok/skip" blocks. -- `store_for(adapter) -> ProfileStore` — the **6** `ProfileStore(adapter.profile_dir)` - constructions in `profile.py`. -*Invariant to preserve:* typed BLOCKED returns (B1), the `capability_gap` trace -event (E2), and each step's public name/behavior. - -### K4 — `state_updates` ergonomics · readability + ~10 LOC · risk: low -**21** call sites write `outputs={"state_updates": {...}, ...}` by hand. Add a -small constructor helper — `published(summary, *, state=None, **outputs)` or a -`StepResult.publishing(...)` classmethod — so a handoff is one clear call. -*Invariant to preserve:* **B2** — every state key a later step consumes is still -published; the helper makes it easier to comply, not optional. - -### K5 — Retire the three engine shims · ~26 LOC + 3 files · risk: low -`engine/{builtin_steps,pr_steps,rebase_native_steps}.py` are re-export shims. -Migrate their importers (each shim's spec lists them), then delete the files and -their specs. *Invariant to preserve:* the `rebase_native_steps._RUNTIME` -"same object" guarantee until its fixtures are migrated. - -### K6 — Deduplicate cli gate+confirm · ~15 LOC · risk: low -`Copilot.run_task` and `run_playbook` repeat the plan-review + `[y/N]` confirm -sequence. Extract `_gate_and_confirm(resolution, spec, assume_yes) -> bool`. -*Invariant to preserve:* plan-review before confirm; confirm fires for -`confirm_required or requires_review`. - -### K7 — Fetch-step "from state" early return · ~10 LOC · risk: none -**5** fetch steps open with `if "X" in ctx.state: return ...(state_updates)`. -A `from_state(ctx, key) -> StepResult | None` helper collapses them. -*Invariant to preserve:* the injected/offline test path stays intact. - -## Estimated total -~160–200 LOC removed and ~4 files deleted, with several repeated patterns -reduced to single call sites — before any cohesion split. The step files shrink -most (K3+K4+K7), which is where new steps are added, so the marginal cost of a -new step drops too. - -## Suggested sequence (each independently shippable, tests green between) -1. K1, K5 (pure deletions — smallest diff, immediate shrink). -2. K3, K4, K7 (`_common.py` helpers — the step-file boilerplate collapse). -3. K2 (`profiles/languages.py` — the cross-file dedup). -4. K6 (cli helper). -5. Only then reconsider cohesion splits (per-file Refactor notes), now against a - smaller, deduped baseline. diff --git a/doc/SPEC/_CONSTRAINTS.md b/doc/SPEC/_CONSTRAINTS.md deleted file mode 100644 index e5fdcc762..000000000 --- a/doc/SPEC/_CONSTRAINTS.md +++ /dev/null @@ -1,117 +0,0 @@ -# 01 — Programming Constraints & Invariant Catalog - -The rules that keep the code clean and safe. Each is stated as an enforceable -constraint with the mechanism/test that pins it. A change that violates one is -wrong even if it "works". - -## A. Structural constraints (organization) - -- **A1 — One responsibility per module.** Each per-file spec names its single - responsibility. Adding an unrelated concern means a new module, not a bigger - one. -- **A2 — Dependency direction is down/outward only.** See `_ARCHITECTURE.md` §4. - Leaf packages - never import `engine/`; interface is never imported downward; step modules - never import each other (shared code → `engine/steps/_common.py`). -- **A3 — Steps compose tools; the planner composes steps.** The planner MUST NOT - compose raw tools; a step MUST NOT be a thin alias for a single tool. A step = - one stable engineering action with declared risk, I/O, and success criteria. -- **A4 — Self-registration, single definition.** A step's name, metadata - (kind/risk/scope/triggers) and handler live together at its definition via - `@step(...)` or `register_step(StepSpec(...))`. No central `add()` block. -- **A5 — Knowledge at the edge.** Repo-specific literals (repo names, module - names, domain prompts, absolute paths, CI wiring) live only under - `adapters//`. Pinned by `test_repo_neutral_core` (a per-file leak ceiling - that can only shrink; the shims and delegation files carry the sole known - exceptions). - -## B. Contract constraints (typed, structured, explicit) - -- **B1 — Typed failures, never bare exceptions.** A step returns - `StepResult(ok=False, failure=FailureKind.X)`. The six kinds — RETRYABLE, - REPLAN, TEST_FAILURE, BLOCKED, FORBIDDEN, ESCALATE — route differently in the - executor. An unhandled exception is coerced to BLOCKED, never swallowed. -- **B2 — State handoffs are published, not just mutated.** Any state key a later - step consumes MUST be published via `outputs.state_updates` (JSON-simple; - serialize dataclasses like `PushPolicy`). Direct `ctx.state[...] = v` alone is - lost on resume. Pinned by the resume-integrity tests. (This was the top v2 - correctness bug.) -- **B3 — `when:` conditions reference known keys only.** They read TaskSpec - fields then published state keys; an unknown key blocks loudly at execution, - never silently evaluates false. -- **B4 — Structured agent I/O.** Every `kind == "agent"` step goes through - `run_agent_step` with an explicit dispatch context and a JSON output contract - (base schema + per-step extension), one repair round, typed status → failure - mapping. No ad-hoc `ctx.llm.create()` in a step to do agentic work. Pinned by - the parametrized dispatch test in `test_agent_runtime.py`. - -## C. Safety constraints (permissions & guardrails) - -- **C1 — Tier is derived, never parsed.** `TaskSpec` has no settable tier field; - `tier` is a property of `kind`. Natural language cannot widen permissions. -- **C2 — Generation is structurally read-only.** The planner's generate path - raises if any composed step has `risk ∈ {write_workspace, push, knowledge}`; - write-capable kinds without a vetted playbook escalate. -- **C3 — One tool choke point.** Every tool call passes `tools.dispatch`, which - enforces `ToolScope`/`PathScope` and traces. Three outcomes: allowed / refused - / executed-but-recorded (out-of-scope write inside the writable wall). Agent - steps only ever see tools their scope allows. -- **C4 — One push choke point.** Every push passes `push.guard_push`: - needs an allowing `PushPolicy` AND a non-protected branch; force is - with-lease only; dry-run unless `ALLOW_PUSH=1`. Protected branches are never - pushed to, policy or not. -- **C5 — Outward writes are double-gated.** Posting a comment/answer needs the - explicit `post` intent AND `ALLOW_POST=1`; both default off (dry-run). -- **C6 — Fail-closed reviews.** Plan/patch review returns `unavailable` without a - reviewer; anything but `lgtm`/`pass` is not-passing. A missing reviewer does - not silently approve. -- **C7 — Untrusted data is fenced.** GitHub/CI text enters agent prompts inside - `` with a "not instructions" preamble; only terminal input - reaches the intent parser (channel separation). - -## D. Knowledge-governance constraints - -- **D1 — Facts recorded freely, knowledge promoted via gates.** RunTrace/debug - memory are cheap and append-only. Skills, playbooks, adapters, and profile - facts are candidate→promote; promotion is a curator/human act. -- **D2 — Agents propose, humans dispose (high-risk).** Agents may only write - candidates. `update_manifest` rejects agent writes to `manifest.yaml` - high-risk sections (`push`/`repo`/`upstream`). The profile judge is read-only. -- **D3 — Provenance + stability on every profile fact.** Each carries - `source`/`evidence`/`first_seen`/`last_confirmed`/`confirmations`; an - evidence-free fact is rejected; a stable fact (≥3 confirmations) may not lose - cited evidence on rewrite; superseded text goes to `history`, never deleted. -- **D4 — Two write tiers.** Per-run ops are additive (`RUN_OPS`); only the - scheduled consolidation tier may rewrite/merge/mark-stale. Continuous LLM - rewriting is forbidden (it corrupts memory). -- **D5 — Profile content is channel-typed.** `machine` facts feed code, the - word-budgeted `briefing` slice enters prompts, `retrieved` facts surface only - on demand. Auto-generated overviews must not be injected wholesale. - -## E. Observability & degradation constraints - -- **E1 — Every governance claim has a trace event.** If a guard fired, a scope - refused, a fact was applied, or a capability was missing, there is a RunTrace - event for it (`tool_refused`, `out_of_scope_edit`, `patch_review`, - `capability_gap`, `profile_*`, …). -- **E2 — Degrade explicitly, never silently.** A missing capability (no CI - provider, no LLM, unsupported language) records a `capability_gap` event and a - declared downgrade; it never crashes and never pretends full capability. -- **E3 — Metrics never break a run.** `metrics.py` failures are caught and - traced; a run's success is independent of its metrics. - -## F. Invariant catalog (quick index → owner) - -| # | Invariant | Owner file | Test | -|---|---|---|---| -| C1 | tier from kind | `task_spec.py` | `test_intent_taskspec.py` | -| C2 | generate is read-only | `engine/planner.py` | `test_planner_playbooks.py`,`test_capabilities.py` | -| B1 | typed failure routing | `engine/executor.py` | `test_engine.py` | -| B2 | state_updates on resume | `engine/executor.py` + steps | `test_v2_p0.py` | -| B3 | `when:` known keys | `engine/executor.py` | `test_v2_p0.py` | -| B4 | governed agent I/O | `engine/agent_runtime.py` | `test_agent_runtime.py` | -| C3 | tool choke point | `tools.py` + `scopes.py` | `test_scopes_tools.py` | -| C4 | push choke point | `push.py` | `test_push_and_steps.py` | -| A5 | repo-neutral core | whole `src/` | `test_v2_p0.py::test_repo_neutral_core` | -| D3 | provenance + stability | `profiles/store.py` | `test_profile_store.py` | -| D2 | judge/agent read-only-to-active | `profiles/*`,`adapters/base.py` | `test_p3_machinery.py` | diff --git a/doc/SPEC/adapters/base.md b/doc/SPEC/adapters/base.md deleted file mode 100644 index 3ca4f11ed..000000000 --- a/doc/SPEC/adapters/base.md +++ /dev/null @@ -1,46 +0,0 @@ -# adapters/base.py — spec - -`LOC ~221 · edge (repo knowledge) · refactor-status: ok` - -## Responsibility -`RepoAdapter` (repo knowledge at the edge), the adapter registry, and -deterministic Phase-0 bootstrap. `RepoAdapter` is the **container** for a repo's -two trust tiers (DESIGN §V2.3.0): its `.manifest` is **Tier 1** (`manifest.yaml`, -human-gated config) and `.profile_dir`/`.skills_dir`/`.debug_memory_db` are -**Tier 2** (agent-established, evidence-gated knowledge). Renamed from "plugin" -2026-07-11 (`RepoPlugin`→`RepoAdapter`, `plugin.yaml`→`manifest.yaml`) — -"plugin" wrongly implied executable extension code (see DESIGN naming note). - -## Public contract -`RepoAdapter` (props `status`, `repo_path`, `protected_branches`, `modules`, -`high_risk_modules`, `capabilities`, `skills_dir`, `debug_memory_db`, -`profile_dir`, `briefing()`; `module_for_path`); `load_adapter`, -`update_manifest`, `AdapterRegistry(resolve/all)`; `fingerprint_repo`, -`draft_adapter`. - -## Invariants -- **D2**: `update_manifest` rejects agent writes to `push`/`repo`/`upstream`. -- `capabilities` derives from the manifest (repo.path/language.*/ci.provider/ - upstream.*/modules) + explicit `capabilities:` — matched by playbook - `requires:`. -- `high_risk_modules` = modules with `risk: high` (feeds patch-review, **A5**). -- `fingerprint_repo` deterministic, never mutates the target repo; - `draft_adapter` stops at `status: draft` (human gate). -- `briefing()` renders the profile's briefing slice (empty when no profile). - -## Scope — not here -No task logic, no LLM, no profile-store internals (delegates to -`profiles/store`). - -## Dependencies (allowed) -`profiles/store` (inside `briefing()`), `pyyaml`; stdlib. - -## Tests -`test_adapters.py`, `test_capabilities.py`. - -## Refactor notes -Two concerns coexist cleanly: `RepoAdapter` (accessor) + registry + bootstrap. -If bootstrap grows (richer fingerprint), consider `adapters/bootstrap.py`. Keep -`capabilities` derivation as the single source the planner trusts — do not -duplicate capability logic in `cli.resolve` (it currently only ADDS repo.path -for REPO_PATHS, which is acceptable). diff --git a/doc/SPEC/agent_loop.md b/doc/SPEC/agent_loop.md deleted file mode 100644 index fcf25243b..000000000 --- a/doc/SPEC/agent_loop.md +++ /dev/null @@ -1,37 +0,0 @@ -# agent_loop.py — spec - -`LOC ~100 · engine (tool loop) · refactor-status: ok` - -## Responsibility -The minimal tool-use loop, ToolScope-constrained and RunTrace-audited. - -## Functionality -`run_agent` iterates: LLM call → collect tool_uses → `tools.dispatch` each → -feed results back; on budget exhaustion, force a final tool-free answer. - -## Public contract -`run_agent(llm, *, system, prompt, scope, trace?, model?, max_iters, -extra_tools?) -> AgentOutcome`; `AgentOutcome`. - -## Invariants -- The loop only ever sees tools its scope allows (via `tool_definitions_for`). -- Every call goes through `tools.dispatch` (**C3**). -- Budget exhaustion forces a final answer, not a discarded investigation. - -## Scope — not here -No planning/step semantics; no output-contract coercion (that is -`agent_runtime`); no prompt construction beyond passing through. - -## Dependencies (allowed) -`llm`, `run_trace`, `scopes`, `tools`. - -## Extension points -None expected — it is the raw substrate `agent_runtime` builds on. - -## Tests -`test_agent_loop.py`. - -## Refactor notes -Correctly minimal. Keep it prompt-agnostic and contract-agnostic — all -governance/structure belongs one level up in `agent_runtime`. If the split of -`agent_runtime` happens, this file stays as-is (it is already the "loop" layer). diff --git a/doc/SPEC/chat.md b/doc/SPEC/chat.md deleted file mode 100644 index a76dc5ee8..000000000 --- a/doc/SPEC/chat.md +++ /dev/null @@ -1,44 +0,0 @@ -# chat.py — spec - -`LOC ~373 · interface (conversational REPL) · refactor-status: split-candidate` - -## Responsibility -The Claude-Code-style conversational REPL (default when an LLM is configured): -a persistent chat that answers questions and executes work through tools. - -## Functionality -`SYSTEM_PROMPT` (repo-neutral, interpolates `default_repo`); `TOOL_DEFS` -(run_task/run_playbook/get_status/get_logs/read_run_report/list_playbooks/ -repo_read/repo_grep/resume_run); `ChatSession` (history trim that never splits -tool pairs, streaming, tool round-trips, session transcript); read jail. - -## Public contract -`chat_repl(copilot, assume_yes, handle_builtin)`; `ChatSession.turn`. - -## Invariants -- Frontend, not a second execution path: funnels into `run_task`/`run_playbook` - with the **same** gates; cannot widen permissions. -- `repo_read`/`repo_grep` jailed to configured repo roots + run root; `.env*` - refused. -- Fetched GitHub content stays data (handled inside steps) (**C7**). - -## Scope — not here -No planning/execution logic of its own; no repo-knowledge literals (prompt is -neutralized via `default_repo`). - -## Dependencies (allowed) -`cli.Copilot` (TYPE_CHECKING only), `run_trace`, `task_spec`, `ui`. - -## Extension points -New chat tool → a `TOOL_DEFS` entry + a `_dispatch_tool` branch, jailed and -gate-respecting. - -## Tests -`test_chat.py`. - -## Refactor notes -Mixes three concerns: tool schema/registry (`TOOL_DEFS`), the read jail, and the -turn loop. **Suggested split**: `chat_tools.py` (defs + `_dispatch_tool` + -jail) and `chat.py` (session/turn loop). The read-jail (`_allowed_roots`/ -`_check_read`) is generically useful — consider hoisting to a small shared -helper if any other surface needs jailed reads. diff --git a/doc/SPEC/ci/normalize.md b/doc/SPEC/ci/normalize.md deleted file mode 100644 index f009ed1f8..000000000 --- a/doc/SPEC/ci/normalize.md +++ /dev/null @@ -1,29 +0,0 @@ -# ci/normalize.py — spec - -`LOC ~31 · edge (CI) · refactor-status: ok` - -## Responsibility -Normalize a CI failure signature before grouping. - -## Public contract -`normalize_signature(signature) -> str`. - -## Invariants -- Strips run-varying noise (timestamps, hashes, addresses, tmp paths, line - numbers, durations); keeps small literal numbers as signal. -- Deliberate non-inheritance of the parent monitor's exact-string-compare bug — - the same failure across runs must collapse to one group. - -## Scope — not here -String normalization only — no grouping (that is `pr.group_failures`), no log -fetching (that is `ci/providers`). - -## Dependencies (allowed) -stdlib `re` only. - -## Tests -`test_ci_and_repo_map.py`. - -## Refactor notes -Tiny and pure — the normalization rules are the whole value. If false-merges -appear in practice, tune the regex table here (one place), not at call sites. diff --git a/doc/SPEC/ci/providers.md b/doc/SPEC/ci/providers.md deleted file mode 100644 index 25bd84b15..000000000 --- a/doc/SPEC/ci/providers.md +++ /dev/null @@ -1,35 +0,0 @@ -# ci/providers.py — spec - -`LOC ~118 · edge (CI) · refactor-status: ok` - -## Responsibility -Profile-selected CI log adapters for pr-debug. - -## Public contract -`provider_for(adapter, settings, gh_runner?) -> (provider|None, gap_reason)`; -`BuildkiteLogs.enrich`, `GithubActionsLogs.enrich`. - -## Invariants -- Provider chosen by `profile.ci.provider`, never hardcoded. -- `enrich` is best-effort **per check** — an API error leaves that check - name-grouped, never crashes. -- Missing provider/token → `(None, reason)`; the calling step records a - `capability_gap` and degrades to name grouping (**E2**). - -## Scope — not here -Log fetching only — no grouping/debugging logic; no signature normalization. - -## Dependencies (allowed) -stdlib `urllib`/`json`/`re`; the `gh_runner` callable is injected (no direct -`._common.gh` import, keeping this package engine-independent). - -## Extension points -New CI system → a `*Logs` class with `enrich` + a `provider_for` branch. - -## Tests -`test_ci_and_repo_map.py`. - -## Refactor notes -Clean adapter pattern. Keep the `enrich(failures) -> count` contract stable so -`pr.py` stays agnostic. The `gh_runner` injection is deliberate — do not import -`engine.steps._common` here (that would invert the dependency direction, **§ARCH.4.1**). diff --git a/doc/SPEC/cli.md b/doc/SPEC/cli.md deleted file mode 100644 index 32b9c6d1f..000000000 --- a/doc/SPEC/cli.md +++ /dev/null @@ -1,54 +0,0 @@ -# cli/ — spec - -`LOC ~410 across 4 files · interface + orchestration façade · refactor-status: split-applied (2026-07-11)` - -## Responsibility -The flag CLI and the `Copilot` façade: resolve → gate → execute; own the run -directory, RunTrace, notifier, and metrics wiring. Formerly one 406-line module; -now a package that separates the argparse/REPL wiring from the orchestrator. - -## Package layout (one concern per file) -- `__init__.py` — re-exports `Copilot`, `main` (surface below); no logic. -- `__main__.py` — `python -m infermatrix_copilot.cli` parity. -- `copilot.py` — the `Copilot` orchestrator (resolve/run_task/run_playbook/ - run_queue/resume_last/_execute + built-ins). -- `entry.py` — `argparse`, `_handle_line`, `main` (turns argv/stdin into calls - on `Copilot`). -- `utils.py` — pure formatters: `parse_task_params`, `format_metrics_line`. - -## Public contract (importable from `infermatrix_copilot.cli`) -`main(argv)`; `Copilot` (`resolve`, `run_task`, `run_playbook`, `run_queue`, -`resume_last`, `status`, `logs`, `playbooks`, `_execute`, `_adapter_for`, -`_resolve_repo_path`). The re-exporting `__init__` keeps `infermatrix_copilot.cli:main` -(entry point) and `from infermatrix_copilot.cli import Copilot` unchanged. - -## Invariants -- `resolve` feeds capabilities (adapter + REPO_PATHS) to the planner. -- Plan-review gate before confirm; confirm fires for - `confirm_required or requires_review` unless `--yes` (`_gate_and_confirm`, K6). -- `_execute` is the single execution path (task / explicit-playbook / resume). -- Repo knowledge (protected branches, high-risk modules) comes from the adapter - into run state (**A5**); blocked → exit 3 (`BLOCKED_EXIT`). -- `--playbook` is the only way to run a candidate. - -## Scope — not here -No step logic, no repo-knowledge literals, no LLM prompts. Orchestration wiring -only. - -## Dependencies (allowed) -`engine/*`, `playbooks/*`, `intent`, `task_spec`, `adapters/base`, -`push`, `review/reviewer`, `notify`, `run_trace`, `config`, `ui`, -`chat`. MUST NOT be imported by any lower layer (**§ARCH.4.2**). - -## Extension points -New REPL command → `_handle_line` (entry.py); new run wiring → `_execute` -(copilot.py); new pure formatter → utils.py. - -## Tests -`test_cli.py`, `test_phase_b.py`, `test_chat.py`, `test_ui.py`. - -## Refactor notes -Split **applied** (was a cohesion-split candidate). The `Copilot` class stays -whole in `copilot.py` so the resolve→execute flow is followed in one file; only -the argparse/REPL front-end (`entry.py`) and the two pure formatters (`utils.py`) -moved out. K6 (`_gate_and_confirm`) is done and lives on the class. diff --git a/doc/SPEC/config.md b/doc/SPEC/config.md deleted file mode 100644 index da688dca4..000000000 --- a/doc/SPEC/config.md +++ /dev/null @@ -1,42 +0,0 @@ -# config.py — spec - -`LOC ~116 · configuration · refactor-status: ok` - -## Responsibility -`Settings` (pydantic-settings) loaded from env / `.env`. - -## Functionality -Typed fields + safe defaults for LLM, repos, engine, push safety, PR debug, -external rebase, agent runtime, ensemble, profiles, patch triggers, metrics, -escalation. Derivation helpers. - -## Public contract -`Settings` with all tunables; `reviewer`, `intent` (fallback to `agent_model`); -`repo_path(name)`. - -## Invariants -- Secrets only via env/`.env` (git-ignored, never committed). -- Repo-specific defaults (`default_repo`, `rebase_agent_root`, - `high_risk_modules`, `cost_ref_*` keys) are **fallbacks only**; adapter/profile - overrides them (**A5**). These are the sole allowed repo literals here - (leak-capped at 3). - -## Scope — not here -No logic beyond derivation helpers; no I/O beyond env loading. - -## Dependencies (allowed) -`pydantic-settings` only. - -## Extension points -New tunable → a typed field with a safe default and a one-line comment stating -meaning/units. - -## Tests -Exercised indirectly across the suite (fixture builds `Settings(_env_file=None, -...)`). - -## Refactor notes -Growing broad but single-concern — acceptable. If it keeps growing, group into -nested settings models (LLMSettings/PushSettings/ProfileSettings) rather than -splitting the file. Do NOT move repo-specific defaults out to adapters yet -(they're fallbacks); the refactor goal is that a profiled repo never reads them. diff --git a/doc/SPEC/engine/agent_runtime.md b/doc/SPEC/engine/agent_runtime.md deleted file mode 100644 index 2e1a471d9..000000000 --- a/doc/SPEC/engine/agent_runtime.md +++ /dev/null @@ -1,71 +0,0 @@ -# engine/agent_runtime/ — spec - -`LOC ~620 across 6 files · engine (governed agent runtime) · refactor-status: split-applied (2026-07-10)` - -## Responsibility -The single governed entry for every `kind == "agent"` step, plus the -review-quality ensemble. The densest, highest-leverage subsystem in the repo — -formerly one 685-line module, now a package whose substrate (dispatch/knowledge/ -utils) is separated from its two entry points (runner/ensemble). - -## Package layout (one concern per file) -- `__init__.py` — public re-exports only (surface below); no logic. -- `dispatch.py` — the agent **input contract**: `AgentDispatchContext` (+ its - `render()`) and `BASE_OUTPUT_SCHEMA`. Prompt shape, isolated from control flow. -- `knowledge.py` — repo-scoped knowledge: `_resolve_adapter`, `_ScopedKnowledge`, - `_knowledge_stores`, `_retrieve_skills`, `_retrieve_memories`, - `_knowledge_tools`, `_repo_map_tool`. -- `utils.py` — stateless helpers: `_build_evidence`, `_permissions_view`, - `_coerce_output`, `_to_step_result` (+ the status→FailureKind maps). -- `runner.py` — `run_agent_step`: assembles dispatch context, packs evidence, - retrieves knowledge, runs the tool loop, coerces output, traces everything. -- `ensemble.py` — `run_agent_step_ensemble`: perspective-diverse lens fan-out + - verify-and-merge reduction. - -## Public contract (importable from `engine.agent_runtime`) -`run_agent_step(...) -> (StepResult, output)`; `run_agent_step_ensemble(...)`; -`AgentDispatchContext`; `BASE_OUTPUT_SCHEMA`; `_resolve_adapter`, -`_retrieve_skills` (used by steps/tests). The re-exporting `__init__` keeps the -pre-split import paths (`from ..agent_runtime import X`) unchanged. - -## Invariants (**B4**) -- Single entry: agent steps do agentic work only through `run_agent_step` — no - ad-hoc `ctx.llm.create()` for investigation. -- Evidence capped per item + archived + fenced `` (**C7**). -- `_ScopedKnowledge`: repo skills+memory before shared pool; proposals land in - repo namespace, candidates only (**D1/D2**). -- `_repo_map_tool`: on-demand, never injected as prose; unsupported language → - `capability_gap`. -- Briefing enters prompt only when `profile_briefing_enabled` (ablation switch). -- Output: base+extension schema, one repair round, status→FailureKind; budget - exhaustion forces a final answer. Full trace (`agent_dispatch`/`agent_output`). -- Ensemble reducer: per-numbered-candidate keep/drop/dup, deterministic - assembly, unmentioned kept (fail-open), consensus-gated fast path. - -## Internal dependency rule -`runner`/`ensemble` depend on `dispatch`/`knowledge`/`utils`, never the reverse; -`ensemble` depends on `runner` (not vice-versa). The substrate files import no -sibling entry point, so there is no cycle. - -## Scope — not here -No task/planning logic; no repo literals (system prompt repo-neutral). Step- -specific prompts/lenses live in the step file (e.g. `steps/review/prompts.py`). - -## Dependencies (allowed) -`agent_loop`, `llm`, `memory/*`, `adapters/base`, `profiles/repo_map`, `scopes`, -`tools`, `engine/step`. - -## Extension points -A new list-output agent step adopts the ensemble by passing lenses + -merge_guidance; a new knowledge source → extend `_ScopedKnowledge`/ -`_knowledge_tools` in `knowledge.py`. - -## Tests -`test_agent_runtime.py`, `test_agent_ensemble.py`, `test_review_step.py`. - -## Refactor notes -Split **applied** (was the highest-priority cohesion target). The inline -eval-citation comments are institutional memory — they moved WITH their code -into the sibling files, never dropped. `ensemble.py` (~290 LOC) remains the -largest file; its reducer is a single cohesive algorithm and is not a further -split target. diff --git a/doc/SPEC/engine/executor.md b/doc/SPEC/engine/executor.md deleted file mode 100644 index b218282da..000000000 --- a/doc/SPEC/engine/executor.md +++ /dev/null @@ -1,46 +0,0 @@ -# engine/executor.py — spec - -`LOC ~203 · engine substrate (the loop) · refactor-status: ok` - -## Responsibility -Run a playbook's steps with task-agnostic guarantees: checkpoint/resume, -foreach fan-out, `when:` conditions, bounded retries, typed failure routing, -escalation. - -## Functionality -`run` iterates steps; skips completed (restoring their `state_updates`); fans -out `foreach`; evaluates `when:`; retries RETRYABLE; routes BLOCKED/ESCALATE/ -FORBIDDEN to the notifier; persists progress + indexes outputs. - -## Public contract -`Executor(registry, settings, run_dir, trace, llm?, notifier?)`; -`run(playbook, state) -> RunOutcome(status, step_results, blocked_reason)`. -Helpers `_eval_when`, `_merge`. - -## Invariants -- **B2**: resume restores `outputs.state_updates` before skipping; on success, - `state.update(state_updates)` and index outputs by step id. -- `_merge` lifts each foreach item's `state_updates` (last-writer-wins). -- **B3**: `when:` reads TaskSpec then state; unknown key → blocked, not silent. -- **B1**: typed routing; unhandled exception → BLOCKED (never swallowed). -- Retries only on RETRYABLE, bounded by `max_step_retries`. - -## Scope — not here -No step logic, no repo knowledge, no planning, no LLM prompts. Execution -guarantees only. - -## Dependencies (allowed) -`engine/{step,registry}`; typing-only: config/llm/notifier/playbook/run_trace. - -## Extension points -A new cross-step guarantee (e.g. per-step timeout) goes here; a new step -behavior does NOT. - -## Tests -`test_engine.py`, `test_v2_p0.py` (resume/foreach/when integrity). - -## Refactor notes -Well-scoped at ~200 lines. The `state_updates` publish/restore/merge logic is -subtle and load-bearing (it was the top v2 bug) — do not "simplify" it without -re-running the resume-integrity tests. If DAG edges are ever added (currently -ordered lists only), keep them here behind the same `RunOutcome` contract. diff --git a/doc/SPEC/engine/planner.md b/doc/SPEC/engine/planner.md deleted file mode 100644 index 2b5bc77b3..000000000 --- a/doc/SPEC/engine/planner.md +++ /dev/null @@ -1,46 +0,0 @@ -# engine/planner.py — spec - -`LOC ~116 · planning · refactor-status: ok` - -## Responsibility -Resolve a `TaskSpec` (+ repo capabilities) to a runnable `Playbook` via -**reuse > adapt > generate**, with capability-gap handling. - -## Functionality -Looks up a playbook (`store.find`); returns reuse (verbatim) / adapt (extra -params on non-locked → review) / generate (read-only kinds, fixed template); -raises `PlanningError` on locked-adapt, write-capable generate, or capability -gaps. - -## Public contract -`Planner(store, registry)`; `resolve(spec, capabilities?) -> Resolution`; -`PlanningError`; `_GENERATE_TEMPLATES`. - -## Invariants -- Reuse: `requires_review=False` when params ⊆ declared surface. -- Locked playbook refuses adaptation (raises). -- **C2**: generate is read-only-kinds-only and re-checks every step is - `risk ∈ {read, report}` (raises otherwise). -- Capability gap (**§ARCH.8**): write-capable + unmet → raise "run repo_profile"; - read-only → generate; `capabilities=None` = v1 behavior. -- Tier comes from `spec.tier` — never invented. - -## Scope — not here -Selection/parameterization only. No execution, no repo knowledge, no LLM, -no raw-tool composition (**A3**). - -## Dependencies (allowed) -`playbooks/store`, `task_spec`, `engine/registry`. - -## Extension points -New read-only kind needing generation → a `_GENERATE_TEMPLATES` entry of -read/report steps. Write-capable kinds must ship a vetted playbook. - -## Tests -`test_planner_playbooks.py`, `test_capabilities.py`. - -## Refactor notes -Small and clean; the three branches are the whole point — do not collapse them. -The capability-gap message text is user-facing guidance ("run repo_profile") — -keep it actionable. If generate ever needs LLM composition, it must still pass -the per-step risk re-check; never bypass C2 for flexibility. diff --git a/doc/SPEC/engine/registry.md b/doc/SPEC/engine/registry.md deleted file mode 100644 index 88da68554..000000000 --- a/doc/SPEC/engine/registry.md +++ /dev/null @@ -1,34 +0,0 @@ -# engine/registry.py — spec - -`LOC ~31 · engine substrate · refactor-status: ok` - -## Responsibility -`StepRegistry`: the name → `StepSpec` map — the single place a name string -resolves to a handler. - -## Functionality -`register`/`get`/`__contains__`/`names`. - -## Public contract -`StepRegistry` with the methods above. - -## Invariants -- Duplicate registration raises; `get` on unknown raises with the registered set - (fail loudly, never silent). - -## Scope — not here -Storage/lookup only — no execution, no policy, no self-population (population is -`steps.register_builtin_steps`). - -## Dependencies (allowed) -`engine/step`. - -## Extension points -None needed; it is a container. - -## Tests -Exercised via `register_builtin_steps` in most tests. - -## Refactor notes -Minimal and correct. Do not add filtering/policy here — that belongs to the -planner (which selects) and the store (which validates references). diff --git a/doc/SPEC/engine/step.md b/doc/SPEC/engine/step.md deleted file mode 100644 index 611c09b95..000000000 --- a/doc/SPEC/engine/step.md +++ /dev/null @@ -1,45 +0,0 @@ -# engine/step.py — spec - -`LOC ~67 · engine base vocabulary · refactor-status: ok` - -## Responsibility -The base execution vocabulary shared by the whole engine. - -## Functionality -Defines the failure taxonomy, the step result/context/spec dataclasses, and the -`Kind`/`Risk` literal sets. - -## Public contract -`FailureKind` (RETRYABLE, REPLAN, TEST_FAILURE, BLOCKED, FORBIDDEN, ESCALATE); -`StepResult(ok, failure?, summary, outputs, changed_files)`; -`StepContext(settings, state, params, run_dir, trace, llm?, item?)`; -`StepSpec(name, kind, risk, handler, description)`. `Kind ∈ {deterministic, -script, agent, validation, report}`; `Risk ∈ {read, write_workspace, push, -knowledge, report}`. - -## Invariants -- Repo- and task-agnostic; `StepSpec` is frozen. -- The six `FailureKind`s are the complete routing vocabulary (**B1**). -- `risk` is enforced (planner C2); `kind` is descriptive (`agent` ⇒ governed - runtime by convention). No decorative fields: a `StepSpec` field must be read - by something (the `tool_scope`/`patch_review_triggers` fields were removed - because nothing enforced them — scope/triggers are handler-local). - -## Scope — not here -Types only — no behavior, no registry, no execution. - -## Dependencies (allowed) -Only `run_trace` (**§ARCH.4.3**). Nothing task/repo-specific. - -## Extension points -A new `Kind`/`Risk` value or `FailureKind` is a deliberate vocabulary change — -update the executor routing and `_CONSTRAINTS` together. - -## Tests -Exercised everywhere; shape pinned implicitly. - -## Refactor notes -Foundational — keep the dependency set at exactly {run_trace}. Adding an -import here would couple the whole engine to it. If `StepContext` grows more -optional fields, that's a signal a step is over-reaching, not that this file -needs restructuring. diff --git a/doc/SPEC/engine/steps/__init__.md b/doc/SPEC/engine/steps/__init__.md deleted file mode 100644 index f05411fd8..000000000 --- a/doc/SPEC/engine/steps/__init__.md +++ /dev/null @@ -1,40 +0,0 @@ -# engine/steps/__init__.py — spec - -`LOC ~31 · step library aggregation · refactor-status: ok` - -## Responsibility -Import every step module for its registration side effects and expose -`register_builtin_steps`. - -## Functionality -Imports `_common` + all 8 domain modules (running their `@step`/`register_step` -decorators, which fill `_common._COLLECTED`); `register_builtin_steps` flushes -the collection into a `StepRegistry`. - -## Public contract -`register_builtin_steps(registry) -> registry`. - -## Invariants -- Importing the package registers all steps exactly once (module cache); - `register_builtin_steps` is idempotent per fresh registry (**A4**). -- The import list is the authoritative set of step modules — a new step module - must be added here to be discovered. - -## Scope — not here -No step logic, no registration policy beyond flushing. - -## Dependencies (allowed) -`engine/registry`, `._common`, the 8 step modules. - -## Extension points -New step domain module → add it to the side-effect import list. - -## Tests -Exercised by every `register_builtin_steps(StepRegistry())` call (smoke: 38 -steps). - -## Refactor notes -Consider auto-discovery (iterate the package) to drop the manual import list — -but explicit imports are grep-able and fail loudly on typos, so the current -form is acceptable. If kept manual, this list + `_COLLECTED` is the single -source of "which steps exist". diff --git a/doc/SPEC/engine/steps/_common.md b/doc/SPEC/engine/steps/_common.md deleted file mode 100644 index e1d5d4fde..000000000 --- a/doc/SPEC/engine/steps/_common.md +++ /dev/null @@ -1,51 +0,0 @@ -# engine/steps/_common.py — spec - -`LOC ~141 · step library infrastructure · refactor-status: ok` - -## Responsibility -The step self-registration surface + cross-module helpers shared by step files. - -## Functionality -`@step`/`register_step` decorator + `_COLLECTED` + `collected()`; helpers -`repo_path`, `task_spec`, `gh`, `git`, `gh_read_tools`, `post_step`. - -## Public contract -`step(name, kind, risk, description)`; `register_step(StepSpec)`; -`collected()`; the helpers above (incl. the K3/K4/K7 guard helpers). - -## Invariants -- Duplicate step name → raise at import (**A4**). -- The **only** home for step-shared helpers — step modules import from here, - never from each other (**A2**). -- Helpers are thin, side-effect-honest, repo-neutral. - -## Scope — not here -No step handlers; no domain logic. Infrastructure + shared IO only. - -## Dependencies (allowed) -`engine/step`; `...tools` (inside `gh_read_tools`); stdlib. - -## Extension points -A helper genuinely shared by ≥2 step files → add here. A helper used by one step -belongs in that step's file. - -## Tests -Exercised via the step tests; `post_step`/`gh`/`git` patched in -`test_ci_and_repo_map.py` etc. - -## Refactor notes -Watch for scope creep — if it accretes many single-user helpers, push them back -to their step. `gh`/`git`/`repo_path` are the true cross-cutting ones; keep the -monkeypatch seam (tests patch `steps.pr._gh`, i.e. the import alias) documented -so a future rename doesn't silently break patching. - -## Concision — home for **K3/K4/K7** helpers -This file is where the step-boilerplate collapse lands. Add: -- `require_repo(ctx) -> Path | StepResult` (K3 — 8 repo-guard sites). -- `adapter_or_result(ctx)` / `@needs_adapter` (K3 — 7 profile guards), - `no_llm_gap(ctx, step, effect)` (K3 — 4 blocks), `store_for(adapter)` (K3 — 6). -- `published(summary, *, state=None, **outputs)` (K4 — 21 `state_updates` - literals collapse to one call). -- `from_state(ctx, key)` (K7 — 5 fetch early-returns). -Each must preserve the guarantee it wraps (B1 typed returns, B2 handoff, E2 -`capability_gap` event). Only extract at ≥2 real sites (all above qualify). diff --git a/doc/SPEC/engine/steps/issue.md b/doc/SPEC/engine/steps/issue.md deleted file mode 100644 index 6f9d2d0da..000000000 --- a/doc/SPEC/engine/steps/issue.md +++ /dev/null @@ -1,34 +0,0 @@ -# engine/steps/issue.py — spec - -`LOC ~131 · step library (issues) · refactor-status: ok` - -## Responsibility -Issue fetch, drafted-answer & triage agent steps, and gated posting. - -## Steps (4) -`issue.fetch` (deterministic/read); `agent.draft_issue_answer`, -`agent.triage_issues` (agent/read, read-only scope); `issue.post_answer` -(script/push). - -## Invariants -- Agent steps ground claims in fetched text/code via the governed runtime. -- `issue.post_answer` double-gated (**C5**). -- `_issue_agent_step` is the shared factory; the two `agent.*` handlers are - registered imperatively (`register_step`). - -## Scope — not here -No agent governance internals; no posting authorization beyond `post_step`. - -## Dependencies (allowed) -`scopes`, `engine/step`, `._common`, `..agent_runtime`. - -## Tests -Covered via issue playbooks + `test_agent_runtime` paths. - -## Refactor notes -Well-sized. The `_issue_agent_step` factory + `_render_answer`/`_render_triage` -pattern is clean; if a third issue agent step appears, keep using the factory. - -## Concision — **K4/K7** -`issue.fetch` uses the "from state" early-return (K7 `from_state`) and verbose -`state_updates` literals (K4 `published(...)`). Small wins; preserve B2. diff --git a/doc/SPEC/engine/steps/pr.md b/doc/SPEC/engine/steps/pr.md deleted file mode 100644 index 2287669f3..000000000 --- a/doc/SPEC/engine/steps/pr.md +++ /dev/null @@ -1,60 +0,0 @@ -# engine/steps/pr/ — spec - -`LOC ~500 across 6 files · step library (PR) · refactor-status: split-applied (2026-07-11)` - -## Responsibility -Guarded push, read-only PR fetch/gate, PR rebase, PR debug, gated review -posting. Formerly one 484-line module; now a package split by concern. - -## Package layout (one concern per file) -- `__init__.py` — side-effect imports of the submodules (`@step`/`register_step` - registration); re-exports `extract_signature`. No logic. -- `fetch.py` — read-only fetches: `pr.fetch_diff`, `pr.gate_check`. -- `rebase.py` — `pr.checkout_branch`, `pr.rebase_onto_base`, `pr.analyze_diff`, - `agent.verify_module`. -- `debug.py` — `pr.fetch_ci_failures` (+ `_enrich_ci_logs`), `pr.group_failures`, - `agent.debug_group`. -- `publish.py` — outward writes (risk=push): `ci.push`, `pr.post_review`. -- `utils.py` — the pure `extract_signature` (+ its regex). - -## Steps (11) -`ci.push` (script/push); `pr.fetch_diff`, `pr.gate_check`, `pr.checkout_branch`, -`pr.analyze_diff`, `pr.fetch_ci_failures`, `pr.group_failures` -(deterministic/read); `pr.rebase_onto_base`, `agent.debug_group` -(agent/write_workspace); `agent.verify_module` (validation/read); -`pr.post_review` (script/push). - -## Public contract (importable from `engine.steps.pr`) -`extract_signature` (re-exported from `utils`; used by tests). - -## Invariants -- `ci.push` delegates all safety to `guard_push` (**C4**). -- `pr.checkout_branch` publishes the derived `PushPolicy` serialized (**B2**) — - resume at push must not see the deny-all default. -- `pr.rebase_onto_base`: governed-agent conflict resolution or abort+escalate - (workspace always restored). -- `pr.fetch_ci_failures` enriches logs via the profile-selected CI provider or - records a `capability_gap` (**E2**); `pr.group_failures` groups by - **normalized** signature. -- `pr.post_review` double-gated (**C5**). - -## Scope — not here -No push authorization logic (that is `push`); no CI log fetching -mechanics (that is `ci/providers`); no agent governance (that is -`agent_runtime`). - -## Dependencies (allowed) -`scopes`, `push`, `ci/*`, `adapters/base` (analyze), `engine/step`, -`.._common`, `..agent_runtime`. - -## Tests -`test_pr_steps.py`, `test_push_and_steps.py`, `test_ci_and_repo_map.py` -(note: `test_ci_and_repo_map` monkeypatches `pr.debug._gh`, the submodule where -`pr.fetch_ci_failures` binds `gh`). - -## Refactor notes -Split **applied**. The submodules share only `_common` helpers, so the split -creates no cross-import. The read/write axis is explicit: `fetch` is read-only, -`publish` holds both risk=push steps. K3/K4/K7 concision (require_repo/published/ -from_state) is in place across the submodules. `_enrich_ci_logs` stays the thin -seam to `ci/providers`. diff --git a/doc/SPEC/engine/steps/profile.md b/doc/SPEC/engine/steps/profile.md deleted file mode 100644 index 11e28e526..000000000 --- a/doc/SPEC/engine/steps/profile.md +++ /dev/null @@ -1,50 +0,0 @@ -# engine/steps/profile.py — spec - -`LOC ~387 · step library (profile pipeline) · refactor-status: split-candidate` - -## Responsibility -The repo-profile establishment (Stages 0–1.5) + Stage-4 maintenance steps. - -## Steps (8) -Establishment: `profile.fingerprint`, `profile.structure_scan`, -`profile.ingest_docs` (deterministic/knowledge), `agent.profile_repo` -(agent/knowledge). Stage-4: `profile.detect_drift` (deterministic/read), -`profile.decay_stale` (deterministic/knowledge), `agent.profile_consolidate` -(agent/knowledge), `profile.judge` (agent/read). - -## Invariants -- `fingerprint` drafts a adapter at `status: draft` for unknown repos (human - gate); `structure_scan` never overwrites declared modules. -- `agent.profile_repo` facts without evidence are rejected by the store; - redundancy-filtered; overviews forbidden. -- `agent.profile_consolidate` is the ONLY rewrite/merge tier; the LLM's ops pass - the store's stability gates (**D3/D4**). -- `profile.judge` never calls `apply_ops` (read-only, **D2**). -- No-LLM paths record a `capability_gap` and run only deterministic stages. - -## Scope — not here -No store internals (that is `profiles/store`); no establishment helpers (those -are `profiles/establish`/`consolidate`). - -## Dependencies (allowed) -`adapters/base`, `profiles/*`, `engine/step`, `._common`, `..agent_runtime`. - -## Tests -`test_profile_steps.py`, `test_p3_machinery.py`. - -## Refactor notes -Two lifecycles in one file. **Suggested split**: `steps/profile_establish.py` -(fingerprint/structure_scan/ingest_docs/profile_repo) and -`steps/profile_maintain.py` (detect_drift/decay_stale/consolidate/judge). They -share only `_adapter_from_state` (→ `_common` or a tiny `profiles` helper) and -the store — no cross-import. Guidance prompt constants could move beside the -review prompts under a `steps/*_prompts.py` convention. - -## Concision — **K3** (biggest boilerplate collapse here) -This file has the densest guard repetition: **7** `_adapter_from_state` + -`isinstance(..., StepResult)` guards, **6** `ProfileStore(adapter.profile_dir)` -constructions, **3** no-LLM `capability_gap` blocks. Replace with `_common` -helpers `adapter_or_result`/`@needs_adapter`, `store_for(adapter)`, -`no_llm_gap(...)`. Expect ~25–35 LOC saved in this file alone. Preserve: each -step's name/behavior, the store's gates (D3/D4), and the `capability_gap` -events (E2). Do NOT let the helper hide the read-only-ness of `profile.judge`. diff --git a/doc/SPEC/engine/steps/rebase_ext.md b/doc/SPEC/engine/steps/rebase_ext.md deleted file mode 100644 index 419fff411..000000000 --- a/doc/SPEC/engine/steps/rebase_ext.md +++ /dev/null @@ -1,31 +0,0 @@ -# engine/steps/rebase_ext.py — spec - -`LOC ~101 · step library (delegation) · refactor-status: ok` - -## Responsibility -`rebase.run_external` — monitored subprocess delegation to the locked 5-phase -orchestrator (wrap-don't-rewrite). - -## Steps -`rebase.run_external` (script/write_workspace). - -## Invariants -- Zero-regression: does NOT reimplement the pipeline. -- Streams parent `state.json` into RunTrace; stale-state guard prevents a prior - run's `phase=done` masking a crash; failures classified into escalation - material. -- Names the parent package (an allowed repo literal, leak-capped at 1). - -## Scope — not here -No rebase logic of its own; no parsing beyond delegating to `rebase/monitor`. - -## Dependencies (allowed) -`rebase/monitor`, `engine/step`, `._common`; stdlib asyncio/subprocess. - -## Tests -`test_rebase_monitor.py` (the monitor it drives). - -## Refactor notes -Acceptable. The one repo literal (`"vllm-omni-rebase-agent"`) is by-design -delegation text — do not templatize it prematurely; if a second external -orchestrator is ever wrapped, then extract the name to the adapter. diff --git a/doc/SPEC/engine/steps/rebase_native.md b/doc/SPEC/engine/steps/rebase_native.md deleted file mode 100644 index 4936fae5c..000000000 --- a/doc/SPEC/engine/steps/rebase_native.md +++ /dev/null @@ -1,37 +0,0 @@ -# engine/steps/rebase_native.py — spec - -`LOC ~397 · step library (candidate native rebase) · refactor-status: ok` - -## Responsibility -The candidate native decomposition of the nightly rebase, importing the parent -package's own phase wrappers + `node_rebase_module`. - -## Steps (9) -`rebase.prelude`, `rebase.phase1..phase5`, `rebase.phase2_prepare`, -`rebase.module_rebase`, `rebase.phase2_finalize`, `rebase.compare_with_locked`. - -## Public contract (importable from `engine.steps.rebase_native`) -`_RUNTIME` (per-process memoized parent runtime; test fixtures clear it). - -## Invariants -- Invisible to the planner (candidate playbook `repo-rebase-native` only). -- Phase-4 push is behind the copilot push guard; env export is traced - (`env_exported`). -- Delegates to the parent's functions — does not reimplement phases. -- Names the parent package (allowed repo literals, leak-capped at 6). - -## Scope — not here -No promotion logic; no rebase reimplementation. Wrapper only. - -## Dependencies (allowed) -`rebase/monitor`, `adapters/base` (wave cross-check), `engine/step`, `._common`; -the external `agent.*` package (lazy imports, ImportError → BLOCKED). - -## Tests -`test_rebase_native.py`. - -## Refactor notes -By design coupled to the parent package — the 6 repo literals and the parent -imports are the delegation surface, not a smell. Registered imperatively -(mixed factory/direct handlers) — that is the sanctioned exception to `@step`. -Only touch when working the promotion path (candidate → active → locked). diff --git a/doc/SPEC/engine/steps/report.md b/doc/SPEC/engine/steps/report.md deleted file mode 100644 index 12703284e..000000000 --- a/doc/SPEC/engine/steps/report.md +++ /dev/null @@ -1,25 +0,0 @@ -# engine/steps/report.py — spec - -`LOC ~24 · step library · refactor-status: ok` - -## Responsibility -`report.final_summary` — write `RUN_REPORT.md` from accumulated step outputs. - -## Steps -`report.final_summary` (report/report). - -## Invariants -- Pure output; no failure paths (always ok). -- Reads `ctx.state["outputs"]`; writes only the run dir. - -## Scope — not here -No analysis, no repo knowledge, no side effects beyond the report file. - -## Dependencies (allowed) -`engine/step`, `._common`. - -## Tests -Exercised in playbook end-to-end tests. - -## Refactor notes -Trivial. If richer reporting is wanted, keep it additive and side-effect-free. diff --git a/doc/SPEC/engine/steps/review.md b/doc/SPEC/engine/steps/review.md deleted file mode 100644 index 971fc2096..000000000 --- a/doc/SPEC/engine/steps/review.md +++ /dev/null @@ -1,47 +0,0 @@ -# engine/steps/review/ — spec - -`LOC ~350 across 4 files · step library (review) · refactor-status: split-applied (2026-07-10)` - -## Responsibility -The conditional patch gate + the PR-review agent step and its repo-neutral -prompt system. Formerly one 341-line module; now a package that separates the -eval-tuned prompt data from the handlers and the deterministic helpers. - -## Package layout (one concern per file) -- `__init__.py` — imports `steps` for `@step` registration side effects; - re-exports the public contract (below). No logic. -- `prompts.py` — eval-derived prompt data: `_REVIEW_SYSTEM`, `_REVIEW_LENSES`, - `_REVIEW_MERGE`. ~120 lines of text kept out of the control flow. -- `utils.py` — deterministic, LLM-free helpers: `_sweep_targets`, - `_render_review_md`, `_SEVERITY_ORDER`. -- `steps.py` — the two `@step` handlers: `review.patch_gate` (validation/read), - `agent.review_diff` (agent/read). - -## Public contract (importable from `engine.steps.review`) -`_REVIEW_LENSES`, `_render_review_md`, `_sweep_targets` — re-exported from the -package `__init__` so the pre-split import paths are unchanged. - -## Invariants -- Patch gate: cheap summary always, LLM review only on a trigger; fail-closed - (**C6**); high-risk modules from the adapter, settings fallback (**A5**). -- Review: domain checklist extends from the profile's `review.md`; - `_sweep_targets` keyed on `repo.language`, degrades honestly; verdict - coherence (any ≥minor comment ⇒ REQUEST CHANGES); deterministic - severity-ordered comment cap. -- Prompts are repo-neutral (**A5**). - -## Scope — not here -No agent-runtime governance (that is `agent_runtime`); no adapter/profile writes. - -## Dependencies (allowed) -`review/*`, `engine/step`, `.._common`, `..agent_runtime`, `profiles/languages`. - -## Tests -`test_review_step.py`, `test_agent_ensemble.py`, -`test_profile_steps.py::test_review_guidance_from_profile`. - -## Concision — **K2** (shared language rules, applied) -`_sweep_targets` consumes `profiles/languages.py::sweep_re` — one of the three -former copies of "per-language rules" (also `profiles/establish`, `repo_map`), -now collapsed to that single source. Unknown-language honest degradation -(file-level sweep only) is preserved. diff --git a/doc/SPEC/engine/steps/workspace.md b/doc/SPEC/engine/steps/workspace.md deleted file mode 100644 index 7769ed3b2..000000000 --- a/doc/SPEC/engine/steps/workspace.md +++ /dev/null @@ -1,32 +0,0 @@ -# engine/steps/workspace.py — spec - -`LOC ~43 · step library · refactor-status: ok` - -## Responsibility -Workspace precondition + cheap diff summary steps. - -## Steps -`workspace.guard_clean` (deterministic/read), `analysis.diff_summary` -(deterministic/read). - -## Invariants -- `guard_clean` refuses to start on a dirty tree (BLOCKED) — the precondition - for any write-capable run. -- Both are deterministic — no LLM. - -## Scope — not here -No mutation of the workspace; no repo knowledge. - -## Dependencies (allowed) -`review/diff_summary`, `engine/step`, `._common`. - -## Tests -`test_push_and_steps.py` (guard), diff-summary via review tests. - -## Refactor notes -Trivial and correct. If more read-only "analysis" steps appear, they can share -this file. - -## Concision — **K3** -`guard_clean`/`diff_summary` each repeat the `repo is None → BLOCKED` guard → -`require_repo(ctx)` (one of the 8 K3 sites). diff --git a/doc/SPEC/intent.md b/doc/SPEC/intent.md deleted file mode 100644 index 180097d1f..000000000 --- a/doc/SPEC/intent.md +++ /dev/null @@ -1,51 +0,0 @@ -# intent.py — spec - -`LOC ~100 · task layer · refactor-status: ok` - -## Responsibility -Classify one NL command into a `TaskSpec` (LLM-only), or a clarifying question; -split compound commands into an ordered list. - -## Functionality -The LLM classifies the command (kind / pr / issue / flags) into a `TaskSpec`; -below the confidence gate or with an explicit clarify → a question, never a -guess. `parse_intents` splits on connectors and carries the prior segment's -PR/issue ("… then review it") — segmentation, not classification. - -## Public contract -`parse_intent(text, llm?, default_repo, model?) -> IntentResult`; -`parse_intents(...) -> list[IntentResult]`; `IntentResult(spec?, clarify)`. - -## Invariants -- **LLM-only classification.** No deterministic keyword parser. Empty command - or no configured LLM → clarify (never guess). -- **Clarify on ambiguity — never guess**: LLM confidence < 0.7, an explicit - clarify, a malformed reply, or an unknown kind all → clarify. -- **C7**: only terminal input reaches here; fetched GitHub text never does. - Injection is defended by channel separation + the LLM's low-confidence signal - (a suspicious command clarifies, never runs). - -## Scope — not here -No execution, no planning, no repo/network reads. Text → TaskSpec only. The -segmentation/carry-over in `parse_intents` is sentence-splitting, not -classification (which is the LLM's). - -## Dependencies (allowed) -`llm.py`, `task_spec.py`. - -## Extension points -New kind → add it to `TaskKind`/`KIND_TIER` (`task_spec.py`) and the -`_LLM_SYSTEM` prompt's kind list here. No hint table to maintain. - -## Tests -`test_intent_taskspec.py` (LLM-path contract: mapping, confidence gate, clarify -passthrough, no-LLM, malformed reply), `test_phase_b.py` (compound split + -carry-over via a fake classifier). - -## Refactor notes -`default_repo="vllm-omni"` default args are the 2 allowed repo literals -(leak-capped) — leave them; do not add more. Production always passes an LLM -(`cli/`); intent now requires one — there is no offline fast-path (the -deliberate cost of LLM-only). If the LLM ever needs to also handle compound -splitting, `parse_intents` could return a list directly and `_COMPOUND_SPLIT`/ -carry-over would go too — a bigger change, not needed yet. diff --git a/doc/SPEC/llm.md b/doc/SPEC/llm.md deleted file mode 100644 index 8323d23e8..000000000 --- a/doc/SPEC/llm.md +++ /dev/null @@ -1,40 +0,0 @@ -# llm.py — spec - -`LOC ~120 · engine substrate (transport) · refactor-status: ok` - -## Responsibility -The provider-neutral LLM client wrapper. It supports Anthropic Messages and -OpenAI Chat Completions, including provider-specific Base URL overrides. - -## Functionality -Wraps `messages.create`, exposes availability, normalizes replies into -`Reply`/`Block`, parses JSON out of model text. - -## Public contract -`LLM(settings)` with `available`, `create(system, messages, tools?, model?, -max_tokens?, on_text?) -> Reply`; `Reply`, `Block`, `parse_json_reply`. - -## Invariants -- `available` is false without a key/endpoint; callers must degrade - (a `capability_gap`), not crash (**E2**). -- Untrusted content fencing is the caller's job, not here (**C7** lives in - agent_runtime). - -## Scope — not here -No prompts, no policy, no retries beyond transport. Not a place for -task/repo logic. - -## Dependencies (allowed) -`anthropic` SDK; `openai` SDK; `config.py`. - -## Extension points -New provider/endpoint → behind this wrapper's constructor; keep `Reply`/`Block` -stable so no caller changes. - -## Tests -Provider selection and OpenAI tool translation are unit-tested; step/agent -tests use `ScriptedLLM`. - -## Refactor notes -Keep the `Reply`/`Block` contract as the seam — callers must never see -provider-specific types. diff --git a/doc/SPEC/memory/debug_memory.md b/doc/SPEC/memory/debug_memory.md deleted file mode 100644 index 2dd1709c8..000000000 --- a/doc/SPEC/memory/debug_memory.md +++ /dev/null @@ -1,31 +0,0 @@ -# memory/debug_memory.py — spec - -`LOC ~106 · memory (failure→fix store) · refactor-status: ok` - -## Responsibility -FTS5 store of failure→fix experience. - -## Public contract -`DebugMemory(db_path)` with `search(query, k)` and a write method taking the -required fields. - -## Invariants (**D1/D3**) -- A write must include repo/module/run_id/symptom/root_cause/fix_summary/files/ - verification/status. -- Retrieval is top-k-by-relevance, summary-first. -- Facts recorded freely; promotion to a skill is a separate gated act. - -## Scope — not here -No per-repo namespacing (the agent runtime's `_ScopedKnowledge` applies that); -no LLM. - -## Dependencies (allowed) -stdlib `sqlite3`. - -## Tests -`test_memory.py`. - -## Refactor notes -The write contract (required fields) is the D3 guarantee — enforce it at write -time, never accept a partial memory. Per-repo DB path is chosen by the caller -(`adapter.debug_memory_db`) — keep this class path-agnostic. diff --git a/doc/SPEC/memory/skills.md b/doc/SPEC/memory/skills.md deleted file mode 100644 index 917f4bac2..000000000 --- a/doc/SPEC/memory/skills.md +++ /dev/null @@ -1,32 +0,0 @@ -# memory/skills.py — spec - -`LOC ~120 · memory (procedural knowledge) · refactor-status: ok` - -## Responsibility -Procedural knowledge, gated harder than debug memory. - -## Public contract -`SkillStore(dir)` with `find`, `propose`, `promote`, `candidates`, -`load_all`, `render_for_prompt`; `Skill` (with a `trigger` for recall). - -## Invariants (**D1**) -- Agents may only `propose` (writes the candidates file); `promote` to an active - `SKILL.md` is a curator/human act. -- `find` ranks by module hit + text hit + run_count; only `status: active` - skills load. - -## Scope — not here -No per-repo namespacing (applied by `_ScopedKnowledge`); no LLM; not the curator -UI. - -## Dependencies (allowed) -`pyyaml`; stdlib. - -## Tests -`test_memory.py`. - -## Refactor notes -Clean propose→promote gate — do not add an agent-callable `promote`. The -`trigger` field is the Devin-style recall cue; keep it first-class as more -trigger-gated retrieval lands. Per-repo dir is caller-chosen -(`adapter.skills_dir`) — keep path-agnostic. diff --git a/doc/SPEC/metrics.md b/doc/SPEC/metrics.md deleted file mode 100644 index e7bdaec52..000000000 --- a/doc/SPEC/metrics.md +++ /dev/null @@ -1,42 +0,0 @@ -# metrics.py — spec - -`LOC ~342 · cross-cutting (measurement) · refactor-status: oversized` - -## Responsibility -Compute and persist per-run `metrics.json` (CATQ = Q·S/C). - -## Functionality -Reads the run's trace/artifacts; computes Q (weighted quality per kind over -known components), S (safety multiplier from incidents), C (RQS3e-style log cost -over USD + wall-clock vs reference budgets); writes `metrics.json`. - -## Public contract -`collect_run_metrics(run_dir, settings, status) -> {quality, risk, cost, catq}`. - -## Invariants -- **E3**: metrics are facts about a run and MUST NEVER break it — every failure - caught and traced (`metrics_error`); a run's success is independent of metrics. -- Q uses KNOWN components only (renormalized, `partial` flagged); judged/GT - merged later, never fabricated; safe-abstain scores on escalated runs. -- Incidents derive from explicit events + existing out_of_scope/tool_refused/ - patch_review-revise. - -## Scope — not here -No influence on control flow. Measurement only. - -## Dependencies (allowed) -`config`, `run_trace` (reads events); the price table lives here. - -## Extension points -New quality component / cost term → extend the respective sub-computation with a -KNOWN-only rule; document the reference budget in `config`. - -## Tests -`test_metrics.py`. - -## Refactor notes -Largest cross-cutting file; three sub-metrics (Q, S, C) + price table + CATQ -assembly in one module. **Suggested split**: `metrics/quality.py`, -`metrics/safety.py`, `metrics/cost.py`, `metrics/__init__.py` (assembly + -`collect_run_metrics`). The `[planned]` gh-feedback/post-push-CI collectors -should land as new files under that package, not appended here. diff --git a/doc/SPEC/notify.md b/doc/SPEC/notify.md deleted file mode 100644 index 8908e0ce5..000000000 --- a/doc/SPEC/notify.md +++ /dev/null @@ -1,37 +0,0 @@ -# notify.py — spec - -`LOC ~112 · cross-cutting (escalation) · refactor-status: ok` - -## Responsibility -The "notify, never guess" exit channel. - -## Functionality -`Notifier.escalate` writes `ESCALATION.md`, emails (Resend or SMTP if -configured), traces the escalation; `BLOCKED_EXIT` = 3. - -## Public contract -`Notifier(settings, run_dir, trace, run_id)` with `escalate(reason, phase, -severity, state_summary, artifacts)`; `BLOCKED_EXIT`. - -## Invariants -- A blocked run writes `ESCALATION.md`, notifies, and the caller exits 3. -- Escalation is a first-class outcome — never swallowed as an error path. -- Email failures are best-effort and must not mask the escalation itself. - -## Scope — not here -Deciding *to* escalate is the executor's (typed-failure routing) — this file -only performs the notification. - -## Dependencies (allowed) -`config`, `run_trace`; stdlib `urllib`/`smtplib`. - -## Extension points -New channel (IM/webhook) → a method here, gated on its own config field; -keep `escalate` the single entry. - -## Tests -Via `test_engine.py` routing + escalation assertions. - -## Refactor notes -Clean. If channels multiply, introduce a small `Channel` strategy list rather -than more branches inside `escalate`. diff --git a/doc/SPEC/playbooks/PLAYBOOKS.md b/doc/SPEC/playbooks/PLAYBOOKS.md deleted file mode 100644 index 167ce9fb1..000000000 --- a/doc/SPEC/playbooks/PLAYBOOKS.md +++ /dev/null @@ -1,40 +0,0 @@ -# playbooks/*.yaml — spec - -`9 files · declarative orchestration data · refactor-status: ok` - -## Responsibility -Declarative, ordered step lists (with `foreach`, `when:`, per-step `params`) -realizing a task kind. - -## Contract per file -`name, version, status, task_kinds, repos, requires?, params, provenance, -success, steps[]`. - -## The registered playbooks -- `repo-rebase` — **locked**, L0, `requires: [orchestrator.external]`. - Byte-identical zero-regression — do NOT edit its step list. -- `pr-rebase`/`pr-debug`/`pr-review`/`issue-answer`/`issue-triage` — active, - repo-neutral (`repos: []`, `requires: [repo.path]`). -- `repo-profile` — active, repo-neutral (onboards a second repo). -- `repo-rebase-native`, `profile-consolidate` — **candidates** (planner- - invisible; run only via `--playbook`). - -## Invariants -- Every step id unique; every `step` name registered (enforced by - `store.validate`). -- Write/push steps appear only in vetted (non-generated) playbooks. -- Locking is for code-modifying/pushing playbooks; promotion - candidate→active→locked is a human act with provenance. - -## Scope — not here -No code, no repo knowledge beyond `repos`/`requires` matching. - -## Extension points -A new task realized as a YAML file; a new repo reuses the repo-neutral playbooks -(zero core change) once its profile satisfies `requires`. - -## Refactor notes -These are the "config" half of the reuse>adapt>generate model — keep them -declarative. Resist adding conditional logic beyond `when:`/`foreach`; anything -richer belongs in a step. When repo #2 onboards, it should need NO new playbook -(that is the invariance test). diff --git a/doc/SPEC/playbooks/store.md b/doc/SPEC/playbooks/store.md deleted file mode 100644 index 76c6c335a..000000000 --- a/doc/SPEC/playbooks/store.md +++ /dev/null @@ -1,39 +0,0 @@ -# playbooks/store.py — spec - -`LOC ~166 · planning (registry) · refactor-status: ok` - -## Responsibility -The versioned playbook registry: load/parse/validate YAML, recall by -kind+repo+capabilities, persist candidates. - -## Public contract -`Playbook`, `PlaybookStep`, `parse_playbook`, `playbook_to_doc`; -`PlaybookStore(dir, registry)` with `find(kind, repo, capabilities?)`, -`missing_capabilities(kind, capabilities)`, `get`, `all`, `save_candidate`, -`validate`. - -## Invariants -- Statuses `candidate | active | locked | retired`; only active/locked recalled - by `find`; candidates run only via explicit `--playbook`. -- `find`: exact-repo wins; repo-neutral match only when `requires ⊆ - capabilities` (when known); locked > active; higher version > lower. -- `validate` refuses a playbook referencing an unregistered step (fail at load). -- `save_candidate` forces `status=candidate` (**D1** — no self-promotion). - -## Scope — not here -No execution, no planning policy (planner's), no step logic. - -## Dependencies (allowed) -`engine/registry`, `pyyaml`. - -## Extension points -New playbook field → extend `Playbook` + `parse_playbook` + `playbook_to_doc` -together. - -## Tests -`test_planner_playbooks.py`, `test_capabilities.py`, `test_review_step.py`. - -## Refactor notes -Clean. `Playbook`/`PlaybookStep` are data; `PlaybookStore` is mechanics — keep -them separable. If DAG playbooks arrive, `PlaybookStep` gains edges but the -`find`/`validate` contract stays. diff --git a/doc/SPEC/profiles/consolidate.md b/doc/SPEC/profiles/consolidate.md deleted file mode 100644 index 5dca7f4fa..000000000 --- a/doc/SPEC/profiles/consolidate.md +++ /dev/null @@ -1,30 +0,0 @@ -# profiles/consolidate.py — spec - -`LOC ~52 · profiles (Stage-4 helpers) · refactor-status: ok` - -## Responsibility -Deterministic Stage-4 helpers: staleness decay + drift detection. - -## Public contract -`decay_stale(store, days) -> stale ids`; `detect_drift(adapter, store) -> -findings`. - -## Invariants -- `decay_stale` flips over-window active facts to `stale` (excluded, not - deleted) via the store's `mark_stale` op (**D4**). -- `detect_drift` is **report-only** — declared module paths that vanished, facts - joined to unknown modules; it never mutates. - -## Scope — not here -Deterministic detection only — no LLM, no auto-fix (consolidation is the -`agent.profile_consolidate` step). - -## Dependencies (allowed) -`adapters/base`, `profiles/store`; stdlib `time`. - -## Tests -`test_p3_machinery.py`. - -## Refactor notes -Small and pure. Keep `detect_drift` report-only — the "findings become refresh -proposals, never auto-fixes" rule is the whole point (**D2**). diff --git a/doc/SPEC/profiles/establish.md b/doc/SPEC/profiles/establish.md deleted file mode 100644 index f06b8c669..000000000 --- a/doc/SPEC/profiles/establish.md +++ /dev/null @@ -1,33 +0,0 @@ -# profiles/establish.py — spec - -`LOC ~107 · profiles (Stage 0–1.5 helpers) · refactor-status: ok` - -## Responsibility -Deterministic establishment helpers. - -## Public contract -`fact_id`, `build_doc_corpus`, `is_redundant`, `extract_directives`, -`scan_modules`, `HUMAN_DOC_NAMES`, `LANGUAGE_SUFFIXES`. - -## Invariants -- `is_redundant` (6-word shingle vs README+docs) drops any briefing line the - repo's own docs already state (the ETH-study rule, **D5**). -- `scan_modules` deterministic, language-keyed, skips non-code dirs. -- `extract_directives` bounds line length (short imperative only). - -## Scope — not here -Pure deterministic helpers — no LLM, no store writes, no step logic. - -## Dependencies (allowed) -stdlib only. - -## Tests -`test_profile_steps.py` (redundancy filter, module scan, directive extraction). - -## Refactor notes -Pure functions — easy to test and reuse. The redundancy filter is the -load-bearing ETH-study defense; keep it deterministic. ## Concision — **K2** (shared language rules) -`LANGUAGE_SUFFIXES` (+ the language use in `scan_modules`) is one of three copies -of the per-language rule set (also `review._sweep_targets`, `repo_map`). Move the -data to a leaf `profiles/languages.py` and consume it here; keep it a plain data -map + tiny accessors. Preserve: unknown language → empty module scan. diff --git a/doc/SPEC/profiles/languages.md b/doc/SPEC/profiles/languages.md deleted file mode 100644 index 2808538fb..000000000 --- a/doc/SPEC/profiles/languages.md +++ /dev/null @@ -1,36 +0,0 @@ -# profiles/languages.py — spec - -`LOC ~50 · edge (shared data) · refactor-status: ok` - -## Responsibility -The single home for per-language rules (concision K2), previously triplicated. - -## Public contract -`suffixes(language) -> tuple[str, ...]`; `symbol_re(language) -> Pattern | None`; -`sweep_re(language) -> (Pattern, Pattern) | None`. - -## Functionality -Plain data maps (`_SUFFIXES`, `_SYMBOL_RE`, `_SWEEP_RE`) + tiny accessors. - -## Invariants -- Unknown language returns empty/None so every consumer degrades honestly - (file-level sweep only / empty module scan / "use grep"). -- Pure data — no I/O, no state. - -## Scope — not here -No scanning/rendering logic — the consumers (`review._sweep_targets`, -`establish.scan_modules`, `repo_map`) apply the rules. - -## Dependencies (allowed) -stdlib `re` only. Must stay a leaf (`_ARCHITECTURE.md` §4) — no imports of -engine/profiles machinery. - -## Tests -Exercised via the three consumers' tests -(`test_ci_and_repo_map.py`, `test_profile_steps.py`). - -## Refactor notes -Adding a language = one row in each of the three maps. Keep it plain data; do -not grow it into a language-detection engine (detection stays in -`fingerprint_repo`). This is the deduplication target of K2 — do not re-inline -the rules into a consumer. diff --git a/doc/SPEC/profiles/repo_map.md b/doc/SPEC/profiles/repo_map.md deleted file mode 100644 index c196009f2..000000000 --- a/doc/SPEC/profiles/repo_map.md +++ /dev/null @@ -1,40 +0,0 @@ -# profiles/repo_map.py — spec - -`LOC ~138 · profiles (on-demand structure) · refactor-status: ok` - -## Responsibility -An on-demand, goal-ranked, budgeted symbol map (design §V2.0.2: structure is -pulled, never pushed). - -## Public contract -`RepoMap(repo, language, cache_dir?)` with `supported`, `index()`, -`render(query, budget_chars)`; `build_index`. - -## Invariants -- Regex symbol index per language; disk-cached keyed by HEAD (one HEAD, one - cache; rebuilds on drift). -- `render` is query-ranked + budget-capped; zero-score tail dropped. -- Unsupported language → honest "use grep" string (agent-runtime records a - `capability_gap`). - -## Scope — not here -Never injected into prompts — surfaced only as the `repo_map` tool -(wired in `agent_runtime._repo_map_tool`). - -## Dependencies (allowed) -stdlib `re`/`json`/`subprocess`. - -## Tests -`test_ci_and_repo_map.py`. - -## Refactor notes -Regex-based (no tree-sitter dependency) — a deliberate simplicity/portability -trade. If precision becomes a problem, a tree-sitter backend can slot behind the -same `RepoMap.render` contract. Keep the "pulled on demand" stance — do not add -a code path that injects the map into a prompt. - -## Concision — **K2** (shared language rules) -`_SYMBOL_RES` + `_SUFFIXES` are the third copy of the per-language rule set -(also `review._sweep_targets`, `profiles/establish`). Consume the shared -`profiles/languages.py` (K2). Preserve: `supported` false + honest "use grep" -for an unknown language. diff --git a/doc/SPEC/profiles/store.md b/doc/SPEC/profiles/store.md deleted file mode 100644 index 407e12645..000000000 --- a/doc/SPEC/profiles/store.md +++ /dev/null @@ -1,41 +0,0 @@ -# profiles/store.py — spec - -`LOC ~268 · profiles (curated layer) · refactor-status: ok` - -## Responsibility -The curated fact store: typed patch ops as the only write surface, provenance/ -stability gates, channel-typed rendering. - -## Public contract -`ProfileStore(root)`; `apply_ops(ops, tier, actor) -> per-op reject reasons`; -`active(channel?, module?)`; `render_briefing(budget)`; `render_report()`. -`Fact`; `RUN_OPS`/`CONSOLIDATE_OPS`; `CHANNELS`, `KINDS`, `SOURCES`, -`STABLE_CONFIRMATIONS`, `BRIEFING_WORD_BUDGET`. - -## Invariants -- Ops are the ONLY mutation path; malformed/forbidden ops rejected individually - (never raise); wrong-tier ops rejected (**D4**). -- `add_fact` needs text + evidence; duplicate id = confirmation (**D3**). -- `rewrite_fact` never leaves a fact evidence-free; stable (≥3) facts may not - drop cited evidence; superseded text → `history` (**D3**). -- `merge_facts` leaves a pointer stub (never deletes); `mark_stale` excludes but - keeps for audit. -- `render_briefing` emits only active briefing-channel facts, most-confirmed - first, under the hard word budget (**D5**). -- Every accepted op → `ops_log.jsonl`; `save()` re-renders `PROFILE_REPORT.md`. - -## Scope — not here -No LLM, no repo scanning, no step logic. - -## Dependencies (allowed) -`pyyaml`; stdlib. - -## Tests -`test_profile_store.py`. - -## Refactor notes -The reference implementation of D3/D4 — do not add a mutation path outside -`apply_ops`. Each `_op_*` is a small validator; keep them individually -rejectable. If op kinds grow, keep `RUN_OPS`/`CONSOLIDATE_OPS` as the tier -gate. This is the transplant of the personal-agent store — keep them -conceptually aligned. diff --git a/doc/SPEC/push.md b/doc/SPEC/push.md deleted file mode 100644 index 7e5ff1d6a..000000000 --- a/doc/SPEC/push.md +++ /dev/null @@ -1,37 +0,0 @@ -# push.py — spec - -`LOC ~46 · safety primitive (push authorization) · refactor-status: ok` - -## Responsibility -The single push authorization choke point + its policy/decision types. - -> History: this was `targets/base.py`, the vestige of a designed "Target layer" -> that never became a real abstraction (its task-definition role is carried by -> `TaskSpec` + `Playbook`). The dead target dataclasses were removed (concision -> K1) and the module renamed to what it actually is — push authorization. There -> is no Target layer. - -## Public contract -`PushPolicy`, `PushDecision`, `guard_push(policy, protected_branches)`. - -## Invariants (**C4**) -- A push happens only when the policy allows it AND the branch is not protected. -- Force is with-lease only; a protected branch is never pushed to (force or - not), regardless of policy. -- The ONLY push authorization point — `ci.push` and native phase-4 defer here. - -## Scope — not here -No git execution (that is the step); no repo knowledge; no dry-run decision -(that is the step reading `ALLOW_PUSH`). - -## Dependencies (allowed) -stdlib only. It is a leaf safety primitive (like `scopes.py`). - -## Tests -`test_push_and_steps.py`. - -## Refactor notes -Security-critical and pure — keep it dependency-free and side-effect-free. Every -push path must route through `guard_push`; a new push site that reimplements the -checks is a defect. A natural sibling of `scopes.py` (both are pure permission -primitives); if a `safety/` package is ever introduced, they belong together. diff --git a/doc/SPEC/rebase/monitor.md b/doc/SPEC/rebase/monitor.md deleted file mode 100644 index 1de56c1f6..000000000 --- a/doc/SPEC/rebase/monitor.md +++ /dev/null @@ -1,33 +0,0 @@ -# rebase/monitor.py — spec - -`LOC ~159 · edge (external pipeline monitor) · refactor-status: ok` - -## Responsibility -Read/classify the parent orchestrator's state for the locked -`rebase.run_external` delegation. - -## Public contract -`build_command`, `parse_parent_state`, `summarize_progress`, `diff_progress`, -`classify_failure`, `build_escalation`. - -## Invariants -- Read-only toward the parent's `state.json`. -- Classifies exit code + state into a typed failure + escalation material. -- Stale-state aware — a prior run's `phase=done` must not mask this run's crash. -- Names the parent package/paths (allowed repo literals, leak-capped at 1). - -## Scope — not here -Does not run or reimplement the rebase; no push logic; no notification (that is -`notify`). - -## Dependencies (allowed) -stdlib `json`/`subprocess` only. - -## Tests -`test_rebase_monitor.py`. - -## Refactor notes -Cohesive monitoring/classification unit. The baseline-signature comparison here -is the known-weak spot the CI-normalize module was written to avoid inheriting — -if this monitor's classification is ever tightened, reuse `ci/normalize` rather -than re-implementing string compare. diff --git a/doc/SPEC/review/diff_summary.md b/doc/SPEC/review/diff_summary.md deleted file mode 100644 index cd9f6594e..000000000 --- a/doc/SPEC/review/diff_summary.md +++ /dev/null @@ -1,29 +0,0 @@ -# review/diff_summary.py — spec - -`LOC ~62 · review (always-on stage) · refactor-status: ok` - -## Responsibility -The cheap, always-on first stage of patch review. - -## Public contract -`build_diff_summary(repo, base_ref, primary_files, trace) -> DiffSummary` -(changed files, insertions/deletions, out-of-scope files, full-file writes, -tests run, push requested). - -## Invariants -- Deterministic; no LLM. -- Reads git diff + RunTrace events (`out_of_scope_edit`, `full_file_write`, - `test_run`, `push_requested`) to build the summary. - -## Scope — not here -No trigger decisions (that is `triggers`), no verdict (that is `reviewer`). - -## Dependencies (allowed) -`run_trace`; stdlib `subprocess`/`fnmatch`. - -## Tests -`test_review.py`. - -## Refactor notes -Clean single stage. Keep it cheap (no LLM) — it runs on every gate; the LLM only -runs when this summary trips a trigger. diff --git a/doc/SPEC/review/reviewer.md b/doc/SPEC/review/reviewer.md deleted file mode 100644 index 4cbb17349..000000000 --- a/doc/SPEC/review/reviewer.md +++ /dev/null @@ -1,31 +0,0 @@ -# review/reviewer.py — spec - -`LOC ~89 · review (verdict) · refactor-status: ok` - -## Responsibility -The read-only patch/plan review verdict. - -## Public contract -`run_patch_review(llm, diff_text, summary, fired_rules, model) -> verdict`; -`run_plan_review(llm, playbook_doc, task, model) -> verdict`. - -## Invariants (**C6**) -- Returns `unavailable` without a reviewer LLM (a missing reviewer does not - silently approve). -- Unparseable reviewer output degrades to `revise`. -- Anything but `lgtm`/`pass` is not-passing. - -## Scope — not here -Not the review *step* (that is `engine/steps/review.py`); no trigger logic; no -diff building. - -## Dependencies (allowed) -`llm`; the review prompt lives here. - -## Tests -`test_review.py`. - -## Refactor notes -Fail-closed is the invariant — never add a path where a missing/failed reviewer -returns passing. If plan and patch review diverge more, split into two files; -today they share the same fail-closed shape and belong together. diff --git a/doc/SPEC/review/triggers.md b/doc/SPEC/review/triggers.md deleted file mode 100644 index e767b403e..000000000 --- a/doc/SPEC/review/triggers.md +++ /dev/null @@ -1,31 +0,0 @@ -# review/triggers.py — spec - -`LOC ~50 · review (trigger rules) · refactor-status: ok` - -## Responsibility -Decide when the LLM patch review fires. - -## Public contract -`evaluate_triggers(summary, settings, *, touched_modules, pre_push, -knowledge_edit, high_risk_modules?) -> fired[]`; `ALL_RULES` (7). - -## Invariants -- Seven rules: `out_of_scope_edits, high_risk_modules, large_diff, - tests_unavailable, full_file_fallback, before_push, knowledge_edit`. -- High-risk modules come from the caller (adapter), settings only as fallback - (**A5**). - -## Scope — not here -No diff building (that is `diff_summary`), no LLM call (that is the step, which -calls `reviewer` when a trigger fires). - -## Dependencies (allowed) -`config`, `review/diff_summary`. - -## Tests -`test_review.py`. - -## Refactor notes -The rule set is the contract — adding/removing a trigger changes the safety -posture (**C6**) and must be a deliberate, documented change. Keep it a pure -function of (summary, settings, caller flags). diff --git a/doc/SPEC/run_trace.md b/doc/SPEC/run_trace.md deleted file mode 100644 index b0028ffd9..000000000 --- a/doc/SPEC/run_trace.md +++ /dev/null @@ -1,38 +0,0 @@ -# run_trace.py — spec - -`LOC ~40 · cross-cutting (audit spine) · refactor-status: ok` - -## Responsibility -Append-only JSONL event log — the immutable audit layer. - -## Functionality -`record(event, **fields)` appends one JSON line; `events(name)` filters by type. - -## Public contract -`RunTrace(path)` with `record`, `events`. - -## Invariants -- **E1**: every governance claim maps to a trace event (`agent_dispatch`/ - `agent_output`, `tool_call`/`tool_refused`/`out_of_scope_edit`/ - `full_file_write`, `patch_review*`, `push_requested`, `capability_gap`, - `env_exported`, `posted_artifact`, `profile_*`). -- Facts are recorded freely (**D1**); this is the immutable layer under the - curated profile. - -## Scope — not here -Recording only. No policy, no filtering of what may be recorded, no reads that -drive control flow (except `events()` for diff-summary/metrics assembly). - -## Dependencies (allowed) -stdlib only. - -## Extension points -New event → just `record("", ...)` at the site; document notable names in -`_CONSTRAINTS.md` §E1 if it backs a guarantee. - -## Tests -Exercised across the suite via `trace.events(...)` assertions. - -## Refactor notes -Deliberately trivial and dependency-free — keep it that way. It underpins every -governance claim, so it must never gain logic that could fail and lose an event. diff --git a/doc/SPEC/scopes.md b/doc/SPEC/scopes.md deleted file mode 100644 index 66ed6f390..000000000 --- a/doc/SPEC/scopes.md +++ /dev/null @@ -1,37 +0,0 @@ -# scopes.py — spec - -`LOC ~94 · engine (permissions) · refactor-status: ok` - -## Responsibility -Path-level tool permissions — the permission vocabulary the dispatcher enforces. - -## Functionality -`ToolScope.check` (tool allowed? write path allowed?); `PathScope.check_write` -(writable hard wall + primary owned-files); scope factories. - -## Public contract -`ToolScope(name, allowed_tools, path_scope?, read_only)`; `PathScope(writable, -primary)`; `Decision`; `read_only_scope`, `pre_plan_scope`, `post_plan_scope`; -tool-set constants (READ/WRITE/EXEC). - -## Invariants -- Three outcomes: allowed / refused (tool not in set, write outside `writable`, - or read-only scope) / out-of-scope (inside `writable` but outside `primary` — - allowed + recorded). -- `writable` is a hard wall; `primary` defines the module's owned files. - -## Scope — not here -Permission decisions only — no execution, no tracing (the dispatcher traces). - -## Dependencies (allowed) -stdlib only. - -## Extension points -New scope shape → a factory function; keep the three-outcome `Decision` contract. - -## Tests -`test_scopes_tools.py`. - -## Refactor notes -Clean, dependency-free, security-critical. Keep it pure (no side effects) so it -stays trivially testable. `engine/step.py` imports it — do not add a back-import. diff --git a/doc/SPEC/task_spec.md b/doc/SPEC/task_spec.md deleted file mode 100644 index 1742243b5..000000000 --- a/doc/SPEC/task_spec.md +++ /dev/null @@ -1,38 +0,0 @@ -# task_spec.py — spec - -`LOC ~70 · task layer, pure data · refactor-status: ok` - -## Responsibility -Define `TaskSpec` (the structured product of intent parsing) and derive its -permission **tier** from the task kind. - -## Functionality -Holds kind/repo/pr/issue/flags; computes `tier`, `read_only`, -`confirm_required`, and a human `describe()`. - -## Public contract -`TaskSpec(kind, repo, pr?, issue?, report_only, post, params)`; properties -`tier`, `read_only`, `confirm_required`; `describe()`. Constants: `TaskKind` -(7 kinds), `READ_ONLY_KINDS`, `KIND_TIER`. - -## Invariants -- **C1**: no settable tier field; `tier = KIND_TIER[kind]` — text can't widen it. -- `read_only` = `not post` for read-only kinds, else `report_only`; - `confirm_required = not read_only`. - -## Scope — not here -No parsing, no I/O, no execution, no repo knowledge. Pure data + derivation. - -## Dependencies (allowed) -`pydantic` only. - -## Extension points -New kind → add to `TaskKind` + `KIND_TIER` (+ `READ_ONLY_KINDS` if read-only). - -## Tests -`test_intent_taskspec.py`. - -## Refactor notes -Clean and minimal — the canonical example of "one responsibility". Do not add -behavior here; keep it a data+derivation module. It is the single source of -truth for C1, so any permission logic elsewhere is a smell. diff --git a/doc/SPEC/tools.md b/doc/SPEC/tools.md deleted file mode 100644 index 9c1659585..000000000 --- a/doc/SPEC/tools.md +++ /dev/null @@ -1,41 +0,0 @@ -# tools.py — spec - -`LOC ~162 · engine (capabilities + choke point) · refactor-status: ok` - -## Responsibility -Atomic capabilities + the single scope-enforcing dispatch choke point. - -## Functionality -Defines builtin tools (read_file, write_file, edit_file, list_dir, grep, -run_shell); `dispatch` scope-checks, runs, and traces each call. - -## Public contract -`ToolDef`; `TOOLS`; `tool_definitions_for(scope, extra?)`; -`dispatch(name, args, *, scope?, trace?, extra?) -> {ok, result|error, -out_of_scope}`. - -## Invariants (**C3**) -- Every builtin call is scope-checked; refused → error return (never raise). -- Out-of-scope write executes but emits `out_of_scope_edit`; full-file `.py` - write emits `full_file_write`. -- Errors are observations, not crashes. -- Extra (step-provided) tools bypass the builtin allowlist but are still traced. - -## Scope — not here -"What can be done" only — tools express capability, not engineering semantics -(that is a step, **A3**). No task/repo logic. - -## Dependencies (allowed) -`run_trace`, `scopes`; stdlib. - -## Extension points -New builtin tool → a `ToolDef` in `TOOLS` (declare `write_path_arg` if it -writes). Step-specific tools are passed as `extra`, not added here. - -## Tests -`test_scopes_tools.py`. - -## Refactor notes -The dispatch choke point is a hard security invariant — every path (builtin and -extra) must remain traced and scope-checked. Do not add a "fast path" that -bypasses `dispatch`. Fine at current size. diff --git a/doc/SPEC/ui.md b/doc/SPEC/ui.md deleted file mode 100644 index c53bc63f8..000000000 --- a/doc/SPEC/ui.md +++ /dev/null @@ -1,36 +0,0 @@ -# ui.py — spec - -`LOC ~164 · presentation · refactor-status: ok` - -## Responsibility -Terminal rendering: streaming, spinners, markdown, color; degrade to plain text -on non-TTY/pipes. - -## Functionality -`make_ui(out?)` returns a rich UI or a `PlainUI`; stream start/delta, styled -step/tool output, markdown rendering, input-history. - -## Public contract -`make_ui`, `style(...)`, the UI object's `stream_*`/print methods. - -## Invariants -- Presentation only — carries no control flow or run state. -- Deterministic plain-text fallback so scripted output stays stable. - -## Scope — not here -No decisions about *what* to do — only *how it looks*. No task/planning/exec -logic; no repo knowledge. - -## Dependencies (allowed) -`rich`; stdlib. - -## Extension points -New rendered element → a method on the UI; keep the `PlainUI` fallback in sync. - -## Tests -`test_ui.py`. - -## Refactor notes -Clean separation. The only smell: `cli/`/`chat.py` format some strings inline -before handing to `ui`; a refactor could move more of that formatting here so -callers pass structured data, not pre-formatted strings. diff --git a/doc/VLLM_OMNI_RELEASE_MAINTENANCE.md b/doc/VLLM_OMNI_RELEASE_MAINTENANCE.md deleted file mode 100644 index aa868c217..000000000 --- a/doc/VLLM_OMNI_RELEASE_MAINTENANCE.md +++ /dev/null @@ -1,80 +0,0 @@ -# vLLM-Omni 发版漂移审计 - -InferMatrixCopilot 的模型清单、代码 owner 路由和源码引用会随着 -vLLM-Omni 发版而过期。这个审计只读取两个 Git 提交和本仓库声明，不会 checkout -目标版本，也不会自动修改知识规则。 - -## 用户入口 - -安装 InferMatrixCopilot 后，在 Codex、Claude Code 或 Cursor 中运行： - -```text -/imupdate D:\path\to\vllm-omni -/imupdate vllm-omni -/imupdate vllmomni -``` - -支持三种用法： - -1. 本地 Git 路径：直接用 baseline 的 `audited_sha` 对比仓库 `HEAD`，机器审计后 - 由 Agent 更新知识。 -2. 仓库名、别名或 URL：Agent 先解析权威仓库与目标版本，寻找已配置 checkout - 或建立临时 clone，再尽量进入同一机器审计流程；无法运行机器审计时必须明确标记 - 为模型分析，不能宣称 `CLEAN`。选择版本时不得因为 GitHub “latest release” - 忽略预发布版本而退回到比当前 baseline 更旧的版本。 -3. 下方完整 Python 命令：纯机器规则对比，不调用 LLM，也不修改文件。 - -`/imupdate` 的可选第二个参数可以指定目标 tag 或 SHA。 - -## 底层命令 - -先在 vLLM-Omni checkout 中 fetch 需要比较的 tag 或 SHA，然后运行： - -```powershell -python tools/audit_vllm_omni_release.py ` - --from 5d44868e ` - --to v0.26.0rc1 ` - --repo D:\path\to\vllm-omni ` - --json-output $env:TEMP\vllm-omni-release-audit.json -``` - -审计覆盖： - -- AR、Diffusion 和 pipeline registry； -- deploy YAML； -- 新增、修改、删除和重命名路径； -- changed path 到 Direct 知识 owner 的机器路由； -- adapter runtime module 未覆盖的 changed path； -- active knowledge 中受删除或重命名影响的 `sources:`； -- component、model catalog 与 canonical baseline 的 source pin。 - -同样的提交和 baseline 会产生等价 JSON；报告不包含时间戳和本机 checkout 路径。 -默认 `--mode enforce`：存在未解释漂移时退出 1，输入或 Git 失败时退出 2。 -`--mode report-only` 仍报告 `DRIFT`，但退出 0，供定时巡检使用。 - -底层实现不 checkout 版本：它用 `git rev-parse` 固定两个提交，用 -`git diff --name-status -M` 找新增、修改、删除和重命名，再用 -`git show :` 直接读取 Git 对象。Python AST 负责提取模型和 -pipeline registry，deploy YAML 直接按提交列举；结果做排序和哈希后与 baseline、 -owner 路由、manifest、知识 `sources:` 和 pin 对账，最后输出稳定 JSON。 -这个命令只报告证据，不编辑任何文件。 - -## 更新一个 release - -1. 用当前 `release_baseline.yaml` 的 audited SHA 作为 `--from`，新 tag 作为 - `--to`，先跑 `report-only`。 -2. 人工确认 registry、deploy、路径 owner 和知识来源变化。 -3. 把旧的 `audited_sha` 移到 `previous_audited_sha`，再只更新报告证明已经漂移的 - baseline、catalog、source map 或 manifest；不要自动生成语义规则。 -4. 用同一组 `--from/--to` 跑 `enforce`，再运行知识 validator 和相关 pytest。 -5. JSON 是临时证据，不提交到 `knowledge/`；完成后删除。 - -## PR 学习与 release 审计是两件事 - -合并 PR 的复盘只提炼可复用、可执行的 owner 规则，且必须等最终修复、review thread -和 CI 结果稳定后再做。Release 审计只对账结构事实，不把 PR 内容、事故过程或审计报告 -写进知识树，也不因为上游发版自动升级 InferMatrixCopilot 版本或依赖。 - -CI 每周对 upstream `main` 运行 `report-only`。修改 baseline 的 PR 会自动用 -`previous_audited_sha → audited_sha` 运行 `enforce`；Actions 手动运行可用 -`report-only` 检查任意区间，`enforce` 只接受当前 baseline 声明的升级区间。 diff --git a/doc/CODE_TOUR.md b/doc/architecture/CODE_TOUR.md similarity index 86% rename from doc/CODE_TOUR.md rename to doc/architecture/CODE_TOUR.md index 0f4929835..0e6339595 100644 --- a/doc/CODE_TOUR.md +++ b/doc/architecture/CODE_TOUR.md @@ -3,11 +3,16 @@ 本文按**数据流**组织:跟着数据对象走——终端字符串 → `TaskSpec` → 共享 `state` 字典 → agent 的 prompt → 交付物与落盘,再看每次 run 读入/写出的 **知识平面**。每个文件在它所处的数据流位置就地讲解;`src/infermatrix_copilot/` -下全部 80 个 `.py` 文件都被覆盖,文末**附录**是 `文件 → 段落` 的查阅索引。 +下全部 **92** 个 `.py` 文件都被覆盖,文末**附录**是 `文件 → 段落` 的查阅索引。 第一次通读建议按 §13 的阶段化阅读路线精读,再回来把本文当查阅手册用。 -包根 `__init__.py` 只声明版本(`__version__`)。配套:`DESIGN.md`(为什么)、 -`IMPLEMENTATION_STATUS.md`(做了什么)、`SPEC/`(规范契约)、 -`eval/dataset/judgments/CAMPAIGN_FINAL.md`(评测结论)。 +包根 `__init__.py` 只声明版本(`__version__`)。 + +**本文只讲"数据怎么流、模块为什么在这一环"。** 清单类内容(七种 task kind、 +9 个 playbook、38 个 step、三个工具面)的唯一归属是 +[`../GUIDE.md`](../GUIDE.md) §2/§5/§6/§7——本文只链接,不复述。 +配套:[`DESIGN.md`](DESIGN.md)(为什么这样设计)、 +[`SPEC/`](SPEC/README.md)(逐文件契约)、 +[`../evaluation/README.md`](../evaluation/README.md)(评测结论)。 ## 0. 数据流全景 @@ -225,7 +230,21 @@ re-export,公开导入面不变)。 在此)。**`review/utils.py`**(193)——`_render_review_md`:**裁决校准** (仅已验证 blocker/major 阻塞,minor→COMMENT,自述 uncertain 永不阻塞)+ `[validated]`/`[upstream-verify]`/`[sweep]` 渲染成 "Validated" 段; - `_sweep_targets`。`review/__init__.py`(17)re-export + 注册。 + `_sweep_targets`。 + **`review/anchor.py`**(350)——**别问模型行号,问它引用哪段代码**。模型给 + 出的行号错得够频繁,以至于发布时要降级处理;修法不是更好的校验器而是换个 + 问题:让模型逐字引用它在说的代码,位置由**程序**自己算(这也是 OCR 的设计 + ——`internal/diff/resolver.go:82` 的评论工具根本不接受行号)。校验器仍然最后 + 跑,"绝不发布错锚点"的保证不变;变的是**只有行号错**的发现不再被降级进正文, + 保住了自己的 inline 位置。 + **`review/repo_tools.py`**(225)——只读的**变更考古**工具组 + (`diff_stat` / `file_at_base` / `show_commit` / `search_history` / `calc`)。 + 它们不是想出来的:wave-2 召回取证测出,基线能做而我们的 pass 做不到的那些 + 判官决定性动作,全都是只读工具集当时**恰好缺少**的一条命令 + ——`git diff --stat base..HEAD` 证明文档 PR 里被要求删除的东西压根没删 + (pr5715 的两条 GT 发现正是这个检查)、读 merge-base 版本的文件做前后对照 + (pr5840 的 FP8 发现)、`git show` / `git log -S` 重建一个修复所针对的回归。 + `review/__init__.py`(17)re-export + 注册。 - **议题支流**(issue_text → draft_answer): - **`engine/steps/issue.py`**(338)——`issue.fetch`、`agent.draft_issue_answer` @@ -393,6 +412,52 @@ infermatrix-copilot --resume # 从首个未完成 step - **改交付/评审行为前先看 trace 取证**:评测教训是裁判扣分约九成来自机械 交付问题而非分析能力;先读 `eval/dataset/judgments/T3_FORENSICS.md`。 +## 11.5 后端平面 —— 同一个 agent step,换一条腿落地 + +与 §4 正交的一层:`run_agent_step` 组装好的 prompt 包(contract `system` + +渲染后的 dispatch context)**要么**交给我们自己的 `agent_loop`(§4), +**要么**整包委托给一个厂商 harness。两条腿共用同一个 prompt、同一份 scope、 +同一套 trace 词汇——分叉点只在"谁拥有工具循环"。 + +- **`providers/registry.py`**(107)——唯一的一张表:`PROVIDERS` 五条 + `ProviderSpec`(`api` / `cursor` / `claude-code` / `codex` / `deepseek`), + `resolve_provider` 读 `STRICT_BACKEND`,`transport_for_id` 取传输实现。 + 未发布的后端登记进 `_UNSHIPPED`,于是 readiness/doctor 报"尚未发布", + 而不是让一次 run 跑到一半才失败。 +- **`providers/base.py`**(161)——契约:`ProviderSpec`(能力位)、 + `AgentSessionRequest`(**与 `run_agent` 拿到的同一个 prompt 包** + scope + + 会话上限 + 工具桥 spec 路径)、`SessionUsage`(harness 报数不齐,缺的就留 0, + `cost_usd` 留 `None`——metrics 记 `subscription`,**绝不编造 USD**)、 + `HarnessTransport`(`run_session` 委托一整个 step;`complete` 一次性无工具)。 + **`sanitized_env()`** 是安全要点:子进程环境是白名单,厂商 CLI 必须保住自己 + HOME 里的订阅认证,但**绝不能继承我们的 `ANTHROPIC_BASE_URL`**——这类机器上 + 它往往指向某个网关,继承过去会悄悄把厂商流量改道。 +- 四个传输,各自的治理姿态不同,差别全在"内置工具能不能关": + **`claude_code.py`**(205)能关(`builtin_tools_off`),`--max-turns` 直接 + 承载我们的迭代预算,并回报 `modelUsage`/成本; + **`codex.py`**(197)关不掉,改用 OS 级 `--sandbox read-only`; + **`cursor.py`**(207)两样都做不到——headless + `cursor-agent --print --force --output-format stream-json`,**prompt 走 + STDIN**(argv 单参数 128KiB 上限,证据包超得轻松),因此额外配事后审计; + **`deepseek.py`**(442)是唯一 **api-keyed 的 harness**(SDK 驱动、PATH 上 + 没有二进制、要 `DEEPSEEK_HARNESS_API_KEY`),用最小 composition 保证沙箱干净。 +- **`providers/audit.py`**(109)——**侦测型**兜底,只服务于关不掉内置工具的 + 后端(今天是 cursor):事后检查文件读取是否越出容纳根(PR-time worktree + + run 目录)、只读 scope 下是否出现 write/edit。从 Composer 评测臂产品化而来 + ——它当初真的抓到过一次越界读 `~/.claude/skills/...`。结论进 RUN_REPORT, + **公开声明,绝不静默丢弃**。 +- **`tool_bridge.py`**(308)——把 copilot 自己的 scoped 工具以 MCP 形式递给 + harness 会话,于是**每一次调用仍然回到 `tools.dispatch`**(§4/§7 的闸不因 + 换后端而失效)。同时供给按需 `repo_map`;skill/memory 检索**刻意不开放** + ——跨进程的 candidate 写入没有开口。 +- **`providers/harness_llm.py`**(66)——`LLM` 形状的适配器,**只允许无工具 + 调用**。intent 分类、ensemble reducer、输出修复、chat 这些持有 `llm.LLM` + 的调用点保持原样;一旦有人带着 tools 调进来就**大声抛错**——这是防止在 + harness 之外再跑一个不受治理的第二工具循环的守卫。 + +使用者视角的选型与配置在 [`../guide/backends.md`](../guide/backends.md), +决策记录在 [`../features/provider-registry.md`](../features/provider-registry.md)。 + ## 12. 第三条数据流 —— MCP 服务端(Claude Code / Codex / Cursor → 只读工具) 叠在核心之上、面向宿主的一条独立入口。宿主非交互(无 `[y/N]`),所以"宿主 @@ -446,7 +511,7 @@ Direct 的数据流: 一个**隔离子进程**(`python -m infermatrix_copilot --execute-reserved `): 子进程 stdout 落 `console.log`,server stdout 只走 JSON-RPC;进程级全局 tracer / `last_run_dir` 因此天然按 run 隔离。它自身的 `build_mcp()` 暴露 - V1 工具面,是 `docs/autonomous-workflow.md` 的独立执行器入口,默认不注册。 + V1 工具面,是 `doc/guide/autonomous-workflow.md` 的独立执行器入口,默认不注册。 - **`run_status.py`**(247)——`run_status.json` 的耐久单写者记录。`reserve_run` (server,子进程存在前)写 `queued`;子进程一启动先写自己的 pid (`mark_child_started`)再 `planning→running→terminal`,是运行期唯一写者; @@ -462,8 +527,8 @@ Direct 的数据流: 专属形状。 - 打包:`plugins/infermatrix-copilot/`(共享 MCP + Skill + 薄市场清单)、根 `.claude-plugin/marketplace.json`、`.cursor-plugin/marketplace.json`、 - `.agents/plugins/marketplace.json`、`docs/codex/config.toml`、 - `docs/cursor/mcp.json`;安装说明见 `doc/MCP.md`。`mcp` 依赖在 `[mcp]` extra + `.agents/plugins/marketplace.json`、`integrations/config-templates/codex/config.toml`、 + `integrations/config-templates/cursor/mcp.json`;安装说明见 `doc/guide/mcp.md`。`mcp` 依赖在 `[mcp]` extra 后,独立 CLI 安装保持零依赖。由 `test_mcp.py` 固定(篡改防御、单写者对账、 分页、只读工具集)。 @@ -510,7 +575,10 @@ Direct 的数据流: `report.py`·`rebase_ext.py`·`rebase_native.py`·`profile.py`§5。 `engine/steps/pr/`:`__init__.py`·`fetch.py`·`debug.py`·`utils.py`·`rebase.py`· `publish.py`§5(`publish.py` 亦 §7)。 -`engine/steps/review/`:`__init__.py`·`steps.py`·`prompts.py`·`utils.py`§5。 +`engine/steps/review/`:`__init__.py`·`steps.py`·`prompts.py`·`utils.py`· +`anchor.py`·`repo_tools.py`§5。 +`providers/`:`registry.py`·`base.py`·`claude_code.py`·`codex.py`·`cursor.py`· +`deepseek.py`·`audit.py`·`harness_llm.py`§11.5;`tool_bridge.py`§11.5。 `memory/`:`__init__.py`·`debug_memory.py`·`skills.py`§6。 `profiles/`:`__init__.py`·`store.py`·`establish.py`·`consolidate.py`· `repo_map.py`·`languages.py`§6。 diff --git a/doc/architecture/DESIGN.md b/doc/architecture/DESIGN.md new file mode 100644 index 000000000..4149db066 --- /dev/null +++ b/doc/architecture/DESIGN.md @@ -0,0 +1,529 @@ +# 设计 + +本仓库实现的是 **InferMatrixCopilot** 的设计 —— 它最初以 "vLLM-Omni Copilot" 之名 +写在 `vllm-omni-rebase-agent/docs/copilot/copilot_design` 里，外加它的 +`implementation/` 里程碑计划。本文档把那份设计映射到这里的代码上；**理据本身住在那些 +源文档里**。 + +下面的 Part I 描述**如实建成**的 v1 架构。**Part II（Design v2，2026-07-10）** 在其上 +扩展三个目标：修掉 v1 的已知问题、让 copilot **仓库不变**（跨仓库保持稳定的任务表现）， +并让它有能力为任何被指向的仓库**建立仓库 profile**（代码格式、skill、约束、模块分析 +及相关产物）。**Part III（Design v3，2026-08）** 记录 Direct 模式与 provider 注册表。 + +--- + +# Part I —— v1 架构（如实建成） + +## 架构 + +``` + 用户自然语言 ──► intent.py ──► TaskSpec (task_spec.py) §3.Y CLI 层 + │ tier 由 kind 固定 (L0/L1/L2)，确认门 + ▼ + engine/planner.py §3.2 reuse>adapt>generate + reuse ─────────┼───────── generate（仅只读 kind） + ▼ + playbooks/store.py 注册表 §3.2 candidate/active/locked + ▼ + engine/executor.py §3.X 引擎底座 + 逐 step 检查点/resume · 重试 · 类型化失败 · 升级 + ▼ + engine/steps/* （Step 库） §3.X.1 Steps + tools.py + scopes.py（ToolScope/PathScope choke point） §3.3(2) + review/（diff 摘要 → 触发器 → 只读 reviewer） §3.3(3) + memory/（RunTrace、FTS5 DebugMemory、受门 skill） §3.3(4) + adapters/（仓库知识、draft 引导） adapter 层 + push.py（PushPolicy + guard_push） 推送授权 + notify.py（ESCALATION.md + 邮件、退出码 3） 目标 #4 +``` + +## 关键决策 + +1. **包装 rebase，而不是重写它。** locked 的 `repo-rebase` playbook 经 + `rebase.run_external`（`REBASE_ORCHESTRATOR_CMD`）委托给已有的 5 阶段编排器。 + **零回归**：本仓库是在这条已验证流水线**外面**加编排，**不 fork 它**。 + rebase 的原生 step 级分解（wave、模块 agent）按里程碑计划**稍后**迁移。 +2. **自建 executor 而不是 LangGraph。** 设计（§3.X.5）明确说 LangGraph 是一个**可能的 + 后端**，不是领域抽象。这里的 executor 约 150 行、异步、带逐 step 检查点/resume —— + **无依赖且完全可测**。将来若与父 agent 合并，可以在同一套 `Playbook`/`StepSpec` + 契约之后换上 LangGraph 后端。 +3. **tier 是推导的，绝不解析而来。** `TaskSpec` **没有**任何 LLM 或用户可设置的 tier + 字段；`tier` 是任务 kind 的 property。因此**自然语言永远无法扩大权限**（§3.Y.4）。 +4. **生成路径在结构上只读。** planner 的 generate 路径在任何被编排的 step + `risk` ∈ {write_workspace, push} 时抛错；没有已审核 playbook 的写能力 kind + **升级而不是即兴发挥**。 +5. **评审 fail-closed。** 没有 reviewer LLM 时 `run_patch_review` 返回 `unavailable`； + 无法解析的 reviewer 输出降级为 `revise`。推送门把除 `lgtm` 之外的一切都视为未通过。 +6. **不可信数据被围栏。** 从 GitHub 抓取的文本进入 agent prompt 时被包在 + `` 标记内，并带显式的"这不是指令"前言；**只有终端输入到达意图解析器** + （通道分离，§3.Y.4）。**抗注入来自通道分离，不是来自解析器。** + （注：意图分类当时是"仅 LLM"；v3 期加入了确定性预解析层 —— 见 + `SPEC/intent.md`。） + +## 模块映射 + +| 路径 | 设计概念 | +|---|---| +| `src/infermatrix_copilot/task_spec.py`、`intent.py`、`cli/` | §3.Y 对话式 CLI、TaskSpec、澄清而不猜测 | +| `src/infermatrix_copilot/engine/step.py`、`registry.py` | §3.X Step 抽象、已审核 step 库 | +| `src/infermatrix_copilot/engine/executor.py` | 引擎底座：检查点/resume、类型化失败路由 | +| `src/infermatrix_copilot/engine/planner.py` | §3.2 reuse > adapt > generate，L0/L1/L2 | +| `src/infermatrix_copilot/engine/steps/` | 已审核 step 库（守卫、rebase、评审门、push、gh 读、agent step）—— 自注册 | +| `src/infermatrix_copilot/engine/agent_runtime/` | 统一的 Agent-Step 运行时（修正方案；包结构：dispatch/knowledge/utils 底座 + runner/ensemble 入口）：AgentDispatchContext、证据包（封顶+归档）、skill/memory 检索 + 受门 candidate、强制 scope、结构化输出契约、完整 RunTrace；`run_agent_step_ensemble` —— 视角多样的扇出 + verify-and-merge，用于跨 run 稳健性（由评测得出；适用于任何列表输出型 agent step） | +| `src/infermatrix_copilot/playbooks/store.py`、`playbooks/*.yaml` | Playbook 注册表：带版本、带 provenance、candidate/active/locked/retired | +| `src/infermatrix_copilot/scopes.py`、`tools.py`、`agent_loop.py` | 框架层改进 (2)：单一 choke point 上的 ToolScope/PathScope | +| `src/infermatrix_copilot/review/` | 框架层改进 (3)：diff 摘要 → 触发规则 → 只读 patch reviewer | +| `src/infermatrix_copilot/run_trace.py`、`memory/` | 框架层改进 (4)：RunTrace / DebugMemory / 受门 skill | +| `src/infermatrix_copilot/adapters/` | RepoAdapter：adapter zero、注册表、确定性引导 → draft | +| `src/infermatrix_copilot/push.py` | 推送授权：`PushPolicy` + 统一的 `guard_push`（没有独立的 Target 层） | +| `src/infermatrix_copilot/notify.py` | 目标 #4：升级通道，被阻塞时退出码 3 | + +## 逐 run 的数据与产物 + +`~/.infermatrix-copilot/runs/run-/`：`run_trace.jsonl`（事实）、`progress.json` +（step 检查点 —— resume 从首个未完成 step 重新进入）、`RUN_REPORT.md`、 +`ESCALATION.md`（仅在被阻塞时）。 + +--- + +# Part II —— Design v2：仓库不变的 copilot（2026-07-10） + +v1 已经写下了多仓库的成功判据（"接入第二个仓库不得触碰核心引擎 —— 只加一个 adapter"）。 +v2 把它从*能跑起来*加强为**跑出稳定、可测量的表现**，并补上当前让这件事不可能达成的 +那些缺口。三条工作流： + +1. **§V2.1 修掉既有问题** —— 正确性缺陷、渗进核心的仓库知识、v1 遗留的开放边界。 +2. **§V2.2 仓库不变性** —— 一份可强制的契约：引擎是仓库中立的，且质量是**逐仓库测量的， + 不是假定的**。 +3. **§V2.3 仓库 profile** —— copilot 能够建立、维护并消费任何仓库的完整 profile： + 代码格式、skill、约束、模块分析，以及任务所需的其他产物。 + +## V2.0 先行工作与证据（调研，2026-07-10） + +关于"给 agent 提供仓库上下文"这件事，业界学到了什么 —— 下面每一条都改变了一个 v2 决策。 +来源列在本节末尾。 + +1. **自动生成的仓库上下文文件是有害的。** 苏黎世联邦理工的研究 + （[arXiv:2602.11988](https://arxiv.org/html/2602.11988v1)，438 个任务、4 个 agent） + 发现 LLM 生成的 AGENTS.md 式文件会**降低**解决率（−0.5 至 −2%），同时增加 20–23% + 的推理成本和 2–4 个额外步骤；agent 会尽责地照做那些额外指令（更多 grep、更多测试）， + **却换不来准确率**。代码库/目录概览 —— 在约 100% 的生成文件里都有 —— + 对"agent 多快够到相关代码"的影响是**零**。人工撰写的文件只有轻微帮助（+4%）， + 而且**仅当**它们承载**非冗余、仓库专属的工具事实**（用哪个包管理器、确切的 + build/test 命令）；当同样的信息在 README/docs 里已经存在时，上下文文件是**纯成本**。 + GitHub 自己对 `copilot-instructions.md` 的建议也一致：从 10–20 条简短祈使指令开始， + 对照观察到的行为迭代。 + → **v2 后果**：profile **不是** prompt 倾倒场。它的大部分内容必须**被代码消费**或 + **按需检索**；always-on 的 prompt 切片必须**最小、精选，并在评测上自证其价值** + （§V2.3.4、§V2.3.5）。 +2. **有用的那张"图"是逐任务算出来的，不是一次写死的。** Aider 的 repo map + （tree-sitter 符号图 + 个性化 PageRank，按每次对话的 token 预算裁剪）和 RepoGraph + 这条线（[arXiv:2410.14684](https://arxiv.org/html/2410.14684v1)，SWE-bench 上相对 + +32.8%；CodexGraph [arXiv:2408.03910](https://arxiv.org/abs/2408.03910)） + 都表明：仓库**结构**作为一张**可查询、按当前目标排序**的图才有回报 —— + 而不是作为一份静态概览文档。 + → **v2 后果**：`modules.yaml` 是**机器产物**（喂扇出、patch 触发器、wave 调度）， + 外加给 agent step 用的按需 `repo_map` 工具；**它绝不被整体注入 prompt**。 +3. **触发式知识胜过 always-on 知识。** OpenHands 的 microagent/skill + （关键词触发的 .md 文件）、Devin 的知识库（每一条 = 内容 + 一段**触发描述**， + 相关时被语义召回）、以及 Agent Workflow Memory + （[arXiv:2409.07429](https://arxiv.org/html/2409.07429v1)，从成功轨迹归纳出工作流， + 在 web-agent 基准上相对 +24.6–51.1% 且跨域稳健）收敛到同一个形状： + **小的知识单元，各自携带自己的召回条件，只在被触发时注入**。 + 我们的 skill store 已经是这个形状 —— v2 为它加上逐仓库命名空间， + 以及从成功 RunTrace 中归纳。 +4. **逐仓库的表现方差是常态，不是例外。** SWE-bench 家族的分析显示，同一个模型在不同 + 仓库上的解决率从 <10% 摆到 >50%，存在系统性的语言效应，且在私有/未见代码库上有 + 显著下滑（例如 Opus 4.1 在 SWE-bench Pro 私有集上 22.7%→17.8%）。 + 仓库属性 —— 架构复杂度、adapter/hook 体系、API 面清晰度 —— 可以预测难度。 + → **v2 后果**：不变性必须**逐仓库测量**（§V2.2.5）；一个只在 vllm-omni 上被评测过的 + agent，**在别处的表现是未知的**，而 profile 质量是那道差距上的一阶杠杆。 +5. **"精选顶层架在证据之上"是正在收敛的记忆架构。** 2026 年的 agent 记忆文献与已上线 + 系统（Zep/Graphiti、Mem0、Letta 的 sleep-time agent；**连续的 LLM 重写会腐蚀记忆** + —— arXiv:2605.12978）收敛到：一个**不可变的证据库**之上架着一层小的、精选的、 + 人类可读的 profile；**类型化 patch 操作是唯一写入面**；巩固是**有节奏的** + （而不是每次交互都做）；溯源与稳定性分数**门控重写**。我们自己的个人 agent 正是这么 + 实现的，且已在生产中运行（§V2.3.2 借用了它的骨架）。 + +来源：[ETH AGENTS.md 研究](https://arxiv.org/html/2602.11988v1) · +[Aider repo map](https://aider.chat/docs/repomap.html) · +[RepoGraph](https://arxiv.org/html/2410.14684v1) · +[CodexGraph](https://arxiv.org/abs/2408.03910) · +[OpenHands repo skills](https://docs.openhands.dev/overview/skills/repo) · +[Devin Knowledge/DeepWiki](https://docs.devin.ai/work-with-devin/deepwiki) · +[Agent Workflow Memory](https://arxiv.org/html/2409.07429v1) · +[SWE-bench Pro](https://scale.com/blog/swe-bench-pro) · +[Cline memory bank](https://docs.cline.bot/best-practices/memory-bank) · +[Copilot custom instructions](https://docs.github.com/copilot/customizing-copilot/adding-custom-instructions-for-github-copilot) + +## V2.1 既有问题与修复 + +### (a) 正确性缺陷（P0 —— 先于一切修掉） + +| # | 问题 | 证据 | 修复 | +|---|---|---|---| +| 1 | **resume 丢失内存态。** step 之间通过直接写 `ctx.state` 交接结果（`pr.fetch_diff` → `diff_text`、`issue.fetch` → `issue_text`、`pr.gate_check` → `gate_report`、`pr.checkout_branch` → `push_policy`/分支引用、`agent.review_diff` → `review_text`），但 executor 的 resume 路径**只恢复 `outputs.state_updates`**。越过一个已完成 fetch 的 `--resume` 会在没有 `diff_text` 的情况下重入 `agent.review_diff` → 假 BLOCKED；在 push step 处 resume pr-rebase 会看到默认（拒绝一切）的 `PushPolicy` → 假 FORBIDDEN。 | `executor.py` 的 resume 分支 vs. `steps/pr/`/`steps/review/` 的 state 写入 | 契约：**任何被后续 step 消费的 state 键都必须经 `outputs.state_updates` 发布**（JSON-simple；`PushPolicy` 序列化）。由逐 playbook 的 resume 完整性测试强制：跑到第 N 步、杀掉进程、resume，断言在任何 N 上都不会出现"因缺 state 而 BLOCKED"。 | +| 2 | **`foreach` 扇出丢掉 `state_updates`。** `_merge` 按下标重新给条目输出打键，于是扇出条目发布的 `state_updates` 永远到不了 `state.update(...)`。 | `executor.py::_merge` | 显式跨条目合并 `state_updates`（同键最后写者胜是可接受的；冲突记 trace）。由同一个 resume 完整性测试覆盖。 | +| 3 | **`when:` 静默地只对 TaskSpec 求值。** 一个基于计算出的 state 键（例如 `has_conflicts`）的条件会**无错地判为 false**。 | `executor.py::_eval_when` | 对 `state` 求值并回退到 TaskSpec；未知键 → **规划期错误**，而不是静默跳过。 | +| 4 | **逐仓库知识被设计了但没接线。** `RepoAdapter.skills_dir` 和 `RepoAdapter.debug_memory_db` 存在，但 `agent_runtime` 读的是**全局**的 `settings.skills_dir` / `settings.memory_db` —— 每个仓库共用一个 skill store 和一个 debug memory。 | `adapters/base.py` vs `engine/agent_runtime/knowledge.py::_retrieve_skills/_retrieve_memories` | 知识库先经当前 adapter/profile 解析，全局共享池其次（§V2.2.6）。 | +| 5 | **高风险模块清单是硬编码在核心 settings 里的 vLLM-Omni 名字**，因此 patch-review 的 `high_risk_modules` 触发器在**任何其他仓库上都静默地永不触发**。 | `config.py::high_risk_modules` | 搬进 profile（`modules.yaml` 的风险分层，§V2.3）；该 setting 只作兜底。 | + +### (b) 核心里的仓库知识泄漏（违反"知识在边缘"；阻碍不变性） + +这些在 vllm-omni 上不是缺陷，但每一处都是**第二个仓库会被静默降级**的地方。 +它们全部由**profile 注入**（§V2.3）解决 —— 核心只保留仓库中立的脚手架： + +- `_REVIEW_SYSTEM` 和 `_REVIEW_LENSES` 在 `engine/steps/review/` 里硬编码了 + "vLLM-Omni" 和一份 Python/ML 仓库的评审 checklist（模块拆成包之后， + prompt/checklist 数据现在住在 `engine/steps/review/prompts.py`）。 + → 拆成仓库中立的核心 prompt + 由 profile 提供的领域段落 + （`review.md`：领域 checklist、lens 扩展、严重度规范）。 +- `_sweep_targets` 假定了 Python 的表层语法（`xs[0]`、`elif`、`tests/` 布局）。 + → 由 `profile.language` 选定的按语言抽取器，外加一个通用的 diff 启发式兜底。 +- `TaskSpec.repo` / `Settings.default_repo` 默认是 `"vllm-omni"`；意图解析器**根本 + 无法抽取仓库**（"review pr 12 in repo-b" 解析不了）。→ intent 获得仓库抽取能力； + 只配置了一个 profile 时，默认值来自它。 +- playbook 声明 `repos: [vllm-omni]`，于是第二个仓库需要复制粘贴的 playbook， + 而它们**必然漂移**。→ 仓库中立的 playbook（§V2.2.3）。 +- 默认 `gh` 是唯一 forge，Buildkite 处理零散且是桩。→ 由 profile 声明的 forge/CI + adapter（§V2.2.4）。 + +### (c) v1 待收口的边界 + +- **Buildkite 日志下载**（是桩；PR-debug 只读 `gh pr checks`）→ 一个 `CIProvider` + adapter 接口（拉取失败 job、下载日志、轮询构建），为 GitHub Actions 和 Buildkite + 各出一个实现，由 `profile.ci.provider` 选定。父 rebase-agent 已知的基线签名弱点 + （精确字符串比较把 flaky/已知失败误分类）**不得被继承**：adapter 里的签名匹配在比较前 + 先做归一化（抹去时间戳/哈希/路径）。 +- **指标反馈收集器**（按 METRICS_RESEARCH 路线图）：在线填充 useful/accepted/conflict + 的 gh 收集器、推送后的 CI 快照。 +- **原生 rebase 的 prelude 环境导出**会修改 copilot 自己的进程环境 → 在子进程里用显式 + 环境跑 prelude/各阶段，并保留被记 trace 的 `env_exported` 增量。 +- **ensemble 覆盖面**：`run_agent_step_ensemble` 目前只接了评审。分流和 debug 假设类 + step 要等**它们自己的跨 run 方差被测量之后**才采用 lens ensemble + （与评审战役同样的 replicate 均值纪律；**不要发布未经测量的"改进"**）。 + +## V2.2 仓库不变性 + +**定义。** 对 copilot 支持的每一种任务 kind，在一个**已建立 profile** 的新仓库上的 +质量，都要落在参考仓库（vllm-omni）的**声明区间**内，且接入一个仓库需要 +**零核心代码改动**。不变性是**契约加测量**，不是愿景 —— 基准文献（§V2.0.4）显示同一个 +模型的解决率在不同仓库间从 <10% 摆到 >50%，所以**未经测量的跨仓库主张一文不值**。 + +1. **仓库中立的核心，且被强制。** `src/infermatrix_copilot/` 里**任何地方**都不得出现 + 仓库专属字面量（仓库名、模块名、领域 prompt、绝对路径）—— 那类知识只住在 + `adapters//`。由 `test_repo_neutral_core` 钉住：它扫描源码里已知的泄漏模式， + 遇到新的就失败。核心里的 prompt 可以陈述*怎么审查/调试/分流*； + **只有 profile 才陈述*这个仓库是什么*。** +2. **单一注入路径。** 一切随仓库变化的东西，都经 DispatchContext 的 `profile` 段 + 到达 step（§V2.3.4）：领域 prompt 段落、模块映射、风险分层、test/format 命令、 + CI/forge adapter、约定。**需要仓库知识却在 profile 里找不到的 step，必须显式降级 + （见第 4 条），绝不猜测。** +3. **仓库中立的 playbook。** playbook 声明 `repos: ["*"]` 加上一份 `requires:` + 的 profile 能力清单（例如 repo-rebase 需要 `ci.provider`、`upstream.fork_tracking`）。 + `PlaybookStore.find()` 匹配的是 **任务 kind + 能力满足**，而不是仓库名； + 存在仓库专属覆盖（精确 repo 匹配）时仍由它获胜。locked 的 `repo-rebase` playbook + **按声明**仍然是 vllm-omni 专属的 —— 它 `requires` 那个只有 vllm-omni profile 提供的 + 外部编排器能力；**零回归不受影响**。 +4. **优雅的能力降级。** 能力缺失 → 一次**被声明**的降级，记为 `capability_gap` + RunTrace 事件并在 run 报告里呈现：没有 CI provider → pr-debug 跑只读分诊； + 没有声明 upstream → repo_rebase 在**规划期**带理由拒绝；语言未知 → 通用 sweep 抽取器， + 并在评审输出里标注。**降级 ≠ 静默变差：每一个缺口都是可见的。** +5. **被测量的不变性。** 把评测 harness 扩展成跨仓库基准：逐仓库跑同样的 arm 和 + RQS3/RQS3e/CATQ 指标，**按 replicate 均值**计分（战役的惨痛教训：单次投掷是 ±0.1 的 + 噪声）。逐任务 kind 报告一个**不变性指数** = min(仓库分) / mean(仓库分)；目标 ≥ 0.8。 + 一个新仓库的 profile **只有在它的评测 run 落进区间之后**才被晋升（draft → active） + —— **profile 的晋升门就是不变性门**。 +6. **知识命名空间。** skill 与 debug memory **先按仓库解析** + （`adapters//skills/`、`adapters//store/debug_memory.db` —— + 即 §(a)4 里那些已设计好的字段），再回落到共享的 general 池。 + 写入**始终**落在当前仓库的命名空间；把一条仓库 skill 晋升到共享池是**策展决定** + （它必须真的与仓库无关，比如"怎么二分一个 flaky 测试" vs. "HunyuanImage 的 SSIM 阈值"）。 + +## V2.3 仓库 profile + +profile 把 adapter zero + Phase-0 引导，推广成 copilot 对一个仓库的**完整认识**。 +下面的一切都受两条**由证据驱动**的设计规则统辖（§V2.0.1–2）： + +- **规则 1 —— profile 不是 prompt。** profile 的大部分内容由**代码**消费（step 逻辑、 + adapter、触发器）或**按需检索**；always-on 的 prompt 切片是一份**精选的最小集**， + 只放非显然、非冗余的指令。**自动生成的概览被整体注入，实测比什么都不放更糟。** +- **规则 2 —— 没有溯源就没有 fact，没有稳定性就不许重写。** 每条 fact 都要写明它是 + 怎么得出的、上次何时被确认；精选摘要**只由一个有节奏的巩固层**重写， + **绝不连续重写**。这就是那套个人 agent 的 profile 架构 —— 它已在生产中运行， + 且它的失败模式（碎片化、证据倾倒、连续重写腐蚀）都已被摸清。 + +### V2.3.0 仓库知识的两个信任层 + +仓库知识按**谁可以写它、它承载多少信任**分成两层 —— 这是对**安全**要紧的那条轴， +与 §V2.3.4 的 机器 / briefing / 检索 这条**消费**轴正交： + +- **Tier 1 —— manifest**（`manifest.yaml`）：*人工撰写、人工门禁*的配置， + 是那些**agent 去猜就不安全**的东西 —— 仓库/上游路径、推送策略（保护分支、`allowed`）、 + CI provider、能力。朴素声明式 YAML；高风险段（`repo`/`upstream`/`push`）**只能人工改** + （`HIGH_RISK_SECTIONS`，在 `update_manifest` 里强制）。 + **必需**（没有它就无法安全地操作一个仓库），低频变动 —— 这是**安全门**（**D2**）。 +- **Tier 2 —— profile**（`profile/`，外加 `skills/`、`store/`、`repo_map/`）： + *agent 建立、证据门禁*的知识，是 agent **挣来**的 —— 逐 fact 溯源、 + 类型化 patch op 作为唯一写入面、稳定性 + 陈旧度门（§V2.3.2）。 + **可选**的增强（缺失时检索回落到共享池），高频变动、自我纠正、可审计 —— + 这是**质量杠杆**（**D1**）。 + +manifest 是这个仓库的*宪法*；profile 是在它之下累积起来的知识。 +两者是**刻意不合并**的：把 Tier 1 折进证据门禁的 Tier-2 写入路径，会让推送策略变成 +**agent 可写**（一次安全回归）；而另一个选项 —— 在 profile 内部划一块"仅人类"区域 —— +不过是把同一条边界重画进一个文件里。两者的数据模型也不兼容 +（`repo.path` 这样的 manifest fact 没有"证据"或"陈旧度"；而**没有证据的 profile fact +会被拒绝**）。容器 `adapters//` 已经在磁盘上把它们统一了 —— +**一个仓库边缘，两个信任层**。agent *可以起草* Tier-1 的结构 +（`profile.structure_scan` 为模块清单播种），但 Tier 1 的高风险段**永远需要人工激活**， +因此任何承载安全的东西都是 *agent 提议 / 人类确认*。 + +### V2.3.1 布局 + +``` +adapters// + manifest.yaml 身份、仓库/上游路径、推送策略 （人工门禁） + profile/ + profile.yaml 精选核心（小、人类可读）：逐 fact 的溯源字段 —— 见 §V2.3.2 + briefing.md 渲染出来的 always-on prompt 切片：10-30 条简短祈使指令 + （工具命令、硬约束、已知陷阱）。有预算（<400 词）， + 与仓库文档非冗余 —— 需经验证，见 §V2.3.5 + format.yaml formatter/linter + 确切命令，命名与注释约定（机器：验证命令） + constraints.md 保护分支、评审/合并规范、必需检查、许可/DCO、 + 永不触碰路径（机器：推送守卫 + PathScope；另供 briefing 行） + modules.yaml 模块映射（路径 -> 模块）、import 图 wave、 + 模块 -> 测试命令、归属、风险分层 + （机器：扇出、触发器、调度；**绝不作为散文注入**） + review.md 领域评审 checklist + lens 扩展 + 严重度规范（由 pr_review 检索） + ci.yaml provider、pipeline、日志访问、必需检查、 + 归一化后的 flaky 签名基线（机器） + PROFILE_REPORT.md 渲染出来的溯源视图：每条 fact 是怎么得出的 + （deterministic | agent | human）、证据、置信度 + skills/ 逐仓库 skill，各带一段 TRIGGER 描述 + （Devin 式：内容 + 什么时候召回它） + store/debug_memory.db 逐仓库 debug memory + repo_map/ 供按需 repo_map 工具使用的缓存符号图 + （Aider/RepoGraph 式，按查询排序、有 token 预算；漂移时重建） +``` + +`manifest.yaml` 保持它的 v1 角色和仅限人工的高风险段（`repo`/`upstream`/`push`）； +`profile/` 装那些**可被建立**的知识。 + +> **命名（2026-07-11 更名，原为 "plugin"）。** 旧名 "plugin" 是个误称 —— +> 这个包是**声明式的仓库知识**，不是可执行的扩展代码，而后者才是 "plugin" 在软件里 +> 到处所暗示的东西。这份设计的头号性质是*仓库不变性*：一个通用引擎按仓库特化 —— +> 也就是 **adapter 模式** —— 所以这些概念现在是自解释的： +> - 容器 `adapters//`（原 `plugins//`）—— +> `RepoAdapter`/`AdapterRegistry`（原 `RepoPlugin`/`PluginRegistry`） +> - Tier-1 文件 `manifest.yaml`（原 `plugin.yaml`）—— 它在内部本来就是 `.manifest` +> 属性，而 "manifest" 诚实地命名了这个配置层 +> +> 读作：**adapter**（容器）装着一份 **manifest**（Tier-1 配置）和一份 **profile** +> （Tier-2 知识）。`ADAPTERS_DIR` 环境变量（原 `PLUGINS_DIR`）指向 adapter 根目录。 +> 这次更名是机械的（git-mv + 符号扫描，全程测试绿）；磁盘上的 +> `adapters//manifest.yaml` 取代了旧的 `plugins//plugin.yaml`。 + +### V2.3.2 profile 记忆架构（借自个人 agent） + +个人 agent（一个兄弟项目 —— `profile_store.py` + `RESEARCH_AGENT_MEMORY_2026.md`） +维护着一份人物 profile，其生命周期**恰好**就是仓库 profile 所需要的。 +我们**整套采纳**它的骨架，翻译成仓库术语： + +| 个人 agent 的机制 | 仓库 profile 的对应 | +|---|---| +| 一个小而精选的 `profile.yaml` 之下的不可变证据库（`events.db`） | RunTrace + Stage-1 证据归档是不可变层；`profile.yaml` 是精选层。**fact 概括证据，绝不取代证据** | +| **类型化 patch op 是唯一写入面**（`add_evidence`、`bump_last_seen`、`merge_*`、`mark_dormant`；畸形 op 被拒） | profile 变更是由 store 校验的类型化 op（`add_fact`、`add_evidence`、`bump_confirmed`、`merge_facts`、`mark_stale`、`update_command`）；**agent 绝不自由编辑 profile 文件** | +| **两个写入层**：每日追加式（不重写）vs. 每周巩固（**唯一**允许 `rewrite_entry` 的层）—— 连续的 LLM 重写**实测**会腐蚀记忆 | 逐 run 层：run 只能**追加**证据/candidate。巩固层（有节奏或评测后）：去重、合并、重写 `briefing.md` —— **散文只在这里被重新生成** | +| **对抗碎片化的 join key**：owner 可编辑的 `aliases.yaml`；每个 op 必须点名它的 initiative | **模块映射就是 join key**：每个 profile op 都要点名它所涉及的模块（或 `repo-wide`），于是知识**按模块收敛**而不是碎片化 | +| 逐条溯源：`evidence[]`、`first_seen`、`last_seen`、`confirmations: n` | 逐 fact 相同的字段，外加 `source: deterministic\|agent\|human` 和来源 commit | +| **稳定性门**：≥3 次确认的条目不得被重写成引用更少的证据 | 同一条规则：巩固重写**不得**从一条稳定 fact 上丢掉已引用的证据；被取代的正文进 `history[]`，**绝不删除** | +| **休眠衰减，绝不删除**（30/60 天窗口） | 超窗口未确认的 fact 翻成 `stale`（排除出注入/消费，**保留以备审计**）；刷新会重新确认或让它退役 | +| LLM 不能触碰的受保护段（`identity`、`preferences`） | `manifest.yaml` 的高风险段（已在 `update_manifest` 里强制）+ 标了 `source: human` 的人工撰写约束行 | +| **只读的 LLM judge**（每周）：报告矛盾/陈旧主张，**绝不自动修** | profile judge 拿 profile 对照最近 N 次 RunTrace 做审计（失败过的命令、搬走了的模块、从不触发的 checklist 项）—— findings 变成刷新提案 | +| 每份 profile 一个 git 仓库：每次保存一个提交，每周把 diff 发邮件，`git revert` 回滚 | `adapters//` 是入库的；每次巩固是**一个被评审的提交**，diff 出现在 run 报告里 —— 回滚就是 `git revert` | + +### V2.3.3 建立流水线（扩展 Phase-0 引导） + +- **Stage 0 —— 确定性指纹**（已存在：`fingerprint_repo`）：语言构成、布局、CI 系统、 + remote、默认分支。扩展出 `modules.yaml` 的确定性部分（目录聚类、经静态分析得到的 + import 图 wave）以及 `repo_map/` 的构建。**无 LLM，不对目标仓库写入。** +- **Stage 1 —— profiling agent**（新）：只读的受治理 agent step，一个 profile 产物一个， + 全部经统一 agent 运行时：*structure*（在 Stage-0 图之上做模块语义 + 测试映射）、 + *format*（检测配置文件、推断未成文约定、产出确切的检查命令）、 + *constraints*（经 forge API 读 CONTRIBUTING/CODEOWNERS/PR 模板/分支保护）、 + *ci*（pipeline、日志访问）、*review*（从文档 + 近期人工评审 thread 提炼领域 checklist）。 + **每条产出的 fact 都必须引用它的证据 —— 未引用的 fact 在契约层被拒。** +- **Stage 1.5 —— 冗余过滤**（新，ETH 研究的教训）：一趟确定性处理，丢弃任何 + 内容已在仓库自己的 README/docs/AGENTS.md 里写过的候选 briefing/checklist 行 + （agent 本来就会读那些），以及任何代码库概览式散文。**剩下的才是非显然的残余** —— + 工具命令、陷阱、不变量。如果存在 `AGENTS.md`/`CLAUDE.md`/ + `.github/copilot-instructions.md`，它会作为**人工撰写**的 briefing 输入被摄入 + （最高信任来源），而不是被重新推导。 +- **Stage 2 —— 人工评审门**（已存在，已扩展）：profile 以 `status: draft` + + `PROFILE_REPORT.md` 落盘；高风险段仅限人工。人工评审把 draft 翻成 candidate。 +- **Stage 3 —— 实时校准**：最初 N 次 run 把低置信 fact 当作**假设**； + 修正（模块映射订正、真的能过的命令、flaky 基线条目）经策展门作为 candidate 提出。 + 成功 run 的 RunTrace 喂给 **skill 归纳**（AWM 式）：反复出现的解决模式变成 + 逐仓库的 skill candidate，带触发描述。通过评测区间（§V2.2.5 + §V2.3.5）即把 + candidate 晋升为 active。 +- **Stage 4 —— 巩固与刷新**（个人 agent 的每周一趟）：有节奏的巩固按模块去重/合并 fact、 + 在词预算内重新生成 `briefing.md`、施加休眠衰减；漂移检测器（模块路径消失、format + 命令失败、CI pipeline 改名、N 次与 profile 矛盾的 run、judge findings）触发对受影响 + Stage-1 agent 的**定向重跑**。**每个周期一个被评审的提交。** + +### V2.3.4 消费：三条通道 + +ETH 的结论**强制**了这套纪律；每个产物都要声明自己的通道： + +1. **机器消费（绝不作为散文注入）**：`modules.yaml` → 扇出、wave 调度、 + patch-review 的 `high_risk_modules`、检索查询；`format.yaml` → 在 patch 门之前 + 运行的编辑后验证命令；`ci.yaml` → adapter 选择、必需检查清单、flaky 过滤； + `constraints.md` 的机器行 → 推送守卫 + PathScope。 +2. **always-on prompt 切片**：**只有** `briefing.md`（有词预算、精选、已冗余过滤） + 会进入该仓库的每一个 DispatchContext，外加与该 step 风险级别相关的那几行约束。 +3. **触发式/按需**：逐仓库 skill 由触发匹配召回；`review.md` **只**被 pr_review step + 检索；`repo_map` 工具在 token 预算内回答按该 step 目标排序的结构查询 —— + **agent 在需要时自己拉结构，而不是被灌一份概览。** + +### V2.3.5 profile 的效力是被测量的，不是被假定的 + +同一份研究还显示，**即使是人工撰写**的上下文文件，在某些仓库上也可能是净负面。 +所以 **profile 自己就是一条评测臂**：在每个已建立 profile 的仓库上，任务 kind 的评测 +都跑 {无 profile} vs {有 profile}（一如既往，按 replicate 均值）。 +一份 profile（或一次 briefing 修订）**只有在质量上非负、且不撑爆成本预算**时才被晋升 +（CATQ 的 C 项已经为上下文文件所诱发的 +2–4 个额外步骤定了价）。 +巩固层的 briefing 重写会**重跑这次消融**。 +这就闭上了 AGENTS.md 的作者们跳过的那个环：**我们从不假定注入的上下文有帮助。** + +### V2.3.6 治理 + +**没有新机器** —— profile 复用既有的那些门：profile 编辑就是知识编辑（本来就是 patch +review 的一个触发器）；agent **只能**提交类型化 op 的 candidate（`update_manifest` +本来就拒绝 agent 对高风险段的写入）；draft/candidate/active/retired 镜像 playbook 的 +生命周期；每条被接受的 fact 都可经 `PROFILE_REPORT.md` 追溯； +**每次巩固都是一个被评审、可回滚的提交**。 + +## V2.4 里程碑与验收 + +- **P0 —— 正确性**：§V2.1(a) 的修复 + 逐 playbook 的 resume 完整性测试 + + `test_repo_neutral_core`（初期以 xfail 列出已知泄漏，使这份清单**只能变短**）。 +- **P1 —— profile 底座**：带类型化 patch op + 溯源字段的 profile store（§V2.3.2）、 + `profile/` schema + loader、Stage-0/1 建立 + Stage-1.5 冗余过滤 + PROFILE_REPORT、 + DispatchContext 的 `briefing` 注入、review/sweep/intent 按 profile 参数化 + （去除 §V2.1(b) 的泄漏）。 +- **P2 —— 仓库中立的执行**：playbook 的 `requires:` 能力匹配、CI/forge adapter 接口 + （GitHub Actions + Buildkite）、能力缺口的降级路径、逐仓库知识命名空间接线、 + 按需 `repo_map` 工具。 +- **P3 —— 被测量的不变性**：跨仓库评测 harness（参考仓库 + 至少一个结构上不同的第二 + 仓库）、不变性指数报告、**profile 消融臂**（§V2.3.5）、与评测区间绑定的晋升门； + Stage-3 实时校准 + AWM 式 skill 归纳；有节奏的 Stage-4 巩固/judge/刷新循环。 + +**验收判据** + +1. 第二个仓库被端到端接入（指纹 → profile → active）， + 且**对 `src/infermatrix_copilot/` 零提交**。 +2. 六种任务 kind 全部能在第二个仓库上执行 —— profile 提供了能力的地方就满能力运行， + 没提供的地方带**被记录的** `capability_gap` 降级；**没有静默的质量损失**。 +3. 第二个仓库上的 pr-review RQS3（replicate 均值）落在 vllm-omni 参考的声明区间内； + 不变性指数 ≥ 0.8。 +4. 每个 playbook 的 resume 完整性测试都绿（**没有由 resume 本身造成的 BLOCKED/FORBIDDEN**）。 +5. locked 的 `repo-rebase` 夜跑行为保持**逐字节一致**（零回归约束原样贯穿 v2）。 +6. profile 挣得它的位置：在每个已建立 profile 的仓库上，{有 profile} 臂在质量上 + 相对 {无 profile} 非负且成本可接受（§V2.3.5），且每份 `briefing.md` 都在预算内、 + **零条与文档冗余的行**。 + +--- + +# Part III —— Design v3：两个前门与五个后端（2026-08） + +v1 建成了执行主脊；v2 让它仓库不变。v3 回答的是**另一个问题** —— +**谁来跑模型，以及 copilot 是给谁用的** —— 它产出了当前代码库里占主导地位的两个结构： +第二种**入口**形状（Direct），和第二种**执行**形状（harness 后端）。 + +**两者都不是为重构而重构。** 每一个都补上了一个让产品对某一类真实用户不可用的缺口。 + +## V3.1 Direct 模式 —— 把 copilot 当作知识提供方 + +**缺口。** Strict 跑完整主脊：5–12 分钟、一个 API key、一份本地 checkout、 +一个后台进程。而已经坐在 Claude Code、Codex 或 Cursor 里的用户， +**既有一个有能力的模型，又已经打开了仓库**。让他们再配置第二个模型来审查眼前的代码， +是一笔**没有回报的税**。 + +**决策。** 发布一种模式，让 MCP 服务端**完全不跑模型**。 +`review(mode="direct")` **同步**返回知识路由和治理契约；阅读与判断由**宿主自己的模型** +完成。执行主脊（Part I §3.X）**完全不参与**。 + +**难的是治理，不是路由。** 服务端**约束不了**宿主的模型 —— 它根本不在那个环里。 +所以"该怎么审"被编码成随返回值一起旅行的**结构化数据**（≤3 条带内嵌 `quick_map` 的 +路由、一个硬性的 `execution_budget`、一份 checklist），而验收是一道**机械的完成门** +（`validate_direct_review`）。那道门**刻意只检查形状，不检查真伪**：恰好一条最终评论、 +`subtraction_signal` 的自洽性、以及证明本次评审读的是固定提交的 `evidence_head_sha`。 +**验证被引用的证据是否真实，超出了一个无状态服务端能做到的范围； +声称能做到，比不声称更糟。** + +**对其余一切的后果。** 现在**一个仓库里住着两个产品**。playbook 和 step **只属于 +Strict**；Direct 从不碰它们。这条区分一旦模糊，读者就会迷路 —— +这正是 [`../GUIDE.md`](../GUIDE.md) §1.1 要以它开篇的原因。 + +## V3.2 Provider 注册表 —— 接入点不可能是 `LLM.create()` + +**缺口。** 每一次模型调用都要经过一个需要 `ANTHROPIC_API_KEY` 或 `OPENAI_API_KEY` +的原始 completions 客户端。**只有编码 agent 订阅**、没有裸 API key 的用户， +**根本跑不了 Strict**。 + +**塑造了整个设计的那条约束。** Claude Code、Codex CLI 和 cursor-agent 在订阅认证下 +**不暴露 completions API**。它们是**自带工具循环的 agent harness**。 +所以接缝不可能是 `LLM.create()`（无状态的 `system+messages+tools → tool_use` 往返） +—— 它必须上移一层，落在 **`run_session()`：委托一整个 agent step**。 +**整个 `providers/` 包都是从这一个决策推出来的。** + +**与维护者锁定的决策（2026-08-14）：** + +1. **产品覆盖面，不是质量。** 保真度按 harness 尽力而为，并**逐 run 标注**。 + 一个 harness 后端测量的是**循环 + 模型的组合**，因此**绝不**被并入生成器消融表 —— + 长期规则是：生成器臂骑我们自己的流水线，**只换模型**。 +2. **显式选择，硬报错。** 由 `STRICT_BACKEND` 选定；未设置或未知的后端会**在前面** + 带着确切缺失项失败。**没有自动探测，没有静默回退** —— 与 `TierNotConfiguredError` + 同一套哲学。 +3. **工具经桥回流，预防优先。** harness 会话经 MCP 收到 copilot 自己的 scoped 工具， + 每次调用仍然过 `tools.dispatch`。厂商内置工具在 harness 支持时被**关闭** + （`builtin_tools_off`）；不支持时改用 OS 级只读沙箱（`codex`、`deepseek`）； + **两者都做不到时**（`cursor-agent`），运行后审计作为**公开声明的侦测型**控制运行。 + 纵深防御，且**控制类别写在 RUN_REPORT 里 —— 绝不留静默缺口**。 +4. **统一注册表。** 既有的 API 路径被收编成同一张表里的 provider `api`： + 一条解析路径、一套 trace 词汇，外加一道**平价棘轮** —— 用 `api` 时行为**逐字节一致**。 + +**值得记住的那处不对称。** 凭据模型是**刻意不均匀**的：`api` 经 Settings 解析 +key + 端点；harness provider 把订阅认证放在厂商 CLI 自己的 HOME 状态里， +**本代码库从不接触**。`deepseek` 打破了这个模式 —— 它是 harness 却**需要 API key** +—— 所以注册表用 `api_keyed` 标出它，而不是让调用方去假定"harness ⇒ 凭据在厂商那边"。 + +**为什么子进程环境是白名单。** 厂商 CLI 必须保住自己的认证，但**绝不能继承我们的**。 +在这类机器上 `ANTHROPIC_BASE_URL` 指向一个网关；一旦被继承， +它会**悄悄把厂商 CLI 的流量改道到用户并未选择的地方**。 +`sanitized_env()` 只保留 `PATH`/`HOME`/locale，其余全部丢弃。 + +## V3.3 双路径 `mode` —— 把成本作为一等的轴，且与权限正交 + +评审质量随模型强度上升，而 4-lens ensemble 要花约 4 倍。但**成本**和**爆炸半径** +绝不能混为一谈：v3 引入 `mode`（默认 `eco` / 仅在用户显式要求时 `performance`）， +它选择 agent 的推理模型，并与 `tier` **保持正交**。 + +那条不变量值得直白写下来：**便宜的模型永远不会扩大一个任务被允许做的事。** +`tier` 仍然由 `kind` 推导（Part I 决策 3）；`mode` **碰不到它**。 + +## V3.4 v3 **没有**解决的问题 + +记录在此，好让后来者既不重新争论已经定论的东西，也不继承没有支撑的主张： + +- **相对 agentic 基线的评审质量，并没有赢。** 已测量的状态是：precision 打平、 + recall 比 CC + Opus 5 低约 .05，成本约为其三分之一。 + 见 [`../evaluation/README.md`](../evaluation/README.md)； + "两项均值都严格超过基线"这个目标**没有达成**， + 而且用修正后的统计看，**它在战役的任何时点都从未达成过**。 +- **harness 的成本优势是假定的，不是测量的。** harness 会话不向 span 记账暴露 token + （每项都是 `tok_out=0`），所以 metrics 记来源为 `subscription`，**拒绝编造 USD**。 +- **跨仓库不变性（§V2.2.5）仍未被测量。** v3 把测量预算花在了参考仓库的评审质量上。 + **不变性指数至今从未被计算过**，所以 Part II 里每一条跨仓库主张都是 + **设计意图，不是结果**。 diff --git a/doc/KNOWLEDGE.md b/doc/architecture/KNOWLEDGE.md similarity index 92% rename from doc/KNOWLEDGE.md rename to doc/architecture/KNOWLEDGE.md index c70e5d5da..d5a0df6a3 100644 --- a/doc/KNOWLEDGE.md +++ b/doc/architecture/KNOWLEDGE.md @@ -36,7 +36,7 @@ extensions merely because they are absent from the common source. carry the author's machine-local grants (including a private host address), which the knowledge tree's own validator policy forbids. Their exact bytes are preserved in the audit baseline. -- **Audit baseline:** `doc/reorg-audit/baseline/` holds the upstream `ls-tree` +- **Audit baseline:** `doc/archive/reorg-audit/baseline/` holds the upstream `ls-tree` manifest, a full `git archive` tarball (+sha256) of `f2dc824`, and `dates.tsv` (per-file upstream created/updated dates, captured before the submodule's git history was removed — used for page frontmatter). @@ -46,8 +46,12 @@ extensions merely because they are absent from the common source. is `adapters/vllm_omni/release_baseline.yaml`. - **AFD code-mirror pin:** the `knowledge/repos/afd-plugin/components/` owner maps and architecture summaries are verified against - `vllm-project/afd-plugin` `main @ - a432692ed7d5dd6437a4755b530ee7aaf2685dad` (vLLM 0.26.0 baseline). + `vllm-project/afd-plugin` default branch at + `a432692ed7d5dd6437a4755b530ee7aaf2685dad` (vLLM 0.26.0 baseline). + Deliberately NOT written as `main @ `: this file is a + `pin_documents` entry, and the release audit requires every marker in + that form to equal `audited_sha`. This SHA belongs to another + repository, so the pin syntax is reserved for vllm-omni pins only. ## Layout: general vs repo-specific The knowledge base separates the two, and so do we — the whole tree is **shared**, diff --git a/doc/architecture/SPEC/README.md b/doc/architecture/SPEC/README.md new file mode 100644 index 000000000..d899f8672 --- /dev/null +++ b/doc/architecture/SPEC/README.md @@ -0,0 +1,73 @@ +# InferMatrixCopilot —— 规范（逐文件） + +一份**规范性的逐文件**说明。`doc/architecture/SPEC/` 这棵树镜像 +`src/infermatrix_copilot/`：**一个源文件一页规范**，相对路径保持一致 +（`engine/steps/pr/` → `doc/architecture/SPEC/engine/steps/pr.md`）。每页覆盖一组 +固定的视角。两份跨切文档放在根部，被每一页逐文件规范**引用**（而不是重复）。 + +这份 SPEC 是为**驱动代码库重构**而写的：对你要改的任何源文件，它的规范会告诉你 +这个文件必须继续保证什么、什么不属于它、哪些全局约束绑定着它、以及它目前哪里乱。 + +## 跨切文档（先读） + +- **[_ARCHITECTURE.md](_ARCHITECTURE.md)** —— 分层、依赖方向规则、功能（7 种任务 + kind）、scope、数据与产物、安全模型、仓库不变性契约。每一页逐文件规范都默认你 + 已经知道这张"大图"。 +- **[_CONSTRAINTS.md](_CONSTRAINTS.md)** —— 全局编程约束（A 结构、B 契约、C 安全、 + D 知识、E 可观测）+ 不变量目录。逐文件规范用 id 引用它们（例如 `A2`、`C4`）， + 而不是重述。 +- **[_CONCISION.md](_CONCISION.md)** —— 把代码库变小的计划：一份有优先级、有 grep + 支撑的死代码、重复和样板清单（K1–K7），每条都给出要引入的共享 helper 和必须保留 + 的不变量。"精简"重构就照它执行；某一页存在对应机会时，会带一个 **Concision** 备注。 + +## 逐文件规范模板（那些视角） + +镜像树下的每个 `*.md` 都按这个顺序使用这些标题： + +1. **头行** —— `LOC ~N · 角色 · refactor-status`。 + `refactor-status ∈ {ok, oversized, split-candidate, shim-to-retire, trivial}`。 +2. **职责** —— 这个文件独占的那一件事。 +3. **功能** —— 它实际做什么（行为），简短。 +4. **公开契约** —— 其他代码可以使用的导出符号 + 它们的保证。 +5. **不变量** —— 每条路径上都必须成立的性质（"不能破坏"清单），各自标注它实现的 + 约束 id。 +6. **边界 —— 不属于这里** —— 明确**不**属于这个文件的东西。 +7. **依赖（允许）** —— 这个文件可以有的 import（强制 §_ARCHITECTURE 的依赖规则）。 + 其余一律是分层违规。 +8. **扩展点** —— 在这里增加能力的正规方式。 +9. **测试** —— 钉住这些不变量的护栏测试。 +10. **重构备注** —— 体量/内聚/耦合的坏味道，以及具体建议的搬移/拆分 + （**内聚**视角 —— 提升可读性，会增加文件数）。 +11. **精简**（仅在存在机会时出现）—— 这个文件里哪些是死的、重复的或样板， + 以及 `_CONCISION.md` 的哪一条（K1–K7）能去掉它。这是"精简"重构消费的视角； + 与内聚拆分冲突时，**以精简为准**。 + +## 重构时怎么用它 + +- **改一个文件之前**：先读它的规范。如果你的改动增加了规范里没列的职责，那这个 + 改动属于**另一个文件**（或一个新文件）—— 先改规范。 +- **拆分一个文件时**：建出新的规范文件，把相关视角搬过去，并让"依赖（允许）"保持 + 诚实 —— 一次制造出规则禁止的交叉 import 的拆分，不算完成。 +- **删除一个 shim 时**（`refactor-status: shim-to-retire`）：先把 importer 迁移到 + 真正的模块（规范里列了谁 import 它），然后把代码和它的规范一起删掉。 +- **不变量就是契约**：重构可以自由搬动代码，但**必须保留**规范里点名的每一条不变量 + 和每一个护栏测试。如果某个测试必须改，那就是不变量变了 —— 请把它显式说出来。 + +## 文件索引（镜像 `src/infermatrix_copilot/`） + +| 分层 | 规范文件 | +|---|---| +| 接口 / 任务 | `task_spec` `intent` `cli` `chat` `ui` `config` | +| 引擎底座 | `engine/step` `engine/registry` `engine/executor` `engine/planner` `engine/agent_runtime` `agent_loop` `tools` `scopes` `llm` | +| Step 库 | `engine/steps/__init__` `engine/steps/_common` `engine/steps/{workspace,rebase_ext,review,report,pr,issue,profile,rebase_native}` | +| 规划数据 | `playbooks/store` `playbooks/PLAYBOOKS`（yaml） | +| 边缘 —— 语言 | `profiles/languages`（按语言的规则，共享） | +| 边缘 | `adapters/base` `ci/normalize` `ci/providers` `rebase/monitor` | +| 安全原语 | `scopes` `push` | +| Profile | `profiles/store` `profiles/establish` `profiles/repo_map` `profiles/consolidate` | +| 跨切 | `review/{diff_summary,triggers,reviewer}` `memory/{debug_memory,skills}` `run_trace` `notify` `metrics` | +| 后端 | `providers/{registry,base,claude_code,codex,cursor,deepseek,audit,harness_llm}` `tool_bridge` | +| MCP 面 | `thin_mcp_server` `mcp_server` `mcp_policy` `run_status` `knowledge_docs` `__main__` | + +平凡的 `__init__.py` 再导出文件不单独出规范 +（`engine/steps/__init__` 例外，因为它定义了 `register_builtin_steps`）。 diff --git a/doc/architecture/SPEC/_ARCHITECTURE.md b/doc/architecture/SPEC/_ARCHITECTURE.md new file mode 100644 index 000000000..c10552cb1 --- /dev/null +++ b/doc/architecture/SPEC/_ARCHITECTURE.md @@ -0,0 +1,129 @@ +# 00 —— 架构、范围、功能 + +## 1. 这个系统是什么 + +一个 playbook 驱动的仓库维护 copilot，服务 vLLM-Omni（按设计，也服务任何建立了 +profile 的仓库）。它把一条自然语言或 flag 命令，变成一条**受治理、可 resume、 +可审计**的已审核 step 流水线，并在被阻塞时**升级而不是猜测**。 + +## 2. 功能（任务 kind） + +**恰好存在七种任务 kind**；每种都有一个**任何输入都无法改变**的爆炸半径 **tier** +（见 `task_spec.py`）： + +| Kind | Tier | 只读 | 做什么 | +|---|---|---|---| +| `repo_rebase` | L0 | 否 | 把 fork rebase 到上游（委托给 locked 的 5 阶段编排器） | +| `pr_rebase` | L1 | 否 | 把 PR 分支重放到它最新的 base 上 | +| `pr_debug` | L1 | 否 | 诊断并修复 PR 上失败的 CI | +| `pr_review` | L2 | 是* | 证据锚定的 inline 评审 | +| `issue_answer` | L2 | 是* | 起草一份 issue 回答 | +| `issue_filter` | L2 | 是* | 分流/打标/路由 issue | +| `repo_profile` | L2 | 否** | 建立/刷新一个仓库的 profile | + +\* 除非设置了显式的 `post` 意图，否则只读。\*\* 读取目标仓库，但把知识写进 +`adapters//`。 + +tier（DESIGN `§3.2`）：**L0** 原样复用 locked playbook；**L1** 改编已审核 playbook +（受 plan-review 门禁）；**L2** 可以退化到生成，**但只对只读 kind**。 + +## 3. 三层 + 引擎底座 + +``` + 接口层 cli/ / chat.py / ui.py 自然语言与 flag -> 一个分派器 + │ + 任务层 task_spec.py / intent.py 自然语言 -> TaskSpec（tier 由 kind 定） + │ + 规划层 engine/planner.py / playbooks/ reuse > adapt > generate + │ + 引擎层 engine/{step,registry,executor} 跑 step：检查点、foreach、 + │ engine/agent_runtime.py 类型化失败、受治理 agent + │ agent_loop.py, tools.py, scopes.py + │ + Step 库 engine/steps/* 已审核的 step 库 + │ + 边缘 adapters/, ci/, profiles/ 仓库知识与能力 + review/, memory/, run_trace, notify, metrics 跨切服务 + scopes.py, push.py 纯安全原语 +``` + +代码上是三层（接口 / 任务+规划 / 引擎+Step），架在一层仓库知识与跨切服务的**边缘** +之上。设计里的 "Target 层" **不是**一个代码层：它的任务定义职责由 `TaskSpec` + +`Playbook` 承担，而它唯一存活下来的产物 —— 推送授权 —— 就是 `push.py` 这个安全原语 +（`scopes.py` 的兄弟）。 + +- **引擎是底座，不是流水线。** 它提供已审核的 step 库、与任务无关的执行保证 + （检查点/resume、类型化失败路由、scope 强制、升级、RunTrace），以及一个 + plan→execute 循环。**它不持有仓库知识，也不允许 planner 去编排原始工具。** +- **知识住在边缘。** 仓库专属事实（模块、CI、prompt、约定、推送策略）住在 + `adapters//`（manifest + profile），**绝不在核心里**。 + **增加一个仓库是边缘的增量，不是核心的改动。** + +## 4. 依赖规则（被强制的分层） + +import **只能向下、向外**。以下是一次改动不得违反的、已被验证的约束： + +1. 叶子边缘包（`profiles/`、`ci/`、`adapters/`、`review/`、`memory/`）和安全原语 + （`scopes.py`、`push.py`）**不得** import `engine/`。是引擎依赖它们。 +2. 接口层（`cli/`、`chat.py`）**不得**被任何下层 import。 + `chat.py` 只可以在 `TYPE_CHECKING` 下引用 `cli.Copilot`。 +3. `engine/step.py` 是基础词汇：它**只能**依赖 `run_trace`。 + 不含任何任务或仓库专属内容。 +4. `engine/steps/*` 编排工具与边缘包；它们经 `engine.agent_runtime` 到达 agent 运行时。 + **step 不得被引擎底座 import**（step 位于其上）。 +5. 跨 step 的共享 helper 住在 `engine/steps/_common.py` —— step 模块 + **不得互相 import**（旧的 `pr_steps -> builtin_steps` 延迟 import 已被移除， + **不得回来**）。 + +## 5. 范围 + +**在范围内：** 上面七种任务 kind；单一编排引擎；仓库 profile 的建立/维护； +对话式 + flag 两种 CLI；经 profile 支持多仓库；安全护栏（plan/patch review、 +scope 强制、推送守卫、升级）；逐 run 指标。 + +**刻意在范围外：** 重新实现夜跑 rebase（它是**被委托的，包装而非重写**）； +一个通用 agent 框架；从应用内部跑付费评测（评测 harness 是 `eval/` 里的离线机器）； +成为 CI/forge adapter 所声明之外的 forge；存储密钥（只有 `.env`，被 git 忽略， +**绝不提交**）。 + +## 6. 数据与产物（ground truth） + +- **逐 run** —— `~/.infermatrix-copilot/runs/run-/`：`run_trace.jsonl` + （仅追加的事实）、`progress.json`（step 检查点）、`RUN_REPORT.md`、`metrics.json`、 + `ESCALATION.md`（仅在被阻塞时），外加 step 专属产物 + （`rebase_status.json`、`COMPARISON.md`、证据归档）。 +- **逐仓库（知识）** —— `adapters//`，**两个信任层**（DESIGN §V2.3.0）： + **Tier 1 —— manifest** `manifest.yaml`（人工撰写、人工门禁：身份/路径/push/ci —— + **agent 不得自行编辑**的安全配置），以及 + **Tier 2 —— profile** `profile/`（agent 建立、证据门禁：`profile.yaml`、 + `PROFILE_REPORT.md`、`JUDGE_REPORT.md`、`ops_log.jsonl`，以及按需建立的 + `format.yaml`/`review.md`/…）+ `skills/`、`store/debug_memory.db`、`repo_map/` 缓存。 + （容器是一个仓库 *adapter*，里面装着一份 *manifest* + 一份 *profile* —— + 2026-07-11 由 "plugin" 更名而来。） +- **治理规则：** RunTrace/证据是**不可变层**；profile 是架在它之上的**精选层**。 + **fact 概括证据，绝不取代证据。** + +## 7. 安全模型（纵深防御） + +五道相互独立、按"与"叠加的门 —— 一次改动可以**加强**但**不得移除**其中任何一道： + +1. **tier 由 kind 定** —— 自然语言/用户文本永远无法扩大权限（`_CONSTRAINTS.md` C1）。 +2. **plan review** —— 改编/生成出来的计划在执行前先被评审； + 生成路径在**结构上**被禁止包含 write/push step。 +3. **ToolScope/PathScope** —— 每次工具调用都过同一个分派器；越界的写**会执行但被记录**， + **绝不静默**。 +4. **patch review** —— 由 7 条风险触发器条件触发，在推送前 **fail-closed**。 +5. **推送守卫** —— 唯一 choke point；受保护分支永不被推送；force 只有 with-lease； + 除非 `ALLOW_PUSH=1` 否则 dry-run。 + +被阻塞的 run 写出 `ESCALATION.md`、发出通知、以退出码 3 结束 —— +**通知，绝不猜测**。 + +## 8. 仓库不变性（多仓库契约）[部分为计划中] + +在一个已建立 profile 的仓库上，质量必须落在参考仓库的**声明区间**内， +且接入一个新仓库需要**对 `src/infermatrix_copilot/` 零改动**。 +由以下机制强制：仓库中立的核心（`test_repo_neutral_core`）、按能力匹配的 playbook、 +显式的 `capability_gap` 降级、逐仓库的知识命名空间。 +由跨仓库评测 + 不变性指数衡量（`eval/invariance.py`）—— +**评测战役本身仍是 `[planned]`。** diff --git a/doc/architecture/SPEC/_CONCISION.md b/doc/architecture/SPEC/_CONCISION.md new file mode 100644 index 000000000..e8c545150 --- /dev/null +++ b/doc/architecture/SPEC/_CONCISION.md @@ -0,0 +1,124 @@ +# 精简计划 —— 在不丢任何一条保证的前提下把代码库变小 + +这是重构所遵循的跨切计划，目标是让代码库**精简**：去重复、删死代码、把重复的样板 +收敛成共享 helper。每一条都点名确切的现场（有 grep 支撑）、预估可省行数、要引入的抽象， +以及**必须存活下来的不变量**。逐文件规范里带有对应的 **精简** 小节。 + +## 状态 —— K1–K7 已在 `main` 上应用（2026-07-10） + +七条**全部完成**（全程 211 个测试绿；每条各自一个提交）。结果：**净 −2 个文件** +（删掉 3 个 shim + 4 个死 dataclass，新增 1 个共享的 `profiles/languages.py`）， +且下面每一处重复都收敛到了单一真相来源。原始行数大致持平，因为 K2 是用**三份互相 +分叉的副本**换来**一个共享模块**（DRY，不是更少的行）；真正的收益是**可维护性** —— +现在一个新 step 直接复用 `._common` 的守卫，语言规则也只住在一处。 +**保留本文档作为那些 helper 为何存在的记录，免得它们被重新内联回去。** + +## 两条轴 —— 不要混淆 + +- **精简**（本文档）：更少的行、更少的重复、更少的死代码。**本次重构优先它。** +- **内聚拆分**（各逐文件规范的"重构备注"）：把过大的文件拆成聚焦的文件 —— + 提升可读性，但会**增加文件数**，**并不减少总行数**。 + +两者冲突时，**优先精简**。例子：`agent_runtime.py`（685 行）曾带着一条内聚拆分备注， +但那里的精简收益更小（把修复轮和证据封顶的代码去重），**不是**拆分 —— +所以先做了去重。等它仍然难以导航时，内聚拆分才跟上。 + +## 已应用的内聚拆分（2026-07-10，在 K1–K7 之后） + +在上面的精简轮之后仍然稠密的两个（后来是四个）过大文件被拆成了包 +（全程 219 个测试绿；公开 import 面经由再导出的 `__init__` 保留， +**没有任何 importer 需要改**）： +- `engine/agent_runtime.py`（685 行）→ `engine/agent_runtime/` —— + `dispatch`/`knowledge`/`utils`（底座）+ `runner`/`ensemble`（入口）。 +- `engine/steps/review.py`（341 行）→ `engine/steps/review/` —— + `prompts`（评测调优过的文本）+ `utils`（确定性 helper）+ `steps`（handler）。 +- `engine/steps/pr.py`（484 行）→ `engine/steps/pr/` —— 按关注点拆： + `fetch`（只读）+ `rebase` + `debug` + `publish`（两个 risk=push 的 step 都在这里） + + `utils`（`extract_signature`）。 +- `cli.py`（406 行）→ `cli/` —— `copilot`（编排器类，**完整保留**）+ + `entry`（argparse/REPL）+ `utils`（纯格式化器）；`__init__`/`__main__` 保住了 + `infermatrix_copilot.cli:main` 入口点和 `python -m` 的对等性。 + +每一次拆分里，无状态/纯 helper 都搬进了 `utils.py`，好让类/handler 文件承载**控制流 +而不是管道**。评测引用注释**跟着它们的代码一起搬**，公开 import 面经再导出的 +`__init__` 保留（只有一处测试的白盒 monkeypatch 目标改到了新子模块 —— `pr.debug._gh` +—— 这是四次拆分里**唯一**的测试改动）。 + +## 精简重构不得破坏的规则 + +- 逐文件规范里标记的**每一条不变量都逐字存活**；每一个护栏测试仍然通过 + （helper 可以改变一条保证**如何**被满足，**绝不能改变它是否**被满足）。 +- 新 helper 必须住在依赖规则允许的地方（step helper 进 `engine/steps/_common.py`； + 共享数据进一个叶子数据模块）。**helper 不得制造被禁止的交叉 import** + （`_ARCHITECTURE.md` §4）。 +- **只会被用一次的 helper 不值得抽** —— 只在 ≥2 个真实调用点时才抽取 + （下面的数字都满足）。 + +## 按优先级排列的目录（价值最高在前） + +### K1 —— 删除死掉的 target dataclass · 约 35 行 · 风险：无 +`targets/base.py`（现在的 `push.py`）里的 `ModuleTask`、`ModuleSchedule`、 +`ValidationPlan`、`RebaseRunSpec` 除了自己的再导出之外**无处使用** +（已验证：0 个其他 importer）。删掉这四个 dataclass 及其再导出。 +**保留** `PushPolicy`/`PushDecision`/`guard_push`（活着的推送守卫）。 +*要保住的不变量：* C4（推送守卫）不受影响。 + +### K2 —— 集中按语言的规则 · 约 30–40 行 + 防止漂移 · 风险：低 +"在这门语言里，什么算源文件 / 符号 / 分支"这份数据一式三份： +`review._sweep_targets`（`line_rules`）、`establish`（`LANGUAGE_SUFFIXES` + +`scan_modules`）、`repo_map`（`_SYMBOL_RES` + `_SUFFIXES`）。 +引入一个叶子数据模块 `profiles/languages.py`（纯数据 + 极小访问器：逐语言的后缀、 +符号正则、分支/索引正则），三个消费者从它 import。 +*要保住的不变量：* 未知语言在**每个**消费者里都诚实降级 +（只做文件级 sweep / 空的模块扫描 / "use grep"）。 + +### K3 —— step handler 的守卫 helper · 约 40–60 行 · 风险：低 +加进 `engine/steps/_common.py`，替换那些重复块： +- `require_repo(ctx) -> Path | StepResult` —— 各 step 文件里那 **8** 处 + `repo is None → BLOCKED` 守卫，各自缩成一行。 +- `adapter_or_result(ctx) -> RepoAdapter | StepResult`（或一个 `@needs_adapter` + 装饰器）—— `profile.py` 里那 **7** 处 `_adapter_from_state` + + `isinstance(..., StepResult)` 守卫。 +- `no_llm_gap(ctx, step, effect) -> StepResult` —— 那 **4** 处一模一样的 + "无 LLM → 记 `capability_gap` → 返回 ok/skip" 块。 +- `store_for(adapter) -> ProfileStore` —— `profile.py` 里那 **6** 处 + `ProfileStore(adapter.profile_dir)` 构造。 +*要保住的不变量：* 类型化的 BLOCKED 返回（B1）、`capability_gap` trace 事件（E2）， +以及每个 step 的公开名字/行为。 + +### K4 —— `state_updates` 的人体工学 · 可读性 + 约 10 行 · 风险：低 +有 **21** 处调用点手写 `outputs={"state_updates": {...}, ...}`。加一个小的构造 helper +—— `published(summary, *, state=None, **outputs)` 或一个 `StepResult.publishing(...)` +类方法 —— 让一次交接变成**一次清晰的调用**。 +*要保住的不变量：* **B2** —— 每个被后续 step 消费的 state 键仍然被发布； +**helper 让"遵守"更容易，而不是让它变成可选。** + +### K5 —— 退役三个引擎 shim · 约 26 行 + 3 个文件 · 风险：低 +`engine/{builtin_steps,pr_steps,rebase_native_steps}.py` 是再导出 shim。 +先迁移它们的 importer（各 shim 的规范里列了名单），然后删掉文件和它们的规范。 +*要保住的不变量：* 在其 fixture 迁移完成之前， +`rebase_native_steps._RUNTIME` 的"同一个对象"保证。 + +### K6 —— 去重 cli 的 过门+确认 · 约 15 行 · 风险：低 +`Copilot.run_task` 和 `run_playbook` 重复了 plan-review + `[y/N]` 确认的序列。 +抽出 `_gate_and_confirm(resolution, spec, assume_yes) -> bool`。 +*要保住的不变量：* 确认之前先 plan-review； +`confirm_required or requires_review` 时触发确认。 + +### K7 —— 抓取类 step 的"从 state"早返回 · 约 10 行 · 风险：无 +有 **5** 个抓取 step 以 `if "X" in ctx.state: return ...(state_updates)` 开头。 +一个 `from_state(ctx, key) -> StepResult | None` helper 就能把它们收掉。 +*要保住的不变量：* 注入式/离线测试路径保持完好。 + +## 总计预估 +约 **160–200 行**被移除、约 **4 个文件**被删除，若干重复模式收敛到单一调用点 —— +而这一切都发生在任何内聚拆分**之前**。缩得最多的是各 step 文件（K3+K4+K7）， +而那正是新 step 被添加的地方，所以**新增一个 step 的边际成本也随之下降**。 + +## 建议顺序（每一步都可独立发布，之间保持测试绿） +1. K1、K5（纯删除 —— diff 最小，立竿见影地变小）。 +2. K3、K4、K7（`._common` helper —— step 文件样板的收敛）。 +3. K2（`profiles/languages.py` —— 跨文件去重）。 +4. K6（cli helper）。 +5. **然后**才重新考虑内聚拆分（各逐文件规范的重构备注）， + 此时面对的已经是一个更小、已去重的基线。 diff --git a/doc/architecture/SPEC/_CONSTRAINTS.md b/doc/architecture/SPEC/_CONSTRAINTS.md new file mode 100644 index 000000000..e4b381915 --- /dev/null +++ b/doc/architecture/SPEC/_CONSTRAINTS.md @@ -0,0 +1,106 @@ +# 01 —— 编程约束与不变量目录 + +让代码保持干净与安全的那些规则。每一条都表述为**可强制的约束**，并给出钉住它的 +机制/测试。**违反其中任何一条的改动都是错的，哪怕它"能跑"。** + +## A. 结构约束（组织） + +- **A1 —— 一个模块一个职责。** 每份逐文件规范都点名了它唯一的职责。 + 引入不相关的关注点意味着**新建一个模块**，而不是把现有模块撑大。 +- **A2 —— 依赖方向只能向下/向外。** 见 `_ARCHITECTURE.md` §4。 + 叶子包**绝不** import `engine/`；接口层**绝不**被向下 import； + step 模块**绝不互相 import**（共享代码 → `engine/steps/_common.py`）。 +- **A3 —— step 编排工具，planner 编排 step。** planner **不得**编排原始工具； + 一个 step **不得**只是某个工具的薄别名。step = **一次稳定的工程动作**， + 带声明的风险、I/O 和成功判据。 +- **A4 —— 自注册，单一定义。** 一个 step 的名字、元数据（kind/risk/scope/triggers） + 和 handler **就在它的定义处**通过 `@step(...)` 或 `register_step(StepSpec(...))` + 写在一起。**不存在集中式的 `add()` 块。** +- **A5 —— 知识在边缘。** 仓库专属字面量（仓库名、模块名、领域 prompt、绝对路径、 + CI 接线）**只**住在 `adapters//` 之下。由 `test_repo_neutral_core` 钉住 + （逐文件的泄漏上限**只能变小**；shim 和委托文件承载着仅有的已知例外）。 + +## B. 契约约束（类型化、结构化、显式） + +- **B1 —— 类型化失败，绝不裸抛异常。** step 返回 + `StepResult(ok=False, failure=FailureKind.X)`。六种 —— RETRYABLE、REPLAN、 + TEST_FAILURE、BLOCKED、FORBIDDEN、ESCALATE —— 在 executor 里走**不同路由**。 + 未处理的异常被强制转为 BLOCKED，**绝不吞掉**。 +- **B2 —— 状态交接必须被发布，而不只是被修改。** 任何被后续 step 消费的 state 键 + **必须**经 `outputs.state_updates` 发布（JSON-simple；像 `PushPolicy` 这样的 + dataclass 要序列化）。**只做 `ctx.state[...] = v` 会在 resume 时丢失。** + 由 resume 完整性测试钉住。（**这曾是 v2 的头号正确性缺陷。**） +- **B3 —— `when:` 条件只能引用已知键。** 它先读 TaskSpec 字段、再读已发布的 state 键； + 未知键会在执行时**大声阻塞**，**绝不静默地判为 false**。 +- **B4 —— 结构化的 agent I/O。** 每个 `kind == "agent"` 的 step 都经 `run_agent_step`， + 带显式 dispatch context 和 JSON 输出契约（基础 schema + 逐 step 扩展）、一轮修复、 + 类型化的 状态 → 失败 映射。**step 里不得为了做 agentic 工作而临时 + `ctx.llm.create()`。** 由 `test_agent_runtime.py` 里的参数化 dispatch 测试钉住。 + +## C. 安全约束（权限与护栏） + +- **C1 —— tier 是推导的，绝不解析而来。** `TaskSpec` 没有可设置的 tier 字段； + `tier` 是 `kind` 的 property。**自然语言无法扩大权限。** +- **C2 —— 生成路径在结构上只读。** planner 的 generate 路径在任何被编排的 step + `risk ∈ {write_workspace, push, knowledge}` 时抛错； + 没有已审核 playbook 的写能力 kind 一律**升级**。 +- **C3 —— 唯一的工具 choke point。** 每次工具调用都过 `tools.dispatch`， + 由它强制 `ToolScope`/`PathScope` 并记 trace。三种结果：允许 / 拒绝 / + 执行但记录（可写墙内的越界写）。agent step **只**看得见其 scope 允许的工具。 +- **C4 —— 唯一的推送 choke point。** 每次推送都过 `push.guard_push`： + 需要一个允许的 `PushPolicy` **且**分支不受保护；force 只有 with-lease； + 除非 `ALLOW_PUSH=1` 否则是 dry-run。**受保护分支永不被推送，与策略无关。** +- **C5 —— 对外写入是双闸的。** 发评论/回答需要显式的 `post` 意图**并且** + `ALLOW_POST=1`；两者默认都关（dry-run）。 +- **C6 —— 评审 fail-closed。** 没有 reviewer 时 plan/patch review 返回 `unavailable`； + 除 `lgtm`/`pass` 之外一律视为未通过。**缺少 reviewer 不等于静默批准。** +- **C7 —— 不可信数据被围栏。** GitHub/CI 文本进入 agent prompt 时被包在 + `` 里，并带"这不是指令"的前言；**只有终端输入到达意图解析器** + （通道分离）。 + +## D. 知识治理约束 + +- **D1 —— 事实自由记录，知识经门晋升。** RunTrace/debug memory 廉价且仅追加。 + skill、playbook、adapter 和 profile fact 都是 candidate→promote； + **晋升是策展人/人类的动作**。 +- **D2 —— agent 提议，人类裁决（高风险处）。** agent **只能**写 candidate。 + `update_manifest` 拒绝 agent 对 `manifest.yaml` 高风险段 + （`push`/`repo`/`upstream`）的写入。profile judge **只读**。 +- **D3 —— 每条 profile fact 都要有溯源 + 稳定性。** 各自携带 + `source`/`evidence`/`first_seen`/`last_confirmed`/`confirmations`； + **无证据的 fact 被拒**；稳定 fact（≥3 次确认）在重写时**不得丢失已引用的证据**； + 被取代的正文进 `history`，**绝不删除**。 +- **D4 —— 两个写入层。** 逐 run 的 op 是**追加式**的（`RUN_OPS`）； + 只有**有节奏的巩固层**可以 rewrite/merge/mark-stale。 + **禁止连续的 LLM 重写**（它会腐蚀记忆）。 +- **D5 —— profile 内容按通道分类。** `machine` fact 喂代码，有词预算的 `briefing` + 切片进 prompt，`retrieved` fact **只在按需时**浮现。 + **自动生成的概览不得被整体注入。** + +## E. 可观测与降级约束 + +- **E1 —— 每条治理主张都有一个 trace 事件。** 只要某道守卫触发过、某个 scope 拒绝过、 + 某条 fact 被应用过、或某项能力缺失过，就有对应的 RunTrace 事件 + （`tool_refused`、`out_of_scope_edit`、`patch_review`、`capability_gap`、 + `profile_*`…）。 +- **E2 —— 显式降级，绝不静默。** 能力缺失（没有 CI provider、没有 LLM、语言不支持） + 会记一条 `capability_gap` 事件和一次**被声明的降级**； + 它**绝不崩溃**，也**绝不假装能力完整**。 +- **E3 —— metrics 绝不搞坏一次 run。** `metrics.py` 的失败被捕获并记 trace； + **run 的成败与它的 metrics 无关**。 + +## F. 不变量目录（快速索引 → 归属） + +| # | 不变量 | 归属文件 | 测试 | +|---|---|---|---| +| C1 | tier 由 kind 推导 | `task_spec.py` | `test_intent_taskspec.py` | +| C2 | generate 只读 | `engine/planner.py` | `test_planner_playbooks.py`,`test_capabilities.py` | +| B1 | 类型化失败路由 | `engine/executor.py` | `test_engine.py` | +| B2 | resume 时的 state_updates | `engine/executor.py` + 各 step | `test_v2_p0.py` | +| B3 | `when:` 只用已知键 | `engine/executor.py` | `test_v2_p0.py` | +| B4 | 受治理的 agent I/O | `engine/agent_runtime.py` | `test_agent_runtime.py` | +| C3 | 工具 choke point | `tools.py` + `scopes.py` | `test_scopes_tools.py` | +| C4 | 推送 choke point | `push.py` | `test_push_and_steps.py` | +| A5 | 仓库中立的核心 | 整个 `src/` | `test_v2_p0.py::test_repo_neutral_core` | +| D3 | 溯源 + 稳定性 | `profiles/store.py` | `test_profile_store.py` | +| D2 | judge/agent 对生效内容只读 | `profiles/*`,`adapters/base.py` | `test_p3_machinery.py` | diff --git a/doc/architecture/SPEC/__main__.md b/doc/architecture/SPEC/__main__.md new file mode 100644 index 000000000..f58ef75c8 --- /dev/null +++ b/doc/architecture/SPEC/__main__.md @@ -0,0 +1,27 @@ +# __main__.py —— 规范 + + + +`LOC ~12 · 模块入口点 · refactor-status: trivial` + +## 职责 +`python -m infermatrix_copilot` —— MCP server 用来拉起一次已预约 run 的入口。 + +## 不变量（**C3**、**E1**） +- server 以 `sys.executable -m infermatrix_copilot --execute-reserved ` + 拉起子进程，即**用当前解释器**，因此子进程继承 server 被安装进去的那套环境。 +- 子进程必须是**独立进程**而不是线程：它的 stdout 落到该 run 的 `console.log` + （server 自己的 stdout 只走 JSON-RPC），并且进程级全局 tracer / `last_run_dir` + **由此天然按 run 隔离**。 + +## 边界 —— 不属于这里 +**没有逻辑。** 它委托给 `cli.main`，并且必须一直这么薄。 + +## 依赖（允许） +`.cli`。 + +## 测试 +由 `test_mcp.py` 覆盖（它会拉起一个真实子进程）。 + +## 重构备注 +在这里加任何东西，都会**先于 `cli.main` 的那些门**执行。 diff --git a/doc/architecture/SPEC/adapters/base.md b/doc/architecture/SPEC/adapters/base.md new file mode 100644 index 000000000..c7de26b1b --- /dev/null +++ b/doc/architecture/SPEC/adapters/base.md @@ -0,0 +1,50 @@ +# adapters/base.py —— 规范 + + + +`LOC ~368 · 边缘（仓库知识） · refactor-status: ok` + +## 职责 +`RepoAdapter`（住在边缘的仓库知识）、adapter 注册表，以及确定性的 Phase-0 引导。 +`RepoAdapter` 是一个仓库**两个信任层**的**容器**（DESIGN §V2.3.0）：它的 `.manifest` +是 **Tier 1**（`manifest.yaml`，人工门禁的配置），而 +`.profile_dir`/`.skills_dir`/`.debug_memory_db` 是 **Tier 2**（agent 建立、证据门禁的 +知识）。2026-07-11 由 "plugin" 更名而来（`RepoPlugin`→`RepoAdapter`、 +`plugin.yaml`→`manifest.yaml`）—— "plugin" 会错误地暗示这是可执行的扩展代码 +（见 DESIGN 的命名说明）。 + +## 公开契约 +`RepoAdapter`（属性 `status`、`repo_path`、`protected_branches`、`modules`、 +`high_risk_modules`、`capabilities`、`skills_dir`、`debug_memory_db`、 +`profile_dir`、`briefing()`；方法 `module_for_path`）；`load_adapter`、 +`update_manifest`、`AdapterRegistry(resolve/all)`；`fingerprint_repo`、 +`draft_adapter`。 + +## 不变量 +- **D2**：`update_manifest` **拒绝** agent 对 `push`/`repo`/`upstream` 的写入。 +- `capabilities` 由 manifest 推导（repo.path/language.*/ci.provider/upstream.*/ + modules）+ 显式的 `capabilities:` —— 与 playbook 的 `requires:` 匹配。 +- `high_risk_modules` = 标了 `risk: high` 的模块（喂给 patch-review，**A5**）。 +- `fingerprint_repo` 是确定性的，**绝不修改目标仓库**；`draft_adapter` 停在 + `status: draft`（人工门）。 +- `briefing()` 渲染 profile 的 briefing 切片（无 profile 时为空）。 +- manifest 可以声明**知识路由键**（例如 `knowledge.review_checklist`），把一篇精选页 + 注入评审；知识 linter 会校验它们。**这正是仓库知识到达引擎、而不让任何仓库字面量 + 进入 `src/` 的方式。** +- skills / debug memory / profile 的解析顺序是**仓库优先、共享池其次**； + 写入**始终**落在当前仓库的命名空间里。 + +## 边界 —— 不属于这里 +不含任务逻辑、不含 LLM、不含 profile store 内部机制（委托给 `profiles/store`）。 + +## 依赖（允许） +`profiles/store`（在 `briefing()` 内部）、`pyyaml`；stdlib。 + +## 测试 +`test_adapters.py`、`test_capabilities.py`。 + +## 重构备注 +两个关注点在这里共存得很干净：`RepoAdapter`（访问器）+ 注册表 + 引导。 +如果引导继续长大（更丰富的指纹），可考虑 `adapters/bootstrap.py`。 +**把 `capabilities` 推导保持为 planner 信任的唯一来源** —— 不要在 `cli.resolve` 里 +复制一份能力逻辑（它目前只为 REPO_PATHS **补上** repo.path，这是可以接受的）。 diff --git a/doc/architecture/SPEC/agent_loop.md b/doc/architecture/SPEC/agent_loop.md new file mode 100644 index 000000000..de37dbb3a --- /dev/null +++ b/doc/architecture/SPEC/agent_loop.md @@ -0,0 +1,43 @@ +# agent_loop.py —— 规范 + + + +`LOC ~126 · 引擎（工具循环） · refactor-status: ok` + +## 职责 +最小的工具使用循环，受 ToolScope 约束、被 RunTrace 审计。 + +## 功能 +`run_agent` 反复执行：调 LLM → 收集 tool_uses → 逐个过 `tools.dispatch` → +把结果喂回去；预算耗尽时，**强制一次不带工具的最终答复**。 + +## 公开契约 +`run_agent(llm, *, system, prompt, scope, trace?, model?, max_iters, +extra_tools?) -> AgentOutcome`；`AgentOutcome`。 + +## 不变量 +- 循环**只**看得见它的 scope 允许的工具（经 `tool_definitions_for`）。 +- 每次调用都过 `tools.dispatch`（**C3**）。 +- 预算耗尽会**强制一次最终答复**，而不是丢弃这场调查。 +- **"FINAL ROUND" 收尾提示必须追加在 tool_result 块之后。** + 把它放在 `tool_use` 与其 `tool_result` 之间，会违反邻接契约并让整个请求以 400 失败。 + 由 `test_agent_loop.py::test_final_round_nudge_follows_tool_results` 钉住。 +- 这个循环**只**为 `api` 后端运行；harness 后端拥有自己的循环， + 被委托的是一整个 step（`providers/base.md`）。 + +## 边界 —— 不属于这里 +不含规划/step 语义；不做输出契约收敛（那是 `agent_runtime`）； +除透传外不构造 prompt。 + +## 依赖（允许） +`llm`、`run_trace`、`scopes`、`tools`。 + +## 扩展点 +预期没有 —— 它是 `agent_runtime` 赖以构建的原始底座。 + +## 测试 +`test_agent_loop.py`。 + +## 重构备注 +**恰到好处地小。** 保持它与 prompt 无关、与契约无关 —— 所有治理/结构都属于上一层的 +`agent_runtime`。如果 `agent_runtime` 发生拆分，本文件保持原样（它已经就是"循环"层）。 diff --git a/doc/architecture/SPEC/chat.md b/doc/architecture/SPEC/chat.md new file mode 100644 index 000000000..7ea8d456b --- /dev/null +++ b/doc/architecture/SPEC/chat.md @@ -0,0 +1,45 @@ +# chat.py —— 规范 + + + +`LOC ~440 · 接口（对话式 REPL） · refactor-status: split-candidate` + +## 职责 +Claude-Code 风格的对话式 REPL（配置了 LLM 时的默认形态）：一个持续的对话， +既回答问题，也通过工具执行工作。 + +## 功能 +`SYSTEM_PROMPT`（仓库中立，插值 `default_repo`）；`TOOL_DEFS` +（run_task/run_playbook/get_status/get_logs/read_run_report/list_playbooks/ +repo_read/repo_grep/resume_run）；`ChatSession`（**永不切开工具对**的历史裁剪、 +流式输出、工具往返、会话记录）；以及读取囚笼。 + +## 公开契约 +`chat_repl(copilot, assume_yes, handle_builtin)`；`ChatSession.turn`。 + +## 不变量 +- 它是**前端，不是第二条执行路径**：最终汇入 `run_task`/`run_playbook`， + 走**同一套**门；**无法扩大权限**。 +- `repo_read`/`repo_grep` 被囚禁在已配置的仓库根 + run 根内；`.env*` 一律拒绝。 +- 抓取回来的 GitHub 内容**始终是数据**（在 step 内部处理）（**C7**）。 +- 历史裁剪**绝不切开 tool_use/tool_result 对** —— 被切开的一对会让整个请求直接失败。 +- 会话记录落在 `~/.infermatrix-copilot/sessions/`；`--no-chat` 回退到朴素命令 REPL。 + +## 边界 —— 不属于这里 +自身不含规划/执行逻辑；不含仓库知识字面量（prompt 经 `default_repo` 中立化）。 + +## 依赖（允许） +`cli.Copilot`（仅 TYPE_CHECKING）、`run_trace`、`task_spec`、`ui`。 + +## 扩展点 +新的对话工具 → 一条 `TOOL_DEFS` 条目 + 一个 `_dispatch_tool` 分支，**受囚笼约束、 +且尊重各道门**。 + +## 测试 +`test_chat.py`。 + +## 重构备注 +混了三个关注点：工具 schema/注册（`TOOL_DEFS`）、读取囚笼、回合循环。 +**建议拆分**：`chat_tools.py`（定义 + `_dispatch_tool` + 囚笼）与 +`chat.py`（会话/回合循环）。读取囚笼（`_allowed_roots`/`_check_read`）具有通用价值 —— +如果还有别的面需要受囚笼的读取，可考虑提升为一个小的共享 helper。 diff --git a/doc/architecture/SPEC/ci/normalize.md b/doc/architecture/SPEC/ci/normalize.md new file mode 100644 index 000000000..6dba2ca02 --- /dev/null +++ b/doc/architecture/SPEC/ci/normalize.md @@ -0,0 +1,31 @@ +# ci/normalize.py —— 规范 + + + +`LOC ~31 · 边缘（CI） · refactor-status: ok` + +## 职责 +在分组之前，把一条 CI 失败签名归一化。 + +## 公开契约 +`normalize_signature(signature) -> str`。 + +## 不变量 +- 抹去逐 run 变化的噪声（时间戳、哈希、地址、tmp 路径、行号、耗时）； + 保留小的字面数字作为信号。 +- **刻意不继承**父 monitor 的精确字符串比较缺陷 —— 跨 run 的同一个失败必须坍缩为 + 同一组。 + +## 边界 —— 不属于这里 +只做字符串归一化 —— 不分组（那是 `pr.group_failures`），不抓日志（那是 +`ci/providers`）。 + +## 依赖（允许） +仅 stdlib 的 `re`。 + +## 测试 +`test_ci_and_repo_map.py`。 + +## 重构备注 +小而纯 —— 归一化规则本身就是它的全部价值。如果实践中出现误合并，请调这里的正则表 +（**唯一一处**），而不是在调用点打补丁。 diff --git a/doc/architecture/SPEC/ci/providers.md b/doc/architecture/SPEC/ci/providers.md new file mode 100644 index 000000000..46f212d84 --- /dev/null +++ b/doc/architecture/SPEC/ci/providers.md @@ -0,0 +1,37 @@ +# ci/providers.py —— 规范 + + + +`LOC ~118 · 边缘（CI） · refactor-status: ok` + +## 职责 +供 pr-debug 使用的、由 profile 选定的 CI 日志适配器。 + +## 公开契约 +`provider_for(adapter, settings, gh_runner?) -> (provider|None, gap_reason)`； +`BuildkiteLogs.enrich`、`GithubActionsLogs.enrich`。 + +## 不变量 +- provider 由 `profile.ci.provider` 选定，**绝不硬编码**。 +- `enrich` 是**逐 check** 尽力而为的 —— 某次 API 出错只会让那个 check 退回按名字分组， + **绝不崩溃**。 +- 缺 provider/token → `(None, reason)`；调用方 step 记一条 `capability_gap` + 并降级为按名字分组（**E2**）。 + +## 边界 —— 不属于这里 +只抓日志 —— 不含分组/调试逻辑；不做签名归一化。 + +## 依赖（允许） +stdlib 的 `urllib`/`json`/`re`；`gh_runner` 可调用对象是**注入**进来的 +（不直接 import `._common.gh`，从而让这个包与引擎无关）。 + +## 扩展点 +新的 CI 系统 → 一个带 `enrich` 的 `*Logs` 类 + `provider_for` 里的一个分支。 + +## 测试 +`test_ci_and_repo_map.py`。 + +## 重构备注 +干净的适配器模式。保持 `enrich(failures) -> count` 契约稳定，好让 `pr.py` 保持无感。 +`gh_runner` 注入是**刻意的** —— 不要在这里 import `engine.steps._common` +（那会反转依赖方向，**§ARCH.4.1**）。 diff --git a/doc/architecture/SPEC/cli.md b/doc/architecture/SPEC/cli.md new file mode 100644 index 000000000..d129baf8c --- /dev/null +++ b/doc/architecture/SPEC/cli.md @@ -0,0 +1,62 @@ +# cli/ —— 规范 + + + +`LOC ~1000（5 个文件） · 接口 + 编排门面 · refactor-status: ok` + +## 职责 +flag CLI 与 `Copilot` 门面：解析 → 过门 → 执行；并持有 run 目录、RunTrace、notifier +和 metrics 的接线。它曾是一个 406 行的模块；现在是一个把 argparse/REPL 接线与编排器 +分开的包。 + +## 包内布局（一个文件一个关注点） +- `__init__.py` —— 只再导出 `Copilot`、`main`（见下方公开面）；无逻辑。 +- `__main__.py` —— `python -m infermatrix_copilot.cli` 的对等入口。 +- `copilot.py` —— `Copilot` 编排器（resolve/run_task/run_playbook/run_queue/ + resume_last/_execute + 内置命令）。 +- `entry.py` —— `argparse`、`_handle_line`、`main`（把 argv/stdin 变成对 `Copilot` + 的调用）。 +- `utils.py` —— 纯格式化器：`parse_task_params`、`format_metrics_line`。 +- `doctor.py` —— 预检诊断（2026-07 新增）：逐项 ✓/✗，每个失败给出**唯一**确切的修复命令。 + +## 公开契约（可从 `infermatrix_copilot.cli` import） +`main(argv)`；`Copilot`（`resolve`、`run_task`、`run_playbook`、`run_queue`、 +`resume_last`、`status`、`logs`、`playbooks`、`_execute`、`_adapter_for`、 +`_resolve_repo_path`）。再导出的 `__init__` 让 `infermatrix_copilot.cli:main` +（entry point）和 `from infermatrix_copilot.cli import Copilot` 保持不变。 + +## 不变量 +- `resolve` 把能力（adapter + REPO_PATHS）喂给 planner。 +- 确认之前先过 plan-review 门；除非 `--yes`，`confirm_required or requires_review` + 时触发确认（`_gate_and_confirm`，K6）。 +- `_execute` 是**唯一**的执行路径（task / 显式 playbook / resume）。 +- 仓库知识（保护分支、高风险模块）由 adapter 进入 run state（**A5**）； + 被阻塞 → 退出码 3（`BLOCKED_EXIT`）。 +- `--playbook` 是运行 candidate 的**唯一**方式。 +- **`doctor` 只读，且永不打印密钥的值** —— 只打印它的名字。除非传 `--probe`， + 否则它不做任何付费 LLM 调用；`--probe` 是唯一的付费检查（每个已配置档位一个 token）。 + `--json` 供 CI 使用，而**在没有凭据时以非零码退出正是 CI 里的预期状态**。 +- **`--performance` 是抬高模型档位的唯一方式**；默认是 `eco`。 + **抬高档位永远不会扩大权限**（`tier` 仍然由 `kind` 推导）。 +- CLI 主路径在**创建 run 目录之前**过门，所以被放弃的计划不留下任何东西。 + MCP 的预约形状（先建、后规划）是**刻意不同**的 —— 见 `mcp_server.md`。 + +## 边界 —— 不属于这里 +不含 step 逻辑、不含仓库知识字面量、不含 LLM prompt。**只做编排接线。** + +## 依赖（允许） +`engine/*`、`playbooks/*`、`intent`、`task_spec`、`adapters/base`、`push`、 +`review/reviewer`、`notify`、`run_trace`、`config`、`ui`、`chat`。 +**任何下层都不得 import 它**（**§ARCH.4.2**）。 + +## 扩展点 +新 REPL 命令 → `_handle_line`（entry.py）；新的 run 接线 → `_execute`（copilot.py）； +新的纯格式化器 → utils.py。 + +## 测试 +`test_cli.py`、`test_phase_b.py`、`test_chat.py`、`test_ui.py`。 + +## 重构备注 +拆分**已完成**（它曾是内聚拆分候选）。`Copilot` 类完整留在 `copilot.py`， +好让 resolve→execute 的流程能在一个文件里读完；只有 argparse/REPL 前端（`entry.py`） +和两个纯格式化器（`utils.py`）搬了出去。K6（`_gate_and_confirm`）已完成，挂在类上。 diff --git a/doc/architecture/SPEC/config.md b/doc/architecture/SPEC/config.md new file mode 100644 index 000000000..eee2916a7 --- /dev/null +++ b/doc/architecture/SPEC/config.md @@ -0,0 +1,57 @@ +# config.py —— 规范 + + + +`LOC ~418 · 配置 · refactor-status: oversized` + +## 职责 +从 env / `.env` 加载的 `Settings`（pydantic-settings），以及把本次 run 的档位与后端 +选择变成具体目标的那些推导 helper。 + +## 功能 +为 LLM 端点与逐档模型、仓库、引擎预算、推送安全、PR debug、外部 rebase、agent 运行时、 +ensemble、MoA、评审深度与按 pass 路由、Strict 后端选择、profile、patch 触发器、 +metrics 与升级，提供带类型字段和安全默认值。 + +## 公开契约 +带全部可调项的 `Settings`；`reviewer` / `intent`（回退到 `agent_model`）； +`repo_path(name)`；`model_for(mode)`；`tier_target(role)` → `ResolvedTarget`； +以及 `strict_backend` 校验器。 + +## 不变量（**A5**、**C2**、**B1**） +- 密钥只经 env / `.env`（被 git 忽略，**绝不提交**）。 +- 仓库专属默认值（`default_repo`、`rebase_agent_root`、`high_risk_modules`、 + `cost_ref_*`）**只是兜底**；adapter/profile 会覆盖它们。它们是这里**唯一被允许**的 + 仓库字面量，泄漏上限为 3，并由 `test_v2_p0.py::test_repo_neutral_core` 钉住。 +- **`STRICT_BACKEND` 在前面就被校验**，失败时列出合法集合。未知后端**绝不能**到达 + `providers.resolve_provider` —— 两层，因为 `.env` 里的一个拼写错误不该启动一次注定 + 失败的 run。 +- **`tier_target` 对 harness provider 返回空 key。** harness 的认证住在厂商 CLI 自己的 + 状态里，本代码库从不接触。**唯一例外是 `deepseek`**：它是 API-keyed 的，凭据在它自己的 + transport 内部解析 —— 见 `providers/deepseek.md`。 +- **fail-closed 的 served-model 守卫**：实际服务的模型与请求不符是**错误**， + 而不是静默替换。 +- **推理模型需要 token 留白。** 灰区评审 planner 曾在两个战役里对每一个灰区条目静默 + 失败，因为思考在任何 JSON 出现之前就吃完了 400-token 上限 —— 对推理模型调用的上限 + 是**正确性设置，不是成本旋钮**。 + +## 边界 —— 不属于这里 +除推导 helper 外无逻辑；除加载 env 外无 I/O；不含 provider transport（`providers/`）； +不调模型。 + +## 依赖（允许） +仅 `pydantic-settings`。 + +## 扩展点 +新可调项 → 一个带安全默认值的类型字段，并用一行注释写明含义与单位。 +新后端 → 注册表条目 + 本文件的校验器集合，**绝不是这里的一条新分支**。 + +## 测试 +`test_llm_providers.py`、`test_tier_split.py`、`test_providers.py`； +全套测试间接覆盖（fixture 会构造 `Settings(_env_file=None, ...)`）。 + +## 重构备注 +在 v3 后端工作中从约 116 行长到约 418 行，现在**确实偏大**，但仍是单一关注点。 +如果再增长，请分组成嵌套 settings 模型（`LLMSettings` / `PushSettings` / +`ReviewSettings` / `BackendSettings`），**而不是拆文件** —— 调用点依赖"一个 `Settings` +对象到达每个 `StepContext`"。 diff --git a/doc/architecture/SPEC/engine/agent_runtime.md b/doc/architecture/SPEC/engine/agent_runtime.md new file mode 100644 index 000000000..c8da7751a --- /dev/null +++ b/doc/architecture/SPEC/engine/agent_runtime.md @@ -0,0 +1,89 @@ +# engine/agent_runtime/ —— 规范 + + + +`LOC ~1100（7 个文件） · 引擎（受治理的 agent 运行时） · refactor-status: ok` + +## 职责 +每个 `kind == "agent"` step 的**唯一**受治理入口，外加评审质量 ensemble。 +它是全库信息密度最高、杠杆最大的子系统 —— 曾经是一个 685 行的模块， +现在是一个把底座（dispatch/knowledge/utils）与两个入口（runner/ensemble）分开的包。 + +## 包内布局（一个文件一个关注点） +- `__init__.py` —— 只做公开再导出（见下方公开面）；无逻辑。 +- `dispatch.py` —— agent 的**输入契约**：`AgentDispatchContext`（及其 `render()`） + 和 `BASE_OUTPUT_SCHEMA`。prompt 形状，与控制流隔离。 +- `knowledge.py` —— 按仓库限定的知识：`_resolve_adapter`、`_ScopedKnowledge`、 + `_knowledge_stores`、`_retrieve_skills`、`_retrieve_memories`、 + `_knowledge_tools`、`_repo_map_tool`。 +- `utils.py` —— 无状态 helper：`_build_evidence`、`_permissions_view`、 + `_coerce_output`、`_to_step_result`（+ 状态→FailureKind 映射）。 +- `runner.py` —— `run_agent_step`：组装 dispatch context、打包证据、检索知识、 + 跑工具循环、收敛输出、全程记 trace。 +- `ensemble.py` —— `run_agent_step_ensemble`：视角多样的 lens 扇出 + verify-and-merge + 归约。 +- `moa.py` —— Mixture-of-Agents（2026-07 新增）：lens/draft 提案可以跑在异构的 + `LLM_MIXTURE` 成员上，而 verify-and-merge reducer 仍留在本次 run 的档位模型上。 + +## 公开契约（可从 `engine.agent_runtime` import） +`run_agent_step(...) -> (StepResult, output)`；`run_agent_step_ensemble(...)`； +`AgentDispatchContext`；`BASE_OUTPUT_SCHEMA`；`_resolve_adapter`、 +`_retrieve_skills`（供 step/测试使用）。再导出的 `__init__` 让拆分前的 import 路径 +（`from ..agent_runtime import X`）保持不变。 + +## 不变量（**B4**） +- **唯一入口**：agent step 只能经 `run_agent_step` 做 agentic 工作 —— + **不允许**为了调查而临时 `ctx.llm.create()`。 +- 证据逐项封顶 + 归档 + `` 围栏（**C7**）。 +- `_ScopedKnowledge`：仓库 skill+memory 优先于共享池；提案落在仓库命名空间， + **且只能是 candidate**（**D1/D2**）。 +- `_repo_map_tool`：按需拉取，**绝不作为散文注入**；语言不支持 → `capability_gap`。 +- briefing 只在 `profile_briefing_enabled` 时进 prompt（消融开关）。 +- 输出：base+extension schema、一轮修复、状态→FailureKind；预算耗尽会**强制最终答复**。 + 全程 trace（`agent_dispatch`/`agent_output`）。 +- ensemble reducer：逐编号候选做 keep/drop/dup、确定性组装、**未提及即保留** + （fail-open）、共识门控的快路径。 +- **零产出的 lens 单独重问一次**（`ensemble_zero_yield_retry`）， + 而不是整组重跑；lens 启动**错峰**（`ensemble_stagger_seconds`）。 +- **非空的最终答复会被抢救，绝不丢弃**：修复轮之后仍无法解析的输出， + 变成 `needs_review` + `_raw_text`。 +- **MoA 预算在代码里强制，不在 prompt 里**（`moa.py`）：每个成员请求先原子 + `reserve()` 一个保守上界，结算时替换为实际用量，**已结算花费永不超过 `moa_max_usd`** + —— 预约失败就降级到档位模型或跳过该成员，因此**不可能出现不封顶的请求**。 + 成员身份是 `model@host`；**密钥只经 env 变量名引用，永不进入日志或 trace**。 +- **按 pass 的后端路由**（`review_lens_backends`）可以把某个 seat 放到另一个 provider 上。 + 路由必须**在重试中保持**，无法履约时必须**大声回退** —— + 一个被静默改道的 seat，会让这条 arm **不再是它标签所声称的配置**。 +- MoA 成员本身也可以骑上 provider 注册表里的某个 harness（`transport_for_id`）， + 与本次 run 自己的 `STRICT_BACKEND` 无关。 + +## 内部依赖规则 +`runner`/`ensemble` 依赖 `dispatch`/`knowledge`/`utils`，**反向绝不允许**； +`ensemble` 依赖 `runner`（反之不成立）。底座文件不 import 任何兄弟入口，因此**无环**。 + +## 边界 —— 不属于这里 +不含任务/规划逻辑；不含仓库字面量（system prompt 仓库中立）。 +step 专属的 prompt/lens 住在各自的 step 文件里（例如 `steps/review/prompts.py`）。 + +## 依赖（允许） +`agent_loop`、`llm`、`memory/*`、`adapters/base`、`profiles/repo_map`、`scopes`、 +`tools`、`engine/step`。 + +## 扩展点 +新的"列表输出型" agent step 通过传入 lenses + merge_guidance 来采用 ensemble； +新的知识来源 → 在 `knowledge.py` 里扩展 `_ScopedKnowledge`/`_knowledge_tools`。 + +## 测试 +`test_agent_runtime.py`（参数化 dispatch —— `kind=="agent"` ⇒ 受治理运行时； +无法解析输出的抢救）、`test_agent_ensemble.py`（裁决校准、零产出重问、lens 后端路由）、 +`test_moa.py`（预算台账封顶；密钥永不进 trace）、`test_review_step.py`。 + +## 重构备注 +拆分**已完成**（它曾是内聚性的头号目标）。那些内联的评测引用注释是**机构记忆** —— +它们**跟着各自的代码**搬进了兄弟文件，一条都没丢。`ensemble.py`（约 387 行）仍是最大的 +文件；它的 reducer 是一个内聚的单一算法，**不是**进一步拆分的目标。 + +这个包吸收了 v3 期的大部分变化（自 2026-07-12 起 10 个提交：深度评审 pass、 +cheap-seat 与按 pass 路由、MoA harness 成员、考古工具）。有两条教训被编码在上面的不变量 +里而不是写成散文：**预算保证属于代码，不属于 prompt**；以及**无法履约的路由决策必须 +大声失败** —— 因为本轮战役已经测到过三条与其标签不符的 arm。 diff --git a/doc/architecture/SPEC/engine/executor.md b/doc/architecture/SPEC/engine/executor.md new file mode 100644 index 000000000..f6783d630 --- /dev/null +++ b/doc/architecture/SPEC/engine/executor.md @@ -0,0 +1,51 @@ +# engine/executor.py —— 规范 + + + +`LOC ~267 · 引擎底座（那个循环） · refactor-status: ok` + +## 职责 +以与任务无关的保证运行一个 playbook 的各个 step：检查点/resume、`foreach` 扇出、 +`when:` 条件、有界重试、类型化失败路由、升级。 + +## 功能 +`run` 逐个迭代 step；跳过已完成的（并**恢复它们的 `state_updates`**）；对 `foreach` +扇出；求值 `when:`；对 RETRYABLE 重试；把 BLOCKED/ESCALATE/FORBIDDEN 路由给 notifier； +持久化进度并按 step id 索引输出。 + +## 公开契约 +`Executor(registry, settings, run_dir, trace, llm?, notifier?)`； +`run(playbook, state) -> RunOutcome(status, step_results, blocked_reason)`。 +helper：`_eval_when`、`_merge`。 + +## 不变量 +- **B2**：resume 在跳过之前先恢复 `outputs.state_updates`；成功后 + `state.update(state_updates)` 并按 step id 索引输出。 +- `_merge` 把每个 foreach 条目的 `state_updates` **提升**到顶层（同键最后写者胜）。 +- **B3**：`when:` 先读 TaskSpec 再读 state；**未知键 → 阻塞，绝不静默**。 +- **B1**：类型化路由；未处理的异常 → BLOCKED（**绝不吞掉**）。 +- 只对 RETRYABLE 重试，受 `max_step_retries` 限制。 +- **run 级 task params 会到达每个 step**：`state["task_spec"]["params"]` 被合并在每个 + step 自己的 params **之下**，所以 `--task-param limit=5` 不会被静默丢弃。 + **playbook 自己的 params 在合并中获胜** —— 其中好几个是安全攸关的 + （`force_push`、`pre_push`），因此被撰写下来的不变量**不得**被命令行覆盖。 +- step 的 span 会记录 playbook 的 step id 和 foreach 条目键：同一个 spec 可能在一个 + playbook 里跑两次、还会再扇出，所以**光靠 `step` 无法标识这份工作**。 + +## 边界 —— 不属于这里 +不含 step 逻辑、不含仓库知识、不做规划、不含 LLM prompt。**只提供执行保证。** + +## 依赖（允许） +`engine/{step,registry}`；仅类型引用：config/llm/notifier/playbook/run_trace。 + +## 扩展点 +新的**跨 step** 保证（例如逐 step 超时）放这里；新的 step **行为**不放这里。 + +## 测试 +`test_engine.py`、`test_v2_p0.py`（resume/foreach/when 的完整性）。 + +## 重构备注 +约 200 行，职责划得很好。`state_updates` 的发布/恢复/合并逻辑**微妙且承重** +（它曾是 v2 的头号缺陷）—— **不要在没有重跑 resume 完整性测试的情况下"简化"它**。 +如果将来加入 DAG 边（目前只有有序列表），请把它们留在这里、藏在同一个 `RunOutcome` +契约之后。 diff --git a/doc/architecture/SPEC/engine/planner.md b/doc/architecture/SPEC/engine/planner.md new file mode 100644 index 000000000..274dd359f --- /dev/null +++ b/doc/architecture/SPEC/engine/planner.md @@ -0,0 +1,45 @@ +# engine/planner.py —— 规范 + + + +`LOC ~116 · 规划 · refactor-status: ok` + +## 职责 +把一个 `TaskSpec`（+ 仓库能力）解析成可运行的 `Playbook`，走 +**reuse > adapt > generate**，并处理能力缺口。 + +## 功能 +查找 playbook（`store.find`）；返回 reuse（原样）/ adapt（在非 locked 上追加参数 → +需评审）/ generate（仅只读 kind，固定模板）；遇到 locked 改编、具备写能力的 generate、 +或能力缺口时抛 `PlanningError`。 + +## 公开契约 +`Planner(store, registry)`；`resolve(spec, capabilities?) -> Resolution`； +`PlanningError`；`_GENERATE_TEMPLATES`。 + +## 不变量 +- reuse：当参数 ⊆ 已声明面时 `requires_review=False`。 +- **locked playbook 拒绝改编**（抛错）。 +- **C2**：generate **只对只读 kind 存在**，并会重新检查每个 step 的 + `risk ∈ {read, report}`（否则抛错）。 +- 能力缺口（**§ARCH.8**）：具备写能力 + 未满足 → 抛出 "run repo_profile"； + 只读 → 走 generate；`capabilities=None` 即 v1 行为。 +- tier 来自 `spec.tier` —— **绝不自己发明**。 + +## 边界 —— 不属于这里 +只做选择/参数化。不执行、不含仓库知识、不含 LLM、不做原始工具编排（**A3**）。 + +## 依赖（允许） +`playbooks/store`、`task_spec`、`engine/registry`。 + +## 扩展点 +需要生成的新只读 kind → 在 `_GENERATE_TEMPLATES` 加一条由 read/report step 组成的 +条目。**具备写能力的 kind 必须自带一份已审核的 playbook。** + +## 测试 +`test_planner_playbooks.py`、`test_capabilities.py`。 + +## 重构备注 +小而干净；**三条分支就是它的全部意义 —— 不要把它们合并**。能力缺口的提示文本是面向 +用户的操作指引（"run repo_profile"）—— 保持它可执行。如果将来 generate 需要 LLM 编排， +它**仍然必须**通过逐 step 的 risk 复检；绝不为了灵活性绕过 C2。 diff --git a/doc/architecture/SPEC/engine/registry.md b/doc/architecture/SPEC/engine/registry.md new file mode 100644 index 000000000..4323f9cd7 --- /dev/null +++ b/doc/architecture/SPEC/engine/registry.md @@ -0,0 +1,35 @@ +# engine/registry.py —— 规范 + + + +`LOC ~31 · 引擎底座 · refactor-status: ok` + +## 职责 +`StepRegistry`：名字 → `StepSpec` 的映射 —— **名字字符串解析到 handler 的唯一地点**。 + +## 功能 +`register` / `get` / `__contains__` / `names`。 + +## 公开契约 +`StepRegistry`，带上述方法。 + +## 不变量 +- 重复注册直接抛错；对未知名字调用 `get` 会连同**已注册集合**一起抛出 + （**大声失败，绝不静默**）。 + +## 边界 —— 不属于这里 +只做存储/查找 —— 不执行、不含策略、不自我填充（填充由 +`steps.register_builtin_steps` 负责）。 + +## 依赖（允许） +`engine/step`。 + +## 扩展点 +不需要；它就是一个容器。 + +## 测试 +多数测试通过 `register_builtin_steps` 间接覆盖。 + +## 重构备注 +极简且正确。**不要**在这里加过滤/策略 —— 那属于 planner（负责选择）和 store +（负责校验引用）。 diff --git a/doc/architecture/SPEC/engine/step.md b/doc/architecture/SPEC/engine/step.md new file mode 100644 index 000000000..9f317197d --- /dev/null +++ b/doc/architecture/SPEC/engine/step.md @@ -0,0 +1,45 @@ +# engine/step.py —— 规范 + + + +`LOC ~67 · 引擎基础词汇 · refactor-status: ok` + +## 职责 +整个引擎共用的基础执行词汇。 + +## 功能 +定义失败分类法、step 的 result/context/spec 数据类，以及 `Kind`/`Risk` 字面量集合。 + +## 公开契约 +`FailureKind`（RETRYABLE、REPLAN、TEST_FAILURE、BLOCKED、FORBIDDEN、ESCALATE）； +`StepResult(ok, failure?, summary, outputs, changed_files)`； +`StepContext(settings, state, params, run_dir, trace, llm?, item?)`； +`StepSpec(name, kind, risk, handler, description)`。`Kind ∈ {deterministic, +script, agent, validation, report}`；`Risk ∈ {read, write_workspace, push, +knowledge, report}`。 + +## 不变量 +- 与仓库、任务无关；`StepSpec` 是 frozen 的。 +- 六种 `FailureKind` 就是完整的路由词汇（**B1**）。 +- `risk` 是被强制的（planner C2）；`kind` 是描述性的（按约定 `agent` ⇒ 受治理运行时）。 + **不要装饰性字段**：`StepSpec` 的每个字段都必须被什么东西读取 + （`tool_scope`/`patch_review_triggers` 两个字段已被移除，因为没有任何东西强制它们 + —— scope 和触发器都是 handler 本地的）。 + +## 边界 —— 不属于这里 +只有类型 —— 无行为、无 registry、不执行。 + +## 依赖（允许） +只有 `run_trace`（**§ARCH.4.3**）。不含任何任务/仓库专属内容。 + +## 扩展点 +新增一个 `Kind`/`Risk` 取值或 `FailureKind` 是**刻意的词汇变更** —— 必须与 +executor 的路由和 `_CONSTRAINTS` 一起更新。 + +## 测试 +到处都在用；形状被隐式钉住。 + +## 重构备注 +地基性文件 —— 把依赖集合**精确**保持在 {run_trace}。在这里多加一个 import，就会把 +整个引擎耦合到它上面。如果 `StepContext` 长出更多可选字段，那是**某个 step 越权**的 +信号，而不是这个文件需要重构的信号。 diff --git a/doc/architecture/SPEC/engine/steps/__init__.md b/doc/architecture/SPEC/engine/steps/__init__.md new file mode 100644 index 000000000..3af4e14b9 --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/__init__.md @@ -0,0 +1,38 @@ +# engine/steps/__init__.py —— 规范 + + + +`LOC ~31 · step 库聚合 · refactor-status: ok` + +## 职责 +为了**注册副作用**而 import 每个 step 模块，并暴露 `register_builtin_steps`。 + +## 功能 +import `_common` + 全部 8 个领域模块（从而运行它们的 `@step`/`register_step` 装饰器， +填充 `_common._COLLECTED`）；`register_builtin_steps` 把这份集合冲刷进一个 +`StepRegistry`。 + +## 公开契约 +`register_builtin_steps(registry) -> registry`。 + +## 不变量 +- import 这个包会把所有 step **恰好注册一次**（靠模块缓存）；对每个全新 registry， + `register_builtin_steps` 是幂等的（**A4**）。 +- **这份 import 列表就是 step 模块的权威集合** —— 新的 step 模块必须加到这里才会被发现。 + +## 边界 —— 不属于这里 +不含 step 逻辑；除冲刷之外不含注册策略。 + +## 依赖（允许） +`engine/registry`、`._common`，以及那 8 个 step 模块。 + +## 扩展点 +新的 step 领域模块 → 加进这份副作用 import 列表。 + +## 测试 +每次 `register_builtin_steps(StepRegistry())` 调用都会覆盖（冒烟：38 个 step）。 + +## 重构备注 +可以考虑自动发现（遍历这个包）以去掉手工 import 列表 —— 但显式 import **可 grep**， +且拼错时会大声失败，所以当前形式是可以接受的。若保持手工，这份列表 + `_COLLECTED` +就是"到底存在哪些 step"的唯一来源。 diff --git a/doc/architecture/SPEC/engine/steps/_common.md b/doc/architecture/SPEC/engine/steps/_common.md new file mode 100644 index 000000000..3b55073c6 --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/_common.md @@ -0,0 +1,53 @@ +# engine/steps/_common.py —— 规范 + + + +`LOC ~141 · step 库基础设施 · refactor-status: ok` + +## 职责 +step 的自注册面，以及各 step 文件共享的跨模块 helper。 + +## 功能 +`@step`/`register_step` 装饰器 + `_COLLECTED` + `collected()`； +helper：`repo_path`、`task_spec`、`gh`、`git`、`gh_read_tools`、`post_step`。 + +## 公开契约 +`step(name, kind, risk, description)`；`register_step(StepSpec)`；`collected()`； +以及上述 helper（含 K3/K4/K7 的守卫 helper）。 + +## 不变量 +- step 名重复 → **在 import 时抛错**（**A4**）。 +- 这是 step 共享 helper 的**唯一**归处 —— step 模块从这里 import， + **绝不互相 import**（**A2**）。 +- helper 保持轻薄、对副作用诚实、且仓库中立。 + +## 边界 —— 不属于这里 +不含 step handler；不含领域逻辑。**只有基础设施 + 共享 IO。** + +## 依赖（允许） +`engine/step`；`...tools`（在 `gh_read_tools` 内部）；stdlib。 + +## 扩展点 +**确实**被 ≥2 个 step 文件共享的 helper → 加到这里。只被一个 step 用的 helper +属于那个 step 自己的文件。 + +## 测试 +经各 step 测试覆盖；`post_step`/`gh`/`git` 在 +`test_ci_and_repo_map.py` 等处被 patch。 + +## 重构备注 +**当心职责蔓延** —— 如果这里堆积了很多只有单一使用者的 helper，把它们推回各自的 step。 +真正跨切的是 `gh`/`git`/`repo_path`；请把 monkeypatch 接缝 +（测试 patch 的是 `steps.pr._gh`，即 import 别名）记录清楚， +免得将来一次改名**静默地**弄坏 patch。 + +## 精简 —— **K3/K4/K7** helper 的归处 +step 样板的收敛就落在这个文件。要加入： +- `require_repo(ctx) -> Path | StepResult`（K3 —— 8 处仓库守卫）。 +- `adapter_or_result(ctx)` / `@needs_adapter`（K3 —— 7 处 profile 守卫）、 + `no_llm_gap(ctx, step, effect)`（K3 —— 4 处）、`store_for(adapter)`（K3 —— 6 处）。 +- `published(summary, *, state=None, **outputs)`（K4 —— 21 处 `state_updates` + 字面量收敛为一次调用）。 +- `from_state(ctx, key)`（K7 —— 5 处 fetch 早返回）。 +每一个都必须**保住它所包裹的那条保证**（B1 类型化返回、B2 交接、E2 `capability_gap` +事件）。**只在 ≥2 个真实现场时才抽取**（上述都满足）。 diff --git a/doc/architecture/SPEC/engine/steps/issue.md b/doc/architecture/SPEC/engine/steps/issue.md new file mode 100644 index 000000000..297b17b16 --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/issue.md @@ -0,0 +1,41 @@ +# engine/steps/issue.py —— 规范 + + + +`LOC ~338 · step 库（issue） · refactor-status: ok` + +## 职责 +issue 抓取、起草回答与分流两个 agent step，以及受门禁的发布。 + +## Steps（4 个） +`issue.fetch`（deterministic/read）；`agent.draft_issue_answer`、 +`agent.triage_issues`（agent/read，只读 scope）；`issue.post_answer`（script/push）。 + +## 不变量 +- agent step 通过受治理运行时，把主张**锚定在抓取到的文本/代码上**。 +- `issue.post_answer` 是**双闸**的（**C5**）。 +- `_issue_agent_step` 是共享工厂；两个 `agent.*` handler 用命令式方式注册 + （`register_step`）。 +- 起草契约带一个 **disposition 槽**（close / keep-open / duplicate-of-#N / reopen） + —— 一份没有说明"这个 issue 接下来该**怎么处理**"的分流回答是**不完整的**。 +- **未经 gh 验证的"已合并"声明必须带认识论 caveat**，并且即使该 step 升级， + 实质性草稿也会**带着那条 caveat 交付** —— 因为把握不足就扣着真实工作不给， + 对谁都没有帮助。 +- `max_agent_iters` 给 grep 密集的分流留出余量。 + +## 边界 —— 不属于这里 +不含 agent 治理内部机制；除 `post_step` 之外不含发布授权。 + +## 依赖（允许） +`scopes`、`engine/step`、`._common`、`..agent_runtime`。 + +## 测试 +经 issue playbook + `test_agent_runtime` 相关路径覆盖。 + +## 重构备注 +体量合适。`_issue_agent_step` 工厂 + `_render_answer`/`_render_triage` 的模式很干净； +如果出现第三个 issue agent step，**继续用那个工厂**。 + +## 精简 —— **K4/K7** +`issue.fetch` 用到了"从 state 取"的早返回（K7 `from_state`）和冗长的 `state_updates` +字面量（K4 `published(...)`）。收益不大；**务必保住 B2**。 diff --git a/doc/architecture/SPEC/engine/steps/pr.md b/doc/architecture/SPEC/engine/steps/pr.md new file mode 100644 index 000000000..8e551e3ea --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/pr.md @@ -0,0 +1,67 @@ +# engine/steps/pr/ —— 规范 + + + +`LOC ~1100（6 个文件） · step 库（PR） · refactor-status: ok` + +## 职责 +受守卫的推送、只读的 PR 抓取/门禁、PR rebase、PR debug、受门禁的评审发布。 +它曾是一个 484 行的模块；现在是一个按关注点拆开的包。 + +## 包内布局（一个文件一个关注点） +- `__init__.py` —— 对各子模块做副作用 import（`@step`/`register_step` 注册）； + 再导出 `extract_signature`。无逻辑。 +- `fetch.py` —— 只读抓取：`pr.fetch_diff`、`pr.gate_check`。 +- `rebase.py` —— `pr.checkout_branch`、`pr.rebase_onto_base`、`pr.analyze_diff`、 + `agent.verify_module`。 +- `debug.py` —— `pr.fetch_ci_failures`（+ `_enrich_ci_logs`）、`pr.group_failures`、 + `agent.debug_group`。 +- `publish.py` —— 对外写入（risk=push）：`ci.push`、`pr.post_review`。 +- `utils.py` —— 纯函数 `extract_signature`（及其正则）。 + +## Steps（11 个） +`ci.push`（script/push）；`pr.fetch_diff`、`pr.gate_check`、`pr.checkout_branch`、 +`pr.analyze_diff`、`pr.fetch_ci_failures`、`pr.group_failures`（deterministic/read）； +`pr.rebase_onto_base`、`agent.debug_group`（agent/write_workspace）； +`agent.verify_module`（validation/read）；`pr.post_review`（script/push）。 + +## 公开契约（可从 `engine.steps.pr` import） +`extract_signature`（由 `utils` 再导出；供测试使用）。 + +## 不变量 +- `ci.push` 把全部安全判断委托给 `guard_push`（**C4**）。 +- **`pr.fetch_diff` 把评审树 pin 到 PR head**（`_pr_time_checkout`）：head sha 取自 + 最后一个 commit 的 oid，`git fetch origin pull/N/head`（对**开放和已合并**的 PR 都 + 有效），detached worktree 复用于 `~/.infermatrix-copilot/worktrees/`。 + 它经 `state_updates` 发布 `repo_path` / `checkout_note`，于是**每个 lens 调查的都是 + PR 那一刻的代码**，而不是本地 checkout 恰好停在哪个分支。失败时降级回 live checkout， + 并带**响亮**的注记加一条 `capability_gap` trace。 +- **`pr.post_review` 只发一条 GitHub review + inline thread**，且先把每条发现的位置 + 对照已抓取的 diff 校验过 —— **绝不是一串独立评论**。 +- `diff_text` 和 `gate_report` 都可以经 state 注入，因此网络之下的每条路径都可离线测试； + `gh` 不可用时降级为 BLOCKED，**绝不崩溃**。 +- `pr.checkout_branch` 把推导出的 `PushPolicy` **序列化后发布**（**B2**）—— + 在推送处 resume 时**绝不能**看到那个 deny-all 的默认值。 +- `pr.rebase_onto_base`：由受治理 agent 解冲突，或 abort+升级（**工作区始终被还原**）。 +- `pr.fetch_ci_failures` 经 profile 选定的 CI provider 富化日志，否则记一条 + `capability_gap`（**E2**）；`pr.group_failures` 按**归一化后**的签名分组。 +- `pr.post_review` 是**双闸**的（**C5**）。 + +## 边界 —— 不属于这里 +不含推送授权逻辑（那是 `push`）；不含 CI 日志抓取机制（那是 `ci/providers`）； +不含 agent 治理（那是 `agent_runtime`）。 + +## 依赖（允许） +`scopes`、`push`、`ci/*`、`adapters/base`（analyze）、`engine/step`、`.._common`、 +`..agent_runtime`。 + +## 测试 +`test_pr_steps.py`、`test_push_and_steps.py`、`test_ci_and_repo_map.py` +（注意：`test_ci_and_repo_map` monkeypatch 的是 `pr.debug._gh`， +即 `pr.fetch_ci_failures` 绑定 `gh` 的那个子模块）。 + +## 重构备注 +拆分**已完成**。各子模块只共享 `._common` 的 helper，所以拆分**没有制造交叉 import**。 +读/写轴是显式的：`fetch` 只读，`publish` 装着两个 risk=push 的 step。 +K3/K4/K7 的精简（require_repo/published/from_state）已在各子模块落地。 +`_enrich_ci_logs` 保持为通往 `ci/providers` 的**薄接缝**。 diff --git a/doc/architecture/SPEC/engine/steps/profile.md b/doc/architecture/SPEC/engine/steps/profile.md new file mode 100644 index 000000000..b9b55c780 --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/profile.md @@ -0,0 +1,52 @@ +# engine/steps/profile.py —— 规范 + + + +`LOC ~387 · step 库（profile 流水线） · refactor-status: split-candidate` + +## 职责 +仓库 profile 的建立（Stage 0–1.5）+ Stage-4 维护 step。 + +## Steps（8 个） +建立期：`profile.fingerprint`、`profile.structure_scan`、`profile.ingest_docs` +（deterministic/knowledge）、`agent.profile_repo`（agent/knowledge）。 +Stage-4：`profile.detect_drift`（deterministic/read）、`profile.decay_stale` +（deterministic/knowledge）、`agent.profile_consolidate`（agent/knowledge）、 +`profile.judge`（agent/read）。 + +## 不变量 +- 对未知仓库，`fingerprint` 起草一个 `status: draft` 的 adapter（人工门）； + `structure_scan` **绝不覆盖**已声明的模块。 +- `agent.profile_repo` 产出的**无证据 fact 会被 store 拒绝**；经冗余过滤； + **禁止概览式内容**。 +- `agent.profile_consolidate` 是**唯一**的 rewrite/merge 层；LLM 的 op 必须通过 + store 的稳定性门（**D3/D4**）。 +- `profile.judge` **绝不调用 `apply_ops`**（只读，**D2**）。 +- 无 LLM 的路径会记一条 `capability_gap`，并只运行确定性阶段。 + +## 边界 —— 不属于这里 +不含 store 内部机制（那是 `profiles/store`）；不含建立期 helper +（那些在 `profiles/establish`/`consolidate`）。 + +## 依赖（允许） +`adapters/base`、`profiles/*`、`engine/step`、`._common`、`..agent_runtime`。 + +## 测试 +`test_profile_steps.py`、`test_p3_machinery.py`。 + +## 重构备注 +一个文件里装了**两条生命周期**。**建议拆分**：`steps/profile_establish.py` +（fingerprint/structure_scan/ingest_docs/profile_repo）与 +`steps/profile_maintain.py`（detect_drift/decay_stale/consolidate/judge）。 +它们只共享 `_adapter_from_state`（→ 挪进 `._common` 或一个小的 `profiles` helper） +和 store —— 没有交叉 import。指导性 prompt 常量可以按 `steps/*_prompts.py` 的约定 +放到评审 prompt 旁边。 + +## 精简 —— **K3**（这里是最大的一处样板收敛） +本文件的守卫重复最密集：**7** 处 `_adapter_from_state` + +`isinstance(..., StepResult)` 守卫、**6** 处 `ProfileStore(adapter.profile_dir)` +构造、**3** 处无 LLM 的 `capability_gap` 块。改用 `._common` 的 +`adapter_or_result`/`@needs_adapter`、`store_for(adapter)`、`no_llm_gap(...)`。 +**仅此一个文件预计可省约 25–35 行。** 必须保留：每个 step 的名字/行为、store 的那些门 +（D3/D4）、以及 `capability_gap` 事件（E2）。 +**不要让 helper 掩盖掉 `profile.judge` 的只读性。** diff --git a/doc/architecture/SPEC/engine/steps/rebase_ext.md b/doc/architecture/SPEC/engine/steps/rebase_ext.md new file mode 100644 index 000000000..ca51f280d --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/rebase_ext.md @@ -0,0 +1,31 @@ +# engine/steps/rebase_ext.py —— 规范 + + + +`LOC ~101 · step 库（委托） · refactor-status: ok` + +## 职责 +`rebase.run_external` —— 向 locked 的 5 阶段编排器做**受监控的子进程委托** +（**包装，而非重写**）。 + +## Steps +`rebase.run_external`（script/write_workspace）。 + +## 不变量 +- **零回归**：它**不重新实现**那条流水线。 +- 把父进程的 `state.json` 流式送进 RunTrace；**陈旧状态守卫**防止上一次 run 的 + `phase=done` 掩盖本次崩溃；失败被分类成升级材料。 +- 点名父包（被允许的仓库字面量，泄漏上限为 1）。 + +## 边界 —— 不属于这里 +自身不含 rebase 逻辑；除委托给 `rebase/monitor` 外不做解析。 + +## 依赖（允许） +`rebase/monitor`、`engine/step`、`._common`；stdlib 的 asyncio/subprocess。 + +## 测试 +`test_rebase_monitor.py`（它驱动的那个 monitor）。 + +## 重构备注 +可以接受。那一个仓库字面量（`"vllm-omni-rebase-agent"`）是**设计使然**的委托文本 —— +**不要过早模板化**；如果将来真的包装了第二个外部编排器，再把名字抽到 adapter。 diff --git a/doc/architecture/SPEC/engine/steps/rebase_native.md b/doc/architecture/SPEC/engine/steps/rebase_native.md new file mode 100644 index 000000000..2f89676e6 --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/rebase_native.md @@ -0,0 +1,37 @@ +# engine/steps/rebase_native.py —— 规范 + + + +`LOC ~397 · step 库（候选的原生 rebase） · refactor-status: ok` + +## 职责 +夜跑 rebase 的**候选**原生分解版本，import 父包自己的阶段 wrapper 与 +`node_rebase_module`。 + +## Steps（9 个） +`rebase.prelude`、`rebase.phase1..phase5`、`rebase.phase2_prepare`、 +`rebase.module_rebase`、`rebase.phase2_finalize`、`rebase.compare_with_locked`。 + +## 公开契约（可从 `engine.steps.rebase_native` import） +`_RUNTIME`（按进程记忆化的父运行时；测试 fixture 会清掉它）。 + +## 不变量 +- **对 planner 不可见**（只存在于 candidate playbook `repo-rebase-native`）。 +- phase-4 的 push 在 copilot 的推送守卫之后；env 导出被记 trace（`env_exported`）。 +- **委托给父包的函数** —— 不重新实现各阶段。 +- 点名父包（被允许的仓库字面量，泄漏上限为 6）。 + +## 边界 —— 不属于这里 +不含晋升逻辑；不重新实现 rebase。**只是 wrapper。** + +## 依赖（允许） +`rebase/monitor`、`adapters/base`（wave 交叉核对）、`engine/step`、`._common`； +以及外部的 `agent.*` 包（惰性 import，ImportError → BLOCKED）。 + +## 测试 +`test_rebase_native.py`。 + +## 重构备注 +**按设计**与父包耦合 —— 那 6 个仓库字面量和那些父包 import 是**委托面，不是坏味道**。 +它用命令式方式注册（工厂/直接 handler 混用）—— 这是 `@step` 被认可的例外。 +**只有在推进晋升路径（candidate → active → locked）时才动它。** diff --git a/doc/architecture/SPEC/engine/steps/report.md b/doc/architecture/SPEC/engine/steps/report.md new file mode 100644 index 000000000..f91b790d7 --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/report.md @@ -0,0 +1,27 @@ +# engine/steps/report.py —— 规范 + + + +`LOC ~24 · step 库 · refactor-status: ok` + +## 职责 +`report.final_summary` —— 用累积的 step 输出写出 `RUN_REPORT.md`。 + +## Steps +`report.final_summary`（report/report）。 + +## 不变量 +- 纯输出；**没有失败路径**（永远 ok）。 +- 读 `ctx.state["outputs"]`；只写 run 目录。 + +## 边界 —— 不属于这里 +不做分析、不含仓库知识、除报告文件外没有任何副作用。 + +## 依赖（允许） +`engine/step`、`._common`。 + +## 测试 +在 playbook 端到端测试中被覆盖。 + +## 重构备注 +平凡。如果想要更丰富的报告，请保持**追加式且无副作用**。 diff --git a/doc/architecture/SPEC/engine/steps/review.md b/doc/architecture/SPEC/engine/steps/review.md new file mode 100644 index 000000000..103b1b2e3 --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/review.md @@ -0,0 +1,76 @@ +# engine/steps/review/ —— 规范 + + + +`LOC ~900（6 个文件） · step 库（评审） · refactor-status: ok` + +## 职责 +条件式 patch 门 + PR 评审 agent step 及其仓库中立的 prompt 体系。 +它曾是一个 341 行的模块；现在是一个把评测调优过的 prompt 数据、handler、 +确定性 helper 三者分开的包。 + +## 包内布局（一个文件一个关注点） +- `__init__.py` —— import `steps` 以触发 `@step` 注册副作用；再导出公开契约（见下）。无逻辑。 +- `prompts.py` —— 由评测得出的 prompt 数据：`_REVIEW_SYSTEM`、`_REVIEW_LENSES`、 + `_REVIEW_MERGE`。约 120 行文本，**移出控制流之外**。 +- `utils.py` —— 确定性、无 LLM 的 helper：`_sweep_targets`、`_render_review_md`、 + `_SEVERITY_ORDER`。 +- `steps.py` —— 两个 `@step` handler：`review.patch_gate`（validation/read）、 + `agent.review_diff`（agent/read）。 +- `anchor.py` —— 基于代码片段的评论锚定（2026-08 新增）。 +- `repo_tools.py` —— 只读的变更考古工具组（2026-08 新增）。 + +## 公开契约（可从 `engine.steps.review` import） +`_REVIEW_LENSES`、`_render_review_md`、`_sweep_targets` —— 由包的 `__init__` 再导出， +使拆分前的 import 路径保持不变。 + +## 不变量 +- patch 门：廉价摘要**常开**，只有触发时才跑 LLM 评审；**fail-closed**（**C6**）； + 高风险模块来自 adapter，settings 只作兜底（**A5**）。 +- 评审：领域 checklist 由 profile 的 `review.md` 扩展；`_sweep_targets` 以 + `repo.language` 为键，**诚实降级**；裁决自洽（任何 ≥minor 的评论 ⇒ REQUEST CHANGES）； + 确定性的按严重度排序的评论上限。 +- prompt 是仓库中立的（**A5**）。 +- **要引用，不要行号**（`anchor.py`）。模型给错行号的频率高到发布时必须降级该发现； + 修法是**换一个问题** —— 让模型引用它在说的代码，位置由程序自己算。校验器**仍然最后 + 跑**，所以"绝不发布错锚点"的保证不变；变的是**只有行号错**的发现能保住自己的 inline + 位置，而不是被降级进正文。 +- **裁决校准**：只有**已验证**的 blocker/major 才阻塞；minor 降级为 COMMENT， + 自述不确定的发现**永不阻塞**。 +- **带着评论的升级会被抢救成一次成功的 REQUEST CHANGES** —— **找到缺陷本来就是一次成功 + 的评审**。 +- **考古工具是召回机器，不是便利设施**（`repo_tools.py`）。wave-2 取证测出：基线能做而 + 我们的 pass 做不到的那些判官决定性动作，**全都是只读工具集当时恰好缺少的一条命令** + （`git diff --stat base..HEAD` 证明被要求删除的东西压根没删；读 merge-base 版本的 + 文件；`git show` / `git log -S`）。 +- **覆盖率驱动的第二轮**由该 run 自己的覆盖空洞播种；**验证账本**会把这次 run 其实已经 + 写下来的残余提升为发现 —— 取证发现，**漏掉的东西往往已经被赞许地记下来了，只是没被 + 提出来**。 +- **重复发现会合并到最丰富的那条陈述，而不是被丢弃**（`_dedupe_comments` + `_richness`， + v20）。丢掉后出现的那条重复，会丢掉其中证据更好的那一份 —— 所以去重必须是**合并**， + 因为两份副本很少同样有据。 +- 主体打分会把路径标识符坍缩成 basename（`_topic`），于是"用完整路径"和"用裸文件名" + 陈述的同一个发现会被识别为一条。 +- **只有当一条评论确实同时具备主张和论证时，才发出 claim headline** + （`_split_claim`，v21 及其修复）。只有一句话的评论会渲染成朴素的 + `file:line [sev] — text` 形状，而不是一个残缺标题；且 headline **永不截断**： + 第一版截到 160 字符，结果 49 条里有 34 条停在悬空的连接词上， + **读起来比它所替代的那段还差**。那次提交里记下了一条长期教训 —— + 主张/论证的切分**无法从并非按此写就的散文里确定性地还原**， + 要可靠地拿到它，需要 reducer 直接产出一个 title 字段， + **那是模型层的改动，不是渲染层的**。 + +## 边界 —— 不属于这里 +不含 agent 运行时治理（那是 `agent_runtime`）；不写 adapter/profile。 + +## 依赖（允许） +`review/*`、`engine/step`、`.._common`、`..agent_runtime`、`profiles/languages`。 + +## 测试 +`test_review_step.py`、`test_agent_ensemble.py`、 +`test_profile_steps.py::test_review_guidance_from_profile`。 + +## 精简 —— **K2**（共享语言规则，已应用） +`_sweep_targets` 消费 `profiles/languages.py::sweep_re` —— 那是"按语言规则"三份旧副本 +之一（另两份在 `profiles/establish`、`repo_map`），现已收敛到那个唯一来源。 +未知语言时的诚实降级（只做文件级 sweep）**被保留**。 diff --git a/doc/architecture/SPEC/engine/steps/workspace.md b/doc/architecture/SPEC/engine/steps/workspace.md new file mode 100644 index 000000000..254ee52d2 --- /dev/null +++ b/doc/architecture/SPEC/engine/steps/workspace.md @@ -0,0 +1,32 @@ +# engine/steps/workspace.py —— 规范 + + + +`LOC ~43 · step 库 · refactor-status: ok` + +## 职责 +工作区前置条件 + 廉价 diff 摘要两个 step。 + +## Steps +`workspace.guard_clean`（deterministic/read）、`analysis.diff_summary` +（deterministic/read）。 + +## 不变量 +- `guard_clean` 在**脏树**上拒绝启动（BLOCKED）—— 这是任何具备写能力的 run 的前置条件。 +- 两者都是确定性的 —— **不含 LLM**。 + +## 边界 —— 不属于这里 +不修改工作区；不含仓库知识。 + +## 依赖（允许） +`review/diff_summary`、`engine/step`、`._common`。 + +## 测试 +`test_push_and_steps.py`（guard），diff-summary 经评审测试覆盖。 + +## 重构备注 +平凡且正确。如果出现更多只读的"分析"类 step，可以共用这个文件。 + +## 精简 —— **K3** +`guard_clean`/`diff_summary` 各自重复了 `repo is None → BLOCKED` 这道守卫 → +应收敛为 `require_repo(ctx)`（K3 的 8 个现场之一）。 diff --git a/doc/architecture/SPEC/intent.md b/doc/architecture/SPEC/intent.md new file mode 100644 index 000000000..315f34449 --- /dev/null +++ b/doc/architecture/SPEC/intent.md @@ -0,0 +1,55 @@ +# intent.py —— 规范 + + + +`LOC ~379 · 任务层 · refactor-status: ok` + +## 职责 +把一条自然语言命令分类成 `TaskSpec`，或者一个澄清问题；并把复合命令切成有序列表。 + +## 功能 +LLM 把命令（kind / pr / issue / flags）分类成 `TaskSpec`；低于置信度门或显式要求澄清时 +→ 提问，**绝不猜测**。`parse_intents` 按连接词切分，并延续前一段的 PR/issue +（"… then review it"）—— 那是**切分**，不是分类。 + +## 公开契约 +`parse_intent(text, llm?, default_repo, model?) -> IntentResult`； +`parse_intents(...) -> list[IntentResult]`；`IntentResult(spec?, clarify)`。 + +## 不变量 +- **确定性优先，然后才是 LLM。** 预解析层用纯代码解决 GitHub URL 和无歧义的 + ` pr/issue N` 命令，**不花一次 LLM 调用**；只有真正自由形式的输入才进分类器。 + （本规范此前写着"仅 LLM，无确定性解析器"——那描述的是 2026-08 之前的设计，已不成立。） +- **仓库身份被完整校验。** URL 必须 owner **和** repo 都匹配；同名但异主的仓库会被 + **拒绝**，而不是静默地跑在本地 checkout 上。`resolve_repo_alias` 同时服务 Strict MCP + 入口。 +- **双路径 `mode` 在这里设定**：默认 `eco`，只有用户显式要求时才是 `performance` + （确定性短语正则 **或** 分类器的 `performance` 标志）。 + **成本敏感的决策绝不靠猜。** +- 空命令或未配置 LLM → 澄清（绝不猜测）。 +- **有歧义就澄清 —— 绝不猜测**：LLM 置信度 < 0.7、显式澄清、畸形回复、未知 kind， + 全都 → 澄清。 +- **C7**：只有终端输入到达这里；抓取回来的 GitHub 文本**永远不会**。抗注入靠的是 + **通道分离** + LLM 的低置信度信号（可疑命令会澄清，绝不执行）。 + +## 边界 —— 不属于这里 +不执行、不规划、不读仓库/网络。**只做 文本 → TaskSpec。** `parse_intents` 里的 +切分与延续是**断句**，不是分类（分类归 LLM）。 + +## 依赖（允许） +`llm.py`、`task_spec.py`。 + +## 扩展点 +新 kind → 加进 `TaskKind`/`KIND_TIER`（`task_spec.py`）以及这里 `_LLM_SYSTEM` prompt +的 kind 列表。**没有 hint 表需要维护。** + +## 测试 +`test_intent_taskspec.py`（LLM 路径契约：映射、置信度门、澄清透传、无 LLM、畸形回复）、 +`test_phase_b.py`（用假分类器测复合切分 + 延续）。 + +## 重构备注 +`default_repo="vllm-omni"` 这个默认参数是被允许的 2 个仓库字面量之一（有泄漏上限） +—— 保留它们，**不要再加**。生产路径总会传入 LLM（`cli/`）；intent 现在**需要**一个 +—— 没有离线快路径（这是"仅 LLM"的刻意代价）。如果将来 LLM 也要负责复合切分， +`parse_intents` 可以直接返回列表，`_COMPOUND_SPLIT` 与延续逻辑一并消失 —— +那是更大的改动，目前不需要。 diff --git a/doc/architecture/SPEC/knowledge_docs.md b/doc/architecture/SPEC/knowledge_docs.md new file mode 100644 index 000000000..91c924272 --- /dev/null +++ b/doc/architecture/SPEC/knowledge_docs.md @@ -0,0 +1,32 @@ +# knowledge_docs.py —— 规范 + + + +`LOC ~122 · 供 MCP 面使用的只读知识检索 · refactor-status: ok` + +## 职责 +对那棵精选 Markdown 知识库的跨平台、按仓库限定的**只读访问** —— +`doc_search`/`doc_read` 的底座。 + +## 公开契约 +`KnowledgeDocs`（search、read）、`KnowledgeDocsError`。 + +## 不变量（**C1**、**D1**） +- **限定在切片内**：general 切片加上**单个**仓库的切片。知识根之外的路径一律拒绝 + （`KnowledgeDocsError`）—— 这就是防路径逃逸的守卫。 +- **只读。** 这里**根本没有写入面**；知识写入走别处的 candidate/类型化 op 路径。 +- 读取是**分页**的（每页 24k），因此一个大页面不会把宿主对话撑爆。 +- 检索是确定性的词重叠打分 —— **不调模型**。 + +## 边界 —— 不属于这里 +不撰写知识、不提 candidate、不在代码里放仓库专属规则（那棵树是数据面）。 + +## 依赖（允许） +仅 stdlib。 + +## 测试 +`test_knowledge_source.py`、`test_thin_mcp_server.py`。 + +## 重构备注 +**保持它不含模型**：这是 Direct MCP 路径与工具桥共用的**唯一**知识读取器， +在这里加一次模型调用，等于把第二个模型塞进了"server 不跑模型"这条保证里面。 diff --git a/doc/architecture/SPEC/llm.md b/doc/architecture/SPEC/llm.md new file mode 100644 index 000000000..0a25cc54c --- /dev/null +++ b/doc/architecture/SPEC/llm.md @@ -0,0 +1,47 @@ +# llm.py —— 规范 + + + +`LOC ~452 · 引擎底座（传输层） · refactor-status: ok` + +## 职责 +provider 中立的 LLM 客户端封装。它支持 Anthropic Messages 和 OpenAI Chat Completions， +包含各自的 Base URL 覆盖。 + +## 功能 +封装 `messages.create`，暴露可用性，把回复归一化成 `Reply`/`Block`， +并从模型文本里解析 JSON。 + +## 公开契约 +`LLM(settings)`，带 `available`、`create(system, messages, tools?, model?, +max_tokens?, on_text?) -> Reply`；以及 `Reply`、`Block`、`parse_json_reply`。 + +## 不变量 +- 没有 key/端点时 `available` 为 false；调用方必须**降级**（记 `capability_gap`）， + 而不是崩溃（**E2**）。 +- 不可信内容的围栏是**调用方**的职责，不在这里（**C7** 住在 agent_runtime）。 +- **逐档端点。** `eco` 和 `performance` 各自解析自己的模型、base URL 和 key， + 所以便宜档完全可以住在另一个网关上。**便宜模型仍然永远不会扩大权限** —— + `mode` 与 `tier` 正交（见 `task_spec.md`）。 +- **fail-closed 的 served-model 守卫**：如果端点报告服务的模型与请求不符，那是**错误**， + 不是静默替换 —— **一条跑在未申明模型上的评测臂，就是一次作废的测量**。 +- 捕获 `cache_read_input_tokens` 供计费/缓存分析；调用方和测试 fake **永不接触 SDK + 类型**，只接触归一化后的 `Reply`/`Block`。 +- **harness 后端不走这里**：它们经 `providers/harness_llm.py` 完全绕过 `create()`， + 而那个类在被传入 tools 时会抛错。 + +## 边界 —— 不属于这里 +不含 prompt、不含策略、除传输层外不做重试。**不是放任务/仓库逻辑的地方。** + +## 依赖（允许） +`anthropic` SDK；`openai` SDK；`config.py`。 + +## 扩展点 +新 provider/端点 → 藏在本封装的构造函数之后；保持 `Reply`/`Block` 稳定， +这样没有任何调用方需要改。 + +## 测试 +provider 选择与 OpenAI 工具翻译有单元测试；step/agent 测试使用 `ScriptedLLM`。 + +## 重构备注 +**把 `Reply`/`Block` 契约当作接缝守住** —— 调用方绝不能看见 provider 专属类型。 diff --git a/doc/architecture/SPEC/mcp_policy.md b/doc/architecture/SPEC/mcp_policy.md new file mode 100644 index 000000000..4269cdeff --- /dev/null +++ b/doc/architecture/SPEC/mcp_policy.md @@ -0,0 +1,37 @@ +# mcp_policy.py —— 规范 + + + +`LOC ~141 · 安全原语（MCP 结构性门） · refactor-status: ok` + +## 职责 +从原始 MCP 输入**重新推导**出一个*安全的* `TaskSpec`，拒绝 MCP 面不被允许做的任何事 +—— 这就是"宿主无法扩大服务端权限"的**结构性**保证。 + +## 公开契约 +`enforce_mcp_policy(raw) -> TaskSpec`（拒绝时抛错）。 + +## 不变量（**C2**、**C3**、**A2**） +- **它跑两次，这是设计如此。** 一次在**边界**（server，工具被调用时），一次在 + **子进程**（**权威**，就在它读完 `request.json` 之后）。子进程这次复检之所以存在， + 是因为这条保证**不能依赖 `request.json` 未被篡改**：同用户的宿主进程完全可以在预约 + 与执行之间把它改写。 +- `kind` 必须 ∈ `READ_ONLY_KINDS`；`post` 被**硬置为 `False`**；`repo` 必须在 allowlist + 内；`pr`/`issue` 必须为正；未知 params 被**剥除**而不是透传。 +- **允许集是 import 进来的，绝不重述。** 它直接引用 `task_spec.READ_ONLY_KINDS`， + 因此策略永远不会与任务模型漂移 —— 新增一个具备写能力的 kind，**无法**静默地把 MCP + 面放宽。 +- MCP 宿主是**非交互**的：这里没有 `[y/N]`，所以本文件里任何逻辑都不得退化成"问用户"。 + +## 边界 —— 不属于这里 +不执行 run；不调模型；不访问知识。 + +## 依赖（允许） +`..task_spec` + stdlib。一个叶子安全原语。 + +## 测试 +`test_mcp.py`（篡改防御、只读工具集）。 + +## 重构备注 +和 `push.guard_push`、`scopes` 一样，这是一个纯权限原语 —— 保持它无依赖。任何新的 +"MCP 可达能力"都必须表达为对 `READ_ONLY_KINDS` 的修改，**而不是这里的一个特例**。 diff --git a/doc/architecture/SPEC/mcp_server.md b/doc/architecture/SPEC/mcp_server.md new file mode 100644 index 000000000..53177c401 --- /dev/null +++ b/doc/architecture/SPEC/mcp_server.md @@ -0,0 +1,48 @@ +# mcp_server.py —— 规范 + + + +`LOC ~392 · Strict 后台机器（start/poll） · refactor-status: ok` + +## 职责 +为 MCP 宿主运行 Strict 工作流：预约、拉起、跟踪、供给结果 —— +**且绝不阻塞一次同步工具调用**。 + +## 功能 +评审要跑 5–12 分钟，任何同步 MCP 调用都会超时，所以接口是**start/poll 工具对**： +`start_review` / `start_issue_answer` / `start_issue_triage` 预约后**立即**返回 +`run_id`；`get_result`（经 `next_offset` 分页，受 `mcp_report_max_bytes` 封顶） +和 `get_status`（`run_status` + `progress`）负责轮询。 + +## 公开契约 +`CopilotMCP`、`build_mcp()`，以及那组 start/poll 工具。 + +## 不变量（**C2**、**C3**、**E1**） +- **安全是结构性的，不是"信任宿主"。** `enforce_mcp_policy` 在这里跑一次， + **并且**在子进程里（权威地）再跑一次，因此只有 `READ_ONLY_KINDS` 会被执行、 + `post` 恒为 `False`、只有 allowlist 内的仓库可达 —— **无论被篡改的 + `request.json` 声称什么**。 +- **每次 run 都是一个隔离子进程** + （`python -m infermatrix_copilot --execute-reserved `）。两个理由都很关键： + copilot 的 stdout 必须离开本进程的 JSON-RPC stdio 通道（子进程 stdout → + `/console.log`），且进程级全局 tracer / `last_run_dir` **由此天然按 run 隔离**。 +- **run 由单个 worker 线程串行执行。** +- 跨重启、跨多个并发 server 的轮询正确性，来自 `run_status.py` 的持久记录 + 按属主 + 对账，**而不是内存状态**。 +- **`mcp` SDK 是可选 import**，藏在 `[mcp]` extra 之后，且**绝不能**被核心包 import + —— 纯 CLI 安装保持零依赖。 +- `build_mcp()` 暴露 V1 工具面（autonomous 工作流执行器）；它**默认不注册**。 + +## 边界 —— 不属于这里 +不含 Direct 模式逻辑（`thin_mcp_server.py`）；不定义策略（`mcp_policy.py`）； +不定义状态文件格式（`run_status.py`）。 + +## 依赖（允许） +stdlib + `mcp`（可选 extra）+ `.mcp_policy` + `.run_status` + `.cli`。 + +## 测试 +`test_mcp.py`（篡改防御、单写者对账、分页、只读工具集）。 + +## 重构备注 +预约形状（先建 run 目录，再规划）是 **MCP 专属**的；CLI 主路径仍然在**建目录之前** +过门，所以被放弃的计划不会留下目录。**不要在没有保住这个差别的前提下把两者统一。** diff --git a/doc/architecture/SPEC/memory/debug_memory.md b/doc/architecture/SPEC/memory/debug_memory.md new file mode 100644 index 000000000..4ef781f0e --- /dev/null +++ b/doc/architecture/SPEC/memory/debug_memory.md @@ -0,0 +1,30 @@ +# memory/debug_memory.py —— 规范 + + + +`LOC ~106 · 记忆（失败→修复库） · refactor-status: ok` + +## 职责 +以 FTS5 存放"失败 → 修复"经验。 + +## 公开契约 +`DebugMemory(db_path)`，带 `search(query, k)` 和一个要求必填字段的写入方法。 + +## 不变量（**D1/D3**） +- 一次写入必须包含 + repo/module/run_id/symptom/root_cause/fix_summary/files/verification/status。 +- 检索是按相关度取 top-k，**摘要优先**。 +- 事实可自由记录；晋升为 skill 是**另一个**受门禁的动作。 + +## 边界 —— 不属于这里 +不做按仓库的命名空间隔离（由 agent 运行时的 `_ScopedKnowledge` 施加）；不含 LLM。 + +## 依赖（允许） +stdlib 的 `sqlite3`。 + +## 测试 +`test_memory.py`。 + +## 重构备注 +写入契约（必填字段）就是 D3 保证 —— **在写入时强制它，绝不接受残缺的记忆**。 +按仓库的 DB 路径由调用方选择（`adapter.debug_memory_db`）—— 本类保持路径无关。 diff --git a/doc/architecture/SPEC/memory/skills.md b/doc/architecture/SPEC/memory/skills.md new file mode 100644 index 000000000..ac7f4b36e --- /dev/null +++ b/doc/architecture/SPEC/memory/skills.md @@ -0,0 +1,32 @@ +# memory/skills.py —— 规范 + + + +`LOC ~120 · 记忆（程序性知识） · refactor-status: ok` + +## 职责 +程序性知识，门禁比 debug memory 更严。 + +## 公开契约 +`SkillStore(dir)`，带 `find`、`propose`、`promote`、`candidates`、`load_all`、 +`render_for_prompt`；以及 `Skill`（带用于召回的 `trigger`）。 + +## 不变量（**D1**） +- agent **只能 `propose`**（写入 candidates 文件）；把它 `promote` 成一个生效的 + `SKILL.md` 是**策展人/人类**的动作。 +- `find` 按 模块命中 + 文本命中 + run_count 排序；只有 `status: active` 的 skill + 才会被加载。 + +## 边界 —— 不属于这里 +不做按仓库的命名空间隔离（由 `_ScopedKnowledge` 施加）；不含 LLM；不是策展 UI。 + +## 依赖（允许） +`pyyaml`；stdlib。 + +## 测试 +`test_memory.py`。 + +## 重构备注 +propose→promote 这道门很干净 —— **不要**加一个 agent 可调用的 `promote`。 +`trigger` 字段是 Devin 式的召回线索；随着更多触发式检索落地，请让它保持一等地位。 +按仓库的目录由调用方选择（`adapter.skills_dir`）—— 本类保持路径无关。 diff --git a/doc/architecture/SPEC/metrics.md b/doc/architecture/SPEC/metrics.md new file mode 100644 index 000000000..b50e9cd7f --- /dev/null +++ b/doc/architecture/SPEC/metrics.md @@ -0,0 +1,45 @@ +# metrics.py —— 规范 + + + +`LOC ~493 · 跨切（测量） · refactor-status: oversized` + +## 职责 +计算并持久化逐 run 的 `metrics.json`（CATQ = Q·S/C）。 + +## 功能 +读取该 run 的 trace/产物；计算 Q（按 kind 在**已知**分量上加权的质量）、 +S（由 incident 得出的安全系数）、C（RQS3e 式的对数成本，基于 USD + 墙钟对参考预算）； +写出 `metrics.json`。 + +## 公开契约 +`collect_run_metrics(run_dir, settings, status) -> {quality, risk, cost, catq}`。 + +## 不变量 +- **E3**：metrics 是关于一次 run 的**事实**，且**绝不能把这次 run 搞坏** —— + 每个失败都被捕获并记 trace（`metrics_error`）；**run 的成败与 metrics 无关**。 +- Q **只使用已知分量**（重新归一化，并标 `partial`）；judged/GT 稍后合并， + **绝不编造**；被升级的 run 记安全弃权分。 +- incident 来自显式事件 + 既有的 out_of_scope/tool_refused/patch_review-revise。 +- **成本绝不编造。** harness 后端不向 span 记账暴露 token（每项都是 `tok_out=0`）， + 所以来源被记为 `subscription`，**不发明 USD** —— 这正是 harness 的成本优势 + **是假设而非测量**的原因（`providers/base.md`）。 + +## 边界 —— 不属于这里 +**不影响控制流。只做测量。** + +## 依赖（允许） +`config`、`run_trace`（读事件）；价格表住在这里。 + +## 扩展点 +新的质量分量 / 成本项 → 用**仅限已知**的规则扩展对应的子计算； +把参考预算记录在 `config` 里。 + +## 测试 +`test_metrics.py`。 + +## 重构备注 +最大的跨切文件；三个子指标（Q、S、C）+ 价格表 + CATQ 组装挤在一个模块里。 +**建议拆分**：`metrics/quality.py`、`metrics/safety.py`、`metrics/cost.py`、 +`metrics/__init__.py`（组装 + `collect_run_metrics`）。那些 `[planned]` 的 +gh 反馈 / 推送后 CI 收集器应当作为**新文件**落进那个包，而不是追加到这里。 diff --git a/doc/architecture/SPEC/notify.md b/doc/architecture/SPEC/notify.md new file mode 100644 index 000000000..367fc1923 --- /dev/null +++ b/doc/architecture/SPEC/notify.md @@ -0,0 +1,38 @@ +# notify.py —— 规范 + + + +`LOC ~112 · 跨切（升级） · refactor-status: ok` + +## 职责 +"通知，而不是猜测"的出口通道。 + +## 功能 +`Notifier.escalate` 写出 `ESCALATION.md`、发邮件（配置了就走 Resend 或 SMTP）、 +把这次升级记进 trace；`BLOCKED_EXIT` = 3。 + +## 公开契约 +`Notifier(settings, run_dir, trace, run_id)`，带 `escalate(reason, phase, +severity, state_summary, artifacts)`；以及 `BLOCKED_EXIT`。 + +## 不变量 +- 被阻塞的 run 会写出 `ESCALATION.md`、发出通知，调用方以退出码 3 结束。 +- **升级是一等结果** —— 绝不被当作错误路径吞掉。 +- 邮件失败是尽力而为的，**不得**掩盖升级本身。 + +## 边界 —— 不属于这里 +"要不要升级"由 executor 决定（类型化失败路由）—— 本文件只负责**执行通知**。 + +## 依赖（允许） +`config`、`run_trace`；stdlib 的 `urllib`/`smtplib`。 + +## 扩展点 +新通道（IM/webhook）→ 在这里加一个方法，由它自己的配置字段开关； +保持 `escalate` 是唯一入口。 + +## 测试 +经 `test_engine.py` 的路由 + 升级断言覆盖。 + +## 重构备注 +干净。如果通道变多，引入一个小的 `Channel` 策略列表，而不是在 `escalate` 内部继续 +加分支。 diff --git a/doc/architecture/SPEC/playbooks/PLAYBOOKS.md b/doc/architecture/SPEC/playbooks/PLAYBOOKS.md new file mode 100644 index 000000000..3bd59e9bc --- /dev/null +++ b/doc/architecture/SPEC/playbooks/PLAYBOOKS.md @@ -0,0 +1,49 @@ +# playbooks/*.yaml —— 规范 + + + +`9 个文件 · 声明式编排数据 · refactor-status: ok` + +> 当前集合：`pr-review`@6、`pr-debug`@2、`pr-rebase`@2、`issue-answer`@2、 +> `issue-triage`@2、`repo-profile`@1（active）；`repo-rebase`@2（**locked**）； +> `repo-rebase-native`@1、`profile-consolidate`@1（candidate）。带 step 链的完整清单 +> 在 [`../../../GUIDE.md`](../../../GUIDE.md) §5 —— **不在这里**。 + +## 职责 +用声明式的有序 step 列表（含 `foreach`、`when:`、逐 step `params`）实现一种任务 kind。 + +## 每个文件的契约 +`name, version, status, task_kinds, repos, requires?, params, provenance, +success, steps[]`。 + +## 已注册的 playbook +- `repo-rebase` —— **locked**，L0，`requires: [orchestrator.external]`。 + 逐字节零回归 —— **不要改它的 step 列表**。 +- `pr-rebase`/`pr-debug`/`pr-review`/`issue-answer`/`issue-triage` —— active， + 仓库中立（`repos: []`、`requires: [repo.path]`）。 +- `repo-profile` —— active，仓库中立（用于接入第二个仓库）。 +- `repo-rebase-native`、`profile-consolidate` —— **candidate** + （planner 不可见；只能经 `--playbook` 运行）。 + +## 不变量 +- 每个 step id 唯一；每个 `step` 名字都必须已注册（由 `store.validate` 强制）。 +- 写/推送 step **只**出现在已审核（非生成）的 playbook 里。 +- locking 面向会改代码/会推送的 playbook；晋升 candidate→active→locked + 是**带 provenance 的人类动作**。 +- **`candidate` 对 `find()` 不可见** —— 只能经 `--playbook ` 到达。 + 正是这一条让 `profile-consolidate` 保持"刻意的节奏"（连续的 LLM 重写**实测**会腐蚀 + 记忆），也让 `repo-rebase-native` 在并排验证通过前**不进夜跑**。 +- 仓库中立的 playbook 声明 `repos: []` + 一份 `requires:` 能力清单； + 存在精确 repo 的 playbook 时仍然由它获胜。 + +## 边界 —— 不属于这里 +不含代码；除 `repos`/`requires` 匹配外不含仓库知识。 + +## 扩展点 +新任务 = 一个 YAML 文件；新仓库在其 profile 满足 `requires` 之后， +**直接复用仓库中立的 playbook**（核心零改动）。 + +## 重构备注 +它们是 reuse>adapt>generate 模型里的"配置"那一半 —— **保持声明式**。 +要顶住在 `when:`/`foreach` 之外加条件逻辑的冲动；更复杂的东西属于某个 step。 +**当第二个仓库接入时，它应该不需要任何新 playbook —— 那就是不变性测试。** diff --git a/doc/architecture/SPEC/playbooks/store.md b/doc/architecture/SPEC/playbooks/store.md new file mode 100644 index 000000000..71de6caee --- /dev/null +++ b/doc/architecture/SPEC/playbooks/store.md @@ -0,0 +1,40 @@ +# playbooks/store.py —— 规范 + + + +`LOC ~166 · 规划（注册表） · refactor-status: ok` + +## 职责 +带版本的 playbook 注册表：加载/解析/校验 YAML，按 kind+repo+capabilities 召回， +持久化 candidate。 + +## 公开契约 +`Playbook`、`PlaybookStep`、`parse_playbook`、`playbook_to_doc`； +`PlaybookStore(dir, registry)`，带 `find(kind, repo, capabilities?)`、 +`missing_capabilities(kind, capabilities)`、`get`、`all`、`save_candidate`、 +`validate`。 + +## 不变量 +- 状态为 `candidate | active | locked | retired`；`find` **只召回 active/locked**； + candidate 只能经显式 `--playbook` 运行。 +- `find`：精确 repo 优先；仓库无关的匹配仅当 `requires ⊆ capabilities`（已知时）； + locked > active；高版本 > 低版本。 +- `validate` 拒绝引用了未注册 step 的 playbook（**加载即失败**）。 +- `save_candidate` 强制 `status=candidate`（**D1** —— 不允许自我晋升）。 + +## 边界 —— 不属于这里 +不执行、不含规划策略（那是 planner 的）、不含 step 逻辑。 + +## 依赖（允许） +`engine/registry`、`pyyaml`。 + +## 扩展点 +新增 playbook 字段 → `Playbook` + `parse_playbook` + `playbook_to_doc` **一起**扩展。 + +## 测试 +`test_planner_playbooks.py`、`test_capabilities.py`、`test_review_step.py`。 + +## 重构备注 +干净。`Playbook`/`PlaybookStep` 是数据，`PlaybookStore` 是机制 —— 保持两者可分离。 +如果将来出现 DAG 形态的 playbook，`PlaybookStep` 会长出边，但 `find`/`validate` +的契约不变。 diff --git a/doc/architecture/SPEC/profiles/consolidate.md b/doc/architecture/SPEC/profiles/consolidate.md new file mode 100644 index 000000000..f05a2e7ab --- /dev/null +++ b/doc/architecture/SPEC/profiles/consolidate.md @@ -0,0 +1,30 @@ +# profiles/consolidate.py —— 规范 + + + +`LOC ~52 · profile（Stage-4 helper） · refactor-status: ok` + +## 职责 +确定性的 Stage-4 helper：陈旧衰减 + 漂移检测。 + +## 公开契约 +`decay_stale(store, days) -> stale ids`；`detect_drift(adapter, store) -> findings`。 + +## 不变量 +- `decay_stale` 通过 store 的 `mark_stale` op，把超窗口的 active fact 翻成 `stale` + （**排除，而不是删除**）（**D4**）。 +- `detect_drift` **只报告**：已声明却消失的模块路径、挂到未知模块上的 fact； + **它绝不做任何变更**。 + +## 边界 —— 不属于这里 +只做确定性检测 —— 不含 LLM，不自动修（巩固是 `agent.profile_consolidate` step 的事）。 + +## 依赖（允许） +`adapters/base`、`profiles/store`；stdlib 的 `time`。 + +## 测试 +`test_p3_machinery.py`。 + +## 重构备注 +小而纯。**保持 `detect_drift` 只报告** —— "findings 变成刷新提案，绝不自动修" +这条规则正是它的全部意义（**D2**）。 diff --git a/doc/architecture/SPEC/profiles/establish.md b/doc/architecture/SPEC/profiles/establish.md new file mode 100644 index 000000000..9a4c0d621 --- /dev/null +++ b/doc/architecture/SPEC/profiles/establish.md @@ -0,0 +1,37 @@ +# profiles/establish.py —— 规范 + + + +`LOC ~107 · profile（Stage 0–1.5 helper） · refactor-status: ok` + +## 职责 +确定性的建立期 helper。 + +## 公开契约 +`fact_id`、`build_doc_corpus`、`is_redundant`、`extract_directives`、 +`scan_modules`、`HUMAN_DOC_NAMES`。 + +## 不变量 +- `is_redundant`（对 README+docs 做 6 词 shingle）会丢弃任何**仓库自己的文档已经写过** + 的 briefing 行（ETH 研究那条规则，**D5**）。 +- `scan_modules` 是确定性的、按语言索引的，跳过非代码目录。 +- `extract_directives` 限制行长（只收短祈使句）。 + +## 边界 —— 不属于这里 +纯确定性 helper —— 不含 LLM、不写 store、不含 step 逻辑。 + +## 依赖（允许） +仅 stdlib。 + +## 测试 +`test_profile_steps.py`（冗余过滤、模块扫描、指令抽取）。 + +## 重构备注 +纯函数 —— 易测、易复用。冗余过滤器是承载 ETH 研究结论的关键防线；**保持它确定性**。 + +## 精简 —— **K2**（共享语言规则）—— 已完成 +本模块曾经拥有 `LANGUAGE_SUFFIXES`，那是按语言规则集的三份副本之一 +（另两份在 `review._sweep_targets` 和 `repo_map`）。数据现在住在叶子模块 +`profiles/languages.py`，藏在小访问器（`suffixes` / `symbol_re` / `sweep_re`） +之后并从那里消费；这个符号已从本模块的公开契约中消失。按设计保留的一点： +**未知语言产出空的模块扫描，而不是猜测**。 diff --git a/doc/architecture/SPEC/profiles/languages.md b/doc/architecture/SPEC/profiles/languages.md new file mode 100644 index 000000000..8c82178bb --- /dev/null +++ b/doc/architecture/SPEC/profiles/languages.md @@ -0,0 +1,37 @@ +# profiles/languages.py —— 规范 + + + +`LOC ~50 · 边缘（共享数据） · refactor-status: ok` + +## 职责 +按语言规则的**唯一归处**（精简 K2），此前曾一式三份。 + +## 公开契约 +`suffixes(language) -> tuple[str, ...]`；`symbol_re(language) -> Pattern | None`； +`sweep_re(language) -> (Pattern, Pattern) | None`。 + +## 功能 +朴素数据映射（`_SUFFIXES`、`_SYMBOL_RE`、`_SWEEP_RE`）+ 极小的访问器。 + +## 不变量 +- 未知语言返回空/None，因此每个消费者都**诚实降级**（只做文件级 sweep / 空的模块扫描 / + "use grep"）。 +- 纯数据 —— 无 I/O、无状态。 + +## 边界 —— 不属于这里 +不含扫描/渲染逻辑 —— 规则由消费者施加（`review._sweep_targets`、 +`establish.scan_modules`、`repo_map`）。 + +## 依赖（允许） +仅 stdlib 的 `re`。**必须保持为叶子**（`_ARCHITECTURE.md` §4）—— 不得 import +engine/profiles 的机器。 + +## 测试 +经三个消费者各自的测试覆盖 +（`test_ci_and_repo_map.py`、`test_profile_steps.py`）。 + +## 重构备注 +新增一种语言 = 在三张映射里各加一行。保持它是**朴素数据**；不要让它长成语言检测引擎 +（检测留在 `fingerprint_repo`）。这是 K2 的去重目标 —— **不要**把规则再内联回某个 +消费者。 diff --git a/doc/architecture/SPEC/profiles/repo_map.md b/doc/architecture/SPEC/profiles/repo_map.md new file mode 100644 index 000000000..ca686397a --- /dev/null +++ b/doc/architecture/SPEC/profiles/repo_map.md @@ -0,0 +1,38 @@ +# profiles/repo_map.py —— 规范 + + + +`LOC ~138 · profile（按需结构） · refactor-status: ok` + +## 职责 +按需、按目标排序、有预算上限的符号索引（设计 §V2.0.2：结构是**被拉取的，永不被推送**）。 + +## 公开契约 +`RepoMap(repo, language, cache_dir?)`，带 `supported`、`index()`、 +`render(query, budget_chars)`；以及 `build_index`。 + +## 不变量 +- 按语言的正则符号索引；按 HEAD 做磁盘缓存（一个 HEAD 一份缓存；漂移时重建）。 +- `render` 按查询排序 + 预算封顶；零分尾部被丢弃。 +- 语言不支持时返回诚实的 "use grep" 字符串（agent 运行时会记一条 `capability_gap`）。 + +## 边界 —— 不属于这里 +**永远不注入 prompt** —— 只以 `repo_map` 工具的形式浮现 +（接线在 `agent_runtime._repo_map_tool`）。 + +## 依赖（允许） +stdlib 的 `re`/`json`/`subprocess`。 + +## 测试 +`test_ci_and_repo_map.py`。 + +## 重构备注 +基于正则（不依赖 tree-sitter）—— 这是**刻意的**简单性/可移植性取舍。如果精度成为问题， +可以让 tree-sitter 后端接在同一个 `RepoMap.render` 契约后面。保持"按需拉取"的姿态 —— +**不要**新增把这张图注入 prompt 的代码路径。 + +## 精简 —— **K2**（共享语言规则） +`_SYMBOL_RES` + `_SUFFIXES` 是按语言规则集的第三份副本（另两份见 +`review._sweep_targets`、`profiles/establish`）。改为消费共享的 +`profiles/languages.py`（K2）。必须保留：未知语言时 `supported` 为 false + 诚实的 +"use grep"。 diff --git a/doc/architecture/SPEC/profiles/store.md b/doc/architecture/SPEC/profiles/store.md new file mode 100644 index 000000000..4bafa3907 --- /dev/null +++ b/doc/architecture/SPEC/profiles/store.md @@ -0,0 +1,40 @@ +# profiles/store.py —— 规范 + + + +`LOC ~268 · profile（精选层） · refactor-status: ok` + +## 职责 +精选事实库：**类型化 patch op 是唯一写入面**，配上溯源/稳定性门禁和按通道分类的渲染。 + +## 公开契约 +`ProfileStore(root)`；`apply_ops(ops, tier, actor) -> 逐 op 的拒绝原因`； +`active(channel?, module?)`；`render_briefing(budget)`；`render_report()`。 +`Fact`；`RUN_OPS`/`CONSOLIDATE_OPS`；`CHANNELS`、`KINDS`、`SOURCES`、 +`STABLE_CONFIRMATIONS`、`BRIEFING_WORD_BUDGET`。 + +## 不变量 +- op 是**唯一**的变更路径；畸形/被禁的 op **逐条**拒绝（绝不抛异常）；档位不对的 op + 被拒（**D4**）。 +- `add_fact` 需要正文 + 证据；重复 id 视为一次**确认**（**D3**）。 +- `rewrite_fact` 绝不让某条 fact 变成无证据；**稳定**（≥3 次确认）的 fact 不得丢弃已 + 引用的证据；被取代的正文进 `history`（**D3**）。 +- `merge_facts` 留下一个指针存根（**绝不删除**）；`mark_stale` 只排除、保留以备审计。 +- `render_briefing` 只发出 active 的 briefing 通道 fact，**确认次数多的在前**， + 并处于硬性词预算之下（**D5**）。 +- 每条被接受的 op → `ops_log.jsonl`；`save()` 会重新渲染 `PROFILE_REPORT.md`。 + +## 边界 —— 不属于这里 +不含 LLM、不扫描仓库、不含 step 逻辑。 + +## 依赖（允许） +`pyyaml`；stdlib。 + +## 测试 +`test_profile_store.py`。 + +## 重构备注 +它是 D3/D4 的参考实现 —— **不要**在 `apply_ops` 之外新增变更路径。每个 `_op_*` 都是 +一个小校验器；保持它们可以被**逐条拒绝**。op 种类增加时，继续用 +`RUN_OPS`/`CONSOLIDATE_OPS` 作为档位门。这是从个人 agent 的 store 移植过来的 —— +保持两者在概念上对齐。 diff --git a/doc/architecture/SPEC/providers/audit.md b/doc/architecture/SPEC/providers/audit.md new file mode 100644 index 000000000..47f3864f8 --- /dev/null +++ b/doc/architecture/SPEC/providers/audit.md @@ -0,0 +1,40 @@ +# providers/audit.py —— 规范 + + + +`LOC ~109 · 侦测型控制（运行后会话审计） · refactor-status: ok` + +## 职责 +对已完成的 harness 会话做容纳性违规审计 —— 服务于那些**没有预防型控制可用**的后端。 + +## 功能 +给定该会话的工具事件，检查文件读取是否停留在容纳根内（PR-time worktree + run 目录）， +以及只读 scope 下是否出现过 write/edit 调用。**返回 findings；从不修改任何东西。** + +## 公开契约 +`SessionAudit`（`ok`、`tool_calls`）、`contained_in(path, roots)`、 +`audit_events(events, *, roots, ...)`。 + +## 不变量（**C1**、**C2**、**E2**） +- **侦测型，不是预防型** —— 它是工具治理决策**公开声明的兜底**。它服务于内置工具无法 + 关闭的 cursor-agent。**它不阻止任何事，它只报告。** +- **findings 必须浮现，绝不静默丢弃。** 违规交给调用方记 trace 并渲染进 RUN_REPORT。 + 一次静默的审计**比没有审计更糟**，因为它暗示存在一个其实并未生效的控制。 +- **只放产品策略。** 评测臂在其之上叠加了一条额外的"不得访问 PR 讨论"规则；那是 + ground-truth 泄漏问题，不是产品问题，**刻意不住在这里**。 +- **一条有记录的豁免：** cursor-agent 会把 MCP 工具的**结果**缓存进它按项目划分的状态 + 目录，再用自己的原生读工具读回来（`_CLI_TOOL_SPOOL`）。那次读回正是桥输出被消费的 + 方式，**不是外泄** —— 没有这条豁免，每个用了桥的会话都会被误报（实测发现）。 + +## 边界 —— 不属于这里 +不做强制；不调用厂商；不含评测专属规则。 + +## 依赖（允许） +仅 stdlib。一个叶子分析模块。 + +## 测试 +经 `test_provider_cursor.py` 覆盖。 + +## 重构备注 +从 `eval/dataset/run_cursor_arm.py` 产品化而来 —— 当初正是这项检查抓到了一次真实的越界 +读取（`~/.claude/skills/...`）。**保持它纯粹**，这样评测臂和产品路径的判断不会分叉。 diff --git a/doc/architecture/SPEC/providers/base.md b/doc/architecture/SPEC/providers/base.md new file mode 100644 index 000000000..0778aecf9 --- /dev/null +++ b/doc/architecture/SPEC/providers/base.md @@ -0,0 +1,50 @@ +# providers/base.py —— 规范 + + + +`LOC ~161 · provider 层契约 + 子进程环境白名单 · refactor-status: ok` + +## 职责 +每个 provider 都要实现的那组契约，以及决定"厂商 CLI 子进程继承什么环境"的安全原语。 + +## 功能 +定义两种 provider kind（`api` —— 无状态 completions，工具循环归**我们的** +`agent_loop`；`harness` —— 自带工具循环的厂商 agent，所以接缝是**一整个 step**）、 +请求/用量数据类、带共享二进制解析的 `HarnessTransport` 基类，以及 `sanitized_env()`。 + +## 公开契约 +`ProviderSpec`、`AgentSessionRequest`、`SessionUsage`、`HarnessTransport` +（`cli_path`、`require_cli`、`auth_gap`、`run_session`、`complete`）、 +`sanitized_env()`、`flatten_messages()`。 + +## 不变量（**C1**、**C4**、**E2**） +- **环境是白名单，不是黑名单**（`_ENV_KEEP` + `LC_`/`XDG_` 前缀）。厂商 CLI 必须保住 + 自己的订阅认证（HOME 状态），但**绝不能继承我们的模型端点**：在这类机器上，被继承的 + `ANTHROPIC_BASE_URL` 指向一个网关，会**悄悄把厂商流量改道**。API key、gh token 和 + `CLAUDECODE` 这类宿主标记，出于同样理由一并丢弃。 +- **绝不编造成本。** harness 报用量的方式参差不齐；缺失的数字保持 `0`，`cost_usd` + 保持 `None`，于是 metrics 记来源为 `"subscription"`，而不是发明一个 USD 数值。 +- **接缝是一整个 step。** `run_session` 收到的是 `run_agent` **本来会收到的同一个 + prompt 包**（契约 `system` + 渲染后的 dispatch context），所以 harness 跑的是同一场 + 评审，而不是另一场。 +- `flatten_messages` **只用于无工具**对话 —— 由调用方保证。把带工具的对话拍平会**静默 + 丢掉 tool result**。 +- `auth_gap()` 返回 `None` 表示"未知或没问题"，**绝不表示"已验证良好"**：没有廉价检查 + 手段的 transport，应当让 run **大声失败**，而不是断言一个它并没有测过的就绪状态。 + +## 边界 —— 不属于这里 +不含厂商专属的调用方式（归各 transport）；不含注册表；不为 `api` 解析凭据 +（那是 `Settings`）。 + +## 依赖（允许） +stdlib + `..scopes.ToolScope`。它是一个叶子契约模块。 + +## 扩展点 +新的能力标志加进 `ProviderSpec.capabilities`；新的会话上限加进 `AgentSessionRequest`。 + +## 测试 +`test_providers.py`（环境白名单、spec 形状）。 + +## 重构备注 +`sanitized_env()` 和 `push.guard_push`、`scopes` 一样是安全原语 —— 保持它纯粹、无依赖。 +**放宽 `_ENV_KEEP` 是一个安全决策，不是便利性决策。** diff --git a/doc/architecture/SPEC/providers/claude_code.md b/doc/architecture/SPEC/providers/claude_code.md new file mode 100644 index 000000000..f4c896115 --- /dev/null +++ b/doc/architecture/SPEC/providers/claude_code.md @@ -0,0 +1,42 @@ +# providers/claude_code.py —— 规范 + + + +`LOC ~205 · harness transport（Claude 订阅） · refactor-status: ok` + +## 职责 +在订阅认证下，通过 `claude` CLI 跑完**一整个** agent step。 + +## 功能 +headless `claude -p`，用 `--max-turns` 承载我们的迭代预算，**按名字禁用**内置工具， +通过生成的 MCP 配置接入 scoped 工具桥，并从 JSON 结果里解析回 `modelUsage`/成本。 + +## 公开契约 +`ClaudeCodeTransport`（`run_session`、`complete`）、`spec = PROVIDERS["claude-code"]`。 + +## 不变量（**C1**、**C2**、**E2**） +- **内置工具是被禁用的，不只是"没用到"**（`_BUILTIN_DENY`）。这就是能力标志 + `builtin_tools_off` 所宣称的预防型控制；一旦这份 deny 列表与 CLI 的工具名对不上， + 控制就会**悄悄变弱**，所以它被写死在一个常量里。 +- 桥 MCP server 是**唯一**被提供的工具面（`_BRIDGE_SERVER = "infermatrix-tools"`）， + 因此每次调用仍然经过 `tools.dispatch`。 +- `--max-turns` 把我们的迭代预算映射到厂商原生的上限 —— 预算由 **harness 强制**， + 而不只是在 prompt 里请求。 +- 用量按模型上报（`modelUsage`），且**只有 CLI 真的报了成本时才记录** + （见 `base.md`：绝不编造）。 +- 桥的活动按行偏移量从 `bridge_trace.jsonl` 读取，于是第二个进程永远不会与父进程的 + `run_trace.jsonl` 交错。 + +## 边界 —— 不属于这里 +不决定注册表归属；不构造 scope（那归 step）；不处理凭据 —— 订阅认证住在 CLI 自己的 +HOME 状态里，**本代码库从不接触它**。 + +## 依赖（允许） +stdlib + `.base` + `..agent_loop.AgentOutcome` + `..llm` 的类型。 + +## 测试 +`test_provider_claude_code.py`。 + +## 重构备注 +三个 transport（`claude_code`、`codex`、`cursor`）形状相似，但**恰恰在治理不同的地方 +不同** —— 要顶住把它们合并成一个参数化类的冲动：**那些差异本身就是安全姿态**。 diff --git a/doc/architecture/SPEC/providers/codex.md b/doc/architecture/SPEC/providers/codex.md new file mode 100644 index 000000000..8d3d7369b --- /dev/null +++ b/doc/architecture/SPEC/providers/codex.md @@ -0,0 +1,41 @@ +# providers/codex.py —— 规范 + + + +`LOC ~197 · harness transport（ChatGPT 订阅） · refactor-status: ok` + +## 职责 +在 ChatGPT 订阅认证下，通过 `codex` CLI 跑完**一整个** agent step。 + +## 功能 +`codex exec`，用 MCP override 指向工具桥，按事件流解析最终文本与用量，并以 OS 级只读 +沙箱作为容纳控制。 + +## 公开契约 +`CodexTransport`（`auth_gap`、`run_session`、`complete`）、`spec = PROVIDERS["codex"]`。 + +## 不变量（**C1**、**C2**） +- **控制手段是沙箱，不是工具列表。** Codex 无法关闭自己的原生 shell，所以容纳靠 OS 级的 + `--sandbox read-only`。因此沙箱内**较宽的读取是可能且被接受的**；沙箱保证的是 + **什么都写不出去**。 +- 经桥的工具调用仍然过 `tools.dispatch`；沙箱是**纵深防御，不是替代**。 +- `_tool_activity` 是尽力而为的活动日志，**明确不是审计** —— 强制点是沙箱。 + 不要让调用方把它当审计用。 +- `complete()` 在一个空的临时 cwd 里无工具运行，所以一次性调用**根本够不到仓库**。 +- `auth_gap()` 会报出 ChatGPT 登录缺口：这个后端**仅经离线测试**（开发机没有登录）， + readiness 必须如实说出来，而不是暗示它已被验证。 + +## 边界 —— 不属于这里 +不发明沙箱策略（CLI 的 flag 就是契约）；不编造用量。 + +## 依赖（允许） +stdlib + `.base` + `..agent_loop.AgentOutcome` + `..llm` 的类型。 + +## 测试 +`test_provider_codex.py`（离线；实网路径**按设计未经验证** —— 见 +`doc/features/provider-registry.md` 里的状态说明）。 + +## 重构备注 +**不要**把 `_tool_activity` "改进"成审计：`providers/audit.py` 的存在正是为了那些没有 +OS 级控制可用的后端；把两者混同，会模糊掉"当前实际生效的是哪一类控制"—— +而这正是 RUN_REPORT 要披露的事实。 diff --git a/doc/architecture/SPEC/providers/cursor.md b/doc/architecture/SPEC/providers/cursor.md new file mode 100644 index 000000000..ce118559d --- /dev/null +++ b/doc/architecture/SPEC/providers/cursor.md @@ -0,0 +1,40 @@ +# providers/cursor.py —— 规范 + + + +`LOC ~207 · harness transport（Cursor 订阅） · refactor-status: ok` + +## 职责 +在订阅认证下通过 `cursor-agent` 跑完**一整个** agent step —— 这是可用预防型控制最弱的 +后端，因此也是**背着侦测型兜底**的那一个。 + +## 功能 +headless `cursor-agent --print --force --output-format stream-json`， +**prompt 走 STDIN**，按行解析事件，每次会话写一份桥 MCP 配置，并强制做一次运行后审计。 + +## 公开契约 +`CursorTransport`（`auth_gap`、`run_session`、`complete`）、`spec = PROVIDERS["cursor"]`。 + +## 不变量（**C1**、**C2**、**E2**） +- **prompt 走 STDIN，绝不走 argv。** Linux 对单个 argv 条目的上限是 128KiB，而证据包 + 会超过它；作为参数传会**静默截断整场评审**。 +- **这里的内置工具无法完全关闭**，所以治理是两层，且**两层都公开声明**：scoped 工具 + 经桥**提供**（用到时即预防），**并且**每次会话都做运行后审计 + （`providers/audit.py`），裁决进 trace 并渲染进 RUN_REPORT。 + **控制类别被明说 —— 绝不留静默缺口。** +- 子进程环境是 `base.sanitized_env()` 的白名单：厂商 CLI 保住自己的订阅认证， + 但不得继承我们的模型端点或仓库凭据。 + +## 边界 —— 不属于这里 +不含审计规则（那些住在 `providers/audit.py`）；不含评测专属策略 —— 评测臂那条额外的 +"不得访问 PR 讨论"规则属于 ground-truth 泄漏问题，**刻意不放进产品路径**。 + +## 依赖（允许） +stdlib + `.base` + `.registry` + `..agent_loop.AgentOutcome` + `..llm` 的类型。 + +## 测试 +`test_provider_cursor.py`。 + +## 重构备注 +这套调用形状是由 Composer 评测臂（`eval/dataset/run_cursor_arm.py`）验证出来的； +**如果那个脚本和这个 transport 发生漂移，评测臂就不再是在测量产品了。** diff --git a/doc/architecture/SPEC/providers/deepseek.md b/doc/architecture/SPEC/providers/deepseek.md new file mode 100644 index 000000000..5c42c3ed1 --- /dev/null +++ b/doc/architecture/SPEC/providers/deepseek.md @@ -0,0 +1,55 @@ +# providers/deepseek.py —— 规范 + + + +`LOC ~502 · harness transport（dsh，API-keyed） · refactor-status: oversized` + +## 职责 +通过 DeepSeek 自己的 agent harness（`dsh`）的 Python SDK 跑完**一整个** agent step +—— 这是**打破了注册表两条 harness 假设**的后端，两处都是刻意为之且都已声明。 + +## 功能 +驱动 `deepseek-harness-sdk`（它本身是个子进程 SDK，拉起捆绑的 `dsh-jsonrpc-agent` +并通过 JSON-RPC/stdio 通信），**逐会话生成 composition**，以便按 scope 钉住沙箱模式。 + +## 公开契约 +`DeepSeekHarnessTransport`（`cli_path`、`require_cli`、`auth_gap`、`run_session`、 +`complete`）、`spec = PROVIDERS["deepseek"]`。 + +## 不变量（**C1**、**C2**、**E2**） +- **它是 API-keyed，不是订阅认证的。** `base.md` 写明 harness 自己持有认证， + `Settings.tier_target` 对 harness 返回空 key 正是出于这个理由。dsh 是例外： + 它需要一个 DeepSeek 凭据，经 `DeepSeekHarnessConfig.api_key` 交进去。 + 对 cursor/claude-code/codex 而言，注入 key 会是一个**缺陷**；在这里它却是 harness + 能跑起来的唯一方式。注册表用 `api_keyed` 标出这一点。 +- **它用不了我们的工具桥 —— 这是实测事实，不是选择。** 捆绑的运行时编译进了 122 个 + 插件，`@deepseek-ai/dsh-mcp-client` 不在其中（扫描可执行文件验证过）。因此会话跑在 + harness 原生的 `bash` + `str_replace_editor` 上，我们的 scoped 工具（含考古工具组） + **作为具名工具不可达**。每个拿到桥 spec 却无法履约的会话都会记一条 + `review.mcp_tool_bridge` 的 `capability_gap`，结果里 `mcp_bridged=False`。 + **注册表不得为这个 provider 声明 `mcp_tools`**（在 2026-08-17 之前它声明了； + 没有任何分支读它，所以这条假声明只起到了误导读者的作用）。 +- **跑在原生 bash 上的 arm，绝不能被标成 "tools bridged"。** 本轮战役已经测到过三个 + 与其标签不符的 arm。 +- **沙箱模式是被钉住的，绝不继承。** 上游 minimal 出厂即 `mode: danger-full-access`， + 它自己的 README 也说只应对一次性 checkout 使用。而这台机器是共享的、`.env` 里放着 + 真实凭据，所以生成的 composition 会把模式钉到该会话的 scope 上。 +- **step 上限是我们进程内预算的 6 倍**（`_STEP_CAP_FACTOR`），不是 1:1：实测健康的 + lens 在预算 14 的情况下用了约 40 步，1:1 会截断正常会话；6× 仍然能在离实测到的 + 558 步一个数量级之外拦住失控。 +- 一旦运行时的插件集合与 composition 的假设不再匹配，`_assert_plugins_bundled` 会 + **大声失败**。 + +## 边界 —— 不属于这里 +不含桥实现；不含评测标注策略（但上面那条标注不变量**约束调用方**）。 + +## 依赖（允许） +stdlib + `.base` + `.registry` + `..agent_loop` + `..llm` 的类型 + dsh SDK（惰性 import）。 + +## 测试 +`test_provider_deepseek.py`。 + +## 重构备注 +约 502 行，是目前**最大**的 transport，因为它要自己生成 composition —— 而基于 CLI 的 +那几个是从厂商那里拿到的。如果继续增长，composition 构建部分（`_composition`、`_env`） +是天然的拆分点。 diff --git a/doc/architecture/SPEC/providers/harness_llm.md b/doc/architecture/SPEC/providers/harness_llm.md new file mode 100644 index 000000000..4ca8299fd --- /dev/null +++ b/doc/architecture/SPEC/providers/harness_llm.md @@ -0,0 +1,36 @@ +# providers/harness_llm.py —— 规范 + + + +`LOC ~66 · 套在 harness 之上的 LLM 形状适配器（仅限无工具） · refactor-status: ok` + +## 职责 +让既有的 `llm.LLM` 调用点在 harness 后端下继续工作 —— **仅限无工具调用**。 + +## 功能 +把 `HarnessTransport` 包成 `LLM` 的形状，于是那些持有 `LLM` 的调用方 +（意图分类、ensemble reducer/merge、输出修复、chat）在 harness 后端下变成一次性的 +CLI 调用。 + +## 公开契约 +`HarnessLLM`（`available`、`for_target`、`for_member`、`create`）。 + +## 不变量（**C1**、**B1**） +- **任何带工具的调用都会抛错。** agent step 必须走 `run_session` —— 工具循环归 harness + —— 而这里的一声大响，正是防止在厂商循环之外**悄悄再跑一个不受治理的第二工具循环**的 + 守卫。**这就是本模块之所以长成这个形状的唯一理由。** +- 调用点不变：`create()` 保持签名，所以没有任何调用方需要知道当前是哪个后端。 +- `for_member` 是 MoA 的接缝（混合成员在 api 后端的 run 内部骑上某个 harness）。 + +## 边界 —— 不属于这里 +不做 agent step 委托（那是 `run_session`）；不做工具桥接。 + +## 依赖（允许） +`.base` + `..llm` 的类型。 + +## 测试 +`test_providers.py`。 + +## 重构备注 +**要顶住"为了方便加一条带工具路径"的冲动** —— 那会把这个类存在的意义（防止不受治理的 +第二循环）原样请回来。 diff --git a/doc/architecture/SPEC/providers/registry.md b/doc/architecture/SPEC/providers/registry.md new file mode 100644 index 000000000..e8d549ead --- /dev/null +++ b/doc/architecture/SPEC/providers/registry.md @@ -0,0 +1,50 @@ +# providers/registry.py —— 规范 + + + +`LOC ~107 · 后端解析（唯一那张表） · refactor-status: ok` + +## 职责 +"够到一个模型"的所有方式的**唯一一张表**，以及从配置到 transport 的唯一解析路径。 + +## 功能 +声明 `PROVIDERS`（五条 `ProviderSpec`：`api`、`cursor`、`claude-code`、`codex`、 +`deepseek`），从 `Settings.strict_backend` 解析出被选中的那个，并为 harness 类型返回 +一个 `HarnessTransport`。已声明但尚未发布的后端住在 `_UNSHIPPED` 里，抛出指向里程碑的 +错误，而不是返回一个根本跑不起来的 transport。 + +## 公开契约 +`PROVIDERS`、`resolve_provider(settings)`、`transport_for(settings)`、 +`transport_for_id(settings, provider_id)`。 + +## 不变量（**C2**、**B1**） +- **唯一一条解析路径。** 原有的裸 API 路径就是这张表里的 provider `api` —— 不是一条 + 平行分支。用 `api` 时，行为与注册表出现之前**逐字节一致**（平价棘轮）。 +- **未知 id 绝不静默解析。** `resolve_provider` 抛错并列出合法集合；`Settings` 也在 + 前面先拒一次。两层，因为 `.env` 里的一个拼写错误**不该**启动一次注定失败的 run。 +- **`api` 没有 transport。** `transport_for_id` 对它抛错是**设计如此**：API 路径的解析 + 仍然归 `Settings.tier_target`/`llm.LLM`。 +- **未发布 ≠ 坏掉。** `_UNSHIPPED` 里的 id 会抛出点名其里程碑的 `NotImplementedError`， + 于是 `strict_readiness`/doctor 报"尚未发布"，而不是跑到一半失败。 +- `transport_for_id` 解析的是**显式** id，与本次 run 的 `STRICT_BACKEND` 无关 —— + 这正是 MoA harness 成员在 api 后端的 run 内部骑上某个 harness 所用的接缝。 + +## 边界 —— 不属于这里 +不含 transport 实现（各自归 `providers/.py`）；不处理凭据；不做模型选择 +（那是 `Settings.tier_target`）。 + +## 依赖（允许） +仅 `.base`。transport 模块在 `transport_for_id` 内部**惰性 import**，这样装某个后端的 +SDK 永远不会成为使用另一个后端的前提。 + +## 扩展点 +加一条 `ProviderSpec` + 一个 `HarnessTransport` 子类 + `transport_for_id` 里的一个分支。 +transport 还没就绪时，先登记进 `_UNSHIPPED`。 + +## 测试 +`test_providers.py`；逐后端的 +`test_provider_{cursor,claude_code,codex,deepseek}.py`。 + +## 重构备注 +`transport_for_id` 的 if 链是唯一会随后端数量增长的地方；用 id → import 路径的 dict +能消掉分支，但会掩盖惰性 import 的意图。**在它超过约 8 条之前，保持链式写法。** diff --git a/doc/architecture/SPEC/push.md b/doc/architecture/SPEC/push.md new file mode 100644 index 000000000..bfd026945 --- /dev/null +++ b/doc/architecture/SPEC/push.md @@ -0,0 +1,35 @@ +# push.py —— 规范 + + + +`LOC ~46 · 安全原语（推送授权） · refactor-status: ok` + +## 职责 +唯一的推送授权 choke point，以及它的 policy/decision 类型。 + +> 历史：它曾是 `targets/base.py`，那是一个从未成为真正抽象的"Target 层"设计的残留 +> （它的任务定义职责由 `TaskSpec` + `Playbook` 承担）。死掉的 target dataclass 已被 +> 移除（精简 K1），模块也改名为它**实际**是的东西 —— 推送授权。**不存在 Target 层。** + +## 公开契约 +`PushPolicy`、`PushDecision`、`guard_push(policy, protected_branches)`。 + +## 不变量（**C4**） +- 只有在策略允许**且**分支不受保护时，推送才会发生。 +- force 只有 with-lease；受保护分支**永不被推送**（无论是否 force），与策略无关。 +- **唯一**的推送授权点 —— `ci.push` 和原生 phase-4 都在这里汇合。 + +## 边界 —— 不属于这里 +不执行 git（那是 step 的事）；不含仓库知识；不做 dry-run 判断（那是 step 读 +`ALLOW_PUSH` 的事）。 + +## 依赖（允许） +仅 stdlib。它是一个叶子安全原语（和 `scopes.py` 一样）。 + +## 测试 +`test_push_and_steps.py`。 + +## 重构备注 +安全攸关且纯粹 —— 保持它无依赖、无副作用。**每一条推送路径都必须经过 +`guard_push`**；一个自己重新实现这些检查的新推送点就是缺陷。它和 `scopes.py` 是天然 +的兄弟（两者都是纯权限原语）；如果将来引入 `safety/` 包，它们应该住在一起。 diff --git a/doc/architecture/SPEC/rebase/monitor.md b/doc/architecture/SPEC/rebase/monitor.md new file mode 100644 index 000000000..0817c5592 --- /dev/null +++ b/doc/architecture/SPEC/rebase/monitor.md @@ -0,0 +1,32 @@ +# rebase/monitor.py —— 规范 + + + +`LOC ~159 · 边缘（外部流水线监控） · refactor-status: ok` + +## 职责 +为 locked 的 `rebase.run_external` 委托，读取并分类父编排器的状态。 + +## 公开契约 +`build_command`、`parse_parent_state`、`summarize_progress`、`diff_progress`、 +`classify_failure`、`build_escalation`。 + +## 不变量 +- 对父进程的 `state.json` **只读**。 +- 把退出码 + 状态分类成**类型化失败** + 升级材料。 +- **感知陈旧状态** —— 上一次 run 的 `phase=done` **绝不能**掩盖这一次 run 的崩溃。 +- 点名父包/路径（被允许的仓库字面量，泄漏上限为 1）。 + +## 边界 —— 不属于这里 +不运行也不重新实现 rebase；不含推送逻辑；不发通知（那是 `notify`）。 + +## 依赖（允许） +仅 stdlib 的 `json`/`subprocess`。 + +## 测试 +`test_rebase_monitor.py`。 + +## 重构备注 +内聚的监控/分类单元。这里的基线签名比较，正是 `ci/normalize` 模块**当初写出来就是为了 +不去继承**的已知弱点 —— 如果将来要收紧这个 monitor 的分类，请**复用 `ci/normalize`**， +而不是再实现一遍字符串比较。 diff --git a/doc/architecture/SPEC/review/diff_summary.md b/doc/architecture/SPEC/review/diff_summary.md new file mode 100644 index 000000000..a8e71000b --- /dev/null +++ b/doc/architecture/SPEC/review/diff_summary.md @@ -0,0 +1,30 @@ +# review/diff_summary.py —— 规范 + + + +`LOC ~62 · 评审（always-on 阶段） · refactor-status: ok` + +## 职责 +patch review 里那个**廉价、常开**的第一阶段。 + +## 公开契约 +`build_diff_summary(repo, base_ref, primary_files, trace) -> DiffSummary` +（改动文件、增删行数、越界文件、整文件写、已跑测试、是否请求推送）。 + +## 不变量 +- 确定性；**不含 LLM**。 +- 读取 git diff + RunTrace 事件（`out_of_scope_edit`、`full_file_write`、 + `test_run`、`push_requested`）来构建摘要。 + +## 边界 —— 不属于这里 +不做触发判断（那是 `triggers`），不出裁决（那是 `reviewer`）。 + +## 依赖（允许） +`run_trace`；stdlib 的 `subprocess`/`fnmatch`。 + +## 测试 +`test_review.py`。 + +## 重构备注 +单阶段，很干净。**保持它便宜（不调 LLM）** —— 它每道门都会跑；只有当这份摘要触发了 +某条规则时，LLM 才会跑。 diff --git a/doc/architecture/SPEC/review/planner.md b/doc/architecture/SPEC/review/planner.md new file mode 100644 index 000000000..cfbd53a42 --- /dev/null +++ b/doc/architecture/SPEC/review/planner.md @@ -0,0 +1,41 @@ +# review/planner.py —— 规范 + + + +`LOC ~307 · 评审深度选择 · refactor-status: ok` + +## 职责 +为一个 PR 选定评审深度（`light` / `standard` / `full`）。 + +## 功能 +4-lens ensemble 是重大改动的**召回下限**，但要花约 4 倍的 token 和时延（实测： +一个 2 文件 /+60 行的 PR 消耗了约 110 万输入 token、耗时 4 分 48 秒，其中 97% 在评审 +阶段）。确定性规则先裁明确案例；只有灰区才花一次小的 LLM 调用。 + +## 公开契约 +`plan_review_depth(...) -> depth`，以及那些确定性分类器。 + +## 不变量（**C2**、**B2**） +- **`light` 永远不可能来自模型输出。** 灰区那次调用**只能**返回 `standard` 或 `full`。 + 这正是这里真正要紧的抗提示注入性质：**PR 里的内容说服不了 planner 少审自己。** +- 深度不变量就地强制：`light` ⇒ 无 lens；`standard` ⇒ 2–3 个 lens；`full` ⇒ 全部。 +- 确定性规则**先跑**，不花模型调用就裁掉明确案例（小 + 低风险 → light； + 大或高风险路径 → full）。 +- playbook 的 `review_depth` 参数可以显式 pin 住深度，覆盖 planner。 +- **灰区调用需要推理留白。** 它曾在两个战役里对**每一个**灰区条目静默失败，因为思考 + 在任何 JSON 出现之前就吃完了 400-token 的上限 —— 这里的 cap 是**正确性设置， + 不是成本旋钮**。 + +## 边界 —— 不属于这里 +不执行 lens（`agent_runtime/ensemble.py`）；不做渲染；不含仓库专属风险清单 +（那些来自 profile / adapter 的模块映射）。 + +## 依赖（允许） +`..config`、`..llm`、stdlib。**不 import 任何 step。** + +## 测试 +`test_review_planner.py`。 + +## 重构备注 +注意包的划分：这个文件管评审**深度**，而 `engine/steps/review/` 装的是评审 step 本身。 +**同一个词，两个包 —— 不要合并它们。** diff --git a/doc/architecture/SPEC/review/reviewer.md b/doc/architecture/SPEC/review/reviewer.md new file mode 100644 index 000000000..669df2752 --- /dev/null +++ b/doc/architecture/SPEC/review/reviewer.md @@ -0,0 +1,31 @@ +# review/reviewer.py —— 规范 + + + +`LOC ~89 · 评审（裁决） · refactor-status: ok` + +## 职责 +只读的 patch/plan 评审裁决。 + +## 公开契约 +`run_patch_review(llm, diff_text, summary, fired_rules, model) -> verdict`； +`run_plan_review(llm, playbook_doc, task, model) -> verdict`。 + +## 不变量（**C6**） +- 没有 reviewer LLM 时返回 `unavailable`（**缺少 reviewer 不等于静默放行**）。 +- reviewer 输出无法解析时降级为 `revise`。 +- 除 `lgtm`/`pass` 之外一律视为**未通过**。 + +## 边界 —— 不属于这里 +它不是评审 **step**（那是 `engine/steps/review.py`）；不含触发逻辑；不构建 diff。 + +## 依赖（允许） +`llm`；评审 prompt 就住在这里。 + +## 测试 +`test_review.py`。 + +## 重构备注 +**fail-closed 就是那条不变量** —— 绝不要新增"reviewer 缺失/失败却返回通过"的路径。 +如果 plan review 和 patch review 分歧变大，再拆成两个文件；今天它们共享同一个 +fail-closed 形状，理应住在一起。 diff --git a/doc/architecture/SPEC/review/triggers.md b/doc/architecture/SPEC/review/triggers.md new file mode 100644 index 000000000..320ea5b1d --- /dev/null +++ b/doc/architecture/SPEC/review/triggers.md @@ -0,0 +1,31 @@ +# review/triggers.py —— 规范 + + + +`LOC ~50 · 评审（触发规则） · refactor-status: ok` + +## 职责 +决定 LLM patch review 什么时候触发。 + +## 公开契约 +`evaluate_triggers(summary, settings, *, touched_modules, pre_push, +knowledge_edit, high_risk_modules?) -> fired[]`；`ALL_RULES`（7 条）。 + +## 不变量 +- 七条规则：`out_of_scope_edits`、`high_risk_modules`、`large_diff`、 + `tests_unavailable`、`full_file_fallback`、`before_push`、`knowledge_edit`。 +- 高风险模块来自**调用方**（adapter），settings 只作为兜底（**A5**）。 + +## 边界 —— 不属于这里 +不构建 diff（那是 `diff_summary`），不调 LLM（那是 step 的事 —— 触发时它才调 +`reviewer`）。 + +## 依赖（允许） +`config`、`review/diff_summary`。 + +## 测试 +`test_review.py`。 + +## 重构备注 +**这套规则集就是契约** —— 增删一条触发器就改变了安全姿态（**C6**），必须是刻意且 +有记录的变更。保持它是 (summary, settings, 调用方标志) 的纯函数。 diff --git a/doc/architecture/SPEC/run_status.md b/doc/architecture/SPEC/run_status.md new file mode 100644 index 000000000..03fb444c4 --- /dev/null +++ b/doc/architecture/SPEC/run_status.md @@ -0,0 +1,43 @@ +# run_status.py —— 规范 + + + +`LOC ~247 · 持久化的单写者 run 生命周期记录 · refactor-status: ok` + +## 职责 +`run_status.json` —— 一次 Strict run 的**持久、无歧义**的生命周期记录，可跨进程观测。 + +## 功能 +server 把每次 run 作为子进程拉起，且**只能通过文件系统**观测它，所以状态必须能挺过 +server 重启，并且必须能把**崩掉的 run 和正在跑的 run 区分开**（靠"文件在不在"这类 +启发式做不到）。 + +## 公开契约 +`reserve_run`、`mark_child_started`、`read_status`，以及状态常量 +（`queued`/`planning`/`running`/终态/`interrupted`/`FAILED`）。 + +## 不变量（**C3**、**E1**） +- **单写者。** `reserve_run`（server，在子进程存在之前）写下 `queued`；一旦拉起， + **子进程就是运行期唯一的写者**：它的第一件事是通过 `mark_child_started` 写下自己的 + pid，然后 `planning → running → 终态`。父进程**只在 `.wait()` 之后**对账 —— + 也就是在子进程已确认死亡之后。 +- **跨进程对账只发生在写者被确认死亡之后**，持 `flock` 进行，并保留 owner 字段。 +- **按属主对账**（`owner_server_id` / `owner_server_pid` / `child_pid`）：只有**属主** + server 被确认死亡，才可以把一个非终态 run 标记为 `interrupted`。在多 server 模型下 + （Claude Code 和 Codex 各自拉起一个 server），正是这一条阻止了某个 server 去抢另一个 + server 仍然活着的 `queued` run。 +- **没有 run 会永远停在非终态**：对账发生在每次 `get_*` 时（惰性）、父进程 `wait()` + 之后、以及启动扫描时 —— **三处**。 + +## 边界 —— 不属于这里 +不拉起进程（那是 `mcp_server`）；不含策略；不渲染报告。 + +## 依赖（允许） +仅 stdlib（`json`、`os`、`fcntl`/`flock`、`pathlib`）。 + +## 测试 +`test_mcp.py`（单写者对账、属主判定）。 + +## 重构备注 +这里的每一处改动，都必须**带着"两个 server + 一个已死子进程"的场景**去推演； +"精简掉 owner 字段" 会把跨 server 抢 run 的问题原样请回来。 diff --git a/doc/architecture/SPEC/run_trace.md b/doc/architecture/SPEC/run_trace.md new file mode 100644 index 000000000..43f647a22 --- /dev/null +++ b/doc/architecture/SPEC/run_trace.md @@ -0,0 +1,39 @@ +# run_trace.py —— 规范 + + + +`LOC ~40 · 跨切（审计脊柱） · refactor-status: ok` + +## 职责 +仅追加的 JSONL 事件日志 —— 不可变的审计层。 + +## 功能 +`record(event, **fields)` 追加一行 JSON；`events(name)` 按类型过滤。 + +## 公开契约 +`RunTrace(path)`，带 `record`、`events`。 + +## 不变量 +- **E1**：每一条治理主张都映射到一个 trace 事件（`agent_dispatch`/`agent_output`、 + `tool_call`/`tool_refused`/`out_of_scope_edit`/`full_file_write`、 + `patch_review*`、`push_requested`、`capability_gap`、`env_exported`、 + `posted_artifact`、`profile_*`）。 +- 事实可以自由记录（**D1**）；这是精选 profile 之下的**不可变层**。 + +## 边界 —— 不属于这里 +只负责记录。不含策略，不过滤"什么可以被记录"，也不做驱动控制流的读取 +（`events()` 仅供 diff-summary / metrics 组装使用）。 + +## 依赖（允许） +仅 stdlib。 + +## 扩展点 +新事件 → 在现场直接 `record("", ...)`；如果它支撑某条保证，就在 +`_CONSTRAINTS.md` §E1 里记下这个名字。 + +## 测试 +全套测试通过 `trace.events(...)` 断言间接覆盖。 + +## 重构备注 +**刻意保持平凡且无依赖 —— 请维持现状。** 它托着每一条治理主张，所以绝不能引入 +可能失败、进而丢事件的逻辑。 diff --git a/doc/architecture/SPEC/scopes.md b/doc/architecture/SPEC/scopes.md new file mode 100644 index 000000000..0a1e077ca --- /dev/null +++ b/doc/architecture/SPEC/scopes.md @@ -0,0 +1,38 @@ +# scopes.py —— 规范 + + + +`LOC ~94 · 引擎（权限） · refactor-status: ok` + +## 职责 +路径级的工具权限 —— dispatcher 强制执行的那套权限词汇。 + +## 功能 +`ToolScope.check`（工具是否允许？写路径是否允许？）；`PathScope.check_write` +（可写硬墙 + primary 归属文件）；scope 工厂函数。 + +## 公开契约 +`ToolScope(name, allowed_tools, path_scope?, read_only)`；`PathScope(writable, +primary)`；`Decision`；`read_only_scope`、`pre_plan_scope`、`post_plan_scope`； +工具集常量（READ/WRITE/EXEC）。 + +## 不变量 +- **三种结果**：允许 / 拒绝（工具不在集合内、写到 `writable` 之外、或处于只读 + scope）/ 越界（在 `writable` 内但在 `primary` 之外 —— **允许并记录**）。 +- `writable` 是一堵硬墙；`primary` 定义该模块归属的文件。 + +## 边界 —— 不属于这里 +只做权限判定 —— 不执行、不记 trace（由 dispatcher 记）。 + +## 依赖（允许） +仅 stdlib。 + +## 扩展点 +新的 scope 形状 → 加一个工厂函数；保持三结果的 `Decision` 契约。 + +## 测试 +`test_scopes_tools.py`。 + +## 重构备注 +干净、无依赖、安全攸关。保持它是纯的（无副作用），这样它才一直是可轻松测试的。 +`engine/step.py` 会 import 它 —— **不要**加反向 import。 diff --git a/doc/architecture/SPEC/task_spec.md b/doc/architecture/SPEC/task_spec.md new file mode 100644 index 000000000..fa408e3bb --- /dev/null +++ b/doc/architecture/SPEC/task_spec.md @@ -0,0 +1,38 @@ +# task_spec.py —— 规范 + + + +`LOC ~70 · 任务层，纯数据 · refactor-status: ok` + +## 职责 +定义 `TaskSpec`（意图解析的结构化产物），并从任务 kind **推导**出它的权限 **tier**。 + +## 功能 +持有 kind/repo/pr/issue/flags；计算 `tier`、`read_only`、`confirm_required`， +以及给人看的 `describe()`。 + +## 公开契约 +`TaskSpec(kind, repo, pr?, issue?, report_only, post, params)`；property +`tier`、`read_only`、`confirm_required`；`describe()`。常量：`TaskKind` +（7 种 kind）、`READ_ONLY_KINDS`、`KIND_TIER`。 + +## 不变量 +- **C1**：**不存在可设置的 tier 字段**；`tier = KIND_TIER[kind]` —— 文本无法把它扩大。 +- 只读 kind 的 `read_only` = `not post`，其余为 `report_only`； + `confirm_required = not read_only`。 + +## 边界 —— 不属于这里 +不解析、不做 I/O、不执行、不含仓库知识。纯数据 + 推导。 + +## 依赖（允许） +仅 `pydantic`。 + +## 扩展点 +新 kind → 加进 `TaskKind` + `KIND_TIER`（只读的话再加 `READ_ONLY_KINDS`）。 + +## 测试 +`test_intent_taskspec.py`。 + +## 重构备注 +干净、极简 —— "单一职责"的范例。**不要**在这里加行为；保持它是"数据 + 推导"模块。 +它是 C1 的唯一真相来源，所以任何**别处**出现的权限逻辑都是坏味道。 diff --git a/doc/architecture/SPEC/thin_mcp_server.md b/doc/architecture/SPEC/thin_mcp_server.md new file mode 100644 index 000000000..e9a4b8c1b --- /dev/null +++ b/doc/architecture/SPEC/thin_mcp_server.md @@ -0,0 +1,56 @@ +# thin_mcp_server.py —— 规范 + + + +`LOC ~806 · 默认 MCP：Direct 路由 + Strict 入口 · refactor-status: oversized` + +## 职责 +安装器**实际注册**的那个 MCP 门面：以**零模型**提供 Direct 模式的知识路由， +并在被要求时桥接到 Strict。 + +## 功能 +七个工具：`review`（按 `mode` 分流）、`validate_direct_review`、 +`get_review_status` / `get_review_result`（转发给 `CopilotMCP`）、 +`update_knowledge`、`doc_search`、`doc_read`。 + +## 公开契约 +上述七个工具；`build_mcp(...)`；`main()`。 + +## 不变量（**C1**、**C2**、**D1**） +- **Direct 在这个 server 里不跑任何模型。** 它返回知识路由和一份治理契约；阅读由 + **宿主自己的模型**完成。执行主脊完全不参与。 +- **治理靠数据，因为 server 管不住宿主。** "该怎么审"被编码成随返回值一起下发的结构化 + 字段：≤3 条路由（内嵌 `quick_map`，3.5k 封顶）、一个硬性的 `execution_budget`、 + 一份 checklist。 +- **`validate_direct_review` 检查的是结构，不是证据真伪**：恰好一条最终评论、 + `subtraction_signal` 的自洽性（`none` 不得附带证据；`triggered` 需要减法项或最小性 + 证明）、以及证明本次评审读的是固定提交的 `evidence_head_sha`。 + **它不能也没有**去验证被引用的证据是否真实 —— 声称它能，比不声称更糟。 +- **路由绝不静默替换。** `title`/`body` 选 owner；`changed_files` 通常只做范围校验。 + 它们只在**最后手段**下选路（当存活路由无一命中它们推导出的 owner 时），且该情况是 + **显式的**：`status="scope_fallback"`、`selected_by="title_body+changed_files"`， + 且每条这样的路由都会说明理由。 +- **仓库守卫先跑。** 不支持的仓库在任何路由推导**之前**就返回 + `unsupported_exact_router` —— 否则一个没有描述的、来自陌生仓库的 PR，会被喂上 + 本仓库的 owner 知识。 +- `_knowledge_path` 阻止逃出知识根；`_guard` 把异常转成 `{"error": ...}` 值返回， + 而不是协议层错误。 +- **Strict 绝不启动注定失败的 run**：Strict 分支先查 `strict_readiness`， + 改为返回缺失项。 +- `update_knowledge` 只返回知识贡献入口 —— 它**不是** `imupdate` 的发版审计器。 + +## 边界 —— 不属于这里 +Direct 路径里不调模型；不含 Strict 后台机器（`mcp_server.py`）；不定义策略 +（`mcp_policy.py`）。 + +## 依赖（允许） +stdlib + `mcp` extra + `.adapters` + `.config` + `.intent.resolve_repo_alias` + +`.knowledge_docs` + `.mcp_policy` + `.mcp_server`。 + +## 测试 +`test_thin_mcp_server.py`、`test_thin_mcp.py`、`test_imreview_output_contract.py`。 + +## 重构备注 +约 806 行，是包里**最大**的模块；Direct 路由 helper（`_direct_*`）是一个内聚的约 350 +行单元，如果再次增长，那就是显而易见的拆分点。**拆分时务必保住"server 不跑模型"这条 +性质 —— 它就是 Direct 模式的产品承诺。** diff --git a/doc/architecture/SPEC/tool_bridge.md b/doc/architecture/SPEC/tool_bridge.md new file mode 100644 index 000000000..62b25557c --- /dev/null +++ b/doc/architecture/SPEC/tool_bridge.md @@ -0,0 +1,50 @@ +# tool_bridge.py —— 规范 + + + +`LOC ~308 · 面向 harness 会话的 scoped 工具 MCP server · refactor-status: ok` + +## 职责 +以 stdio MCP server 的形式，把本次 run 的 scoped 工具暴露给 harness 会话 —— +好让"把一个 step 委托给厂商 agent"**不等于把权限闸也一并委托出去**。 + +## 功能 +`python -m infermatrix_copilot.tool_bridge --spec ` —— +一个由 harness 从它的 MCP 配置里拉起的 server。它提供本次 run 的内置工具、 +`doc_search`/`doc_read`、按需的 `repo_map`，以及只读的变更考古工具组 +（`diff_stat`、`file_at_base`、`show_commit`、`search_history`、`calc`）。 + +## 公开契约 +`write_bridge_spec(...)`、`load_bridge_spec(path)`、`make_dispatcher(...)`、 +`build_server(spec_path)`、`main(argv)`。 + +## 不变量（**C1**、**C2**、**E2**） +- **每一次内置工具调用都经过 `tools.dispatch`**，带着该 step 反序列化出来的 + `ToolScope`，因此拒绝行为、相对路径对 PR-time worktree 的解析、以及结果上限， + 都与进程内循环**表现一致**。 +- **读取受容纳约束 —— 比进程内还严。** `ToolScope` 只对**写**做路径守卫；而 harness + 是持有不可信 PR diff 的**低信任调用方**，所以桥会拒绝容纳根（scope 根 + run 目录） + 之外的 read/list/grep 目标。**这就是防 `.env` 外泄的那道守卫。** +- **独立的 trace 文件。** 工具事件追加到 spec 旁边的 `bridge_trace.jsonl`； + 第二个进程**绝不能**与父进程的 `run_trace.jsonl` 交错。 +- **`repo_map` 失败只降级，绝不崩溃** —— 记一条 `capability_gap`（`bridge.repo_map`）。 +- **skill/memory 检索刻意不桥接。** 那两个工具会提出知识 candidate； + 开一条跨进程写入路径的方案**已被否决**。harness 会话可以**读**本仓库的知识， + **永远不能往里加**。 + +## 边界 —— 不属于这里 +不调用 transport（那归各 `providers/.py`）；不构造 scope（那归 step）； +**永不写知识**。 + +## 依赖（允许） +stdlib + `mcp`（`[mcp]` extra）+ `..tools` + `..scopes` + `..run_trace` + +knowledge/repo-map 工厂。 + +## 测试 +`test_tool_bridge.py`。 + +## 重构备注 +**并非每个后端都够得到这个 server**：`providers/deepseek.py` 跑在 harness 原生 bash 上， +因为它捆绑的运行时不带 MCP client，拿到无法履约的 spec 时会记一条 `capability_gap`。 +任何"这次 run 走了工具桥"的说法，都必须**对照那个标志核实**， +而不是从"后端是 harness"推定。 diff --git a/doc/architecture/SPEC/tools.md b/doc/architecture/SPEC/tools.md new file mode 100644 index 000000000..0df0de246 --- /dev/null +++ b/doc/architecture/SPEC/tools.md @@ -0,0 +1,47 @@ +# tools.py —— 规范 + + + +`LOC ~228 · 引擎（能力 + choke point） · refactor-status: ok` + +## 职责 +原子能力，以及**唯一那个强制 scope 的 dispatch choke point**。 + +## 功能 +定义内置工具（read_file、write_file、edit_file、list_dir、grep、run_shell）； +`dispatch` 对每次调用做 scope 检查、执行并记 trace。 + +## 公开契约 +`ToolDef`；`TOOLS`；`tool_definitions_for(scope, extra?)`； +`dispatch(name, args, *, scope?, trace?, extra?) -> {ok, result|error, out_of_scope}`。 + +## 不变量（**C3**） +- 每次内置调用都做 scope 检查；被拒 → **返回错误值**（绝不抛异常）。 +- 越界的写**会执行**但发出 `out_of_scope_edit`；整文件写 `.py` 发出 `full_file_write`。 +- **错误是观测结果，不是崩溃。** +- 额外（由 step 提供的）工具绕过内置 allowlist，但**仍然被记 trace**。 +- **`read_file` 是窗口化的（48k 字符，用 `offset` 翻页），不是整文件读。** + 整文件读会吹爆会话历史、成倍增加未缓存 token，并把会话推出可靠缓存长度 —— + 这是**实测出来的成本，不是谨慎**。 +- **`grep` 默认按字面量匹配**（要正则得传 `regex:true`），所以搜 `items[0]` 不需要转义。 + 一个默认正则的工具，会对**最常见的查询形状**静默给出错误答案。 +- 同一个 `dispatch` 通过 `tool_bridge.py` 服务 harness 后端， + 所以**在这里改强制逻辑，等于对所有后端同时改**。 + +## 边界 —— 不属于这里 +只回答"能做什么" —— 工具表达**能力**，不表达工程语义（那是 step，**A3**）。 +不含任务/仓库逻辑。 + +## 依赖（允许） +`run_trace`、`scopes`；stdlib。 + +## 扩展点 +新内置工具 → 在 `TOOLS` 里加一个 `ToolDef`（会写就声明 `write_path_arg`）。 +step 专属工具作为 `extra` 传入，**不加在这里**。 + +## 测试 +`test_scopes_tools.py`。 + +## 重构备注 +dispatch choke point 是一条**硬安全不变量** —— 每条路径（内置和 extra）都必须保持 +被记 trace 且被 scope 检查。**不要新增绕过 `dispatch` 的"快路径"。** 当前体量没问题。 diff --git a/doc/architecture/SPEC/tracing.md b/doc/architecture/SPEC/tracing.md new file mode 100644 index 000000000..87af2ea90 --- /dev/null +++ b/doc/architecture/SPEC/tracing.md @@ -0,0 +1,38 @@ +# tracing.py —— 规范 + + + +`LOC ~652 · 可移植的 span 树记录器 · refactor-status: oversized` + +## 职责 +把一次 run 记录成一棵计时 span 树，**零外部依赖**。 + +## 功能 +OpenTelemetry **形状**的 span（`trace_id`、`span_id`、`parent`、`start`、`end`、 +`attributes`）追加进一个 JSONL 文件，**一行一个 span，在 span 关闭时写出**。 + +## 公开契约 +`span(...)`（上下文管理器）、模块级 tracer 访问器，以及那套 JSONL 记录形状。 + +## 不变量（**E1**、**E3**） +- **在 span 关闭时写、仅追加。** 被杀掉的 run 会保留**每一个已完成**的 span —— + 这正是它不是"内存里建树、退出时冲刷"的原因。 +- **同步与 asyncio 双安全。** `span()` 是可以裹在 `await` 外面用的普通上下文管理器； + 父子嵌套经 `contextvars` 承载并**按 task 复制**，因此**并行的 agent 会得到各自独立 + 且正确的树**，而不是交错成一棵。 +- **零外部依赖** —— **刻意不用** OTel SDK。形状是兼容的，依赖不引入。 +- `create()` 被裹在 `span("llm")` 里，用于记录 TTFT、token 和并发。 +- 与 `run_trace.py` 不同：这里是**计时**，那里是**事实**。两者默认都不进 prompt。 + +## 边界 —— 不属于这里 +不记录事实（`run_trace.py`）；不计算指标（`metrics.py`）；不含导出协议。 + +## 依赖（允许） +仅 stdlib（`contextvars`、`json`、`time`、`threading`）。 + +## 测试 +`test_tracing.py`、`test_trace_pack.py`。 + +## 重构备注 +约 652 行，是最大的叶子模块。如果继续增长，span 模型、JSONL 写入器、进程级 tracer +管道是三个可分离的关注点。 diff --git a/doc/architecture/SPEC/ui.md b/doc/architecture/SPEC/ui.md new file mode 100644 index 000000000..33a3951be --- /dev/null +++ b/doc/architecture/SPEC/ui.md @@ -0,0 +1,35 @@ +# ui.py —— 规范 + + + +`LOC ~164 · 呈现 · refactor-status: ok` + +## 职责 +终端渲染：流式输出、spinner、markdown、颜色；在非 TTY / 管道下降级为纯文本。 + +## 功能 +`make_ui(out?)` 返回 rich UI 或 `PlainUI`；流的 start/delta、带样式的 step/tool 输出、 +markdown 渲染、输入历史。 + +## 公开契约 +`make_ui`、`style(...)`，以及 UI 对象的 `stream_*` / print 方法。 + +## 不变量 +- **只做呈现** —— 不携带控制流或 run 状态。 +- 确定性的纯文本回退，因此脚本化输出保持稳定。 + +## 边界 —— 不属于这里 +不决定**做什么** —— 只决定**长什么样**。不含任务/规划/执行逻辑；不含仓库知识。 + +## 依赖（允许） +`rich`；stdlib。 + +## 扩展点 +新的渲染元素 → 在 UI 上加一个方法；**同步维护 `PlainUI` 回退**。 + +## 测试 +`test_ui.py`。 + +## 重构备注 +职责分离得很干净。唯一的坏味道：`cli/`/`chat.py` 会先内联格式化一些字符串再交给 `ui`； +一次重构可以把更多格式化搬到这里，让调用方传**结构化数据**而不是预格式化字符串。 diff --git a/doc/IMPLEMENTATION_STATUS.md b/doc/archive/IMPLEMENTATION_STATUS.md similarity index 97% rename from doc/IMPLEMENTATION_STATUS.md rename to doc/archive/IMPLEMENTATION_STATUS.md index 4fa7bffbb..b6014e67b 100644 --- a/doc/IMPLEMENTATION_STATUS.md +++ b/doc/archive/IMPLEMENTATION_STATUS.md @@ -1,5 +1,12 @@ # Implementation status vs the design's 15 tasks +> **冻结（2026-08-17）。** 停在 2026-07-23 的状态，**不代表现状**——它写着 +> pr-review v4 / 64 tests / 20 steps，实际是 v6 / 430+ 用例 / 38 steps。 +> 现状看 [`../GUIDE.md`](../GUIDE.md)。保留它是作为设计 15 项任务的交付记录。 + +> **冻结存档，保持英文原文。** 内容停在各自的日期上，不代表现状；翻译一份存档等于改动它。现行文档统一为中文 —— 见 [`../README.md`](../README.md)。 + + Task numbers from `copilot_design` §四 (milestones from `docs/copilot/implementation/`). "Here" = implemented in this repo, offline-tested (64 tests). @@ -149,7 +156,7 @@ and `omni-rebase-orchestrator --resume` concurrently (copilot progress.json is authoritative inside a copilot run; parent markers are still written so the parent's resume works after abandoning the copilot run). -## Design v2 (doc/DESIGN.md Part II) — status +## Design v2 (doc/architecture/DESIGN.md Part II) — status **P0 — correctness (2026-07-10, done):** the §V2.1(a) fixes, pinned by `test/test_v2_p0.py`: diff --git a/doc/PLAN-knowledge-reorg.md b/doc/archive/PLAN-knowledge-reorg.md similarity index 77% rename from doc/PLAN-knowledge-reorg.md rename to doc/archive/PLAN-knowledge-reorg.md index b77a0e9be..91ccbd295 100644 --- a/doc/PLAN-knowledge-reorg.md +++ b/doc/archive/PLAN-knowledge-reorg.md @@ -1,15 +1,21 @@ # Knowledge reorganization v2 — vendored, code-mirrored, detail-preserving +> **冻结（2026-08-17）。** 历史迁移计划，迁移已完成。它仍是溯源规则与 +> `_archive/` 约定的出处，因此保留；不要照它规划新工作。 + +> **冻结存档，保持英文原文。** 内容停在各自的日期上，不代表现状；翻译一份存档等于改动它。现行文档统一为中文 —— 见 [`../README.md`](../README.md)。 + + > Historical migration plan. Its `_archive/`-never-delete rule is superseded by > `knowledge/SCHEMA.md`: merge unique live content into the surviving owner, > delete the duplicate, and use Git history for recovery. ## Context -The copilot's knowledge base is currently a git submodule `knowledge/` = `zuiho-kai/claude-workflow-starter` @ `f2dc824`, consumed unmodified. The author has **authorized copying the content** into this repo, so the submodule indirection (and everything it forced: upstream PRs, pin bumps, lockstep commits, cryptographic authorization) is dropped. The tree becomes ordinary tracked files we edit directly, delivered as normal copilot PRs. +The copilot's knowledge base is currently a git submodule `knowledge/` = the upstream knowledge repository @ `f2dc824`, consumed unmodified. Copying the content into this repo was authorized, so the submodule indirection (and everything it forced: upstream PRs, pin bumps, lockstep commits, cryptographic authorization) is dropped. The tree becomes ordinary tracked files we edit directly, delivered as normal copilot PRs. Three-level goal (unchanged): 1. **Folder** — rename `framework/` → `general/`; content and skills each split general (cross-repo) vs `repos//` (repo-specific). -2. **Module** — organize `repos/vllm-omni/` knowledge to mirror the `vllm_omni/` codespace via the `components/` code-owner axis (work-topics `git/ci/remote/benchmark/review/dev/docs` stay parallel — zuiho's no-nesting rule). +2. **Module** — organize `repos/vllm-omni/` knowledge to mirror the `vllm_omni/` codespace via the `components/` code-owner axis (work-topics `git/ci/remote/benchmark/review/dev/docs` stay parallel — the no-nesting rule). 3. **Information** — reorganize page content with **maximum detail retention**: merge as union, add detail, minimize loss. Every content batch is reviewed by **gpt-5.6-sol-high via cursor-agent**. Out of scope / untouched: `repos/jianghan-roleplay-data-pipeline/` (byte-identical); full manifest `modules:` enumeration (deferred — it changes `module_for_path()` classification and PR-verify fan-out; separate tested PR). @@ -30,22 +36,22 @@ Plan history: 10 gpt-5.6-sol-high review rounds (9 on the earlier upstream-PR de - Adding detail, cross-links, and structure is always preferred over shortening. - Incident `编号` values are preserved across any move; `incidents/`, `history/`, `results/` archives are never pruned. -## Page format & lifecycle — LLMWiki mechanics adopted onto zuiho's taxonomy -The wiki pattern's *mechanics* layer onto zuiho's *owner taxonomy* (general vs `repos/` vs `components/` vs `models/` — which plays the role of LLMWiki's type sections, better suited to repo knowledge than entities/concepts): +## Page format & lifecycle — LLMWiki mechanics adopted onto the owner taxonomy +The wiki pattern's *mechanics* layer onto the *owner taxonomy* (general vs `repos/` vs `components/` vs `models/` — which plays the role of LLMWiki's type sections, better suited to repo knowledge than entities/concepts): - **Two-layer mapping:** `incidents/`, `history/`, `results/` = the immutable **raw evidence layer** (LLMWiki `raw/` — read, cited, never pruned). **Immutability means the substance — findings, evidence, numbers, narrative — is never edited.** One narrow exception: Step 1's mechanical path-token migration (`framework/`→`general/` in links and `归属：` values) touches raw files too, verified by the per-file allowlist/normalized-equality check to prove no other byte changed. `rules.md`, `guides/`, `architecture.md`, `_index.md` = the **synthesized layer** that cites it. - **YAML frontmatter on every synthesized page** (added mechanically, enriched in Step 3): `title, created, updated, type (rule|guide|architecture|index|incident|history|result), tags (from the SCHEMA taxonomy), sources ([incident/PR/file:line refs]),` optional `confidence: high|medium|low` and `contested/contradictions` for disputed conclusions. Incident pages keep their existing `编号/归属/状态/搜索词/影响范围` body fields (validator contract) — frontmatter is additive. - **Provenance markers:** synthesized paragraphs citing a specific incident/PR get a trailing `^[incidents/…]`-style source ref, so every rule traces to its evidence. - **SCHEMA.md** at the knowledge root: the tag taxonomy (10–20 tags: modules, models, ci, benchmark, remote, debug, git, review…; new tags added to SCHEMA first), frontmatter spec, page thresholds, update/contradiction policy — thin, harmonized with (not duplicating) `contributing/`. - **Cross-link rule:** every synthesized page links to ≥2 related pages (relative markdown links). - **Extended lint** (sibling of `check_knowledge_tree.py`, run with it): orphan pages (no inbound links), frontmatter completeness, tags ∉ taxonomy, staleness (`updated` far older than newer sources touching the same module), `contested`/`confidence: low` surfacing, index completeness. -- **Deliberately NOT adopted:** `[[wikilinks]]` (keep relative md links — the validator and the copilot `doc_read` tooling resolve them; Obsidian handles both); `log.md` (git history serves this); LLMWiki's 200-line split (zuiho's validator-enforced 300/500-line thresholds win); a separate `raw/` dir (incidents/history/results already are that layer). +- **Deliberately NOT adopted:** `[[wikilinks]]` (keep relative md links — the validator and the copilot `doc_read` tooling resolve them; Obsidian handles both); `log.md` (git history serves this); LLMWiki's 200-line split (the validator-enforced 300/500-line thresholds win); a separate `raw/` dir (incidents/history/results already are that layer). - **Validator tweak:** add `"_archive"` to `check_knowledge_tree.py` `excluded_parts` so archived pages aren't index/link-enforced; `doc_search` hits still show the `_archive/` path prefix, so agents see the status. ## Step 0 — Vendor the tree (non-destructive order, one commit) - **Preflight:** clean worktree; submodule exactly at `f2dc824` with no local changes (`git -C knowledge status --porcelain` empty). -- **Archive BEFORE removing anything:** save into `doc/reorg-audit/baseline/`: the manifest `git -C knowledge ls-tree -r HEAD`, a `git -C knowledge archive f2dc824` tarball + its sha256, **and a per-file upstream date table** (`git -C knowledge log --format=... --name-only` → `dates.tsv`: path → first-commit date, last-commit date) — deleting `.git/modules/knowledge` destroys this history, and Step 2.5's `created`/`updated` frontmatter must reflect upstream authorship dates, not the vendoring commit. This baseline is also what later "byte-identical to f2dc824" checks (jianghan, moved bodies) diff against. The upstream GitHub repo remains a second durable source. +- **Archive BEFORE removing anything:** save into `doc/archive/reorg-audit/baseline/`: the manifest `git -C knowledge ls-tree -r HEAD`, a `git -C knowledge archive f2dc824` tarball + its sha256, **and a per-file upstream date table** (`git -C knowledge log --format=... --name-only` → `dates.tsv`: path → first-commit date, last-commit date) — deleting `.git/modules/knowledge` destroys this history, and Step 2.5's `created`/`updated` frontmatter must reflect upstream authorship dates, not the vendoring commit. This baseline is also what later "byte-identical to f2dc824" checks (jianghan, moved bodies) diff against. The upstream GitHub repo remains a second durable source. - Export the tree from the archive to a temp dir, **verify checksums against the manifest**, then `git submodule deinit -f knowledge && git rm -f knowledge && rm -rf .git/modules/knowledge`, move the verified tree into `knowledge/`, `git add`. -- Provenance + authorization: rewrite `doc/KNOWLEDGE.md` — vendored copy derived from `zuiho-kai/claude-workflow-starter@f2dc824`, reorganized in-repo; **record the author's copy authorization (who/when/scope) verbatim** (upstream ships no LICENSE file, so this recorded authorization is the licensing artifact); keep upstream README attribution; drop the submodule-update instructions. +- Provenance + authorization: rewrite `doc/architecture/KNOWLEDGE.md` — vendored copy derived from `the upstream knowledge repository@f2dc824`, reorganized in-repo; **record the author's copy authorization (who/when/scope) verbatim** (upstream ships no LICENSE file, so this recorded authorization is the licensing artifact); keep upstream README attribution; drop the submodule-update instructions. - `tools/check_knowledge_tree.py` keeps working unchanged (paths derive from `__file__`; the `local/`-untracked check now runs against this repo's git). - Gates now unify: the knowledge validator and the copilot `pytest` run in the **same repo/branch** — no cross-repo sequencing. Rollback of any step = revert its commit(s). @@ -53,13 +59,13 @@ The wiki pattern's *mechanics* layer onto zuiho's *owner taxonomy* (general vs ` - `git mv knowledge/framework knowledge/general`. - Validator literals: `KNOWLEDGE_ROOTS` (L14), the `"framework"` member of `excluded_parts` (~L327), L1 docstring. - Anchored `\bframework/` → `general/` across all knowledge markdown (README/CLAUDE/AGENTS/CONTRIBUTING, `contributing/*`, all `_index.md` navs/cross-links, `skills/**`, `.claude/commands/*`), plus a manual per-occurrence pass for bare folder mentions (`### framework/<主题>/`, `归属：framework/remote`, "framework tree" prose). Never blind-replace — the English ML-"framework" concept (e.g. `algorithm-vs-framework-fix.md`) survives. -- Copilot lockstep in the same branch: `config.py:56 knowledge_general_docs=["general/_index.md"]` (+comments), `test/test_knowledge_source.py` fixtures/asserts, docstrings in `adapters/base.py` + `engine/agent_runtime/{knowledge,runner}.py`, `doc/KNOWLEDGE.md`, stale comment in `adapters/vllm_omni/manifest.yaml`; then a repo-wide `git grep -nE '\bframework/'` sweep to catch stragglers. +- Copilot lockstep in the same branch: `config.py:56 knowledge_general_docs=["general/_index.md"]` (+comments), `test/test_knowledge_source.py` fixtures/asserts, docstrings in `adapters/base.py` + `engine/agent_runtime/{knowledge,runner}.py`, `doc/architecture/KNOWLEDGE.md`, stale comment in `adapters/vllm_omni/manifest.yaml`; then a repo-wide `git grep -nE '\bframework/'` sweep to catch stragglers. - Gate: validator 0 errors + full `pytest` green + grep returns only concept hits. ## Step 2 — Skills split + module mirror - **Skills:** `git mv skills/model-adaptation-review adapters/vllm_omni/skills/` (vllm-omni-specific → adapter store; `_knowledge_stores` already searches it first). Add a retrieval regression test (runs with the full copilot suite as this step's gate): repo-priority retrieval for vllm-omni; no cross-repo bleed. In-tree `knowledge/skills/*` (claudeception/reflect-system/clean-thinking) are **archival/reference content — NOT loaded by the copilot's SkillStore** (which reads only `skills/` + adapter stores); they stay in place as part of the vendored tree for humans/agents working *on* the knowledge repo itself. `code-quality-review` stays put for now. -- **Module mirror (direct, human-curated — no owner map):** pin a `vllm-omni` SHA and **record it in `doc/KNOWLEDGE.md` + each component `_index.md`**; `ls` the real `vllm_omni/` tree (verify every referenced dir actually exists — globs like `platforms/*/worker/` are expanded and checked literally); expand `knowledge/repos/vllm-omni/components/` so every module *with actual knowledge* has a component dir whose `_index.md` states its owned source paths explicitly (e.g. `model-executor` owns `model_executor/ worker/ platforms/*/worker/ inputs/`; `serving` owns `entrypoints/ engine/`; `diffusion` owns `vllm_omni/diffusion/**`). Create new components (`scheduler`, `config`, …) only when a page lands there — no empty dirs. Work-topics stay parallel. -- **Knowledge components ≠ runtime manifest modules (documented, deliberate):** zuiho's component granularity (e.g. `model-executor` spanning `worker/`) already differs from `manifest.modules` (`worker_runner`, `platform`, `input_output`) today; the two serve different purposes (documentation grouping vs `module_for_path()` runtime routing). State this in `components/_index.md`; reconciling the manifest is the separate deferred F1 PR. +- **Module mirror (direct, human-curated — no owner map):** pin a `vllm-omni` SHA and **record it in `doc/architecture/KNOWLEDGE.md` + each component `_index.md`**; `ls` the real `vllm_omni/` tree (verify every referenced dir actually exists — globs like `platforms/*/worker/` are expanded and checked literally); expand `knowledge/repos/vllm-omni/components/` so every module *with actual knowledge* has a component dir whose `_index.md` states its owned source paths explicitly (e.g. `model-executor` owns `model_executor/ worker/ platforms/*/worker/ inputs/`; `serving` owns `entrypoints/ engine/`; `diffusion` owns `vllm_omni/diffusion/**`). Create new components (`scheduler`, `config`, …) only when a page lands there — no empty dirs. Work-topics stay parallel. +- **Knowledge components ≠ runtime manifest modules (documented, deliberate):** the upstream component granularity (e.g. `model-executor` spanning `worker/`) already differs from `manifest.modules` (`worker_runner`, `platform`, `input_output`) today; the two serve different purposes (documentation grouping vs `module_for_path()` runtime routing). State this in `components/_index.md`; reconciling the manifest is the separate deferred F1 PR. - **Step 2.5 — LLMWiki bootstrap (mechanical, scripted):** write `knowledge/SCHEMA.md` (taxonomy + frontmatter spec); add `"_archive"` to the validator's `excluded_parts` + create the extended lint script; scripted frontmatter backfill across synthesized pages — **excluding `repos/jianghan-*` entirely** (that subtree gets no frontmatter, is skipped by the extended lint, and stays byte-identical per the out-of-scope declaration). `created`/`updated` come **from the upstream per-file date table captured in Step 0** (not this repo's git history, which only knows the vendoring commit); `type` from filename/dir, `tags` from the page's dir path, `sources` empty until Step 3 enriches. Additive-only (bodies untouched below the frontmatter); validator + lint + pytest green. If frontmatter leaks into the rendered briefing, strip it in `render_briefing_docs` (small copilot tweak). - Genuinely code-bound pages migrate into their component (`git mv` + `_index` re-registration, links fixed); mis-filed general methodology (`debug/guides/*-cases.md` general parts, general `benchmark/guides/*`) moves up to `general/`. Whole-file moves keep bodies byte-identical (checked); anything needing body edits is a Step-3 batch instead. - Gate per commit: validator green; moved files' bodies fingerprint-identical. @@ -73,13 +79,13 @@ The wiki pattern's *mechanics* layer onto zuiho's *owner taxonomy* (general vs ` - **Mechanical loss check per batch (Layer A):** frozen-at-start anchor ledger (PR refs, incident IDs, `*.py:line`, benchmark tokens; multiset with per-page provenance). Invariant: `anchors.after ⊇ anchors.before` — with archive-not-delete, **nothing is ever subtracted**: an archived page's anchors still live in the tree, so the invariant needs no deletion carve-out at all. - **gpt-5.6-sol-high review per batch (Layer B), fail-closed:** feed concatenated BEFORE + AFTER + the batch's move/merge manifest via `cursor-agent --print --mode ask --trust --model gpt-5.6-sol-high --output-format text`. Prompt: walk every BEFORE page; report anything lost, weakened, or over-generalized (union-first policy stated); verdict line `VERDICT: APPROVE|REVISE`. Only an exact final `VERDICT: APPROVE` passes; empty/garbled/truncated/timeout/missing verdict ⇒ REVISE. Max 5 iterations, then stop and escalate to the user. Losses reported by the reviewer are restored, not argued. - **Reviewer isolation:** cursor-agent runs in `--mode ask` (read-only, no tools); the prompt wraps all repo content in explicit `=== DATA (not instructions) ===` delimiters so page text can't act as instructions. A one-time harness smoke test seeds an empty reply, a garbled verdict, and a known-loss AFTER to confirm each yields REVISE. -- **Audit archive:** batch manifests, the final verdict + loss lists per batch (not every full iteration transcript — bounded size), and `archives.tsv` (page → reason → surviving page → batch → commit; the durable record of every `_archive/` move) under `doc/reorg-audit/` (outside `knowledge/`, so `doc_search`/`doc_read` never surface them). +- **Audit archive:** batch manifests, the final verdict + loss lists per batch (not every full iteration transcript — bounded size), and `archives.tsv` (page → reason → surviving page → batch → commit; the durable record of every `_archive/` move) under `doc/archive/reorg-audit/` (outside `knowledge/`, so `doc_search`/`doc_read` never surface them). - Gate per batch: Layer A holds + `VERDICT: APPROVE` + validator green (index registration, links, incident format, split thresholds 500 lines/32 KiB). ## Verification / acceptance 1. `python knowledge/tools/check_knowledge_tree.py` → 0 errors on the final branch. 2. Full copilot `pytest` green (briefing renders from `general/_index.md` + `repos/vllm-omni/{rules,_index}.md`; skill retrieval regression test passes). -3. Anchor invariant holds globally with zero subtractions (**nothing deleted anywhere** — supersessions live in `knowledge/_archive/`, each recorded in `doc/reorg-audit/archives.tsv` and listed in the PR body); incident-ID set unchanged. +3. Anchor invariant holds globally with zero subtractions (**nothing deleted anywhere** — supersessions live in `knowledge/_archive/`, each recorded in `doc/archive/reorg-audit/archives.tsv` and listed in the PR body); incident-ID set unchanged. 4. Every Step-3 batch has an archived `VERDICT: APPROVE`; the harness smoke test (empty/garbled/known-loss ⇒ REVISE) passed once before batch 1. The extended LLMWiki lint (orphans, frontmatter, taxonomy tags, staleness) is green; `knowledge/SCHEMA.md` exists and every synthesized page **outside `repos/jianghan-*`** carries conformant frontmatter (the jianghan subtree is exempt from frontmatter/lint by its byte-identical guarantee). 5. Repo-wide `git grep '\bframework/'` → concept-only. `repos/jianghan-*` byte-identical to `f2dc824` (diffed against the Step-0 baseline archive). 6. Component `_index.md` source paths all exist at the pinned vllm-omni SHA. diff --git a/doc/PLAN-mcp-plugin-and-community-merge.md b/doc/archive/PLAN-mcp-plugin-and-community-merge.md similarity index 94% rename from doc/PLAN-mcp-plugin-and-community-merge.md rename to doc/archive/PLAN-mcp-plugin-and-community-merge.md index f231d3e05..54ea3ebed 100644 --- a/doc/PLAN-mcp-plugin-and-community-merge.md +++ b/doc/archive/PLAN-mcp-plugin-and-community-merge.md @@ -1,5 +1,12 @@ # Plan — (1) Copilot as an MCP plugin for Claude Code + Codex, and (2) community-repo adapter contribution +> **冻结（2026-08-17）。** 自称 PROPOSED rev 6，**从未执行**，停在 2026-07-22。 +> MCP 与插件的实际形态见 [`../guide/mcp.md`](../guide/mcp.md) 与 +> [`../guide/hosts/README.md`](../guide/hosts/README.md)。 + +> **冻结存档，保持英文原文。** 内容停在各自的日期上，不代表现状；翻译一份存档等于改动它。现行文档统一为中文 —— 见 [`../README.md`](../README.md)。 + + Status: PROPOSED, revision 6 (planning only — nothing here is executed yet). Rev 6 folds in a fifth external review (GPT). All three blockers were real; two were verified against the source / official docs: @@ -143,7 +150,7 @@ touched (the rev-4/5 multi-server bug). git+…@` is an explicit opt-in fallback only (network latency + supply-chain drift + startup-timeout risk). - Host-agnostic escape hatch remains `claude mcp add`. -4. **Codex**: `docs/codex/config.toml` `[mcp_servers.infermatrix_copilot]` +4. **Codex**: `integrations/config-templates/codex/config.toml` `[mcp_servers.infermatrix_copilot]` (`command`/`args`); **forward secrets via Codex `env_vars`**, not literal `env` values; optional `AGENTS.md`. No plugin. 5. **Tests** (offline): `reserve_run` → id + `queued`, no LLM; @@ -156,7 +163,7 @@ touched (the rev-4/5 multi-server bug). parent+child race); CLI abort leaves no dir; stdout hygiene; boundary (bad id/off-list repo/negative pr; capped report + `next_offset`); no post/push/debug tool in V1 schema. Plus a **live smoke** `start_review`. -6. **Docs**: `CODE_TOUR.md` §11 MCP entry; new `doc/MCP.md`. +6. **Docs**: `CODE_TOUR.md` §11 MCP entry; new `doc/guide/mcp.md`. ### Acceptance - CLI unchanged/in-process; aborts leave no run dir. @@ -172,7 +179,7 @@ touched (the rev-4/5 multi-server bug). --- -## Workstream 2 — contribute the copilot's net-new knowledge to `zuiho-kai/claude-workflow-starter` (fork → PR) +## Workstream 2 — contribute the copilot's net-new knowledge to the upstream knowledge repository (fork → PR) *Declared ready by the review; unchanged from rev 4. Summary:* @@ -183,14 +190,14 @@ touched (the rev-4/5 multi-server bug). - **Content preserved**: a **navigation allowlist** exempts only those `_index.md` files (additive rows); a pre/post SHA-256 manifest hash-gates everything else. - **Dedup + public provenance** (hard gates): never re-contribute - `community:zuiho-kai/…`-sourced facts; cite **public** vllm-omni commit/PR/ + upstream-sourced facts; cite **public** vllm-omni commit/PR/ issue/source links (not internal `eval GT #n`). - **Book form only** — no parallel machine tree. - **Hygiene**: DCO `git commit -s` **mandatory** (verified); Chinese PR body; the PR body carries the hash **verify command + summary + compact allowlisted-index list** (full manifest + dedup matrix are local artifacts, not committed). - Phases: local prep (no mutations) → audit/dedup → author pages → minimal nav → - hygiene → open PR → `zuiho-kai:master`. + hygiene → open PR → upstream `master`. --- diff --git a/doc/archive/reorg-audit/README.md b/doc/archive/reorg-audit/README.md new file mode 100644 index 000000000..c71cb5709 --- /dev/null +++ b/doc/archive/reorg-audit/README.md @@ -0,0 +1,11 @@ +# reorg-audit —— 知识重组的证据基线（冻结） + +> **冻结（2026-08-17）。** 2026-07-16 一次性知识重组留下的取证材料，127 个文件。 +> **不要删除**：`knowledge/SCHEMA.md` 与 +> `knowledge/repos/vllm-omni/rebase/workflow.md` 把它当作溯源基线引用， +> `../../architecture/KNOWLEDGE.md` 也指向这里。 + +内容：`baseline/`（上游 ls-tree + tarball + sha256 + 逐文件日期表）、 +`enrichment-baseline/`（配置事实、skill 快照、rebase 计划）、 +`reviews/`（逐批审阅记录）、`tools/`（当时用的三个校验脚本）、 +`anchors.before.tsv`、`archives.tsv`、`step3-inventory.tsv`。 diff --git a/doc/reorg-audit/anchors.before.tsv b/doc/archive/reorg-audit/anchors.before.tsv similarity index 100% rename from doc/reorg-audit/anchors.before.tsv rename to doc/archive/reorg-audit/anchors.before.tsv diff --git a/doc/reorg-audit/archives.tsv b/doc/archive/reorg-audit/archives.tsv similarity index 100% rename from doc/reorg-audit/archives.tsv rename to doc/archive/reorg-audit/archives.tsv diff --git a/doc/reorg-audit/baseline/dates.tsv b/doc/archive/reorg-audit/baseline/dates.tsv similarity index 100% rename from doc/reorg-audit/baseline/dates.tsv rename to doc/archive/reorg-audit/baseline/dates.tsv diff --git a/doc/reorg-audit/baseline/knowledge-f2dc824.tar.gz b/doc/archive/reorg-audit/baseline/knowledge-f2dc824.tar.gz similarity index 100% rename from doc/reorg-audit/baseline/knowledge-f2dc824.tar.gz rename to doc/archive/reorg-audit/baseline/knowledge-f2dc824.tar.gz diff --git a/doc/reorg-audit/baseline/knowledge-f2dc824.tar.gz.sha256 b/doc/archive/reorg-audit/baseline/knowledge-f2dc824.tar.gz.sha256 similarity index 100% rename from doc/reorg-audit/baseline/knowledge-f2dc824.tar.gz.sha256 rename to doc/archive/reorg-audit/baseline/knowledge-f2dc824.tar.gz.sha256 diff --git a/doc/reorg-audit/baseline/ls-tree-f2dc824.txt b/doc/archive/reorg-audit/baseline/ls-tree-f2dc824.txt similarity index 100% rename from doc/reorg-audit/baseline/ls-tree-f2dc824.txt rename to doc/archive/reorg-audit/baseline/ls-tree-f2dc824.txt diff --git a/doc/reorg-audit/enrichment-baseline/PINS.txt b/doc/archive/reorg-audit/enrichment-baseline/PINS.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/PINS.txt rename to doc/archive/reorg-audit/enrichment-baseline/PINS.txt diff --git a/doc/reorg-audit/enrichment-baseline/cfg-buildkite.txt b/doc/archive/reorg-audit/enrichment-baseline/cfg-buildkite.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/cfg-buildkite.txt rename to doc/archive/reorg-audit/enrichment-baseline/cfg-buildkite.txt diff --git a/doc/reorg-audit/enrichment-baseline/cfg-paths.txt b/doc/archive/reorg-audit/enrichment-baseline/cfg-paths.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/cfg-paths.txt rename to doc/archive/reorg-audit/enrichment-baseline/cfg-paths.txt diff --git a/doc/reorg-audit/enrichment-baseline/cfg-queue.txt b/doc/archive/reorg-audit/enrichment-baseline/cfg-queue.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/cfg-queue.txt rename to doc/archive/reorg-audit/enrichment-baseline/cfg-queue.txt diff --git a/doc/reorg-audit/enrichment-baseline/cfg-waves.txt b/doc/archive/reorg-audit/enrichment-baseline/cfg-waves.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/cfg-waves.txt rename to doc/archive/reorg-audit/enrichment-baseline/cfg-waves.txt diff --git a/doc/reorg-audit/enrichment-baseline/config-facts.txt b/doc/archive/reorg-audit/enrichment-baseline/config-facts.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/config-facts.txt rename to doc/archive/reorg-audit/enrichment-baseline/config-facts.txt diff --git a/doc/reorg-audit/enrichment-baseline/debug-record-447.txt b/doc/archive/reorg-audit/enrichment-baseline/debug-record-447.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/debug-record-447.txt rename to doc/archive/reorg-audit/enrichment-baseline/debug-record-447.txt diff --git a/doc/reorg-audit/enrichment-baseline/debug-record-457.txt b/doc/archive/reorg-audit/enrichment-baseline/debug-record-457.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/debug-record-457.txt rename to doc/archive/reorg-audit/enrichment-baseline/debug-record-457.txt diff --git a/doc/reorg-audit/enrichment-baseline/debug-records-b12a.txt b/doc/archive/reorg-audit/enrichment-baseline/debug-records-b12a.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/debug-records-b12a.txt rename to doc/archive/reorg-audit/enrichment-baseline/debug-records-b12a.txt diff --git a/doc/reorg-audit/enrichment-baseline/debug-records-b12b.txt b/doc/archive/reorg-audit/enrichment-baseline/debug-records-b12b.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/debug-records-b12b.txt rename to doc/archive/reorg-audit/enrichment-baseline/debug-records-b12b.txt diff --git a/doc/reorg-audit/enrichment-baseline/debug-records-b8.txt b/doc/archive/reorg-audit/enrichment-baseline/debug-records-b8.txt similarity index 100% rename from doc/reorg-audit/enrichment-baseline/debug-records-b8.txt rename to doc/archive/reorg-audit/enrichment-baseline/debug-records-b8.txt diff --git a/doc/reorg-audit/enrichment-baseline/debug-records.tsv b/doc/archive/reorg-audit/enrichment-baseline/debug-records.tsv similarity index 100% rename from doc/reorg-audit/enrichment-baseline/debug-records.tsv rename to doc/archive/reorg-audit/enrichment-baseline/debug-records.tsv diff --git a/doc/reorg-audit/enrichment-baseline/skill-manifest.tsv b/doc/archive/reorg-audit/enrichment-baseline/skill-manifest.tsv similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skill-manifest.tsv rename to doc/archive/reorg-audit/enrichment-baseline/skill-manifest.tsv diff --git a/doc/reorg-audit/enrichment-baseline/skills/benchmark-datasets-module-restructured.md b/doc/archive/reorg-audit/enrichment-baseline/skills/benchmark-datasets-module-restructured.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/benchmark-datasets-module-restructured.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/benchmark-datasets-module-restructured.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/buildkite-skipped-build-rebuild-fratricide.md b/doc/archive/reorg-audit/enrichment-baseline/skills/buildkite-skipped-build-rebuild-fratricide.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/buildkite-skipped-build-rebuild-fratricide.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/buildkite-skipped-build-rebuild-fratricide.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/entrypoint-h100-timeout-1800s-runner-mismatch.md b/doc/archive/reorg-audit/enrichment-baseline/skills/entrypoint-h100-timeout-1800s-runner-mismatch.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/entrypoint-h100-timeout-1800s-runner-mismatch.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/entrypoint-h100-timeout-1800s-runner-mismatch.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-accuracy-golden-hardware-mismatch.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-accuracy-golden-hardware-mismatch.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-accuracy-golden-hardware-mismatch.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-accuracy-golden-hardware-mismatch.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-bitsandbytes-libnvjitlink-ld-library-path.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-bitsandbytes-libnvjitlink-ld-library-path.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-bitsandbytes-libnvjitlink-ld-library-path.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-bitsandbytes-libnvjitlink-ld-library-path.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-cumem-allocator-sleep-double-free.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-cumem-allocator-sleep-double-free.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-cumem-allocator-sleep-double-free.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-cumem-allocator-sleep-double-free.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-cumem-cuda-double-free-gc-atexit.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-cumem-cuda-double-free-gc-atexit.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-cumem-cuda-double-free-gc-atexit.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-cumem-cuda-double-free-gc-atexit.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-diffusion-fp8-meta-tensor-weight-loading.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-diffusion-fp8-meta-tensor-weight-loading.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-diffusion-fp8-meta-tensor-weight-loading.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-diffusion-fp8-meta-tensor-weight-loading.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-flashinfer-jit-cache-version-mismatch.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-flashinfer-jit-cache-version-mismatch.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-flashinfer-jit-cache-version-mismatch.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-flashinfer-jit-cache-version-mismatch.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-hunyuanimage3-moe-weight-loading-keyerror.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-hunyuanimage3-moe-weight-loading-keyerror.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-hunyuanimage3-moe-weight-loading-keyerror.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-hunyuanimage3-moe-weight-loading-keyerror.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-missing-gpu-memory-utilization-diffusion-stage.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-missing-gpu-memory-utilization-diffusion-stage.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-missing-gpu-memory-utilization-diffusion-stage.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-missing-gpu-memory-utilization-diffusion-stage.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-mooncake-put-concurrency.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-mooncake-put-concurrency.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-mooncake-put-concurrency.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-mooncake-put-concurrency.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-mooncake-tcp-deserialize-leading-byte.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-mooncake-tcp-deserialize-leading-byte.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-mooncake-tcp-deserialize-leading-byte.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-mooncake-tcp-deserialize-leading-byte.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-omni-master-server-zmq-port-collision.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-omni-master-server-zmq-port-collision.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-omni-master-server-zmq-port-collision.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-omni-master-server-zmq-port-collision.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-omni-serving-chat-upstream-harmony-refactor.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-omni-serving-chat-upstream-harmony-refactor.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-omni-serving-chat-upstream-harmony-refactor.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-omni-serving-chat-upstream-harmony-refactor.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-stale-test-path-collection-error-after-rename.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-stale-test-path-collection-error-after-rename.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-stale-test-path-collection-error-after-rename.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-stale-test-path-collection-error-after-rename.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-talker-truncated-prefill-prefix-cache-key-cap.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-talker-truncated-prefill-prefix-cache-key-cap.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-talker-truncated-prefill-prefix-cache-key-cap.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-talker-truncated-prefill-prefix-cache-key-cap.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-test-parser_cls-missing.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-test-parser_cls-missing.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-test-parser_cls-missing.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-test-parser_cls-missing.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-tts-realtime-asr-accuracy-flake-whisper-small.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-tts-realtime-asr-accuracy-flake-whisper-small.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-tts-realtime-asr-accuracy-flake-whisper-small.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-tts-realtime-asr-accuracy-flake-whisper-small.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-tts-speed-adjustment-phase-vocoder-quality.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-tts-speed-adjustment-phase-vocoder-quality.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-tts-speed-adjustment-phase-vocoder-quality.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-tts-speed-adjustment-phase-vocoder-quality.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-voxcpm2-l4-oom-after-rebase.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-voxcpm2-l4-oom-after-rebase.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-voxcpm2-l4-oom-after-rebase.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-voxcpm2-l4-oom-after-rebase.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/fix-watchdog-non-fatal-false-positive.md b/doc/archive/reorg-audit/enrichment-baseline/skills/fix-watchdog-non-fatal-false-positive.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/fix-watchdog-non-fatal-false-positive.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/fix-watchdog-non-fatal-false-positive.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/gpu-hang-low-max-num-batched-tokens.md b/doc/archive/reorg-audit/enrichment-baseline/skills/gpu-hang-low-max-num-batched-tokens.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/gpu-hang-low-max-num-batched-tokens.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/gpu-hang-low-max-num-batched-tokens.md diff --git a/doc/reorg-audit/enrichment-baseline/skills/resolve-moss-tts-nano-load-weights-conflict.md b/doc/archive/reorg-audit/enrichment-baseline/skills/resolve-moss-tts-nano-load-weights-conflict.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/skills/resolve-moss-tts-nano-load-weights-conflict.md rename to doc/archive/reorg-audit/enrichment-baseline/skills/resolve-moss-tts-nano-load-weights-conflict.md diff --git a/doc/reorg-audit/enrichment-baseline/sources.sha256 b/doc/archive/reorg-audit/enrichment-baseline/sources.sha256 similarity index 100% rename from doc/reorg-audit/enrichment-baseline/sources.sha256 rename to doc/archive/reorg-audit/enrichment-baseline/sources.sha256 diff --git a/doc/reorg-audit/enrichment-baseline/vllm-mrv2-commits-since-v0.20.0.md b/doc/archive/reorg-audit/enrichment-baseline/vllm-mrv2-commits-since-v0.20.0.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/vllm-mrv2-commits-since-v0.20.0.md rename to doc/archive/reorg-audit/enrichment-baseline/vllm-mrv2-commits-since-v0.20.0.md diff --git a/doc/reorg-audit/enrichment-baseline/vllm-omni-mrv2-rebase-plan.md b/doc/archive/reorg-audit/enrichment-baseline/vllm-omni-mrv2-rebase-plan.md similarity index 100% rename from doc/reorg-audit/enrichment-baseline/vllm-omni-mrv2-rebase-plan.md rename to doc/archive/reorg-audit/enrichment-baseline/vllm-omni-mrv2-rebase-plan.md diff --git a/doc/reorg-audit/reviews/B10-agents-planning-env/i1-approve.txt b/doc/archive/reorg-audit/reviews/B10-agents-planning-env/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B10-agents-planning-env/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B10-agents-planning-env/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B1a-debug/i1-approve.txt b/doc/archive/reorg-audit/reviews/B1a-debug/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B1a-debug/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B1a-debug/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B1b-debug/i1-approve.txt b/doc/archive/reorg-audit/reviews/B1b-debug/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B1b-debug/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B1b-debug/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B1c-debug/i1-approve.txt b/doc/archive/reorg-audit/reviews/B1c-debug/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B1c-debug/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B1c-debug/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B1d-debug/i1-approve.txt b/doc/archive/reorg-audit/reviews/B1d-debug/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B1d-debug/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B1d-debug/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B1e-debug/i1-approve.txt b/doc/archive/reorg-audit/reviews/B1e-debug/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B1e-debug/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B1e-debug/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B2a-benchmark/i1-approve.txt b/doc/archive/reorg-audit/reviews/B2a-benchmark/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B2a-benchmark/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B2a-benchmark/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B2b-benchmark/i1-approve.txt b/doc/archive/reorg-audit/reviews/B2b-benchmark/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B2b-benchmark/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B2b-benchmark/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B2c-benchmark/i1-approve.txt b/doc/archive/reorg-audit/reviews/B2c-benchmark/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B2c-benchmark/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B2c-benchmark/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B2d-benchmark/i1-approve.txt b/doc/archive/reorg-audit/reviews/B2d-benchmark/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B2d-benchmark/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B2d-benchmark/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B2e-benchmark/i1-approve.txt b/doc/archive/reorg-audit/reviews/B2e-benchmark/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B2e-benchmark/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B2e-benchmark/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B3a-review/i1-approve.txt b/doc/archive/reorg-audit/reviews/B3a-review/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B3a-review/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B3a-review/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B3b-review/i1-approve.txt b/doc/archive/reorg-audit/reviews/B3b-review/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B3b-review/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B3b-review/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B3c-review/i1-approve.txt b/doc/archive/reorg-audit/reviews/B3c-review/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B3c-review/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B3c-review/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B3d-review/i1-approve.txt b/doc/archive/reorg-audit/reviews/B3d-review/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B3d-review/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B3d-review/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B3e-review/i1-approve.txt b/doc/archive/reorg-audit/reviews/B3e-review/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B3e-review/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B3e-review/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B4a-remote/i1-revise.txt b/doc/archive/reorg-audit/reviews/B4a-remote/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/B4a-remote/i1-revise.txt rename to doc/archive/reorg-audit/reviews/B4a-remote/i1-revise.txt diff --git a/doc/reorg-audit/reviews/B4a-remote/i2-approve.txt b/doc/archive/reorg-audit/reviews/B4a-remote/i2-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B4a-remote/i2-approve.txt rename to doc/archive/reorg-audit/reviews/B4a-remote/i2-approve.txt diff --git a/doc/reorg-audit/reviews/B4b-remote/i1-approve.txt b/doc/archive/reorg-audit/reviews/B4b-remote/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B4b-remote/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B4b-remote/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B4c-remote/i1-approve.txt b/doc/archive/reorg-audit/reviews/B4c-remote/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B4c-remote/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B4c-remote/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B5a-docs-dev/i1-approve.txt b/doc/archive/reorg-audit/reviews/B5a-docs-dev/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B5a-docs-dev/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B5a-docs-dev/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B5b-review/i1-approve.txt b/doc/archive/reorg-audit/reviews/B5b-review/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B5b-review/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B5b-review/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B5c-git-pr-body/i1-approve.txt b/doc/archive/reorg-audit/reviews/B5c-git-pr-body/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B5c-git-pr-body/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B5c-git-pr-body/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B5d-git-branch/i1-approve.txt b/doc/archive/reorg-audit/reviews/B5d-git-branch/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B5d-git-branch/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B5d-git-branch/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B5e-git-debug-integration/i1-approve.txt b/doc/archive/reorg-audit/reviews/B5e-git-debug-integration/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B5e-git-debug-integration/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B5e-git-debug-integration/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B5f-git-scope/i1-approve.txt b/doc/archive/reorg-audit/reviews/B5f-git-scope/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B5f-git-scope/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B5f-git-scope/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B6-ci/i1-approve.txt b/doc/archive/reorg-audit/reviews/B6-ci/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B6-ci/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B6-ci/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B7-components/i1-approve.txt b/doc/archive/reorg-audit/reviews/B7-components/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B7-components/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B7-components/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B8a-hunyuan-core/i1-approve.txt b/doc/archive/reorg-audit/reviews/B8a-hunyuan-core/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B8a-hunyuan-core/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B8a-hunyuan-core/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B8b-hunyuan-guides1/i1-approve.txt b/doc/archive/reorg-audit/reviews/B8b-hunyuan-guides1/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B8b-hunyuan-guides1/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B8b-hunyuan-guides1/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B8c-hunyuan-guides2/i1-approve.txt b/doc/archive/reorg-audit/reviews/B8c-hunyuan-guides2/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B8c-hunyuan-guides2/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B8c-hunyuan-guides2/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B8d-hunyuan-guides3/i1-approve.txt b/doc/archive/reorg-audit/reviews/B8d-hunyuan-guides3/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B8d-hunyuan-guides3/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B8d-hunyuan-guides3/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B8e-hunyuan-guides4/i1-approve.txt b/doc/archive/reorg-audit/reviews/B8e-hunyuan-guides4/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B8e-hunyuan-guides4/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B8e-hunyuan-guides4/i1-approve.txt diff --git a/doc/reorg-audit/reviews/B9-roots/i1-approve.txt b/doc/archive/reorg-audit/reviews/B9-roots/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/B9-roots/i1-approve.txt rename to doc/archive/reorg-audit/reviews/B9-roots/i1-approve.txt diff --git a/doc/reorg-audit/reviews/E-B1-schema-tags/i1-approve.txt b/doc/archive/reorg-audit/reviews/E-B1-schema-tags/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B1-schema-tags/i1-approve.txt rename to doc/archive/reorg-audit/reviews/E-B1-schema-tags/i1-approve.txt diff --git a/doc/reorg-audit/reviews/E-B1-schema-tags/i2-approve.txt b/doc/archive/reorg-audit/reviews/E-B1-schema-tags/i2-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B1-schema-tags/i2-approve.txt rename to doc/archive/reorg-audit/reviews/E-B1-schema-tags/i2-approve.txt diff --git a/doc/reorg-audit/reviews/E-B10-distributed-guides/i1-approve.txt b/doc/archive/reorg-audit/reviews/E-B10-distributed-guides/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B10-distributed-guides/i1-approve.txt rename to doc/archive/reorg-audit/reviews/E-B10-distributed-guides/i1-approve.txt diff --git a/doc/reorg-audit/reviews/E-B11-dev-docs/i1-approve.txt b/doc/archive/reorg-audit/reviews/E-B11-dev-docs/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B11-dev-docs/i1-approve.txt rename to doc/archive/reorg-audit/reviews/E-B11-dev-docs/i1-approve.txt diff --git a/doc/reorg-audit/reviews/E-B12a-rebase-workflow/i1-approve.txt b/doc/archive/reorg-audit/reviews/E-B12a-rebase-workflow/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12a-rebase-workflow/i1-approve.txt rename to doc/archive/reorg-audit/reviews/E-B12a-rebase-workflow/i1-approve.txt diff --git a/doc/reorg-audit/reviews/E-B12b-drift-serving/ESCALATED.txt b/doc/archive/reorg-audit/reviews/E-B12b-drift-serving/ESCALATED.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12b-drift-serving/ESCALATED.txt rename to doc/archive/reorg-audit/reviews/E-B12b-drift-serving/ESCALATED.txt diff --git a/doc/reorg-audit/reviews/E-B12b-drift-serving/i1-revise.txt b/doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12b-drift-serving/i1-revise.txt rename to doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i1-revise.txt diff --git a/doc/reorg-audit/reviews/E-B12b-drift-serving/i2-revise.txt b/doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i2-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12b-drift-serving/i2-revise.txt rename to doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i2-revise.txt diff --git a/doc/reorg-audit/reviews/E-B12b-drift-serving/i3-revise.txt b/doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i3-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12b-drift-serving/i3-revise.txt rename to doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i3-revise.txt diff --git a/doc/reorg-audit/reviews/E-B12b-drift-serving/i4-revise.txt b/doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i4-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12b-drift-serving/i4-revise.txt rename to doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i4-revise.txt diff --git a/doc/reorg-audit/reviews/E-B12b-drift-serving/i5-revise.txt b/doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i5-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12b-drift-serving/i5-revise.txt rename to doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i5-revise.txt diff --git a/doc/reorg-audit/reviews/E-B12b-drift-serving/i6-revise.txt b/doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i6-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12b-drift-serving/i6-revise.txt rename to doc/archive/reorg-audit/reviews/E-B12b-drift-serving/i6-revise.txt diff --git a/doc/reorg-audit/reviews/E-B12c-drift-loading/i1-revise.txt b/doc/archive/reorg-audit/reviews/E-B12c-drift-loading/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12c-drift-loading/i1-revise.txt rename to doc/archive/reorg-audit/reviews/E-B12c-drift-loading/i1-revise.txt diff --git a/doc/reorg-audit/reviews/E-B12c-drift-loading/i2-revise.txt b/doc/archive/reorg-audit/reviews/E-B12c-drift-loading/i2-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12c-drift-loading/i2-revise.txt rename to doc/archive/reorg-audit/reviews/E-B12c-drift-loading/i2-revise.txt diff --git a/doc/reorg-audit/reviews/E-B12c-drift-loading/i3-approve.txt b/doc/archive/reorg-audit/reviews/E-B12c-drift-loading/i3-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B12c-drift-loading/i3-approve.txt rename to doc/archive/reorg-audit/reviews/E-B12c-drift-loading/i3-approve.txt diff --git a/doc/reorg-audit/reviews/E-B13-wiring/i1-approve.txt b/doc/archive/reorg-audit/reviews/E-B13-wiring/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B13-wiring/i1-approve.txt rename to doc/archive/reorg-audit/reviews/E-B13-wiring/i1-approve.txt diff --git a/doc/reorg-audit/reviews/E-B2-scheduler/i1-revise.txt b/doc/archive/reorg-audit/reviews/E-B2-scheduler/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B2-scheduler/i1-revise.txt rename to doc/archive/reorg-audit/reviews/E-B2-scheduler/i1-revise.txt diff --git a/doc/reorg-audit/reviews/E-B2-scheduler/i2-revise.txt b/doc/archive/reorg-audit/reviews/E-B2-scheduler/i2-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B2-scheduler/i2-revise.txt rename to doc/archive/reorg-audit/reviews/E-B2-scheduler/i2-revise.txt diff --git a/doc/reorg-audit/reviews/E-B2-scheduler/i3-approve.txt b/doc/archive/reorg-audit/reviews/E-B2-scheduler/i3-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B2-scheduler/i3-approve.txt rename to doc/archive/reorg-audit/reviews/E-B2-scheduler/i3-approve.txt diff --git a/doc/reorg-audit/reviews/E-B3-distributed/i1-revise.txt b/doc/archive/reorg-audit/reviews/E-B3-distributed/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B3-distributed/i1-revise.txt rename to doc/archive/reorg-audit/reviews/E-B3-distributed/i1-revise.txt diff --git a/doc/reorg-audit/reviews/E-B3-distributed/i2-revise.txt b/doc/archive/reorg-audit/reviews/E-B3-distributed/i2-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B3-distributed/i2-revise.txt rename to doc/archive/reorg-audit/reviews/E-B3-distributed/i2-revise.txt diff --git a/doc/reorg-audit/reviews/E-B3-distributed/i3-approve.txt b/doc/archive/reorg-audit/reviews/E-B3-distributed/i3-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B3-distributed/i3-approve.txt rename to doc/archive/reorg-audit/reviews/E-B3-distributed/i3-approve.txt diff --git a/doc/reorg-audit/reviews/E-B4-config/i1-revise.txt b/doc/archive/reorg-audit/reviews/E-B4-config/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B4-config/i1-revise.txt rename to doc/archive/reorg-audit/reviews/E-B4-config/i1-revise.txt diff --git a/doc/reorg-audit/reviews/E-B4-config/i2-revise.txt b/doc/archive/reorg-audit/reviews/E-B4-config/i2-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B4-config/i2-revise.txt rename to doc/archive/reorg-audit/reviews/E-B4-config/i2-revise.txt diff --git a/doc/reorg-audit/reviews/E-B4-config/i3-revise.txt b/doc/archive/reorg-audit/reviews/E-B4-config/i3-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B4-config/i3-revise.txt rename to doc/archive/reorg-audit/reviews/E-B4-config/i3-revise.txt diff --git a/doc/reorg-audit/reviews/E-B4-config/i4-approve.txt b/doc/archive/reorg-audit/reviews/E-B4-config/i4-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B4-config/i4-approve.txt rename to doc/archive/reorg-audit/reviews/E-B4-config/i4-approve.txt diff --git a/doc/reorg-audit/reviews/E-B5-models/i1-approve.txt b/doc/archive/reorg-audit/reviews/E-B5-models/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B5-models/i1-approve.txt rename to doc/archive/reorg-audit/reviews/E-B5-models/i1-approve.txt diff --git a/doc/reorg-audit/reviews/E-B6-qwen-omni/i1-approve.txt b/doc/archive/reorg-audit/reviews/E-B6-qwen-omni/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B6-qwen-omni/i1-approve.txt rename to doc/archive/reorg-audit/reviews/E-B6-qwen-omni/i1-approve.txt diff --git a/doc/reorg-audit/reviews/E-B7-ci-structure/i1-revise.txt b/doc/archive/reorg-audit/reviews/E-B7-ci-structure/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B7-ci-structure/i1-revise.txt rename to doc/archive/reorg-audit/reviews/E-B7-ci-structure/i1-revise.txt diff --git a/doc/reorg-audit/reviews/E-B7-ci-structure/i2-approve.txt b/doc/archive/reorg-audit/reviews/E-B7-ci-structure/i2-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B7-ci-structure/i2-approve.txt rename to doc/archive/reorg-audit/reviews/E-B7-ci-structure/i2-approve.txt diff --git a/doc/reorg-audit/reviews/E-B8a-env-gotchas/i1-revise.txt b/doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B8a-env-gotchas/i1-revise.txt rename to doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i1-revise.txt diff --git a/doc/reorg-audit/reviews/E-B8a-env-gotchas/i2-revise.txt b/doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i2-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B8a-env-gotchas/i2-revise.txt rename to doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i2-revise.txt diff --git a/doc/reorg-audit/reviews/E-B8a-env-gotchas/i3-revise.txt b/doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i3-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B8a-env-gotchas/i3-revise.txt rename to doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i3-revise.txt diff --git a/doc/reorg-audit/reviews/E-B8a-env-gotchas/i4-revise.txt b/doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i4-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B8a-env-gotchas/i4-revise.txt rename to doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i4-revise.txt diff --git a/doc/reorg-audit/reviews/E-B8a-env-gotchas/i5-approve.txt b/doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i5-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B8a-env-gotchas/i5-approve.txt rename to doc/archive/reorg-audit/reviews/E-B8a-env-gotchas/i5-approve.txt diff --git a/doc/reorg-audit/reviews/E-B8b-accuracy/i1-revise.txt b/doc/archive/reorg-audit/reviews/E-B8b-accuracy/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B8b-accuracy/i1-revise.txt rename to doc/archive/reorg-audit/reviews/E-B8b-accuracy/i1-revise.txt diff --git a/doc/reorg-audit/reviews/E-B8b-accuracy/i2-revise.txt b/doc/archive/reorg-audit/reviews/E-B8b-accuracy/i2-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B8b-accuracy/i2-revise.txt rename to doc/archive/reorg-audit/reviews/E-B8b-accuracy/i2-revise.txt diff --git a/doc/reorg-audit/reviews/E-B8b-accuracy/i3-approve.txt b/doc/archive/reorg-audit/reviews/E-B8b-accuracy/i3-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B8b-accuracy/i3-approve.txt rename to doc/archive/reorg-audit/reviews/E-B8b-accuracy/i3-approve.txt diff --git a/doc/reorg-audit/reviews/E-B9-diffusion-guides/i1-approve.txt b/doc/archive/reorg-audit/reviews/E-B9-diffusion-guides/i1-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/E-B9-diffusion-guides/i1-approve.txt rename to doc/archive/reorg-audit/reviews/E-B9-diffusion-guides/i1-approve.txt diff --git a/doc/reorg-audit/reviews/repin-main/i1-revise.txt b/doc/archive/reorg-audit/reviews/repin-main/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/repin-main/i1-revise.txt rename to doc/archive/reorg-audit/reviews/repin-main/i1-revise.txt diff --git a/doc/reorg-audit/reviews/repin-main/i2-revise.txt b/doc/archive/reorg-audit/reviews/repin-main/i2-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/repin-main/i2-revise.txt rename to doc/archive/reorg-audit/reviews/repin-main/i2-revise.txt diff --git a/doc/reorg-audit/reviews/repin-main/i3-approve.txt b/doc/archive/reorg-audit/reviews/repin-main/i3-approve.txt similarity index 100% rename from doc/reorg-audit/reviews/repin-main/i3-approve.txt rename to doc/archive/reorg-audit/reviews/repin-main/i3-approve.txt diff --git a/doc/reorg-audit/reviews/smoke/i1-revise.txt b/doc/archive/reorg-audit/reviews/smoke/i1-revise.txt similarity index 100% rename from doc/reorg-audit/reviews/smoke/i1-revise.txt rename to doc/archive/reorg-audit/reviews/smoke/i1-revise.txt diff --git a/doc/reorg-audit/step3-inventory.tsv b/doc/archive/reorg-audit/step3-inventory.tsv similarity index 100% rename from doc/reorg-audit/step3-inventory.tsv rename to doc/archive/reorg-audit/step3-inventory.tsv diff --git a/doc/reorg-audit/tools/check_manifest.py b/doc/archive/reorg-audit/tools/check_manifest.py similarity index 100% rename from doc/reorg-audit/tools/check_manifest.py rename to doc/archive/reorg-audit/tools/check_manifest.py diff --git a/doc/reorg-audit/tools/layer_a_check.py b/doc/archive/reorg-audit/tools/layer_a_check.py similarity index 96% rename from doc/reorg-audit/tools/layer_a_check.py rename to doc/archive/reorg-audit/tools/layer_a_check.py index 826ddb968..69d507661 100644 --- a/doc/reorg-audit/tools/layer_a_check.py +++ b/doc/archive/reorg-audit/tools/layer_a_check.py @@ -4,7 +4,7 @@ Recomputes the anchor multiset over the in-scope tree and asserts Counter(after) >= Counter(before) elementwise (nothing subtracted — Step 3 is add-detail/union-first with archive-never-delete, so anchors only accumulate), -plus incident-ID set equality. Baseline: doc/reorg-audit/anchors.before.tsv. +plus incident-ID set equality. Baseline: doc/archive/reorg-audit/anchors.before.tsv. Exit 0 = holds; exit 1 = a loss, listed. """ diff --git a/doc/reorg-audit/tools/review_batch.py b/doc/archive/reorg-audit/tools/review_batch.py similarity index 98% rename from doc/reorg-audit/tools/review_batch.py rename to doc/archive/reorg-audit/tools/review_batch.py index 369216cd5..269d66d7b 100644 --- a/doc/reorg-audit/tools/review_batch.py +++ b/doc/archive/reorg-audit/tools/review_batch.py @@ -12,7 +12,7 @@ Fail-closed: empty/garbled/truncated output, timeout, non-zero exit, missing verdict, or an oversized prompt (> CAP chars — split the batch, never truncate) all count as REVISE (exit 1). Exit 0 only on an exact APPROVE. -Transcripts + prompt sha256 are archived under doc/reorg-audit/reviews/. +Transcripts + prompt sha256 are archived under doc/archive/reorg-audit/reviews/. """ from __future__ import annotations diff --git a/doc/assets/strict-review-vs-opus5.svg b/doc/assets/strict-review-vs-opus5.svg new file mode 100644 index 000000000..edac5d762 --- /dev/null +++ b/doc/assets/strict-review-vs-opus5.svg @@ -0,0 +1,208 @@ + + +Strict PR review — paired Δ vs Claude Code + Opus 5 +blind judge claude-sonnet-5 · 3 replicates · paired inside each verdict, clustered by item · whisker = 95% CI +shaded band = effects this dataset cannot resolve at n=10 (±0.099 recall, ±0.115 precision, from the observed item-level spread) + + +Δ RECALL + +-0.4 + +-0.3 + +-0.2 + +-0.1 + +0 + +0.1 + +0.2 + + +Δ PRECISION + +-0.4 + +-0.3 + +-0.2 + +-0.1 + +0 + +0.1 + +0.2 +THIS PR · PIPELINE VERSIONS (Composer 2.5, train n=10) + +v17cb +Composer 2.5 on v17 n=10 + + + + +-0.052 + + + + +-0.097 +v19cb +dedupe + anchor fix n=10 + + + + +-0.092 + + + + ++0.060 +v20cb ★ +merge-richest n=10 + + + + +-0.005 + + + + ++0.046 +v21cb ↵ +claim headlines n=10 · reverted + + + + +-0.093 + + + + ++0.079 +v22cb ↵ +mechanism depth n=10 · reverted + + + + +-0.082 + + + + ++0.040 +v23cb ↵ +breadth sweep lens n=10 · reverted + + + + +-0.093 + + + + ++0.024 +OTHER GENERATORS (same pipeline family, same judge) + +DeepSeek v4-pro +v17 clean · train n=10 + + + + ++0.024 + + + + ++0.016 +DeepSeek + dsh +v18 native tools · val n=5 · best recall + + + + ++0.038 + + + + ++0.023 +DeepSeek v4-pro +v13 · wave-2 n=10 + + + + +-0.115 + + + + ++0.040 +grok-4.6 +wave-2 n=10 + + + + +-0.080 + + + + ++0.004 +Composer 2.5 +wave-2 standalone n=10 + + + + +-0.127 + + + + +-0.048 +grok-4.5 +wave-2 n=10 + + + + +-0.164 + + + + ++0.030 +MiMo v2.5 +wave-2 n=10 + + + + +-0.212 + + + + +-0.084 + +above baseline + +below baseline + +current champion + +unresolvable at this n + \ No newline at end of file diff --git a/doc/contributing/DOCSTRING_STYLE.md b/doc/contributing/DOCSTRING_STYLE.md new file mode 100644 index 000000000..c9b3704ca --- /dev/null +++ b/doc/contributing/DOCSTRING_STYLE.md @@ -0,0 +1,79 @@ +# Docstring 规范 —— infermatrix-copilot + +本仓库的 docstring 约定，结构以 [PEP 257](https://peps.python.org/pep-0257/) 为准， +"写什么"以 [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html) +为准，再按本代码库"简洁、先讲理由"的语气做了调整。 + +**每个 module、class、function、method 都要有 docstring。** 它解释的是*逻辑* +（做什么，以及那些不显然的*为什么*）和*输入/输出* —— 而不是把签名再抄一遍。 + +## 形式（PEP 257） + +- 三个双引号 `"""..."""`。module docstring 是文件的**第一条语句** + （排在 `from __future__ import ...` 和所有 import 之前）。 +- **摘要行**：一句祈使语气，≤ 约 88 列，以句号结尾。显然的情况下，摘要行本身就是 + 整个 docstring（单行式）。 +- 不平凡的情况：摘要行、空行、然后正文。 +- 多行 docstring 的收尾 `"""` 单独占一行。 + +## 各类分别该写什么 + +- **Module** —— 这个文件独占的那一件事，加一行说明它导出什么。对于只做再导出的 + `__init__.py`，说明这个包聚合了什么、以及公开面是再导出的。 +- **Class** —— 一行说清它的行为/角色；当不显然时，点出关键属性或它维持的不变量。 + `@dataclass` 的字段含义写在类 docstring 里（或用行内 `#` 注释），除非字段很多， + 否则不要单开 `Attributes` 段。 +- **Function / method** —— 先摘要行为，再用**散文**写清 输入 → 输出 和那些不显然的 + 逻辑。点名那些真正承载语义的参数，并说明返回值代表什么。只有参数很多、散文反而 + 更难读时，才启用显式的 `Args:`/`Returns:` 段。 + +## 语气（与现有代码保持一致） + +- **理由优先**：与其逐行复述代码已经写明的*做了什么*，不如解释*为什么*。当某个选择 + 由设计或评测决定时，把它引出来（例如"single untooled reduction call: a + tool-looped reducer over-dropped in live runs"）。 +- 保留 SPEC 使用的那套安全/契约措辞：说明某个函数是不是 fail-closed 门、是否为后续 + step 发布状态（**B2**）、是否降级为类型化的 BLOCKED 结果、是否记录 `capability_gap`。 +- **不要编造行为。** 拿不准某个分支做什么时，就按代码能支撑的粒度描述 —— 绝不猜测 + 代码里并不存在的具体量（行号、阈值）。 + +## 示例（取自本仓库） + +Function —— 输入 → 输出 + 为什么： + +```python +def guard_push(policy: PushPolicy, protected_branches: list[str]) -> PushDecision: + """Authorize a push: allow only when `policy.allowed` AND the target branch + is not in `protected_branches`. Returns a PushDecision carrying the concrete + git command (never run here) or a deny reason — the single C4 choke point, so + every push path is authorized in one place.""" +``` + +Class —— 角色 + 关键属性： + +```python +class StepResult: + """The outcome of one step: `ok` plus, on failure, a typed `failure_kind` + (drives the engine's retry/replan/escalate branch) and the `state_updates` + a later step consumes. Never raises across the step boundary — failures are + values, not exceptions.""" +``` + +只做再导出的 `__init__.py`： + +```python +"""Review subsystem: diff summarization, trigger evaluation, and the patch/plan +reviewers. Re-exports the public surface (`build_diff_summary`, `ReviewVerdict`, +`run_patch_review`) so callers import from the package, not its modules.""" +``` + +> 示例里的 docstring 保持英文原样 —— **代码内的 docstring 依然用英文写**，本页统一 +> 中文说的是这份规范文档本身，不是被规范的对象。 + +## 不要这样做 + +- 不要给平凡的访问器套上 `Args:`/`Returns:`/`Raises:` 脚手架 —— 单行式才是正确的， + 也是 PEP 257 所偏好的。 +- 补写新 docstring 时，**不要**顺手改代码、改签名，或改动已有的（正确的）docstring。 + docstring 是**追加式**的。 +- 不要复述逐文件 SPEC —— docstring 是代码内的局部视角，SPEC 才是规范性契约。 diff --git a/doc/contributing/release-maintenance.md b/doc/contributing/release-maintenance.md new file mode 100644 index 000000000..decd6aa19 --- /dev/null +++ b/doc/contributing/release-maintenance.md @@ -0,0 +1,82 @@ +# vLLM-Omni 发版漂移审计 + +InferMatrixCopilot 的模型清单、代码 owner 路由和源码引用会随着 +vLLM-Omni 发版而过期。这个审计只读取两个 Git 提交和本仓库声明，不会 checkout +目标版本，也不会自动修改知识规则。 + +## 用户入口 + +安装 InferMatrixCopilot 后，在 Codex、Claude Code 或 Cursor 中运行： + +```text +/imupdate D:\path\to\vllm-omni +/imupdate vllm-omni +/imupdate vllmomni +``` + +支持三种用法： + +1. 本地 Git 路径：直接用 baseline 的 `audited_sha` 对比仓库 `HEAD`，机器审计后 + 由 Agent 更新知识。 +2. 仓库名、别名或 URL：Agent 先解析权威仓库与目标版本，寻找已配置 checkout + 或建立临时 clone，再尽量进入同一机器审计流程；无法运行机器审计时必须明确标记 + 为模型分析，不能宣称 `CLEAN`。选择版本时不得因为 GitHub “latest release” + 忽略预发布版本而退回到比当前 baseline 更旧的版本。 +3. 下方完整 Python 命令：纯机器规则对比，不调用 LLM，也不修改文件。 + +`/imupdate` 的可选第二个参数可以指定目标 tag 或 SHA。 + +## 底层命令 + +先在 vLLM-Omni checkout 中 fetch 需要比较的 tag 或 SHA，然后运行： + +```powershell +python tools/audit_vllm_omni_release.py ` + --from 5d44868e ` + --to v0.26.0rc1 ` + --repo D:\path\to\vllm-omni ` + --json-output $env:TEMP\vllm-omni-release-audit.json +``` + +审计覆盖： + +- AR、Diffusion 和 pipeline registry； +- deploy YAML； +- 新增、修改、删除和重命名路径； +- changed path 到 Direct 知识 owner 的机器路由； +- adapter runtime module 未覆盖的 changed path； +- active knowledge 中受删除或重命名影响的 `sources:`； +- component、model catalog 与 canonical baseline 的 source pin。 + +同样的提交和 baseline 会产生等价 JSON；报告不包含时间戳和本机 checkout 路径。 +默认 `--mode enforce`：存在未解释漂移时退出 1，输入或 Git 失败时退出 2。 +`--mode report-only` 仍报告 `DRIFT`，但退出 0，供定时巡检使用。 + +底层实现不 checkout 版本：它用 `git rev-parse` 固定两个提交，用 +`git diff --name-status -M` 找新增、修改、删除和重命名，再用 +`git show :` 直接读取 Git 对象。Python AST 负责提取模型和 +pipeline registry，deploy YAML 直接按提交列举；结果做排序和哈希后与 baseline、 +owner 路由、manifest、知识 `sources:` 和 pin 对账，最后输出稳定 JSON。 +这个命令只报告证据，不编辑任何文件。 + +## 更新一个 release + +1. 用当前 `release_baseline.yaml` 的 audited SHA 作为 `--from`，新 tag 作为 + `--to`，先跑 `report-only`。 +2. 人工确认 registry、deploy、路径 owner 和知识来源变化。 +3. 把旧的 `audited_sha` 移到 `previous_audited_sha`，再只更新报告证明已经漂移的 + baseline、catalog、source map 或 manifest；不要自动生成语义规则。 +4. 用同一组 `--from/--to` 跑 `enforce`，再运行知识 validator 和相关 pytest。 +5. JSON 是临时证据，不提交到 `knowledge/`；完成后删除。 + +## PR 学习与 release 审计是两件事 + +合并 PR 的复盘只提炼可复用、可执行的 owner 规则，且必须等最终修复、review thread +和 CI 结果稳定后再做。Release 审计只对账结构事实，不把 PR 内容、事故过程或审计报告 +写进知识树，也不因为上游发版自动升级 InferMatrixCopilot 版本或依赖。 + +CI 每周一对 upstream `main` 运行 `report-only`。**任何**改到 baseline、adapter +manifest、`imupdate` skill、本页、`knowledge/repos/vllm-omni/**` 或两个审计脚本的 PR， +都会自动用 `previous_audited_sha → audited_sha` 运行 `enforce` —— 知识页的改动同样触发。 +Actions 手动运行可用 `report-only` 检查任意区间，`enforce` 只接受当前 baseline 声明的 +升级区间。 diff --git a/doc/EVAL-PR20-report.md b/doc/evaluation/EVAL-PR20-report.md similarity index 92% rename from doc/EVAL-PR20-report.md rename to doc/evaluation/EVAL-PR20-report.md index c104f82e8..762044167 100644 --- a/doc/EVAL-PR20-report.md +++ b/doc/evaluation/EVAL-PR20-report.md @@ -1,8 +1,11 @@ # 20-case PR-review evaluation — full rerun with complete traces +> **冻结记录，保持英文原文。** 本页是当时被评审/被判分的文本本身；翻译它就是**改动一份记录**。文档树的其余部分统一为中文 —— 见 [`../README.md`](../README.md)。 + + 3 generation replicates x 20 `pr_review` items (10 train / 5 val / 5 test) x 3 judge replicates = 180 blind pairwise verdicts against the recorded `claudecode_opus48` baseline (never rerun). Judge: `claude-sonnet-5`, tool-less, randomized X/Y order — a third model, distinct from both arms. Metric definitions reused unchanged from `eval/dataset/judge_val.py`. -**Configuration**: `MOA_WHEN=off`, `PR_CONTEXT_MODE=no_discussion`, `REVIEW_DEPTH=auto`, `ALLOW_POST=0`/`ALLOW_PUSH=0`, full trace capture (`AGENT_TRACE_IO_FULL=1`). MoA off matches the A1 arm in `doc/EVAL-goal-report.md`, so the val slice stays comparable. +**Configuration**: `MOA_WHEN=off`, `PR_CONTEXT_MODE=no_discussion`, `REVIEW_DEPTH=auto`, `ALLOW_POST=0`/`ALLOW_PUSH=0`, full trace capture (`AGENT_TRACE_IO_FULL=1`). MoA off matches the A1 arm in `doc/evaluation/EVAL-goal-report.md`, so the val slice stays comparable. ## Leakage controls (asserted per item, not assumed) @@ -22,7 +25,7 @@ Per split, never pooled: train is the adaptation stream, val the promotion gate, | val | 5 | 0.468 ± 0.044 | 0.745 ± 0.049 | 0.693 ± 0.043 | | test | 5 | 0.392 ± 0.053 | 0.790 ± 0.038 | 0.792 ± 0.037 | -### vs the recorded A1 val numbers (`doc/EVAL-goal-report.md`) +### vs the recorded A1 val numbers (`doc/evaluation/EVAL-goal-report.md`) | dim | this campaign (val) | A1 (val) | Δ | |---|---|---|---| diff --git a/doc/EVAL-goal-report.md b/doc/evaluation/EVAL-goal-report.md similarity index 96% rename from doc/EVAL-goal-report.md rename to doc/evaluation/EVAL-goal-report.md index 028a33c2f..b3737c3d1 100644 --- a/doc/EVAL-goal-report.md +++ b/doc/evaluation/EVAL-goal-report.md @@ -1,5 +1,8 @@ # Goal evaluation report — one-click NL copilot vs the CC+Opus baseline +> **冻结记录，保持英文原文。** 本页是当时被评审/被判分的文本本身；翻译它就是**改动一份记录**。文档树的其余部分统一为中文 —— 见 [`../README.md`](../README.md)。 + + 2026-07-18 · hook-reviewed eval plan v3 (6 review rounds; plan + trail + all tooling in `eval/dataset/goal-eval/`) · metric definitions reused unchanged from `eval/dataset/judge_val.py` (7 dims, blind pairwise, 3 judge replicates, diff --git a/doc/evaluation/EVAL-goal-strict-vs-opus5.md b/doc/evaluation/EVAL-goal-strict-vs-opus5.md new file mode 100644 index 000000000..2edf05430 --- /dev/null +++ b/doc/evaluation/EVAL-goal-strict-vs-opus5.md @@ -0,0 +1,100 @@ +# Goal campaign — DeepSeek-v4-pro Strict vs Claude Code + Opus 5 (2026-08-12/13) + +> **冻结记录，保持英文原文。** 本页是当时被评审/被判分的文本本身；翻译它就是**改动一份记录**。文档树的其余部分统一为中文 —— 见 [`../README.md`](../README.md)。 + + +> **Final update (2026-08-13, official-model rework):** after the official +> `deepseek-v4-pro` release the pipeline was rebuilt for a strong generator +> (deep investigator+adversary passes alongside the behavior/verification +> breadth lenses; a per-comment agentic verify pass; self-proving verbatim +> evidence quotes end-to-end). Result — **val: ARM 8—7 (first head-to-head +> win); frozen test, one-shot confirmatory: ARM 7—8. Combined 15—15** vs +> the Opus 5 baseline at roughly 1/3 the cost, from 1—4 (val) pre-campaign. +> Test means: r .689 vs .762, p .687 vs .719 (arm's best precision on any +> split); the GOLD gap item pr4834 was swept 3/3 clear. Rubric-mean gaps +> (.03-.07) sit at the magnitude of measured judge drift on identical +> baseline reviews (±.05-.10 across gates). Honest verdict: the copilot +> moved from clearly-worse to statistically indistinguishable from the +> agentic Opus 5 baseline; "strictly better on both means" is NOT +> demonstrated. Both tuning splits are now spent (8 val submissions, 1 +> test shot) — any further iteration requires fresh items (wave2 snapshots +> exist under `eval/dataset/gt/` for exactly this). Key structural lesson +> for successors: the judge is tool-less — repo-side findings score as +> speculation unless the comment carries verbatim-quoted proof; making +> evidence self-proving was the single biggest win (4—11 → 8—7). + +Goal: make the Strict pipeline (DeepSeek-v4-pro generation) beat the recorded +`baselines/claudecode_opus5` agentic review on the 5-PR val split. Protocol: +adaptation on train only; judge `gpt-5.6-sol-high` via cursor (blind pairwise, +`judge_val.py`); PR-time trees pinned to `expected_pr_heads.json`; +`PR_CONTEXT_MODE=no_discussion`. All arms/judgments under +`eval/dataset/arms/copilot_v{7,8,9}_*` and `judgments/goal_*`. + +## Verdict — the gate did NOT pass + +| gate | arm | model | wins (arm—opus) | arm r/p | opus r/p | +|---|---|---|---|---|---| +| pre-campaign (g56) | copilot_v4_pr20_r1 | v4-pro[1m] preview | 1—4 | 0.42/0.51 | 0.82/0.54 | +| val gate 1 | copilot_v8_official_r1 | v4-pro official | 3—12 (×3 reps) | **0.86**/0.56 | 0.79/0.66 | +| val gate 2 (precision trim) | copilot_v9_official_r1 | v4-pro official | 2—13 (×3 reps) | 0.80/0.54 | 0.82/0.65 | + +What the campaign did achieve, measured: + +- **Val recall 0.42 → 0.80–0.86** — at or above the Opus 5 baseline (0.79–0.82). + The pre-campaign recall deficit (−0.40) is gone. +- Gate 1 **swept the GOLD latent-gap item pr4810 3/3** (the arm engaged the + gap both gates; verdict wins depend on the noisy precision margin). +- Train hardest-6: recall 0.42 → 0.50 monotonically over calibration rounds; + pr4870 flipped to a consistent 2/2 win (r≈0.83, p≈0.84). +- Cost ~$0.26–0.53/item ≈ 5× cheaper than the baseline. +- The loss driver flipped: pre-campaign the arm lost on recall; now it loses + on **precision** (0.54–0.56 vs 0.65) — the judge discounts individually + under-grounded or speculative comments, an agentic-verification property of + the generator that two calibration rounds of budget/reducer tuning did not + buy. Head-to-head margins are mostly "slight". + +## What changed (all in-tree, tests + knowledge validators green) + +Workflow: planner `value_flips` signal (default flips can't rule light); +verification lens in the fallback set; evidence caps 120k→260k / reducer +60k→280k (a 170k diff was losing 30% of its hunks); commit timeline + +deterministic changed-symbol consumer sweep (`git grep -nw`) in the evidence +pack; hunk-location paging targets; cross-lens corroboration tags + reducer +protected-class drop rules; comment budget 6 (+≤3 coverage promotions, +overflow only on evidence-rich reviews); coverage-promotion pass converting +lens findings/blockers into grounded comments; severity + red-CI calibration. + +Knowledge: `repos/vllm-omni/review/guides/strict-review-checklist.md` +(train-distilled triggered checks) injected via new manifest key +`knowledge.review_checklist` (linter extended); module map 8→17 entries, +risk tiers +platform/diffusion/distributed (small diffs there now get the +full ensemble — this alone fixed the pr5009/4977 class). + +## Forensic lessons (train, 3-agent trace analysis + 5 judged rounds) + +1. Depth misfires and evidence truncation were real and fixed (verified live). +2. The 5-comment severity-only cap deleted reducer-kept GT concerns; but + widening budgets converts to noise — the judge is recall-dominant between + arms yet punishes volume within an arm. The stable operating point is a + small budget plus class-protected promotion. +3. Generation variance and judge variance are both large: single-replicate + verdicts flipped items across identical configs. Never calibrate on one + replicate (this campaign's r2 was wasted that way). +4. The residual deficit is per-comment grounding depth (the baseline + verifies each claim agentically before asserting it). Workflow scaffolding + moved recall dramatically; it did not move groundedness. + +## Next levers, ranked + +1. Per-comment agentic verify pass (tools, one comment at a time) before the + budget — directly attacks the precision deficit; ~+30% cost. +2. `ensemble_lens_max_iters` ↑ for the drafting lenses on the official model + (the baseline reads 3–10× more code per finding). +3. MoA re-test with the official pro model as reducer over stronger members + (the old regression used weak mimo/qwen members). +4. Re-baseline: the official v4-pro replaced the [1m] preview mid-campaign; + train-side numbers predate it. + +Judgments for the two gates: `judgments/goal_v8_val`, `judgments/goal_v9_val` +(the 402-outage set is quarantined as `INVALID_apierror_goal_v7_val`). +Test split remains frozen and untouched. diff --git a/doc/evaluation/EVAL-v14-v16-recall-campaign.md b/doc/evaluation/EVAL-v14-v16-recall-campaign.md new file mode 100644 index 000000000..9387fd6ff --- /dev/null +++ b/doc/evaluation/EVAL-v14-v16-recall-campaign.md @@ -0,0 +1,154 @@ +# Campaign report — v14→v16 recall attack (2026-08-15) + +> **冻结记录，保持英文原文。** 本页是当时被评审/被判分的文本本身；翻译它就是**改动一份记录**。文档树的其余部分统一为中文 —— 见 [`../README.md`](../README.md)。 + + +Successor to `doc/evaluation/EVAL-goal-strict-vs-opus5.md` (the v7→v13 campaign, which +ended at 15—15 combined fresh verdicts under a GPT judge and a −.16 recall +ratio under the Sonnet judge). This campaign's goal, set by the owner: make +the Strict pipeline beat the recorded `baselines/claudecode_opus5` arm on +**both** recall and precision, using cursor-agent-based configurations and +distillation from Fable traces. + +**Result in one line: the arm matches CC+Opus 5 on precision and trails it on +recall by about five rubric points, at roughly a third of the cost — settled +by a pre-registered 20-item measurement, after an earlier claim of superiority +had to be withdrawn as a statistical artifact.** + +## What was built + +Method: forensics on spent splits (per-GT-finding coverage matrices with +stage-of-loss classification from the arms' own traces), plus a Claude Fable 5 +*teacher arm* run through the identical pinned harness as the baseline on the +train split — which caught the pr4870 GOLD latent gap exactly (the #4910 +dual-axis mechanism human review missed), giving a concrete distillation +target rather than a stylistic one. + +Landed (all default-on, kill-switched, `doc/features/review-recall.md`): + +* **Investigation duties** (checklist 13–23): claim ledger, sibling contrast, + falsification of the PR's own numbers, semantic-merge audit, producer/ + consumer census, test/gate epistemics + CI economy, differential parity, + dead-knob recompute, cache pathology. +* **Docs claims-audit/IA pass** replacing the code-shaped breadth lenses on + docs-heavy diffs, plus planner depth scaling for docs. +* **Archaeology tools** — `diff_stat`, `file_at_base`, `show_commit`, + `search_history`, `calc` — read-only, bridged to harness backends. +* **Coverage-driven second round** (the v13 RFC's open q3), seeded per-hunk + from the run's own coverage holes with uncapped per-file diff slices. +* **Ranked verification ledger** with `[resolved]`/`[claim-*]` channels, and + ledger-residue promotion (forensics found the misses were often already + written down by the arm, recorded approvingly instead of raised). +* **Machinery fixes**, two of them long-latent: pass finals dying at the token + ceiling with a crippled repair round, and the gray-zone planner silently + failing on every gray item since the official reasoning model landed (its + thinking consumed a 400-token cap before any JSON could appear). +* **Per-pass backend routing** (`review_lens_backends`), which turns the + measured model complementarity into one arm. + +## What was measured + +Judge: claude-sonnet-5, blind pairwise, 3 replicates, against the pinned +Opus 5 baseline on identical frozen snapshots. Gates: wave 3 (spent, two +disclosed attempts), wave 4 (clean, three DS-core configs), wave 5 (power +extension, pre-registered). + +| config | raw wins | Δrecall [95% CI] | Δprecision [95% CI] | +|---|---|---|---| +| v15 (all-DS) r1 | 10—20 | −.075 [−.184, +.035] | +.029 [−.022, +.080] | +| v15 r2 (flash cheap seats) | 11—19 | −.056 [−.175, +.064] | +.022 [−.027, +.071] | +| v16 (Fable adversary/round-2) | 6—23—1 | −.080 [−.169, +.010] | −.006 [−.069, +.058] | +| Composer cursor-backend | 4—24—2 | −.081 [−.206, +.044] | −.045 [−.095, +.005] | +| **pooled DS-core, 90 verdicts** | 27—62—1 | **−.070 [−.161, +.021]** | **+.015 [−.025, +.055]** | + +Δ = arm − baseline, paired inside each verdict, clustered by item +(`eval/dataset/paired_analysis.py`). Cost: $0.97/item vs $3.09. + +## The measurement lesson (the part worth keeping) + +Mid-campaign this report would have claimed "precision above baseline on a +fresh holdout" — computed from each side's raw rubric mean. That claim was +**withdrawn**, because the raw means cannot support it: three wave-4 judgment +sets scoring the *same* baseline reviews returned baseline recall means of +.335 / .338 / .416 — ±.08 of pure judge drift, larger than every effect under +study. Pairing within the verdict (one judge call scores both candidates, so +its leniency cancels) and clustering replicates by item gives intervals that +all span zero. + +Three consequences, all of which generalize past this campaign: + +1. **Never compare an arm's mean from one judgment set against a baseline's + mean from another.** Pair inside the verdict. +2. **Replicates of one item are not independent observations.** Ten items × 3 + replicates is n=10 for the CI, not n=30; treating it as 30 understates the + standard error roughly two-fold. +3. **Check power before spending a holdout.** At the observed item-level sd + (.127), resolving a .07 difference needs ~32 items. Wave 4's 10-item gates + never could have answered the question they were run to answer — three + configs were submitted against a bar the design could not measure. + +## The pre-registered answer (wave 5) + +Wave 4's 10-item gates could not resolve the effect they were run against, so +wave 5 was drawn, frozen, and **pre-registered before any wave-5 number +existed** (`eval/dataset/goal-eval/PREREG-wave5.md`): arm, items, judge, +analysis and decision rule fixed in advance, including the instruction not to +run a third wave if the interval still contained zero. + +Primary analysis, v16 pooled over wave 4 + wave 5 (20 items, 60 verdicts): + +| statistic | value | rule triggered | +|---|---|---| +| **Δrecall** | **−.049 [−.097, −.001]** | CI below 0 → the deficit is REAL; stop claiming parity on recall | +| **Δprecision** | **−.000 [−.041, +.040]** | CI contains 0 → parity | + +The recall result is marginal (upper bound −.001) and should be read as *a +real but small deficit*, not a large one. It is also concentrated: of the 20 +pooled items, 6 favour the arm, 4 are inside ±.02, and one item (pr5978, +−.37) supplies roughly a third of the gap. Per the pre-registration, **no +third wave was run.** + +## Honest verdict against the goal + +The goal — both means strictly above the baseline — is **not achieved**, and +with corrected statistics it was never achieved at any point in the campaign. +What the data supports: + +* the arm **matches the baseline on precision and trails on recall by ~.05**, + at roughly a third of the baseline's cost; +* the recall deficit is concentrated, not diffuse — a minority of items carry + the aggregate on both wave 4 and the pooled set; +* **configuration effects replicate in sign** even where individual deltas are + not significant: v15 leans precision (+.029/+.022 across independent + generations), v16 leans recall (best fresh-split recall, .336) and is the + only configuration that reproduced the GOLD-gap catch inside the pipeline; +* Composer-2.5 is a breadth proposer, not a closer (Δprecision −.045): best + challenger recall, measurably worse precision, on three pipeline versions. + +## Product recommendation + +Both configurations are supported and documented in `.env.template`: + +* **v15 (default, all-DS)** — precision-leaning, ~$1/item. The right default + for volume review. +* **v16 (`REVIEW_LENS_BACKENDS` routing Fable into adversary + second round)** + — recall-leaning, adds two harness sessions per item. The right choice for + high-stakes PRs where a missed defect costs more than a spurious comment. + +## Where a successor should start + +The measurement is settled, so the next campaign should not re-litigate it. +Three things follow from the data: + +1. **The gap is one item class, not a level.** A third of the pooled deficit + comes from a single item; wave-3 forensics named the recurring classes + (test/gate epistemics, docs information architecture, CI economy). Fixing + the class beats another round of general duties. +2. **Budget the gate before the pipeline.** At the observed item-level sd + (.103), a .05 effect needs ~38 items. Anything measured on 10 items is + uninterpretable in this range — three configurations were submitted + against a bar wave 4 could not measure. +3. **The remaining untried lever is a strong generator throughout.** Fable in + two seats produced the campaign's best recall and the only in-pipeline + GOLD-gap catch; Fable in every seat was scoped (~$10–20/item, inverting + the cost story) and deliberately not run. diff --git a/doc/evaluation/README.md b/doc/evaluation/README.md new file mode 100644 index 000000000..2263dc6ac --- /dev/null +++ b/doc/evaluation/README.md @@ -0,0 +1,45 @@ +# 评测记录 + +**当前结论（2026-08-17）：Strict 评审在 recall 上落后 CC + Opus 5 基线约五个 +rubric 点（Δrecall −.049 [−.097, −.001]），precision 打平（−.000），成本约为 +基线的三分之一（$0.97 vs $3.09 每项）。** + +这个数字来自预注册的 20 项测量。此前一次"优于基线"的宣称已被撤回——原始均值 +支撑不了它。 + +## 边界：这里放什么 + +- **这里**（`doc/evaluation/`）：叙述性报告——一次战役做了什么、测出什么、 + 该怎么读。**冻结**，各自停在自己的日期上。 +- **[`eval/`](../../eval/README.md)**（顶层）：数据集、生成/判分脚本、arms、 + judgments、结果表。数字有出入时**以那边为准**，尤其是 + [`eval/dataset/results/model_comparison.md`](../../eval/dataset/results/model_comparison.md)。 + +结论摘要与方法论浓缩版在 [`GUIDE.md §8`](../GUIDE.md#8-性能对比)。 + +## 报告（新→旧） + +| 报告 | 时间 | 结论 | +|---|---|---| +| [`EVAL-v14-v16-recall-campaign.md`](EVAL-v14-v16-recall-campaign.md) | 2026-08-15 | **最新**。召回攻坚：precision 打平，recall −.049，成本 1/3。附带三条会超出本战役的测量纪律 | +| [`EVAL-goal-strict-vs-opus5.md`](EVAL-goal-strict-vs-opus5.md) | 2026-08-13 | v7→v13：val 首胜 8—7，冻结 test 7—8，combined **15—15**。从"明显更差"走到"统计上无法区分" | +| [`EVAL-goal-report.md`](EVAL-goal-report.md) | 2026-07-20 | 早期一键 NL copilot vs CC+Opus 基线 | +| [`EVAL-PR20-report.md`](EVAL-PR20-report.md) | 2026-07-23 | 20 例 PR 评审全量重跑，含完整 trace | +| [`RESEARCH-reference-agents.md`](RESEARCH-reference-agents.md) | 2026-07-22 | 参考 agent 的机制调研（PR 评审与 issue 回答） | + +## 读这些报告之前必须知道的三条 + +这三条是 v14→v16 战役用一次被撤回的结论换来的，且**超出该战役本身**： + +1. **绝不拿一组 judgment 里 arm 的均值去比另一组里 baseline 的均值。** + 三组 wave-4 judgment 对**同一批**基线评审打分，基线 recall 均值分别是 + .335 / .338 / .416——±.08 的纯判官漂移，比所有被研究的效应都大。正确做法是 + 在 verdict 内部配对（同一次判官调用同时给两个候选打分，宽严自然抵消）。 +2. **同一个 item 的多个 replicate 不是独立观测。** 10 items × 3 replicates + 对 CI 来说是 n=10 而不是 n=30；当成 30 会把标准误低估约一倍。 +3. **花掉 holdout 之前先算功效。** 在实测的 item 级 sd（.103–.127）下，分辨 + .05 的差异需要约 32–38 个 item。10-item 的门根本无法回答它被用来回答的问题。 + +配套的三道流程闸（2026-08-17 落地，每道对应一次真实事故）：manifest 记录 +**解析后**的设置而非 env 字符串；余额预检**拒绝启动**而不是跑到一半死； +混合门覆盖 MoA 路径。 diff --git a/doc/RESEARCH-reference-agents.md b/doc/evaluation/RESEARCH-reference-agents.md similarity index 98% rename from doc/RESEARCH-reference-agents.md rename to doc/evaluation/RESEARCH-reference-agents.md index 5b6f7e2c6..4948fefbe 100644 --- a/doc/RESEARCH-reference-agents.md +++ b/doc/evaluation/RESEARCH-reference-agents.md @@ -1,5 +1,8 @@ # Reference-agent research — mechanisms for PR review & issue answering +> **冻结记录，保持英文原文。** 本页是当时被评审/被判分的文本本身；翻译它就是**改动一份记录**。文档树的其余部分统一为中文 —— 见 [`../README.md`](../README.md)。 + + Goal-phase-3 deliverable (hook-approved research plan, 2026-07-17). Evidence classes: **source-backed** (clone file:line), **documented** (official doc URL), **inferred** (labeled; never grounds an "adopt"). Every entry cross-checks what the copilot already has (adaptive depth `review/planner.py`, single-model lens ensemble `agent_runtime/ensemble.py`, on-demand `gh_pr_view`/`gh_issue_view`/`gh_ci_read` `_common.py:133-173`, knowledge briefing + doc tools). ## A. Baseline behavior autopsy (output-level; no turn traces exist — `cost.json` has aggregate `calls` only, so tool-usage claims are inference) diff --git a/doc/features/auto-run.md b/doc/features/auto-run.md new file mode 100644 index 000000000..a4a1b8c6d --- /dev/null +++ b/doc/features/auto-run.md @@ -0,0 +1,214 @@ +# 事件触发自动运行（GitHub events） + +> **一览** +> | | | +> |---|---| +> | **状态** | ⚠️ **draft v3 —— 未实现。** 本页描述的是一个提案，不是现有行为。仓库里**没有**任何触发层 | +> | **想做什么** | 给 copilot 一个触发层，让它对 GitHub 事件无人值守地运行（先做 PR 自动评审，再做 issue 回答/分流） | +> | **今天的实际状态** | 每一次运行都是**手动发起**的。headless 构件已具备，触发器、幂等/归属规则、安全加固与放量门禁都还没有 | +> | **落地前必须先有** | 安全加固（本文档列出的那几项）+ 允许发布的分阶段门禁；发布本身仍受 `ALLOW_POST` 约束 | +> | **讨论** | https://github.com/JiusiServe/InferMatrixCopilot/issues/3 | +> +> 读这一页时请记住：它是**设计讨论**，不是使用说明。别照着它配置任何东西。 + +--- + +## 原始 RFC + +- **状态**：draft v3，供团队讨论（已折入两轮 gpt-5.6 评审） +- **作者**：tzhouam（+ copilot 分析） +- **日期**：2026-07-23 +- **讨论**：https://github.com/JiusiServe/InferMatrixCopilot/issues/3 + +## 摘要 + +给 copilot 一个触发层，让它对我们的 GitHub 仓库**无人值守**地运行 —— +先做自动 PR 评审，之后再做 issue 回答/分流。今天每一次 run 都是手动发起的。 +headless 的构件已经具备；本 RFC 覆盖触发器、幂等/触发归属规则、 +**必须先落地**的安全加固，以及允许它发布的放量门。 + +## 动机 + +- copilot 现在已被用在 vllm-omni 之外（AgentInfer、afd-plugin）；issue #1/#2 就来自 + 这次采用，且都已修复。**手动发起是仅剩的摩擦。** +- 实测包络（20-PR 战役，同一批 20 个 PR 上的 3 次生成 replicate —— + **不是 60 个独立条目**）：eco 路径下每次评审约 $0.19、6–13 分钟； + precision 0.777 ± 0.026、recall 0.430 ± 0.018，且与已录制基线之间存在**有记录的 + 上下文不对称**。读法：**评论通常是对的，但覆盖是部分的** —— + 它是一个**额外的**评审者，便宜到可以对每个 PR 都跑；**绝不是人工评审的替代品**。 +- 在 PR 打开后几分钟就到达的评审会改变作者行为；**必须被请求才会发生的评审， + 大多不会发生。** + +## 已经存在的东西 + +- headless 一次性：`-p "review and post the review" --yes`； + URL/裸引用的路由是确定性的（不花 LLM 意图调用），且显式的 post 意图能穿过它 + （issue #1 的修复）。 +- **双闸的对外写入**：命令必须携带 post 意图**并且** `ALLOW_POST=1`； + 达不到就是 dry run。推送安全（只有 force-with-lease、保护 `main`）不变， + 且不在本 RFC 范围内。 +- `pr.gate_check` 报告 draft/合并态/失败 CI —— 作为评审**上下文**， + **刻意非阻塞**。触发层必须做**自己的**资格过滤（draft/closed/merged）； + **它不能倚靠这个 step。** +- 无人值守的失败路径：`ESCALATION.md`、退出码 3、升级邮件。 + +## 前置加固（任何自动运行之前的阻塞项） + +自动运行把"敌意的 PR/issue 内容"从理论输入变成**日常输入**，而当前姿态存在真实缺口。 +这些**必须先落地**： + +- **H1 —— 读取受仓库边界约束。** 今天 agent 的读取工具会把相对路径解析到该 run 的仓库根， + 但**绝对路径原样透传**，而 `PathScope` **只约束写**。一个被提示注入的评审 agent + 可以读取**任意宿主文件**（密钥、`.env`）并把它们引用进输出。 + 修法：对 `read_file`/`list_dir`/`grep`/`run_shell` 的 cwd， + **拒绝该 run 根/worktree 之外的绝对路径**，并拒绝 `.env*`/密钥材料模式 + （对话侧的 `repo_read` 已经有这套，把它应用到 step 运行时）。 +- **H2 —— 出站清洗器。** 在 `pr.post_review` 发布之前，扫描正文里凭据形状的字符串 + （key 前缀、PEM 头、高熵 token），拒绝或脱敏并升级。这是"经评论外泄"的纵深防御； + 注意 **`` 围栏是 prompt 指导，不是执行边界 —— H1/H2 才是边界。** +- **H3 —— 最小权限身份与隔离。** 轮询器**不得**使用环境里的 `gh` 认证 + （它可能持有很宽的权限）。要用**专用机器账号 + 细粒度 PAT**：仅限选定仓库、 + PR 读写、别无其他 —— 且轮询器以一个**专用 OS 用户**身份运行， + 它的环境里**只有**那个 PAT 和它自己的 LLM key，**没有**宿主的其他凭据。 + 这也让每一条被发布的评论都带上一个 bot 名字。 +- **H4 —— 自动运行中不允许 agent 控制的持久化写入。** 评审 agent 平时可以提出 skill + candidate（`skill_update_candidate`）；在自动运行下那是一条**从注入通向持久状态**的 + 路径，所以**自动触发的 run 禁用知识写入工具**。自动 run 是**纯读 + 报告**； + 知识累积仍然是人工发起的流程。 + +## 设计选项 + +### A. 在每个目标仓库里放一个 GitHub Actions workflow + +`pull_request`（opened / ready_for_review）→ 安装 copilot → 一次性运行 → 用仓库 token 发布。 + +- 优点：事件驱动，团队可见、可重跑，逐仓库选择加入。 +- 缺点：fork PR 拿不到 secret，且 `pull_request` 上是只读 token； + `pull_request_target` 会把 secret 交给**由不可信 PR 触发**的 run —— + **在 H1/H2 充分浸泡之前不可接受**。临时 runner 会**重置知识平面** + （debug memory、skill 先验）；只有已入库的 profile briefing 会留存。 + LLM key 会扩散进每个仓库的 Actions secret。 + +### B. 在 copilot 宿主上放一个轮询器（推荐的起点） + +在那台已经跑着夜跑 rebase 的机器上，放一个由 systemd timer 驱动的 `copilot-watch` +脚本：逐个受管仓库，列出开放 PR → 资格过滤 → 去重（见下）→ 一次性运行 → +记录 + 发布（Phase 1+）。 + +- 优点：继承整套现成设置（checkout、worktree 缓存）和**持久的知识平面**； + **没有 LLM secret 被导出到 GitHub**；约 100–200 行；与夜跑 rebase 同一套运维模式。 +- 缺点：触发对团队不可见（结果是可见的）；单宿主；轮询延迟（见下）。 + +### C. GitHub App / webhook 服务 —— 明确的非目标 + +那是产品化的形态；真实的托管 + 认证面。**不适合我们这个规模。** + +## 提案 + +**Phase 0 —— 只报告的影子运行（≥1 周）。** 在团队提名的仓库上跑选项 B，H1–H4 已落地。 +影子 run **完全不带 post 意图**（一个**真正只读**的 TaskSpec —— +`ALLOW_POST=0` 是保险带，**不是机制**）。评审累积在 run 目录里；升级邮件发给运维者。 +由人类裁决输出（见下方的门）。 + +**Phase 1 —— 发布。** 对通过门的仓库逐个翻开 `ALLOW_POST=1`。 +以 bot 身份经 `gh pr comment` 发布，带签名页脚 +（`copilot-review: # @ / `）。 +**已知且刻意的限制**：这是一条**评论**，不是 GitHub review 提交 —— +没有 APPROVE/REQUEST_CHANGES 状态，没有 inline thread。 +升级到 `gh pr review`（以及 bot 是否**可以**设置阻塞状态）是**另一份后续提案**。 + +**Phase 2 —— 可选的 Actions 前端（只是草图）。** 面向想要团队可见触发器的仓库： +**仅同仓库 PR**（**不用 `pull_request_target`**），`/copilot review` 评论命令用于重跑， +且在使用任何 secret **之前**先做发起者权限校验。 +**每个仓库只有一个触发归属**：一个仓库要么由轮询器服务、要么由 Actions 服务， +**绝不同时**；迁移即转移归属。Phase 2 在这里**刻意写得不完整** —— +它会有自己的 RFC（触发安全、权限检查、workflow 溯源）**再谈实现**； +Phase 0/1 的任何东西都**不依赖它**。 + +### 触发与幂等规范（Phase 0/1） + +- **资格**：开放、非 draft、未合并/未关闭、**在该仓库启用时间戳之后**打开 + （**不做历史积压扫荡**）。 +- **去重键**：`(repo, pr_number)` —— 一个 PR **只被自动评审一次**，在它首次符合资格时。 + 新的 push（`synchronize`、force-push）**不会**重新触发；重评审**只按需**发生 + （`/copilot review`，绑定新的 head SHA，且**仅限有写权限的评论者**）。 + 这让成本和评论噪声**随 PR 数线性增长，而不是随 push 数**。 +- **账本是缓存；GitHub 才是真相来源。** 事务顺序：记 `started` → 运行 → + **就在发布之前**重新解析 head SHA 与 PR 状态（已推进/已关闭 ⇒ 标 `stale`，不发布）→ + 发布 → 记 `posted` + 评论 URL。启动时以及任何崩溃不确定时， + 通过查询 bot 的签名评论**对账** —— 因此"发布与记录之间崩溃"**不会重复发布**， + 而一条 `started` 却死掉的条目会**重试**而不是被抑制。 + 把被评审的 SHA 写进页脚，好让人类看得见 review-vs-head 的漂移。 +- **run 状态与上限**：账本条目按 + `started → reviewed → posted | stale | failed | post_unknown` 流转。 + `failed` 最多带退避重试两次，然后升级 —— **一个持续崩溃的 PR 不能每个周期都烧预算**； + `post_unknown`（已发布但未记录）**只能**由签名评论查询来解决。 + 每次 run 硬超时（30 分钟杀掉）、每日成本熔断（可配，例如跨仓库 $10/天）、 + run 目录保留上限。 +- **排队要诚实**：一次评审要 6–13 分钟；**一个 worker**、每周期 ≤4 次新评审、 + 跨仓库 FIFO。延迟是"轻载下约 15 分钟"，在突发时**线性增长** —— + 队列年龄告警（不只是邮件）是健康叙事的一部分，与既有的 `/status` 和每周人工瞥一眼 + 账本并列。API/限流失败**跳过该周期**；连续 ≥3 次失败**发邮件升级**。 +- **防回环**：触发器忽略由 bot 账号或任何 `[bot]` 发起者撰写的 PR 与评论 —— + **copilot 绝不因自己（或别的 bot）的输出而触发。** + +### Phase-1 晋升门（逐仓库） + +只报告的输出由**仓库 owner + 另一位评审者**裁决（分歧向**更严格**的标签收敛）： +每条发现被标为 有用 / 中性 / 错误。在 **≥15 个 PR 且 ≥30 条发现**上满足以下条件才晋升： +有用率 ≥ 0.75；**≥50% 的 PR 至少收到一条有用发现**（一个大多沉默的评审者不得通过）； +**零条严重错误发现**（那种会误导合并的"错误阻塞项"）；且没有未解决的注入/外泄事故。 +recall 被跟踪但**不设门** —— copilot 是**加法**。签字：仓库 owner。 +**这是一道判断门，不是显著性检验** —— 那些下限的存在是为了让"有用率"有意义， +而 owner 可以在任何疑虑下让一个仓库**继续留在影子期**。 + +### 策略默认值（可辩论） + +- 评审深度 `auto`；eco 模型路径。 +- issue 自动回答：Phase 0/1 **关闭**；单独提案、单独的门 —— + **错误的 PR 评论浪费评审者几分钟，错误的 issue 回答误导用户。** +- 具备写能力的 kind（pr_debug 的修复、rebase 推送）：**永久排除**在自动运行之外。 + +## 安全考量 + +- 发布**仍然是双闸的**；自动运行改变的是评审**何时**跑，不是它**被允许做什么**。 +- 注入的最坏情况（在 H1/H2 之后）：一条错误或敌意的**评论** —— + 它本身可能触发别的仓库的评论 bot；bot 身份（H3）让其他自动化能过滤它， + 而清洗器为外泄划定了边界。 +- fork PR：**仅轮询器**（H3 身份只持有评论权限）； + `pull_request_target` **保持禁用**，除非被显式重新讨论。 +- 密钥：Phase 0/1 把所有 LLM key 留在 copilot 宿主上； + Phase 2 才把它们加进参与仓库的 Actions secret。 + +## 回滚 + +- `systemctl stop` **timer 和 service unit**（终止任何活动 worker/run）， + 然后吊销 bot PAT ⇒ 没有新 run，也没有在途的发布。 +- **已发布的评论不会被自动撤回**：当原因是质量/事故时，回滚流程包括批量最小化 bot 的 + 评论（GraphQL `minimizeComment`），并在配置里关闭 `/copilot review` 处理。 +- Phase 2：删掉 workflow 文件；触发归属逐仓库回退。 + +## 给团队的未决问题 + +1. Phase 0 覆盖哪些仓库？ +2. 晋升门（≥10 PR / ≥20 条发现上有用率 ≥0.75、零严重错误）是否合适， + 除仓库 owner 之外还由谁裁决？ +3. "每 PR 一次 + 按命令重评审"是否可接受，还是有人希望**每次 push 都重评审** + （不顾成本/噪声）？ +4. `/copilot review` 是否应该落在 Phase 1（轮询器读评论）而不是等 Phase 2？ +5. 谁运维轮询器宿主、谁负责升级收件箱？ +6. 评论 vs GitHub review 提交：Phase 1 里"没有状态"这个限制是否可接受， + 还是在任何人依赖它之前，inline/review 状态支持就是必需项？ + +## 评审记录 + +先由 copilot 的分析起草，然后**两次**经过我们的外部 plan-review 钩子（gpt-5.6）。 +第 1 轮（裁决 REVISE）浮现、对照源码核实并折入了：绝对路径读取缺口（→ H1）、 +gate_check 非阻塞的表述错误、评论 vs review 提交、synchronize/账本的矛盾 +（→ 每 PR 一次的键控）、评测告诫的诚实性，以及事务性发布顺序。 +第 2 轮（裁决 REVISE）补充了：skill candidate 写入作为一条注入路径（→ H4）、 +不带 post 意图的影子 run、重试/超时/成本上限、防回环、门的下限， +以及"杀掉活动 worker"的回滚 —— 全部折入。 +**未采纳**（记录以备讨论，判定在我们这个规模上不成比例）： +带受限出网的完整容器隔离，以及把 Phase 1 阻塞在 GitHub-review 状态发布上。 +评审者对这两点的立场可按需索取。 diff --git a/doc/features/provider-registry.md b/doc/features/provider-registry.md new file mode 100644 index 000000000..8aa3c671a --- /dev/null +++ b/doc/features/provider-registry.md @@ -0,0 +1,311 @@ +# Provider registry —— Strict 的五种后端 + +> **一览** +> | | | +> |---|---| +> | **状态** | ✅ 已实现。`api` / `cursor` / `claude-code` / `deepseek` 已实测；`codex` 仅离线测试（开发机无 ChatGPT 登录，readiness 会报出登录缺口） | +> | **做什么** | 让只有编码 Agent 订阅、没有裸 API Key 的用户也能跑 Strict；顺带把原有 api 路径收编进同一张注册表 | +> | **怎么开关** | `STRICT_BACKEND=api\|cursor\|claude-code\|codex\|deepseek`（另有 `STRICT_BACKEND_MODEL` / `_CONCURRENCY` / `_CLI` / `_TIMEOUT_S`） | +> | **怎么用** | 面向使用者的说明在 [`../guide/backends.md`](../guide/backends.md) | +> | **实测** | 后端对比见 [`eval/dataset/results/model_comparison.md`](../../eval/dataset/results/model_comparison.md)；结论摘要在 [`../GUIDE.md §8`](../GUIDE.md#8-性能对比) | +> | **硬边界** | 用 `api` 时行为**逐字节不变**（平价棘轮）；harness 会话的工具一律经 `tools.dispatch`；子进程环境是白名单 | +> +> 以下是原始决策记录（RFC），保留它是因为其中的取舍——为什么接入点必须是 +> `run_session()` 而不是 `LLM.create()`、为什么工具治理分预防/侦测两级—— +> 至今仍在约束这块代码。 + +--- + +## 原始 RFC + +- 状态：已接受（2026-08-14 的 grilling 会话）；M1（注册表、api 平价、工具桥、 + cursor transport）已在 PR #81 合入并实网冒烟；M2 claude-code 与 M3 codex 已在本分支 + 实现 —— claude-code 已在订阅认证下实网冒烟，codex **仅离线测试** + （开发机没有 ChatGPT 登录；readiness 会报出这个登录缺口）。 + 工具桥另外提供按需的 `repo_map`；skill/memory 检索工具**仍然仅限进程内** + （跨进程的 candidate 写入**刻意未开放**）。 +- 归属：LLM/后端层（`llm.py`、`config.py`）、agent 运行时 + （`engine/agent_runtime/runner.py`）、新增的 `src/infermatrix_copilot/providers/` +- 研究过的先行工作：Hermes Agent 的 `api_mode` transport + （hermes-agent.nousresearch.com/docs/developer-guide/adding-providers）· + opencode 的 provider 注册表（opencode.ai/v2/docs/providers） + +## 动机 + +Strict 模式跑完整执行主脊，而每一次模型调用都经过 `llm.py::LLM` —— +一个需要 `ANTHROPIC_API_KEY` 或 `OPENAI_API_KEY` 的原始 Anthropic/OpenAI 兼容 +completions 客户端。**只有编码 agent 订阅**（Claude Code Pro/Max、Codex CLI 用的 +ChatGPT 套餐、Cursor）作为模型访问方式的用户，**根本跑不了 Strict**。 + +Claude Code、Codex CLI 和 cursor-agent 在订阅认证下**不暴露原始 completions API** +—— 它们是**自带工具循环的 agent harness**。所以接入点不可能是 `LLM.create()` +（无状态的 `system+messages+tools → tool_use` 往返）；它必须上移一层， +落在**委托一整个 agent step** 的位置。 + +Hermes 用一个 `api_mode` transport 抽象解决了相邻的问题（规范化的内部消息形状； +逐模式的请求构建、响应归一化、用量抽取适配器；一条标准化的运行期解析记录）。 +本代码库其实已经走了一半：Anthropic 的 block 协议就是我们的规范形状， +`_openai_messages()` 就是一个 transport 适配器，`Settings.tier_target()` → +`ResolvedTarget` 就是运行期解析。本 RFC 把这些**形式化成一张 provider 注册表**， +并加上 Hermes 只在它的跨进程 Codex 路径上才需要的那条轴： +**transport kind**（`api` vs `harness`）。 + +## 决策（与维护者锁定，2026-08-14） + +1. **目的：产品覆盖面。** 让没有裸 API key 的用户能在订阅认证下跑 Strict。 + 保真度按 harness **尽力而为**，并**逐 run 标注**。评测/战役表格不受影响 —— + 一个 harness 后端测量的是**循环 + 模型**，**绝不**被并入生成器消融表 + （长期规则：生成器臂骑我们的流水线，**只换模型**）。 +2. **显式选择，硬报错。** 由一个 `.env` 键选定后端。对于**没有选定后端**的 Strict run， + 服务端会在**前面**带着确切缺失项报错（与 `TierNotConfiguredError` / + `strict_readiness` 同一套哲学）。**没有自动探测，没有静默回退。** +3. **工具经桥回流，预防优先。** harness 会话经 MCP 拿到 copilot 自己的工具， + 每次调用都过 `tools.dispatch`（ToolScope/PathScope 这个 choke point 被保住）。 + harness 支持时**关闭厂商内置工具**；cursor-agent 另外配上产品化的运行后审计 + （纵深防御），并把**控制类别**披露在 RUN_REPORT 里。 +4. **统一注册表。** 既有的 API 路径成为同一张表下的 provider `api` —— + **一条解析路径、一套 trace 词汇**。平价棘轮：用 provider `api` 时行为**逐字节一致**， + 而既有测试套件就是那道门。 +5. **分期：cursor-agent 先行**，然后 Claude Code，然后 Codex。 + +## 设计 + +### 术语 + +- **provider id** —— 注册表的键，也是选择配置的取值： + `api`、`cursor`、`claude-code`、`codex`。 +- **kind** —— `api`（无状态 completions；实现 `complete`）或 + `harness`（自带工具循环；实现 `run_session` + 一次性的 `complete`）。 +- **api_mode** —— `api` provider **内部**的线上协议： + `anthropic_messages` | `chat_completions`。这**恰好**就是今天的 + `Settings.resolved_llm_provider`；既有代码里的 "provider" 一词 + （`llm_provider`、`ResolvedTarget.provider`）**继续**指 API 厂商协议， + 原样映射到 api_mode。 + +### 包布局 + +``` +src/infermatrix_copilot/providers/ + __init__.py 再导出；register_builtin_providers() + base.py ProviderSpec + Transport 协议 + 会话数据类 + api.py provider "api"：包住既有的 LLM 类（两种 api_mode）。 + 包装，不是重写 —— LLM 内部、tier_target、served-model + 守卫、tracing 全部不变。 + cursor.py M1 harness transport（cursor-agent CLI） + claude_code.py M2 harness transport（claude -p） + codex.py M3 harness transport（codex exec） + audit.py 运行后会话审计（从 eval/dataset/run_cursor_arm.py 产品化） + registry.py PROVIDER_REGISTRY + resolve_provider(settings) +src/infermatrix_copilot/tool_bridge.py + 暴露该 run scoped 工具的 stdio MCP server + （入口：python -m infermatrix_copilot.tool_bridge） +``` + +### base.py —— 那些契约 + +```python +@dataclass(frozen=True) +class ProviderSpec: + id: str # "api" | "cursor" | "claude-code" | "codex" + kind: Literal["api", "harness"] + display: str + cli_names: tuple[str, ...] = () # 要探测的二进制（harness） + capabilities: frozenset[str] = frozenset() + # 能力标志: "mcp_tools", "builtin_tools_off", "max_turns", + # "system_prompt", "usage_reporting", "cost_reporting" + +class Transport(Protocol): + # api kind + harness 的无工具调用；签名镜像 LLM.create + def complete(self, *, system, messages, tools=None, model=None, + max_tokens=None, on_text=None, role="") -> Reply: ... + # 仅 harness kind：一整个 agent step + def run_session(self, req: AgentSessionRequest) -> AgentOutcome: ... +``` + +`run_session` 返回既有的 `agent_loop.AgentOutcome`（text、iterations、 +tool_calls、truncated、refusals、token 用量、tools_used）， +于是 runner 分叉**之后**的一切 —— `_coerce_output`、`agent_output` trace 事件、 +`_to_step_result` —— **原封不动**。harness 无从得知的字段（iterations）尽力而为； +`tools_used` 来自桥的 trace。 + +### 解析 —— 一个地方，做了扩展 + +`ResolvedTarget`（config.py）新增两个带默认值的字段，使**每一处既有构造点仍然有效**： + +```python +provider_id: str = "api" +kind: Literal["api", "harness"] = "api" +``` + +`Settings.tier_target()` 仍然是**唯一**把 模型×端点×凭据 配对起来的地方； +当 `strict_backend` 选中一个 harness provider 时，它返回带该 provider id/kind 和 harness +模型（`STRICT_BACKEND_MODEL` 或该 provider 默认值）的目标 —— +**base_url/api_key 为空**（订阅认证住在 harness CLI 里，**绝不在我们的配置里**）。 +`LLM.for_target()` **只**对 `kind == "api"` 的目标被咨询。 + +选择范围：**Strict run 必须有 `STRICT_BACKEND`** —— 在 +`CopilotMCP.strict_readiness()` 里检查（与当前的 `shared_api_key` 检查并列， +后者变成 `api` provider 的就绪项），并在子进程的 +`--execute-strict-reserved` 启动时**再检查一次**。CLI 路径（`run_task`）把空值当作 +`api` —— 对维护者**没有破坏性变更** —— 将来可能长出一个 `--backend` flag。 + +### runner 分叉 —— 唯一的接入点 + +在 `run_agent_step`（engine/agent_runtime/runner.py:170）里，当前这次调用 + +```python +outcome = await asyncio.to_thread(run_agent, step_llm, system=..., ...) +``` + +变成按解析出的目标 kind 做**二选一分叉**。`harness` 目标把桥 spec（见下）写进 run 目录， +并在**同一个 worker 线程**里调用 `provider.run_session(...)`。 +分叉**之前**的一切（dispatch context、证据包、briefing、把 scope 绑到 PR-time worktree +根、`agent_dispatch` trace）和分叉**之后**的一切（`_coerce_output` 及其一轮修复与升级 +抢救、`agent_output` trace、skill touch）都是**共享的** —— +**harness 骑的是完全相同的 v13 prompt 包和输出契约。** + +无工具的角色（评审 planner 的灰区调用、ensemble reducer/merge、覆盖率提升、 +`_coerce_output` 修复）经一个实现了 `LLM.create()` 签名的 `HarnessLLM` 适配器： +**传入非空 `tools` 时它抛错**，否则跑一次性 CLI 调用，并返回带 CLI JSON 用量的 +归一化 `Reply`。它的 `available` 属性镜像 doctor 的 CLI 探测， +于是 `run_agent_step` 的 `ctx.llm.available` 门继续有效。对 cursor， +一次性 completion 在一个**空的临时 cwd** 里运行，好让原生工具**无物可读**。 + +MoA（`for_member`）在 v1 里**拒绝**把 harness provider 作为混合成员 —— +预约台账需要**逐请求定价**，而订阅制 CLI 给不出。 + +### 工具桥 —— 保住 choke point + +`python -m infermatrix_copilot.tool_bridge --spec /bridge/.json` +是一个 stdio MCP server（FastMCP，已在 `[mcp]` extra 里）， +**恰好**暴露该 step 的 scope 所允许的那些工具。spec 文件序列化： + +- 那个 `ToolScope`（frozen dataclass → JSON：name、allowed_tools、read_only、 + root、path_scope 模式）—— 可轻松往返； +- 仓库名 + repo_path（PR-time worktree）、run_dir； +- 要重建哪些额外工具族（knowledge/doc/repo_map）—— 它们在桥进程里由 + `Settings()` + adapter 重建，用的是**与 runner 相同的工厂** + （`_knowledge_tools`、`_repo_map_tool`、`_repo_docs_tool`）。 + +每个 MCP 工具 handler 都是套在 +`tools.dispatch(name, args, scope=scope, trace=bridge_trace, extra=extra)` +之上的**薄壳** —— 拒绝、越界记录、相对 worktree 根的路径解析、结果封顶， +**全部与进程内循环表现一致**。桥追加写 run 目录里的 `bridge_trace.jsonl` +（自己的文件：来自第二个进程的仅追加 jsonl **绝不能**与父进程的 `run_trace.jsonl` +交错）；会话结束后 runner 把摘要（tool_calls、tools_used、refusals）折进正常 trace 事件。 + +守卫测试（外泄用例）：经桥 `read_file` 读取 +`~/.infermatrix-copilot/.env` —— 位于 scope 根之外 —— **被拒绝并记 trace**。 + +### 环境净化 —— 承重 + +harness 子进程拿到的是**构造出来的白名单环境**（PATH、HOME、TERM、LANG/LC_*、 +按需的 PYTHON*，加上 provider 专属变量），**绝不是** `os.environ` 透传。 +就在这台机器上的具体危险： + +- 我们的 `.env` 把 `ANTHROPIC_BASE_URL` 指向一个 DeepSeek 兼容网关 —— + 一旦泄漏进 `claude`，订阅调用会被**路由到错误端点并计到错误账户上**； +- 环境里存在的 `ANTHROPIC_API_KEY`/`OPENAI_API_KEY` 会**悄悄**把这些 CLI + 从订阅认证切换成 API 计费； +- 当 Strict 的宿主**本身就是** Claude Code 时，`CLAUDECODE`/宿主标记会让嵌套的 CC 混乱。 + +Strict 子进程本身已经从 server worker 那里收到一份精选环境 +（mcp_server.py:100–115）；**同一套纪律再向下延伸一层**。 + +### 守卫与记账 + +- **served-model 守卫**：harness CLI 在它们的 JSON 输出里报告实际使用的模型； + 经既有的 `llm.served` 事件记录，裁决逻辑不变 + （不匹配 → 按 `MODEL_MISMATCH_POLICY` 失败；缺失 → `unverified`，告警）。 +- **用量/成本**：token 计数从 CLI JSON 解析（claude `-p --output-format json` + 含用量 + 成本；codex `--json` 发出 token 事件；cursor stream-json 含用量）。 + `metrics.py` 把成本来源记为 `subscription`，且**对 harness provider 不查 + `MODEL_PRICES`** —— **CATQ 的 C 项绝不能编造 USD。** +- **ensemble 错峰**（`ensemble_stagger_seconds`）对 harness 目标**禁用**： + 跨 CLI 会话**不存在**可预热的共享 prompt-cache 前缀。每个被委托的 pass 就是一次 + harness 会话，所以 32 轮深度 pass 内部的上下文缓存是 harness 自己的事。 +- **并发**：`STRICT_BACKEND_CONCURRENCY`（默认 2）是并发会话上的信号量 + （深度 pass + 验证扇出）—— **订阅的速率窗口是真实存在的**； + 认证/限流错误浮现为 BLOCKED，**绝不重试到被封**。 + +### 配置 schema（.env） + +``` +STRICT_BACKEND= # Strict 必需：api | cursor | claude-code | codex +STRICT_BACKEND_MODEL= # harness 内部的模型 id（可选） +STRICT_BACKEND_CONCURRENCY=2 # 并发 harness 会话数 +STRICT_BACKEND_CLI= # 二进制路径覆盖（否则走 PATH 查找） +STRICT_BACKEND_TIMEOUT_S=1800 # 单会话墙钟上限 +``` + +安装器和 `.env.template` 写入 `STRICT_BACKEND=api`，于是既有环境随文件刷新即可迁移； +缺失时 `strict_readiness` 会点名**确切的修复行**。`doctor` 增加逐 provider 检查： +二进制是否找到 + 版本、认证状态、桥自检（拉起、列工具、一次受 scope 的读取）； +`doctor --probe` 经选定 provider 做一次**廉价往返**。 + +### Cursor transport（M1 细节） + +- 调用：`cursor-agent --print --output-format stream-json --force`， + prompt 走 stdin，`--model` 来自配置，MCP 配置文件指向工具桥， + cwd = PR-time worktree。 +- 解析、会话记账和边界 prompt 行**复用 Composer 评测臂已经学到的东西** + （`eval/dataset/run_cursor_arm.py`）—— 包括那次冒烟 run 读了 + `~/.claude/skills/` 的事故。 +- **要先做 spike 的未决问题**：在配置了 MCP 工具时，cursor-agent 能否关闭它的内置工具。 + 能 → 预防型；不能 → 桥是**附加的**，而 `providers/audit.py` 就是执行层： + 每次文件访问都做 realpath 归一化的 worktree 边界、写入禁令、讨论访问正则； + 裁决 + 标志进 trace 并渲染进 RUN_REPORT + （"backend: cursor —— 原生工具可能可用，审计：干净/N 个标志"）。 +- 没有 `--max-turns` 的对应物 → 墙钟超时 + 既有的预算纪律 prompt 行 + （把最后几轮留给契约）。 + +### Claude Code（M2）与 Codex（M3） + +- `claude -p --output-format json --mcp-config --allowedTools + <仅桥工具> --max-turns <预算> --model `；**最守规矩的 harness 公民** + （内置工具可完全关闭、原生 max-turns、支持 system prompt、输出含用量 + 成本）。 +- `codex exec --json --sandbox read-only` + 配置里的 MCP server； + **没有 system-prompt 通道**（契约前置到 prompt 里）；预算靠超时。 + +## 里程碑与验收 + +**M1 —— provider 层 + cursor-agent** + +1. `providers/` + 注册表 + `ResolvedTarget` 扩展 + 包住 `LLM` 的 `api` provider。 + 验收：**不改任何配置**的情况下既有全套测试绿；`STRICT_BACKEND=api` **逐字节一致** + （平价棘轮）。 +2. 工具桥 + 桥测试。验收：dispatch 平价（与进程内相同的拒绝/上限）、 + `.env` 外泄读取被拒并记 trace。 +3. cursor transport + 审计 + doctor 检查 + readiness 接线。验收：假 CLI 离线测试绿； + 在一个小 PR 上做一次实网 Strict 冒烟，RUN_REPORT 里带后端标签、用量和审计裁决； + Strict 启动时未设 `STRICT_BACKEND` 会给出**确切的修复行**报错。 + +**M2 —— claude-code** · **M3 —— codex**：同样的验收形状（离线假 CLI 测试 + 各一次实网 +冒烟）；M2 另外要证明在净化环境下**嵌套 CC**（Strict 宿主 = Claude Code）能工作。 + +## 测试计划（离线优先，本仓库纪律） + +- `test_providers.py` —— 注册表解析；Strict 与 CLI 在空选择下的行为差异； + `HarnessLLM` 在带工具时抛错；MoA 成员拒绝；档位交互 + （`tier_target` 返回 harness 目标）。 +- `test_tool_bridge.py` —— spec 往返；经桥的 scope 强制；桥 trace 内容。 +- `test_provider_cursor.py` —— 用发出预置 stream-json 的 fixture 脚本： + 最终文本抽取、用量解析、超时 → truncated 结果、审计标志浮现。 +- doctor：逐 provider 的检查渲染 + 修复行。 +- 棘轮不受影响：`test_repo_neutral_core`、`test_llm_providers.py`、 + `test_tier_split.py`、`test_thin_mcp_server.py`、`test_mcp.py`。 +- 实现之后，在 `doc/architecture/SPEC/` 下为 `providers/` 和 `tool_bridge.py` + 新增 SPEC 页（本仓库规矩：**逐文件约束住在那里**）。 + +## 风险与未决问题 + +1. cursor-agent 能否限制内置工具**未知** → M1 做 spike；治理兜底**已经定好** + （审计作为执行层）。 +2. 在没有 system-prompt 通道时的契约遵从度（cursor/codex）—— + 由既有的修复 + 升级抢救路径吸收；**先在冒烟里观察，再决定是否信任**。 +3. harness 的内部推理对我们的 trace 是**不可见的**；只有经桥的工具调用被审计。 + **逐 run 声明 —— 绝不把它呈现为全保真的 Strict tracing。** +4. 订阅 ToS/速率限制：并发封顶，限流错误**大声 BLOCK**。 + 用户通过显式选择后端，来自行决定把哪个账户暴露出去。 +5. harness 内部的模型命名（composer id、codex 模型 slug）—— + `doctor --probe` 报告 CLI **实际服务**的是什么；served-model 守卫逐次调用记录它。 diff --git a/doc/features/review-recall.md b/doc/features/review-recall.md new file mode 100644 index 000000000..abf7f69dd --- /dev/null +++ b/doc/features/review-recall.md @@ -0,0 +1,164 @@ +# 评审召回攻坚（v14/v15） + +> **一览** +> | | | +> |---|---| +> | **状态** | ✅ 已实现，**默认开**，全部带 kill-switch | +> | **做什么** | 补齐召回：调查 duties（清单 13–23）、文档 claims-audit pass、考古工具、覆盖率驱动的第二轮、排序验证账本 | +> | **怎么开关** | `review_second_round`、`review_second_round_max_iters`；按 pass 路由用 `REVIEW_LENS_BACKENDS` | +> | **新增工具** | `diff_stat`、`file_at_base`、`show_commit`、`search_history`、`calc`——只读，经工具桥同样对 harness 后端可用 | +> | **实测** | [`../evaluation/EVAL-v14-v16-recall-campaign.md`](../evaluation/EVAL-v14-v16-recall-campaign.md) —— **Δrecall −.049 [−.097, −.001]，precision 打平**，成本约基线 1/3 | +> | **诚实结论** | 目标是"两项均超基线"，**没有达成**。召回缺口真实但小且集中：20 项里 6 项偏向 arm，单个 item（pr5978）贡献约三分之一 | +> | **前身** | [`strict-review-deep-engine.md`](strict-review-deep-engine.md)（v13） | + +--- + +## 原始 RFC + +- 状态：已实现，藏在 setting 之后（默认开）；经 wave-4 门测量 —— 见下方"实测结果"与 + `eval/dataset/results/model_comparison.md` +- 归属：评审流水线（`engine/steps/review/`、`review/planner.py`） +- 证据：wave-2 取证（3 个 trace 分析 agent 跑过已花光的 holdout）、 + train 分片上的 Claude Fable 5 **teacher 臂** + （`eval/dataset/baselines/teacher_fable5/`）、 + `doc/evaluation/EVAL-goal-strict-vs-opus5.md` +- 前身：`doc/features/strict-review-deep-engine.md`（v13） + +## 动机 + +在长期使用的 claude-sonnet-5 判官下，**每一个** v13 生成器（DS api、经 cursor 后端的 +Composer/Grok、MoA）在严格计分的 recall 上都输给 CC+Opus 5 +（约 0.29–0.35 vs 约 0.42–0.56），而 precision 打平。逐项分析显示损失集中在五个 wave-2 +条目上（三个文档 PR + 5884 + 5550）；结合 GT 交叉引用的 trace 取证**精确定位**了损失： + +1. **在 pass 阶段"读到了却没提出来"**（主因）：在每一个被诊断的漏掉里， + **决定性证据就摆在某个 pass 面前** —— reducer/预算/验证均被排除嫌疑 + （到处都是 `dropped: 0`）。pass 接受了作者的框定（pr5550）、 + 以确认模式重新推导数字（pr5863 的内存模型被放行；pr5840 自己的 R²=0.095 无人置评）、 + 且**从未打开**那个新代码丢掉了其不变量的树内兄弟文件（pr5550 L1、pr5884 F2）。 +2. **报告组装环节的切割**：Validated 渲染在 50+ 行里只按到达顺序留了 8 行； + 在修复后的快照上，**判官的召回质量恰好落在被切掉的 resolved-thread 确认里**。 + 三条已验证确认的评论死在 cap-8 + 仅证据丰富才溢出的门上（pr5957）。 + 锚点未解析的仓库侧评论渲染成 `file:?`，被判为含糊而扣分。 +3. **文档饥饿**：纯文档 diff 被规划为 light（单 pass、无协议），而 holdout 的 30% 是 + 文档 PR，其 GT 正是**主张订正**；**每一条臂在那里都只跑到基线约一半的 recall**。 +4. **pass 死亡**：耗尽预算的深度 pass 有时会以**空消息**回答那次被强制的最终答复 + （pr5976：两个 pass、各 50 次工具调用、**交付零个候选**）。 +5. **工具缺口**：那些判官决定性的基线动作 —— `git diff --stat base..HEAD`、 + 读 base 版本的文件、`git show`/`log -S` 考古、评估 PR 自己的算术 —— + **在只读工具集里根本无法表达**。 + +在 train 分片上跑的 Claude Fable 5 **teacher 臂**（与基线用同一套 pinned CC harness） +确认了这些类别并补充了更多；它的共享契约 producer 普查抓到了 pr4870 的 GOLD 潜在缺口 +（正是人工评审漏掉的 #4910 双轴机制）。teacher 和 Opus **都表现出这些职责**； +而我们的 v13 prompt **一条也没有编码** —— 这支持"**流水线缺口**"而非"模型能力缺口"。 + +## 设计 + +### 1. 调查职责（prompts.py checklist 13–21 + pass 聚焦） + +从 Opus/Fable 动作的**并集**蒸馏而来，每条都配一个实测范例： +主张账本（核验 PR 正文里每一条可核查的主张；`[claim-verified]`/`[claim-refuted]` +findings 行）；兄弟对照（新增类/文件 → 读树内的孪生体；不变量增量；具体的去重提案）； +PR 自己的数字（用 `calc` **证伪**，**绝不确认模式**）；合并态重校验（语义合并审计）； +producer/consumer 契约普查（含 dummy-run/平台孪生体）；模式/变体矩阵与大声失败检查； +死旋钮重算 + 注释审计；缓存病理；分发链路守卫（teacher 唯一一次误报， +被编码成一条**precision 保护**）。resolved-thread 确认成为一等输出（`[resolved]` 前缀）： +在被修订过的 head 上，**读者拿评审去核对的大部分正是它们**。 + +### 2. 文档 pass（`_REVIEW_DOCS_PASS`）+ planner 深度 + +纯文档/文档为主的 diff：**只有确实很小时**才 light；中等 → standard，大 → full +（planner 规则；资产/配置附带项不再把文档 PR 推进灰区）。深度引擎把代码形状的广度 lens +换成一个文档 pass：主张审计（对着代码树**证伪**事实性/量化陈述）、 +用户路径走查（命令端到端、下载模式 vs 重启碰撞）、 +两种渲染约定下的链接/pin/导航机制，以及缺失警告的适用范围。 + +### 3. 考古 + 数值工具（`review/repo_tools.py`） + +`diff_stat`、`file_at_base`、`show_commit`、`search_history`（git，固定 argv、 +输入受校验、输出有界）与 `calc`（AST 白名单算术）。它们作为 step 提供的额外工具出现在 +**每个评审 pass 和验证 pass** 上；对 harness 后端则由桥侧从 `scope.root` 重建 +（补上了 provider-registry M1 那块"额外工具"缺口的这一片）。 + +### 4. 覆盖率驱动的第二轮（v13 RFC 未决问题 3；`review_second_round`） + +在 reduce+promote 之后，那些**既无评论、也无 findings 行**的改动文件 —— +外加一个"主张未核验"信号 —— 播种**一次**有界的 pass +（`review_second_round_max_iters` 16），它必须对每个空洞**要么提出一条已验证评论， +要么记下 `[validated]`/`[resolved]`**。对已保留评论有近重复守卫；新增项同样要过逐条 +评论的验证 pass。 + +### 5. 报告组装修复 + +Validated 账本改为**排序**（`[resolved]`/`[claim-*]` 在前）并封顶 14， +而不是按到达顺序在 8 处截断；即使评审不"丰富"，溢出也会渲染那些被切掉的已验证确认评论 +（未验证的尾部仍需过丰富门）；锚点未解析的评论渲染成 `file:~declared` 而不是 `file:?` +（发布时仍按仅正文处理）；仓库侧发现的验证 pass 证据**必须携带** +"本 diff 未改动、存在于 PR-time 树中"的证明，好让**只看 diff 的判官**能对它分类。 + +### 6. 循环稳健性 + +在**真实发生过工具调用之后**出现的空模型回复，会得到**恰好一次**大声的追问 +（循环中途以及预算耗尽的强制最终答复处）—— +**一个空的最终答复，是唯一严格劣于任何部分答案的结果**。 +覆盖率提升的 prompt 去掉了模棱两可：被提升的缺陷保持**指令性力度** +（"could you confirm…?" 这种措辞**实测**把一个命中 GT 的关切变成了一个不计分的提问）。 + +## 成本 + +文档 PR 从 light 移到 standard/full（约 15% 的条目上约 2–3 倍）； +第二轮在有覆盖空洞的条目上至多加 1 个 pass；工具是子进程级的廉价。 +**这条臂仍然远低于基线 $/项的一半。** + +## 上线 + +一切都藏在 setting 之后（`review_second_round`，以及既有的深度引擎/验证开关）； +**旧路径原封不动**。离线测试覆盖新的 planner 规则、文档 pass 选择、 +第二轮（触发/跳过/近重复）、溢出门、账本排序、锚点回退、循环追问， +以及每一个新工具（含 calc 的沙箱）。知识 checklist 页按 teacher 蒸馏出的仓库先验重写 +（注入上限 4k→7k 字符）。 + +## 实测结果（在各道门之后补记） + +train 探针（v14，2 个 sonnet replicate）：7—12—1，arm r .608 vs opus .609 +（**recall 打平** —— 此前被 sonnet 判分的臂都在约 .28–.35），arm p .785 vs .765。 +**wave-3 门第 1 次尝试测到的是坏掉的机器，不是这套设计**：pass 的最终答复死在 16k +补全上限上（一个文档 pass 恰好吐出 16,000 token），修复轮在 8k 处**再次截断**且只有 +头部窗口，而灰区 LLM planner **自官方推理模型上线以来一直在静默失败** +（它的思考吃完了 400-token 上限、一个字都没吐出来 —— +**两个战役里每一个灰区条目都掉进了 2-pass 的 standard**）。 +第 2 次尝试（机器已修）：5—25，损失被重新定位到"职责 vs GT 不匹配"上， +随后被 wave-3 取证刻画出来（**验证偏置倒置**：主张账本**赞许地**记下了那些 GT 事实； +test/gate 认知论、文档信息架构、CI 经济性这几类无人认领）。 +v15 编码了那些类别（checklist 22–23、文档 IA 扩展、账本残余提升、逐 hunk 第二轮）， +外加 reducer 失败路径的坍缩。 + +wave-4 干净门，三次 DS-core 提交（v15 r1、把 cheap seat 放在 v4-flash 上的 v15 r2、 +把 Fable 放进 adversary 和第二轮座位的 v16），外加一行 Composer cursor 后端。 +原始胜负计数 10—20、11—19、6—23—1、4—24—2。 + +**统计上正确的读法（见 `eval/dataset/paired_analysis.py` 和结果表）是 +"在测量精度内打平"，不是"更优"。** 在这里跨判分集比较原始 rubric 均值是**无效的**： +基线自己的 recall 均值在**给同一批基线评审打分**的三组里读作 .335 / .338 / .416 +（±.08 的判官漂移）。在每条 verdict 内部配对、并按 item 聚类 replicate 之后， +在 90 条 verdict 上汇总得到 Δrecall −.070 [−.161, +.021]、 +Δprecision +.015 [−.025, +.055] —— **两个区间都跨零**。 +本 RFC 的一个早期修订版曾从原始均值宣称 precision 高于基线；**该主张已撤回**。 + +这轮战役**确实**建立的：这条臂在约 1/3 的成本下达到打平（$0.97/项 vs $3.09）； +总体 recall 差距由 10 个条目里的 3 个承担（其余 7 个在 ±.06 内）； +v15 偏 precision（在两次独立 replicate 上 Δp +.029/+.022，**符号一致**）， +而 v16 偏 recall（新鲜分片上最好的 recall .336）并在**流水线内部**复现了 pr4870 的 +GOLD 缺口捕获；以及在实测的 item 级 sd（.127）下，分辨 .07 的差异需要约 32 个条目 +—— 这正是 wave 5 去扩充新鲜池、而不是再跑一次 10 条目裁决的原因。 +一次 v15 replicate 曾因 DeepSeek 402 余额中断而作废（已隔离），充值后重跑。 + +## 未决问题 + +1. wave-3（`build_wave3.py`，分片 `holdout3`）是晋升门；wave-2 已花光 + （为这轮取证打开了 GT/理由），**只能做冒烟**。 +2. 第二轮目前按 文件覆盖 + 主张覆盖 播种；**reducer 的丢弃原因仍未接线** + （未来某轮的候选种子）。 +3. 修复后快照上的 GT 测的是**对 resolved thread 的参与度**，不是找 bug； + `[resolved]` 通道正是瞄准这一点 —— **判官在规模上是否给它计分，正是那道门要测的。** diff --git a/doc/features/strict-review-deep-engine.md b/doc/features/strict-review-deep-engine.md new file mode 100644 index 000000000..7939654ac --- /dev/null +++ b/doc/features/strict-review-deep-engine.md @@ -0,0 +1,120 @@ +# Strict 评审：深度 pass、逐条验证、自证据 + +> **一览** +> | | | +> |---|---| +> | **状态** | ✅ 已实现，**默认全开**（v13 期） | +> | **做什么** | 面向强生成器重做评审流水线：深度调查/对抗 pass、每条评论一次 agentic 验证、端到端的逐字引用自证据 | +> | **怎么开关** | `review_deep_engine`、`review_deep_max_iters`、`review_verify_comments` | +> | **实测** | [`../evaluation/EVAL-goal-strict-vs-opus5.md`](../evaluation/EVAL-goal-strict-vs-opus5.md) —— val 首胜 8—7，冻结 test 7—8，combined **15—15** | +> | **最大的一课** | 判官是**无工具**的：仓库侧的发现若不带逐字引用的证明，一律被当作臆测扣分。把证据变成自证据是本轮最大的单点收益（4—11 → 8—7） | +> | **后继** | [`review-recall.md`](review-recall.md)（v14/v15） | + +--- + +## 原始 RFC + +- 状态：已实现，藏在开关之后（全部默认开）；在 20-PR 战役分片上测量 + （val + 一次性冻结 test） +- 归属：评审流水线（`engine/steps/review/`、`review/planner.py`） +- 证据：`doc/evaluation/EVAL-goal-strict-vs-opus5.md`、 + `eval/dataset/judgments/goal_v13_val/`、`eval/dataset/judgments/goal_v13_test/` + +## 动机 + +Strict 评审流水线当初是**围绕一个弱生成器**设计的：四个窄 lens 在模板里逼出枚举、 +紧的工具预算、一个无工具的 reducer，以及 5 条评论的严重度上限。换上官方的 +`deepseek-v4-pro`（相对 `[1m]` preview 是一次大的能力跃升）之后， +**那套脚手架本身变成了瓶颈** —— 跨多轮判分扫描测到的症状： + +1. **召回损失是机械性的，不是分析性的**：预算上限**删掉了 reducer 已保留的维护者关切**； + 深度规划把小而爆炸半径大的 diff 误路由到不带 ensemble 的 light 档； + 一个 170k 字符的 diff 因 120k 证据上限**丢了 30% 的 hunk**； + 整类问题（爆炸半径、依赖下限、测试完整性、abort 时的生命周期）**没有任何 lens 认领**。 +2. **precision 在五种不同配置下都卡在 0.54–0.56**，而 recall 在 0.55–0.86 之间摆动 + —— 因为盲评判官是**无工具**的：它只看得见 diff 和人类 thread， + 所以任何仓库侧的发现（别处的 consumer、CI 通道规则、版本下限） + **只要评论本身不携带可核验的证明，就会被判为臆测**。 + +## 设计 + +三个机制，与既有的 ensemble/reducer 机器组合： + +### 1. 混合 pass 集合（`review_deep_engine`，默认开） + +full 深度跑两个**深度 pass** —— `investigator`（核心改动优先的自由调查；先验证再断言； +预算纪律：把最后几轮留给归档）和 `adversary`（对那些**系统性被漏掉的类别**做独立搜捕） +—— **外加**两个价值最高的广度 lens（`behavior`、`verification`）。 +standard 深度跑 investigator+behavior。light 不变。深度 pass 拿到 +`review_deep_max_iters`（32）轮工具回合；**单独测**时它们把主张锚定得很好 +（最难的 train 条目上 precision .82）但**覆盖不足**（val recall .55）； +**与广度 lens 一起测**时能守住 recall .71–.86。 + +### 2. 逐条评论的 agentic 验证（`review_verify_comments`，默认开） + +每条合并后的草稿评论都会走一个小工具循环（4 轮、并发 6、共享 diff 缓存前缀）， +它必须在 PR-time 树上**重新推导**那条主张：`refuted` 丢弃、`unverifiable` 降一级严重度、 +`confirmed` 可以收紧措辞/位置 —— 而**验证失败会保留这条评论** +（这一趟**只能提高 precision，绝不静默地删掉召回**）。 +跳过/失败原因被记 trace（`review_comments_verified` / `review_coverage_skipped`）。 + +### 3. 自证据（起草 + 验证 + 渲染） + +**测出来最大的单点杠杆**（val 门从 4—11 变成 8—7）：每条评论的 `evidence` 字段 +必须带 file:line **逐字引用**那几行决定性代码，好让一个**只拿着评审和 diff** 的读者 +就能核对这条主张。它在起草契约里被强制，由验证器**从它真正读过的代码**重新发出， +渲染时原样保留。 + +随本 RFC 一起发布的配套改动：`value_flips` diff 信号（**默认值翻转永远不能裁为 light**）、 +把 verification lens 放进 planner 的兜底集合、证据上限 120k→260k / reducer 60k→280k、 +证据包里加入 commit 时间线 + 确定性的改动符号 consumer sweep（`git grep -nw`）+ +逐文件 hunk 定位分页目标、跨 lens 佐证标签与 reducer 的受保护类丢弃规则、 +评论预算 8 + 仅证据丰富时才溢出、覆盖率提升 pass（从本次 run 自己的 +findings/blockers 里挖出未被覆盖的受保护类关切）、经新的 adapter manifest 键 +`knowledge.review_checklist` 注入的知识页 +`repos/vllm-omni/review/guides/strict-review-checklist.md`， +以及把 vllm-omni 的模块映射扩到 17 个模块、7 个风险分层。 + +## 实测结果（盲评 gpt-5.6 判官，3 replicate，对照已录制的 CC+Opus 5） + +| 门 | 配置 | 正面交锋（arm—opus） | arm r/p | opus r/p | +|---|---|---|---|---| +| 战役前 | preview 模型，4 lens | 1—4 | .42/.51 | .82/.54 | +| v8 | 官方模型，lens | 3—12 | .86/.56 | .79/.66 | +| v12 | 混合，叙述式证据 | 4—11 | .71/.55 | .81/.66 | +| **v13 val** | 混合 + 自证据 | **8—7** | .857/.555 | .889/.589 | +| **v13 冻结 test**（一次性） | 同上 | 7—8 | .689/.687 | .762/.719 | + +新鲜裁决合计：**15—15**，成本约为基线的 1/3 +（约 $0.75–1.2/项 vs 约 $1.5–3）。GOLD 潜在缺口条目：pr4810 2/3、 +pr4834 三次全清。rubric 均值的差距（.03–.07）**处于同一批基线评审上实测判官漂移的 +量级**（±.05–.10）。 + +## 考虑过的替代方案 + +- **用更强成员重测 MoA** —— 按 owner 指示排除。 +- **只跑深度 pass**（不带广度 lens）：precision 迁移过来了，但 val recall 塌到 .55； + 已否决（门 v11，2—13）。 +- **只调预算/reducer**：五种配置把 precision 挪动了 ≤.02； + **那道缺口是结构性的（无工具判官），不是校准性的**。 + +## 上线 / 兼容性 + +三个机制都藏在带 kill switch 的 setting 之后（`review_deep_engine`、 +`review_verify_comments`；旧的 lens 集合**逐字保留**，深度引擎关闭时使用）。 +离线测试覆盖 pass 选择、验证裁决处理（drop/demote/tighten/fail-open）、 +预算/溢出、佐证排序、提升、以及 planner 信号；全套测试和两个知识 validator 都绿。 +成本约为旧 lens 流水线的 2–3 倍每项，且**仍比基线便宜约 3 倍**。 +`run_copilot_arm.py` 里的 `ARM_JOBS` 支持条目级并行扫描 +（端点在 v4-pro 上允许 500 并发请求）。 + +## 未决问题 + +1. **两个调参分片都已花光**（8 次 val 提交、1 次冻结 test）。进一步迭代需要 wave-2 + 条目（`gt/pr5957*`、`gt/pr5976*`、`build_wave2.py`）在**同一套协议**下判分。 +2. 判官协议的**无工具性**如今已经在**塑造评审本身**（冗长的引用式证据）。 + 如果产品的真实受众是**维护者**而不是一个没有仓库的判官， + 那么证据的冗长程度应当变成一个**渲染选项**，而不是固定行为。 +3. `pr4762`/`pr4954` 那一类损失（宽泛的多关切 PR）仍然存在：**基线在逐条发现上仍然读得 + 比 arm 深**。下一个结构性杠杆，是一次**由覆盖率驱动的第二轮调查** —— + 由 reducer 那份"未覆盖 GT 类"checklist 播种，而不是固定的 pass 数。 diff --git a/doc/guide/autonomous-workflow.md b/doc/guide/autonomous-workflow.md new file mode 100644 index 000000000..46afd8fea --- /dev/null +++ b/doc/guide/autonomous-workflow.md @@ -0,0 +1,50 @@ +# 独立执行器（Autonomous workflow） + +本页介绍可选的 autonomous CLI 与 workflow MCP。它与仓库 README 描述的**默认 Direct +MCP 是两回事**。 + +autonomous 工作流**跑自己的模型**，支持更长的仓库维护 playbook —— 评审、issue 处理、 +CI 调试、rebase。因此它需要模型凭据和仓库配置。 + +## 安装 + +```bash +bash install.sh +``` + +编辑 `.env`，填好必需的模型凭据和 `REPO_PATHS`，然后运行： + +```bash +./infermatrix-copilot doctor +``` + +## 使用 + +```bash +./infermatrix-copilot +./infermatrix-copilot -p "review pr 4830" --yes +./infermatrix-copilot -p "answer issue 4842, do not post" +./infermatrix-copilot -p "rebase pr 4830, then review it" +./infermatrix-copilot --resume +``` + +autonomous 的 MCP 命令是： + +```text +infermatrix-copilot-workflow-mcp +``` + +**默认的 Codex 安装器不会注册它。** + +## 安全 + +- push 需要策略允许，未显式开启前一律 dry-run。 +- 保护分支永不被直接推送。 +- 发布需要明确意图加上配置开关。 +- 被阻塞的 run 写出升级产物，而不是猜测继续。 + +实现细节： + +- [`../../QUICKSTART.md`](../../QUICKSTART.md) +- [`../architecture/DESIGN.md`](../architecture/DESIGN.md) +- [`../archive/IMPLEMENTATION_STATUS.md`](../archive/IMPLEMENTATION_STATUS.md) diff --git a/doc/guide/backends.md b/doc/guide/backends.md new file mode 100644 index 000000000..c61988258 --- /dev/null +++ b/doc/guide/backends.md @@ -0,0 +1,189 @@ +# 后端（Backends）—— 谁跑在 copilot 里面 + +> **你要找的可能是另一篇。** 这里讲的是 **Strict 模式**下 copilot **拿去用**的 +> 模型。如果你想知道怎么在 Codex / Claude Code / Cursor **里面**使用 copilot， +> 那是**宿主端**，看 [`hosts/`](hosts/README.md)。 +> +> | 名字 | 作为**宿主**（copilot 跑在它里面） | 作为**后端**（它被 copilot 拉起） | +> |---|---|---| +> | Claude Code | ✅ `/imreview` → [`hosts/claude-code.md`](hosts/claude-code.md) | ✅ `STRICT_BACKEND=claude-code` | +> | Codex | ✅ `$imreview` → [`hosts/codex.md`](hosts/codex.md) | ✅ `STRICT_BACKEND=codex` | +> | Cursor | ✅ `/imreview` → [`hosts/cursor.md`](hosts/cursor.md) | ✅ `STRICT_BACKEND=cursor` | +> | DeepSeek（dsh） | ✗ | ✅ `STRICT_BACKEND=deepseek` | +> | api（Anthropic/OpenAI 兼容） | ✗ | ✅ 默认 | +> +> 一句话判据：**宿主提供模型给你用；后端是 copilot 拿去用的模型。** +> 宿主端不需要 API Key，后端端要么要 Key、要么要订阅登录。 + +--- + +## 1. 为什么会有"后端"这个概念 + +Strict 模式跑完整执行主脊，每一次模型调用原本都走 `llm.py::LLM`——一个需要 +`ANTHROPIC_API_KEY` 或 `OPENAI_API_KEY` 的原始 completions 客户端。只有编码 +Agent 订阅（Claude Pro/Max、ChatGPT、Cursor）而没有裸 API Key 的用户，根本 +跑不了 Strict。 + +但 Claude Code、Codex CLI、cursor-agent 在订阅认证下**不暴露 completions +接口**——它们是**自带工具循环的 agent harness**。所以接入点不可能是 +`LLM.create()`（无状态的 `system+messages+tools → tool_use` 往返），只能上移 +一层：**委托一整个 agent step**。 + +于是有了两种 provider kind： + +| kind | 是什么 | 谁拥有工具循环 | 接入点 | +|---|---|---|---| +| `api` | 原始 Anthropic/OpenAI 兼容端点 | **我们的** `agent_loop` | `LLM.create()` | +| `harness` | 订阅制（或 SDK 制）厂商 agent | **厂商的** | `run_session()`——一整个 step | + +两者住在同一张注册表 `providers/registry.py` 里，走同一条解析路径、同一套 +trace 词汇。**用 `api` 时行为逐字节不变**（平价棘轮）。 + +## 2. 五个后端 + +| id | kind | 凭据 | 能力 | 验证程度 | +|---|---|---|---|---| +| `api` | api | `.env` 里的 Key | 完整——我们自己的工具循环 | 基线路径 | +| `cursor` | harness | 订阅（`cursor-agent` CLI） | `mcp_tools` `usage_reporting` | 已实测 | +| `claude-code` | harness | 订阅（`claude` CLI） | `mcp_tools` `builtin_tools_off` `max_turns` `system_prompt` `usage_reporting` `cost_reporting` | 订阅认证下实测 | +| `codex` | harness | 订阅（`codex` CLI） | `mcp_tools` `sandbox_read_only` `usage_reporting` | **仅离线测试**（开发机无 ChatGPT 登录，readiness 会报出登录缺口） | +| `deepseek`（dsh） | harness | **API Key** | `sandbox_read_only` `system_prompt` `api_keyed`——**注意没有 `mcp_tools`**，见下 | 最新加入 | + +`claude-code` 的能力集最全；`codex` 与 `deepseek` 靠 OS 级只读沙箱而不是关闭 +内置工具；`cursor` 两者都做不到，因此额外配了事后审计（见 §5）。 + +**`deepseek` 是个例外，有两处打破常规，都要记住**： + +1. **它是 harness，却需要 API Key。** 通过 Python SDK 驱动（PATH 上没有二进制， + 运行时随 wheel 发布），凭据是 `DEEPSEEK_HARNESS_API_KEY`（缺省回退 + `ANTHROPIC_API_KEY`）。注册表用 `api_keyed` 标出这个不对称，供那些默认 + "harness ⇒ 凭据在厂商那边"的调用方判断。 +2. **它用不了我们的工具桥**——这是实测结论不是选择。它捆绑的运行时编译进了 122 + 个插件，`@deepseek-ai/dsh-mcp-client` 不在其中，所以 dsh 的 lens 跑在 harness + 自带的 `bash` + `str_replace_editor` 上，我们的 scoped 工具（含考古工具组） + **作为具名工具不可达**。bash 能覆盖其中大部分（它们本就是薄薄的 git 包装）， + 但那些调用**落在我们的审计轨迹之外**，因此每个拿到桥 spec 却无法履约的会话都会 + 记一条 `capability_gap`。 + + > **由此而来的一条硬规矩：跑在原生 bash 上的 arm，绝不能被标成"tools + > bridged"。** 本轮战役已经测到过三个 arm 与其标签不符。 + +## 3. 怎么选 + +**默认用 `api`。** 它是唯一行为完全确定、成本可测、且所有评测结论都建立其上的 +后端。选别的只有三个理由： + +1. **你只有订阅，没有裸 Key** —— 用 `cursor` / `claude-code` / `codex`。 +2. **你要按 pass 混搭模型** —— 见 §4 的 `REVIEW_LENS_BACKENDS`。 +3. **你在做后端对比实验** —— 先读 §6 的实测结果，别重复已有结论。 + +反过来，有一条**不该**选 harness 的理由值得写下来：harness 会话 +**不向 span 记账暴露 token**（每一项都记 `tok_out=0`），所以它们的成本优势是 +**假定的，不是测出来的**。metrics 会把来源记成 `subscription`，绝不编造 USD。 + +## 4. 怎么配 + +```bash +# ~/.infermatrix-copilot/.env +STRICT_BACKEND=api # api | cursor | claude-code | codex | deepseek +# STRICT_BACKEND_MODEL= # harness 内部的模型 id（可选） +# STRICT_BACKEND_CONCURRENCY=2 # 并发 harness 会话数 +# STRICT_BACKEND_CLI= # 二进制路径覆盖（否则走 PATH 查找） +# STRICT_BACKEND_TIMEOUT_S=1800 # 单会话墙钟上限 +``` + +**显式选择，硬报错。** 未知 id 在 `Settings` 校验阶段就被拒（会列出合法取值）； +Strict 启动前 `strict_readiness` 先查缺项，**绝不启动一个注定失败的后台任务**。 +没有自动探测，没有静默回退——这与 `TierNotConfiguredError` 是同一套哲学。 + +**按 pass 路由**（把测出来的模型互补性变成一条 arm）： + +```bash +REVIEW_LENS_BACKENDS=adversary=claude-code:...,round2=claude-code:... +``` + +把某个评审 pass 映射到 `provider:model`，让那个 seat 跑在别的后端上。改之前先读 +[`eval/dataset/results/model_comparison.md`](../../eval/dataset/results/model_comparison.md)。 + +## 5. 换后端不会绕过权限闸 + +这是接入 harness 时最要紧的一条：**厂商 agent 自带工具循环，但它拿到的工具是 +我们给的**。 + +- **工具经 MCP tool bridge 回流**（`tool_bridge.py`）：harness 会话拿到的是 + copilot 自己的 scoped 工具，每一次调用仍然经过 `tools.dispatch`， + `ToolScope` / `PathScope` 检查与 trace 记录一个不少。桥比进程内循环还**多两条 + 收紧**：读取也受容纳根约束（`ToolScope` 本身只管写；harness 是持有不可信 diff + 的低信任调用方，这是防 `.env` 外泄的那道），且工具事件写独立的 + `bridge_trace.jsonl`，不与父进程的 `run_trace.jsonl` 交错。 + 桥提供内置工具 + `doc_search`/`doc_read` + 按需 `repo_map` + 考古工具组； + **skill/memory 检索刻意不开放**——那两个能提知识 candidate，跨进程写入口没有开。 + **例外：`deepseek` 用不了这座桥**（见 §2），它的调用落在审计轨迹之外。 +- **能关内置工具的就关**（`builtin_tools_off`，claude-code）； + **关不掉的用 OS 级只读沙箱**（`sandbox_read_only`，codex / deepseek）。 +- **两者都做不到的（cursor-agent）额外上事后审计**（`providers/audit.py`）： + 检查文件读取是否越出会话的容纳根（PR-time worktree + run 目录），只读 scope + 下是否出现了 write/edit 调用。这是**侦测型而非预防型**的兜底， + 结论会写进 RUN_REPORT——**公开声明，绝不静默**。 + （这套检查是从 Composer 评测臂产品化过来的，它当初真的抓到过一次越界读 + `~/.claude/skills/...`。） +- **子进程环境是白名单**（`providers/base.py::sanitized_env`）：只保留 + `PATH` `HOME` `TERM` `COLORTERM` `LANG` `USER` `LOGNAME` `SHELL` `TMPDIR` + 以及 `LC_*` / `XDG_*`。厂商 CLI 必须保住自己的订阅认证（存在 HOME 状态里）， + 但**绝不能继承我们的模型端点变量**——这类机器上的 `ANTHROPIC_BASE_URL` + 往往指向某个网关，继承过去会**悄悄把厂商 CLI 的流量改道**。API Key、 + gh token、`CLAUDECODE` 这类宿主标记也一并丢弃。 + +## 6. 实测特点（2026-08-17） + +> 数据以 [`eval/dataset/results/model_comparison.md`](../../eval/dataset/results/model_comparison.md) +> 为准，它比这张表更新得勤。方法论与完整结论见 +> [`GUIDE.md §8`](../GUIDE.md#8-性能对比)。 + +同一批 10 个 train item 上的后端冠军是 **api/DeepSeek 核心，不是 Composer**： + +| 配置 | Δrecall [95% CI] | Δprecision [95% CI] | +|---|---|---| +| v17ds train（api/DeepSeek） | **+.024 [−.030, +.077]** | **+.016 [−.061, +.093]** | +| v17cb train（Composer 2.5 / cursor） | −.052 [−.165, +.061] | −.097 [−.215, +.021] | + +val 一度把 Composer 排前面，那是 n=5、区间 ±.22 的小样本幻觉；train 的 item +方差紧四倍（sd .075 vs .158）。**每个 CI 都跨零**——结论是"测量精度内打平"， +不是"更优"。 + +速度与成本：harness 后端快 2–4 倍（Composer 350–680s vs api 1000–2900s 每项） +且吃订阅，但如 §3 所述，**成本优势未经测量**。 + +一条安全提醒：**grok 家族（4.5 / 4.6，Composer 从未出现）会在评审会话中主动 +搜索并读取 copilot 自己的 `imreview` 方法论 skill**。相关评测臂已作废并进污染 +台账。跑 cursor 系评测前，三份 skill 副本必须移出 `$HOME` 并在事后恢复。 + +## 7. 排查 + +```bash +infermatrix-copilot doctor # 逐项 ✓/✗，失败即给出唯一修复命令 +infermatrix-copilot doctor --probe # 每档模型 1-token 实探（唯一付费检查） +``` + +| 症状 | 原因与修法 | +|---|---| +| `unknown STRICT_BACKEND 'x'` | 配置校验拒绝了未知 id；合法取值已在报错里列出 | +| `... backend selected but no CLI found` | 厂商 CLI 不在 PATH；装上，或设 `STRICT_BACKEND_CLI=/path/to/cli` | +| readiness 报登录缺口 | 该 harness 的订阅未登录（`codex` 在开发机上就是这个状态）；用厂商 CLI 自己的登录命令 | +| `deepseek` 报缺凭据 | 设 `DEEPSEEK_HARNESS_API_KEY`（或 `ANTHROPIC_API_KEY`） | +| Strict 直接返回缺项而没起任务 | 这是**设计行为**：`strict_readiness` 不会启动注定失败的后台任务 | + +各后端的判定逻辑在 `providers/.py::auth_gap()`——它只做一次廉价的状态调用， +拿不准就返回 `None` 让运行期大声失败，而不是猜。 + +## 8. 加一个新后端 + +1. 在 `providers/registry.py::PROVIDERS` 加一条 `ProviderSpec`（id / kind / + display / cli_names / capabilities）。 +2. 实现 `HarnessTransport` 子类：`run_session()`（一整个 agent step）+ + `complete()`（一次性无工具调用，供 `HarnessLLM`），可选 `auth_gap()`。 +3. 在 `transport_for_id()` 里接上。未实现的后端登记进 `_UNSHIPPED`， + 这样 readiness/doctor 会报"尚未发布"而不是运行到一半失败。 +4. 测试：`test_providers.py` + 对应的 `test_provider_.py`。 + +设计决策记录在 [`../features/provider-registry.md`](../features/provider-registry.md)。 diff --git a/doc/guide/hosts/README.md b/doc/guide/hosts/README.md new file mode 100644 index 000000000..45598f824 --- /dev/null +++ b/doc/guide/hosts/README.md @@ -0,0 +1,101 @@ +# 宿主（Hosts）—— copilot 跑在谁里面 + +**宿主**是你日常使用的编码 Agent。装好之后，copilot 以 MCP + Skill 的形式住在 +它里面，用**它自己的模型**读代码做判断——这就是 **Direct 模式**，不需要任何 +API Key，服务端不跑模型。 + +| 宿主 | 调用形式 | 安装页 | +|---|---|---| +| Claude Code | `/imreview` | [`claude-code.md`](claude-code.md) | +| Codex | `$imreview`（**不是** `/imreview`） | [`codex.md`](codex.md) | +| Cursor | `/imreview` | [`cursor.md`](cursor.md) | +| 其他 MCP 客户端 | 手动导入 `infermatrix-copilot.mcp.json` | 见下方 §3 | + +--- + +## 1. 你要找的可能是另一篇 + +`claude-code`、`codex`、`cursor` 这三个名字**在本项目里出现两次，方向相反**： + +| | **宿主**（本目录） | **后端**（[`../backends.md`](../backends.md)） | +|---|---|---| +| 谁调用谁 | copilot 跑在宿主里面 | copilot 把厂商 CLI 拉起来 | +| 谁的模型在读代码 | **宿主的** | copilot 配置的 | +| 模式 | Direct | Strict | +| 需要 API Key | **不需要** | 要 Key 或订阅登录 | +| 配置项 | 安装器自动写入 | `STRICT_BACKEND=` | + +判据一句话：**宿主提供模型给你用；后端是 copilot 拿去用的模型。** + +举个具体例子——同一个 `codex` 二进制： + +- **作为宿主**：你在 Codex 里敲 `$imreview`，Codex 用它当前选中的模型审代码， + copilot 只负责返回知识路由和治理契约。 +- **作为后端**：你在 CLI 里跑 Strict，copilot **反过来**把 `codex exec` 作为 + 子进程拉起，让它执行一个 agent step。 + +## 2. 一次装完所有识别到的宿主 + +```bash +git clone git@github.com:JiusiServe/InferMatrixCopilot.git +cd InferMatrixCopilot + +./install-mcp.sh # macOS / Linux +./install-mcp.sh --repo-path /path/to/vllm-omni + +install.cmd # Windows（双击也可以） +install.cmd --repo-path D:\path\to\vllm-omni +``` + +安装器**自动识别本机的 Codex、Claude Code 和 Cursor**，为识别到的每一个注册 +MCP 与四个 Skill（`imreview` / `imdesign` / `imcifix` / `imupdate`），并创建 +`~/.infermatrix-copilot/.env`。 + +**装完必须重启宿主。** + +`--repo-path` 只影响 Strict（它需要一个本地 vLLM-Omni checkout）；纯 Direct +使用可以不传。 + +## 3. 没被识别到的宿主 + +安装器会生成一份标准的 `infermatrix-copilot.mcp.json`，任何兼容 MCP 的客户端 +都可以导入。现成的配置模板在 +[`integrations/config-templates/`](../../../integrations/config-templates/)： + +- `codex/config.toml` —— Codex 的 `[mcp_servers.infermatrix_copilot]` 段 +- `cursor/mcp.json` —— Cursor 的 MCP 配置 + +接口清单见 [`../mcp.md`](../mcp.md)。 + +## 4. 确认装好了 + +在宿主的 MCP 页面里应该能看到 `infermatrix-copilot` 已连接。Codex 还可以： + +```powershell +codex mcp list +``` + +宿主没有主动调用它时，直接说出来即可： + +```text +Use InferMatrixCopilot in Direct mode to review this PR. +``` + +## 5. Direct 一次审查会发生什么 + +```text +你发起 imreview + → 固定 PR 快照，60 秒内先报 head SHA / CI / mergeability + → 调 review(mode="direct")，传 title / body / changed_files + → 拿到 ≤3 条知识路由（内嵌 quick_map，3.5k 封顶） + + execution_budget（硬顶）+ checklist + → 宿主自己的模型并行读源码、跑验证 + → validate_direct_review 完成门（机械的结构检查） + → 输出单条合并结论 +``` + +结论**只出现在当前对话**，不会自动发到 GitHub。要发布需要显式要求，且服务端 +配置 `ALLOW_POST=1`。 + +想要完整后台工作流时明说 Strict——那会用到后端，见 +[`../backends.md`](../backends.md)。 diff --git a/doc/guide/hosts/claude-code.md b/doc/guide/hosts/claude-code.md new file mode 100644 index 000000000..25ab6c478 --- /dev/null +++ b/doc/guide/hosts/claude-code.md @@ -0,0 +1,77 @@ +# 在 Claude Code 里使用 InferMatrixCopilot + +> **宿主端页面。** 这里讲 copilot 跑在 Claude Code **里面**（Direct 模式，用 +> Claude Code 自己的模型）。如果你要的是让 copilot **反过来**把 `claude` CLI +> 拉起来执行 Strict 的 agent step，那是**后端**，看 +> [`../backends.md`](../backends.md)。对照表见 [`README.md`](README.md)。 + +默认的 MCP 是一个**知识提供方**：Claude Code 用它当前选中的模型审代码，所以 +这条路径**不需要 API Key，也不需要配置模型**。 + +## 安装 + +```bash +git clone git@github.com:JiusiServe/InferMatrixCopilot.git +cd InferMatrixCopilot + +./install-mcp.sh # macOS / Linux +./install-mcp.sh --repo-path /path/to/vllm-omni # 顺带配置 Strict 的本地 checkout +install.cmd # Windows +``` + +安装器自动识别 Claude Code，注册 MCP 与四个 Skill，用的是与 Codex、Cursor +**同一份** MCP 描述符和 Skill。 + +**重启 Claude Code。** + +## 用法 + +Claude Code 用 slash 形式（Codex 用 `$`）： + +```text +/imreview https://github.com/vllm-project/vllm-omni/pull/5172 +/imreview # 当前 PR；没有 PR 时审当前本地改动 +/imdesign <目标 / issue / PR> +/imcifix https://github.com/vllm-project/vllm-omni/issues/5023 +/imupdate vllm-omni +``` + +也可以直接用自然语言： + +```text +帮我审核一下这个 PR xxxx，用知识库 +Use InferMatrixCopilot in Direct mode to review this PR. +``` + +## 四个 Skill 的边界 + +| Skill | 做什么 | 不做什么 | +|---|---|---| +| `imreview` | 审 PR 或本地改动，输出带文件行号的结论 | 不自动发 GitHub | +| `imdesign` | 产出协同设计包（方案、边界、验证计划） | 不自动改代码 | +| `imcifix` | 本地复现 → 最小修复 → 针对性验证 | 不 commit / push / 开 PR / 发评论 | +| `imupdate` | 同步上游发版带来的结构事实 | 不修改目标仓库 | + +`imcifix` 是**宿主 Agent 的本地补丁流程**——它不意味着 MCP 会去改你的仓库。 + +## 确认装好了 + +在 Claude Code 的 MCP 页面里确认 `infermatrix-copilot` 已连接。如果对话里 +Agent 没有主动调用它，直接说： + +```text +Use InferMatrixCopilot in Direct mode to review this PR. +``` + +## 想跑 Strict + +明确说出来： + +```text +Use InferMatrixCopilot in Strict mode to review +https://github.com/vllm-project/vllm-omni/pull/5172. +``` + +Strict 会返回 `run_id`，再用 `get_review_status` 看进度、`get_review_result` +取报告。它需要一个后端和一个本地 checkout——见 +[`../backends.md`](../backends.md)。 diff --git a/doc/guide/hosts/codex.md b/doc/guide/hosts/codex.md new file mode 100644 index 000000000..bccb1c272 --- /dev/null +++ b/doc/guide/hosts/codex.md @@ -0,0 +1,179 @@ +# 在 Codex 里使用 InferMatrixCopilot + +> **宿主端页面 —— 你要找的可能是另一篇。** 这里讲 Codex 作为**宿主**：copilot 跑在 +> Codex **里面**（Direct 模式），由 Codex 自己的模型读代码。如果你要的是让 copilot +> 把 `codex exec` 作为子进程拉起来执行一个 Strict agent step，那是 Codex 作为 +> **后端**，见 [`../backends.md`](../backends.md)。对照表见 [`README.md`](README.md)。 +> +> 一句话判据：**宿主提供模型给你用；后端是 copilot 拿去用的模型。** + +默认的 MCP 是一个**知识提供方**。Codex 用它当前选中的模型审代码，所以这条路径 +**不需要 API Key，也不需要配置模型**。 + +## 安装 + +```text +# Windows +install.cmd + +# macOS / Linux +./install-mcp.sh +``` + +引导程序会自动识别 Codex。它与 Claude、Cursor 及其他兼容宿主使用**同一份** MCP +描述符和 Agent Skill。 + +重启 Codex，然后粘贴： + +```text +Use InferMatrixCopilot to review +https://github.com/vllm-project/vllm-omni/pull/5172. +``` + +插件还会装上 `imreview`、`imdesign`、`imcifix`、`imupdate` 四个 skill，所以最短的 +评审写法是： + +```text +$imreview https://github.com/vllm-project/vllm-omni/pull/5172 +``` + +Codex 的 skill 用 `$name` 而不是 `/name`；没看到就先跑 `/skills`。修 issue 用： + +```text +$imcifix https://github.com/vllm-project/vllm-omni/issues/5023 +``` + +`imcifix` 是**宿主 agent 的本地补丁流程**。它不意味着 MCP 服务端暴露了什么 +`issue_fix` 后台工具，也不会 commit、push、开 PR 或发评论 —— 除非你明确要求。 + +Codex 调用 `review`，拿到本地 `knowledge/AGENTS.md` 路径和一份精简的首轮评审 +checklist，然后**自己**按那份文档的路由图走。MCP 不猜哪个 owner 适用，也不注入 +完整规则页。 + +固定快照之后，Codex 会**立即**在宿主对话里报告 pinned head SHA、当前 CI、 +mergeability 和初步发现。这一步发生在读知识、搜源码、跑测试**之前**，并且不等 +CI 跑完、不等 mergeability 落定。随后 Codex 用收集到的 title、body 和 changed +files 调用一次 Direct，使用每条精确路由里内嵌的 `quick_map`（不打开完整规则页）， +并让知识/源码与验证两条线并行推进。它复用同一份评审内证据包，并在 pytest 之前先 +跑一次 import/版本兼容性预检。验证结果绑定到 head SHA 和环境指纹。 + +那条进度更新**不是** GitHub 上的中间评论。在唯一一条最终评审评论之前，Codex 会调用 +`validate_direct_review`。常规小修改用 `subtraction_signal="none"`，不需要最小性 +证明。只有当 diff 新增或扩张了 helper、类、fallback、兼容分支或公开行为时才用 +`"triggered"`，并且必须提供减法证据。 + +工具返回**刻意做得很小**： + +```json +{ + "mode": "direct", + "knowledge_entry": "C:\\...\\knowledge\\repos\\vllm-omni\\components\\serving\\rules.md", + "knowledge_routes": [ + { + "owner": "serving", + "path": "C:\\...\\knowledge\\repos\\vllm-omni\\components\\serving\\rules.md", + "reason": "title/body: endpoint, request", + "quick_map": "## Direct 代码快速入口\n...", + "read_required": false + } + ], + "navigation_policy": { + "progress_before_knowledge": true, + "use_embedded_quick_maps": true, + "open_route_file_only_for_concrete_ambiguity": true, + "max_routes": 3, + "stop_after_routes": true + }, + "execution_budget": { + "profile": "code", + "knowledge_file_reads": 0, + "validation_commands": 4, + "total_command_calls": 20, + "hard_ceiling": true, + "extension_command_calls": 4 + }, + "first_review_checklist": ["...", "Run subtraction only when the diff has a subtraction signal ..."], + "progress_update": { + "deadline_seconds": 60, + "channel": "host_conversation", + "required_fields": ["head_sha", "ci_status", "mergeability", "early_findings"], + "early_findings_status": "preliminary", + "continue_review": true, + "github_comment": false + }, + "completion_gate": { + "tool": "validate_direct_review", + "subtraction_signal": { + "none": "no subtraction evidence required", + "triggered": "require subtraction evidence" + }, + "triggered_require_one_of": [ + "subtraction[{anchor, action, risk}]", + "minimality_proof{scope_ledger, abstraction_census, why_no_safe_deletion}" + ], + "final_comment_count": 1, + "if_missing": "partial_review" + } +} +``` + +Strict 用的是**同一个**已安装的 MCP。wheel 里已经带了它需要的 playbook、adapter +和 skill。安装时配置本地 checkout： + +```text +install.cmd --repo-path D:\path\to\vllm-omni +``` + +然后在 `~/.infermatrix-copilot/.env` 里填 `ANTHROPIC_API_KEY` 或 `OPENAI_API_KEY` +之一。两个 provider 各自还有可选的 `ANTHROPIC_BASE_URL` / `OPENAI_BASE_URL`，用于 +代理或兼容网关。只填一个 key 时自动选择 provider；两个都填时用 +`LLM_PROVIDER=anthropic` 或 `LLM_PROVIDER=openai` 指定。重启 Codex，然后说： + +```text +Use InferMatrixCopilot in Strict mode to review +https://github.com/vllm-project/vllm-omni/pull/5172. +Do not return a review until the strict run is complete. +``` + +Strict 会用它配置的模型运行打包好的 `pr-review` playbook，带进度跟踪、报告生成和 +发布门禁。轮询 `get_review_result` 直到 run 进入终态。 + +**Strict 永不隐式发布。** 发布仍然需要显式的 `post=true` 调用参数**加上**服务端的 +`ALLOW_POST=1`。 + +安装后重启 Codex。用 `/mcp` 或 `codex mcp list` 确认 `infermatrix-copilot` 已连接， +用 `/skills` 确认 `imreview`、`imdesign`、`imcifix`、`imupdate` 可用。 + +## 默认 MCP 暴露了什么 + +- `review(target, repo?, mode="direct", post=false, title="", body="", + changed_files=[], review_depth="", repo_path="")`：在宿主发出进度更新之后， + Direct 用 title/body 返回**至多三条**精确的 owner/model 路由，各自带内嵌的精简 + `quick_map` 摘录。changed files 通常只做范围校验；但当返回的路由**没有一条**命中 + 它们推导出的 owner 时，改由 changed files 选路，并在响应里明说 + （`status: scope_fallback`）。宿主不打开完整规则页 —— 除非有具体歧义卡住了源码 + 评审：`quick_map_status` 不是 `ok` 的路由（`unavailable`，或章节超过摘录上限时的 + `truncated`）正是这种情况，预算会为每一条放行一次读取。返回的 docs/code + `execution_budget` 是**硬顶**；唯一一次有界扩展，只留给一个明确陈述的、尚未解决的 + P1/高风险契约。Strict 分支启动打包好的工作流，接受 `review_depth`，并可通过 + `repo_path` 临时覆盖本地 checkout。 +- `validate_direct_review(subtraction_signal, subtraction?, minimality_proof?, + final_comment_count=1, evidence_head_sha)`：`evidence_head_sha` 必须是**本次固定的 + head 提交**——每一个被引用的源文件和验证结果都读自它；读自其他本地版本的证据 + 不能完成评审。`none` 用于收尾常规小修改，不需要最小性证明；`triggered` 需要带锚点 + 的减法动作，或"所审范围已经最小"的具体证据。 +- `update_knowledge(repo?)`：`repo` 仅为调用兼容保留，返回 + `doc/knowledge/CONTRIBUTING.md`；由宿主模型照那份文档自己改 Markdown。 +- `get_review_result(run_id, offset?)`：轮询并分页读取 Strict 报告。 +- `get_review_status(run_id)`：返回 Strict run 的持久化进度。 +- `doc_search(query, repo?)`：查找更深的模型/组件规则。 +- `doc_read(path, repo?)`：读取选中的知识页。 + +Direct 模式**不跑第二个模型、不改知识、不发评论、不推代码**。它的确定性路由器从 PR +描述里选出有界的知识 owner；范围校验和被引用代码证据的真伪，仍然由 Codex 自己负责。 +完成校验器检查的是**评审结构**。 + +## 可选：autonomous BYOK 工作流 + +autonomous 工作流有独立的配置和文档： +[`../autonomous-workflow.md`](../autonomous-workflow.md)。 diff --git a/doc/guide/hosts/cursor.md b/doc/guide/hosts/cursor.md new file mode 100644 index 000000000..b803ed752 --- /dev/null +++ b/doc/guide/hosts/cursor.md @@ -0,0 +1,61 @@ +# 在 Cursor 里使用 InferMatrixCopilot + +> **宿主端页面。** 这里讲 copilot 跑在 Cursor **里面**（Direct 模式，用 Cursor +> 自己的模型）。如果你要的是让 copilot **反过来**把 `cursor-agent` 拉起来执行 +> Strict 的 agent step，那是**后端**，看 [`../backends.md`](../backends.md)。 +> 对照表见 [`README.md`](README.md)。 + +默认的 MCP 是一个**知识提供方**：Cursor 用它当前选中的模型审代码，这条路径 +**不需要 API Key，也不需要配置模型**。 + +## 安装 + +```bash +git clone git@github.com:JiusiServe/InferMatrixCopilot.git +cd InferMatrixCopilot + +./install-mcp.sh # macOS / Linux +./install-mcp.sh --repo-path /path/to/vllm-omni +install.cmd # Windows +``` + +安装器自动识别 Cursor 并注册 MCP 与四个 Skill。**装完重启 Cursor。** + +手动配置（安装器没识别到，或你要自己接管）时，模板在 +[`integrations/config-templates/cursor/mcp.json`](../../../integrations/config-templates/cursor/mcp.json)。 + +## 用法 + +Cursor 用 slash 形式： + +```text +/imreview https://github.com/vllm-project/vllm-omni/pull/5172 +/imreview # 当前 PR；没有 PR 时审当前本地改动 +/imdesign <目标 / issue / PR> +/imcifix +/imupdate vllm-omni +``` + +`integrations/cursor/` 下另有 `imreview.md` / `imcifix.md` 两份提示词文件， +供直接粘贴或自定义命令使用。 + +## 一条 Cursor 特有的注意事项 + +这一条属于**后端**侧，但用 Cursor 的人最容易撞上，所以在这里也写一次： + +**grok 家族模型（4.5 / 4.6）会在评审会话中主动搜索并读取 copilot 自己的 +`imreview` 方法论 skill。** Composer 从未出现这个行为。如果你在用 cursor 后端 +跑评测，三份 skill 副本必须先移出 `$HOME` 并在事后恢复，否则结果会被污染。 + +也因为 cursor-agent 无法完全关闭自己的内置工具，凡是走 cursor **后端**的会话 +都会额外过一遍事后审计（`providers/audit.py`），审计结论写进 RUN_REPORT。 +细节见 [`../backends.md` §5](../backends.md#5-换后端不会绕过权限闸)。 + +## 确认装好了 + +在 Cursor 的 MCP 设置里确认 `infermatrix-copilot` 已连接。Agent 没主动调用时 +直接说： + +```text +Use InferMatrixCopilot in Direct mode to review this PR. +``` diff --git a/doc/guide/mcp.md b/doc/guide/mcp.md new file mode 100644 index 000000000..865dc4671 --- /dev/null +++ b/doc/guide/mcp.md @@ -0,0 +1,93 @@ +# 安装 InferMatrixCopilot + +InferMatrixCopilot 的核心发布物保持一份： + +- `plugins/infermatrix-copilot/.mcp.json`：标准 stdio MCP 描述； +- `plugins/infermatrix-copilot/skills/`：开放 Agent Skills； +- Claude、Codex、Cursor 的市场文件只负责把同一个插件展示出来，不包含各自的安装逻辑。 + +## 私有仓库过渡安装 + +```text +git clone git@github.com:JiusiServe/InferMatrixCopilot.git +cd InferMatrixCopilot + +# Windows cmd、PowerShell 或双击 +install.cmd +install.cmd --repo-path D:\path\to\vllm-omni + +# macOS / Linux +./install-mcp.sh +./install-mcp.sh --repo-path /path/to/vllm-omni +``` + +两个入口共用 `scripts/install_mcp.py`，但用户不需要安装或直接运行 Python： +入口会自动安装 `uv`，再由 `uv` 提供隔离运行环境。安装器会： + +- 自动识别 Codex、Claude Code 和 Cursor，检测到几个就安装几个； +- 保留 Cursor 已有配置并先备份； +- 没识别到已知 Agent 时生成 `infermatrix-copilot.mcp.json`，不会直接失败。 +- 创建 `~/.infermatrix-copilot/.env` 作为 Strict 的稳定配置入口；安装时传入 + `--repo-path` 会写入本地 vLLM-Omni checkout。模型密钥不会自动复制，使用 + Strict 前在该文件填写 `ANTHROPIC_API_KEY` 或 `OPENAI_API_KEY`。两者分别支持 + 可选的 `ANTHROPIC_BASE_URL` 和 `OPENAI_BASE_URL`；只填一种 Key 时自动选择， + 两种都填时用 `LLM_PROVIDER` 指定。 + +## 公共市场上线后 + +在 Agent 的插件市场搜索 `infermatrix-copilot` 并点击安装，然后直接运行： + +```text +# Codex +$imreview +$imupdate + +# Claude Code / Cursor +/imreview +/imupdate +``` + +市场只安装同一份 MCP 与 Skill；不会把逐 Agent 逻辑带回核心包。 + +## 其他 MCP Agent + +安装器生成的 `infermatrix-copilot.mcp.json` 与 +[`plugins/infermatrix-copilot/.mcp.json`](../../plugins/infermatrix-copilot/.mcp.json) +内容相同，可直接导入支持 stdio MCP 的客户端。开放 Skill 位于： + +```text +plugins/infermatrix-copilot/skills/imreview/SKILL.md +plugins/infermatrix-copilot/skills/imupdate/SKILL.md +``` + +核心 MCP 配置： + +```json +{ + "mcpServers": { + "infermatrix-copilot": { + "type": "stdio", + "command": "uvx", + "args": [ + "--from", + "infermatrix-copilot[mcp] @ git+https://github.com/JiusiServe/InferMatrixCopilot.git@main", + "infermatrix-copilot-mcp" + ] + } + } +} +``` + +这里用 `uvx` 自动拉取并缓存隔离运行环境。宿主如果能消费 MCP Registry，则应优先 +使用 Registry 条目；项目不为每个新 Agent 增加安装分支。 + +## 使用 + +```text +# Codex +$imreview https://github.com/vllm-project/vllm-omni/pull/5172 +$imupdate D:\path\to\vllm-omni +$imupdate vllmomni +``` + +默认 Direct 模式只提供知识库，由 Agent 当前模型完成审查；不会自动评论或推送代码。 diff --git a/doc/knowledge-templates/README.md b/doc/knowledge-templates/README.md deleted file mode 100644 index a56ce360d..000000000 --- a/doc/knowledge-templates/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Knowledge-base page templates - -Ready-to-copy skeletons for the curated wiki at repo-root `knowledge/`. Full -workflow, decision tree, and gate rules: [`../EXTENDING-KNOWLEDGE.md`](../EXTENDING-KNOWLEDGE.md). - -These files live **outside** `knowledge/`, so they don't run through the wiki gate -(their placeholders would otherwise fail it). Copy one into the right owner -directory, fill in the `<...>` placeholders, delete the leading `` -comment, register the page in the nearest `_index.md`, then validate. - -## Which file for what - -| Copy this | To create | Into | -|---|---|---| -| [`_index.md`](_index.md) | a directory's routing table | any new topic/component/model dir | -| [`repo-index.md`](repo-index.md) | a **repo** entry index | `knowledge/repos//_index.md` | -| [`rules.md`](rules.md) | an always-on gate page | nearest owner dir | -| [`architecture-component.md`](architecture-component.md) | a shared-module architecture page | `repos//components//architecture.md` | -| [`architecture-model.md`](architecture-model.md) | a model architecture page | `repos//models//architecture.md` | -| [`guide.md`](guide.md) | a method/how-to page | component/model owner root; work-topic collections may use `guides/` | -| [`incident.md`](incident.md) | a retro / historical write-up | an `incidents/` dir — **rename to `YYYY-MM-DD-short-name.md`** | - -## Copy + register + validate - -```bash -# 1. copy (example: a new component page) -cp doc/knowledge-templates/architecture-component.md \ - knowledge/repos/vllm-omni/components//architecture.md - -# 2. edit it: fill <...>, delete the line -# 3. add a row for it in the sibling _index.md (遇到什么 → 查看哪里) -# 4. validate — both must print 0 errors -python knowledge/tools/check_knowledge_tree.py # structure / index / links / incidents -python knowledge/tools/check_wiki_lint.py # synthesis-layer frontmatter + tag taxonomy -``` - -Reminders the gates enforce: every non-index page and child dir is linked -**exactly once** from the nearest `_index.md`; relative links must resolve; no real -host/IP/user-path/token/key in tracked pages (those go in git-ignored `local/`); -split a page at ≥500 non-empty lines or 32 KiB. Synthesis-layer pages -(`rule/guide/architecture/index` under `general/`+`repos/`) need frontmatter with a -`tags` value from [`SCHEMA.md`](../../knowledge/SCHEMA.md); the incident template is -evidence-layer and takes none. Deliver via PR — the tree is vendored; never edit -upstream. diff --git a/doc/knowledge-templates/_index.md b/doc/knowledge-templates/_index.md deleted file mode 100644 index 50eea0919..000000000 --- a/doc/knowledge-templates/_index.md +++ /dev/null @@ -1,28 +0,0 @@ - ---- -title: "" -created: -updated: -type: index -tags: [] -sources: [] ---- - -# - -## 什么时候查这里 - -- - -## 不放什么 - -- → `` - -## 目录内容 - -| 遇到什么 | 查看哪里 | 说明 | -|---|---|---| -| | [](.md) | | -| | [](/_index.md) | | diff --git a/doc/knowledge-templates/architecture-component.md b/doc/knowledge-templates/architecture-component.md deleted file mode 100644 index e6a024a84..000000000 --- a/doc/knowledge-templates/architecture-component.md +++ /dev/null @@ -1,32 +0,0 @@ - ---- -title: " 架构" -created: -updated: -type: architecture -tags: [] -sources: [] ---- - -# 架构 - -## 职责和边界 - -- - -## 主要源码和调用入口 - -- `` — -- 测试入口：`` - -## 数据怎样流动 - -- → → - -## 怎样验证 - -- diff --git a/doc/knowledge-templates/architecture-model.md b/doc/knowledge-templates/architecture-model.md deleted file mode 100644 index d711b2b08..000000000 --- a/doc/knowledge-templates/architecture-model.md +++ /dev/null @@ -1,35 +0,0 @@ - ---- -title: " 架构" -created: -updated: -type: architecture -tags: [] -sources: [] ---- - -# 架构 - -## 模型专有部分与共享模块的边界 - -- 专有： -- 依赖的共享模块：[](../../components//_index.md) - -## 配置、checkpoint 和兼容范围 - -- 正式名称 / 别名： / -- checkpoint、尺寸、量化差异：<...> - -## 从输入到输出的主要流程 - -- → → → - -## 怎样验证功能、精度和性能 - -- 功能： -- 精度： -- 性能： diff --git a/doc/knowledge-templates/guide.md b/doc/knowledge-templates/guide.md deleted file mode 100644 index 63758590f..000000000 --- a/doc/knowledge-templates/guide.md +++ /dev/null @@ -1,26 +0,0 @@ - ---- -title: "" -created: -updated: -type: guide -tags: [] -sources: [] ---- - -# - -## 什么时候用 - -- - -## 步骤 / 方法 - -1. -2. - -## 怎样验证 - -- diff --git a/doc/knowledge-templates/incident.md b/doc/knowledge-templates/incident.md deleted file mode 100644 index 9d9b0b1da..000000000 --- a/doc/knowledge-templates/incident.md +++ /dev/null @@ -1,34 +0,0 @@ - -# <人能读懂的现象标题> - -- 编号：`inc--` -- 归属：`/ or general/>` -- 状态：处理中 -- 搜索词：、、 -- 影响范围： - -## 现象 - -- - -## 根因（live 证据） - -- - -## 修复 - -- - -## 验收 - -- - -## 已提炼的规则 - -- 见 [ rules](../rules.md#) diff --git a/doc/knowledge-templates/repo-index.md b/doc/knowledge-templates/repo-index.md deleted file mode 100644 index c69ffc3b6..000000000 --- a/doc/knowledge-templates/repo-index.md +++ /dev/null @@ -1,35 +0,0 @@ - ---- -title: "" -created: -updated: -type: index -tags: [] -sources: [] ---- - -# - -- 上游仓库：`/` -- 常用分支：默认分支 `
`； -- 适用范围： -- 组件源码映射已按 ` main @ ` 校验 - -## 什么时候查这里 - -- 当前 Git 仓库或用户明确目标是 。 - -## 不放什么 - -- 跨仓库通用方法（放 `general/`）。 -- 其他仓库的规则。 - -## 当前入口 - -| 遇到什么 | 查看哪里 | 说明 | -|---|---|---| -| 开始任何 修改、测试、发布任务 | [硬门禁](rules.md) | 仓库硬规则 | -| 查看共享代码模块 | [components](components/_index.md) | 模块职责地图 | -| 查看某个模型 | [models](models/_index.md) | 模型入口 | diff --git a/doc/knowledge-templates/rules.md b/doc/knowledge-templates/rules.md deleted file mode 100644 index 2cfe07baf..000000000 --- a/doc/knowledge-templates/rules.md +++ /dev/null @@ -1,30 +0,0 @@ - ---- -title: " 硬门禁" -created: -updated: -type: rule -tags: [] -sources: [] ---- - -# 硬门禁 - -只在当前任务明确属于 时应用本页。先遵守根 `CLAUDE.md` 的通用 P0，再执行这里的专属门禁。 - -## 场景触发器 - -| 用户提到 | 必读 | 硬约束 | -|---|---|---| -| | [](.md) | | - -## 规则 - -### - -- 触发： -- 必须： -- 禁止： -- 验收： diff --git a/doc/knowledge/CONTRIBUTING.md b/doc/knowledge/CONTRIBUTING.md new file mode 100644 index 000000000..2479c023a --- /dev/null +++ b/doc/knowledge/CONTRIBUTING.md @@ -0,0 +1,76 @@ +# 知识写入入口 + +本页是**人工写入**知识树的短入口，不承载全部细则。先按任务选一篇专题，不要每次落盘都读完整套规范。 + +## 先确认你在哪条路径上 + +知识只由这三条路径更新。**目标仓库不会向本仓库提 PR** —— 它的 owner 走 `[Knowledge]` +issue，由维护者转成第 3 条路径。 + +| 路径 | 触发 | 谁写 | 细则在哪 | +|---|---|---|---| +| **`imupdate`** | 上游发版或目录变化 | agent 产出、人工审核 | [发版漂移审计](../contributing/release-maintenance.md) | +| **运行中的 agent** | 复盘、记录一条教训 | agent **只能提 candidate**，人工晋升 | [writing.md §2](writing.md) + `SPEC/memory/skills.md` | +| **人工** | 维护者沉淀一条规则 | 维护者（可源自一条 `[Knowledge]` issue） | **本页以下全部内容** | + +**下面的「最短落盘流程」是第 3 条路径的流程。** 前两条各有自己的产出与门禁： +`imupdate` 只更新机器已经证明的结构事实（release baseline、model catalog、source +map、SHA pin、adapter manifest），不做 owner 归属判断，也不生成或改写规则；运行中的 +agent 只提 candidate。落盘位置和页型选择属于人工路径，两个校验器**三条路径共用**。 + +现行契约只有四样：本页、[contributing/](contributing/_index.md)、[SCHEMA.md](SCHEMA.md) +和两个校验器。[知识重组计划](../archive/PLAN-knowledge-reorg.md) 是已归档的历史记录， +不是目录契约；批量整理仍按 owner 分批、同义结论 union-first 合并。SCHEMA 只补 +frontmatter、标签和溯源元数据，不能据此发明第二套知识结构。 + +## 正在做什么 + +| 任务 | 只需继续读 | +|---|---| +| 判断内容放 `general/`、`repos/`、component、model 还是 `local/` | [目录与归属](contributing/layout.md) | +| 新增 `_index.md`、`rules.md`、`architecture.md` 或普通页面 | [页面与索引](contributing/page-rules.md) | +| 从 PR/review 复盘并提炼规则 | [复盘与规则摄取](contributing/incidents.md) | +| 文件太长、目录太挤、dev 要拆前后端或模块/模型长大 | [何时拆分](contributing/scaling.md) | +| 新增、移动、重命名、删除 Markdown，或准备提交 | [同步与校验](contributing/validation.md) | +| 改动 vLLM-Omni 的 owner 页、SHA pin 或 `sources:` | [同步与校验](contributing/validation.md) 的发版审计一节 | + +所有专题入口见 [contributing 索引](contributing/_index.md)。 + +## 最短落盘流程 + +1. 先确定内容的最近 owner 和目标目录，再选载体：硬门禁进 `rules.md`，稳定职责和数据流进 `architecture.md`，展开方法进 `guides/`。已有文件不能反过来证明 owner。 +2. 按上表只选一篇与当前动作匹配的专题；位置不确定时选择 [目录与归属](contributing/layout.md)，不要为了保险横向通读。 +3. 写正文，同时更新最近 `_index.md`；父级只增加路由，不复制正文，不预建空目录或占位页面。 +4. 用户要求复盘时，动笔前先完成[语义分流台账](contributing/incidents.md#动笔前的语义分流台账)，不能先打开 `rules.md` 再把不适合的内容往外搬。硬门禁进 `rules.md`，稳定职责和数据流进 `architecture.md`，非硬门禁方法进 `guides/`，一次性历史默认不落盘；incident 只有通过三项准入门禁才创建。 +5. 对照本页“交付前五项”检查链接、索引、隐私和拆分阈值，然后运行： + +```powershell +python knowledge/tools/check_knowledge_tree.py +python knowledge/tools/check_wiki_lint.py +``` + +## 不能放宽的 P0 + +- 只保留一套正式知识树，不建 `_private/`、兼容副本或第二套私人索引。 +- 机器地址、账号、cache、venv、token、密钥和用户绝对路径只属于 Git 忽略的 `local/`；密码、token 和私钥正文不写入文件。 +- 一条知识只有一份正文，其他入口只链接；不复制类似规则。 +- 长期知识只写本仓库的 `general/`、`repos/` 和贡献规范，不推进系统、全局或个人 memory。 +- 仓库、模块、模型和机器规则不互相继承；当前任务只加载真正命中的 owner。 +- 新增、移动、重命名、拆分或删除 Markdown 时，必须在同一修改中更新索引和所有链接。 +- PR/review 学习只能落可执行规则：完整 diff、评论、thread、commit 和回放输出只在 + 整理时进入 Git 忽略的临时目录；批次结束必须删除。正式树只更新最近 component 或 + model owner 的 `rules.md` 及必要索引，不新增 PR case/history/result/incident 页面。 +- 非 PR 学习的既有 guide/architecture 继续按原 owner 维护；不得把不同模型、组件、 + benchmark 和 review 内容压成一篇 catch-all 页面。 +- 评测 cases、hidden labels、predictions、judgments 和 run reports 属于 `eval/`， + 不得混入产品知识树或 adapter 默认 briefing。 + +## 交付前五项 + +- [ ] 位置落在内容的**最近 owner**，owner 证据是稳定边界（源码目录 / 维护职责 / + 测试 / 运行流程），没有把工作主题和代码模块套娃。 +- [ ] 最近 `_index.md` 能找到唯一正文，旧路径没有残留有效链接；父级和其他入口只链接，没有复制正文。 +- [ ] 语义分流台账已完成；`rules.md` 候选逐条通过 purity audit，方法、架构和历史证据没有混入。 +- [ ] 没有公开机器信息、凭据、私人路径或本地临时产物。 +- [ ] `python knowledge/tools/check_knowledge_tree.py` 和 `python knowledge/tools/check_wiki_lint.py` + 都通过，并已检查当前完整 diff。 diff --git a/doc/knowledge/README.md b/doc/knowledge/README.md new file mode 100644 index 000000000..74626fa13 --- /dev/null +++ b/doc/knowledge/README.md @@ -0,0 +1,55 @@ +# 知识库文档 + +关于**怎么写和维护** `knowledge/` 那棵精选 wiki 的一切，都在这个目录里。 +**知识本身不在这里** —— 它在仓库根的 [`knowledge/`](../../knowledge/)， +那里现在只剩数据面：`general/`（通用经验）+ `repos/`（逐仓库知识）。 + +> **为什么分开：** `knowledge/` 曾经既装知识页、又装"怎么写知识页"的规范。 +> 前者是**数据**（被 `doc_search`/`doc_read` 检索、随 wheel 发布、由校验器扫描）， +> 后者是**文档**（人类读一遍就去动手）。两者的读者、生命周期和校验规则都不同， +> 混在一起会让每个打开 `knowledge/` 的人先踩过一层不该读的规范。 + +## 知识由谁更新 + +只有三条路径。**目标仓库（vLLM-Omni）不会向本仓库提 PR** —— 它的模块/模型 owner 提 +`[Knowledge]` issue（`.github/ISSUE_TEMPLATE/knowledge-rule.yml`），由维护者转成第 3 条。 + +| 路径 | 什么时候 | 谁写 | 入口 | +|---|---|---|---| +| **`imupdate`** | 上游发版、目录变化、registry/deploy/路径路由/source pin 过期 | agent 产出、人工审核 | [`../contributing/release-maintenance.md`](../contributing/release-maintenance.md) | +| **运行中的 agent** | 复盘、"记一条教训" | **只能提 candidate**，晋升是人工动作 | [`writing.md`](writing.md) | +| **人工** | 维护者沉淀一条规则或修订页面 | 维护者 | [`CONTRIBUTING.md`](CONTRIBUTING.md) | + +三条路径**共用**同一套落盘位置判断、页型选择和两个校验器 —— 差别只在谁触发、 +以及产出在生效前要过哪道门。 + +## 该读哪一篇 + +| 你要做什么 | 看这篇 | +|---|---| +| 第一次写知识页 —— 决策树、页型、copilot 怎么消费、两道门禁 | [`writing.md`](writing.md) | +| 需要**有约束力**的那条规则（与上面冲突时以它为准） | [`CONTRIBUTING.md`](CONTRIBUTING.md) → [`contributing/`](contributing/_index.md) | +| 直接开写，要现成骨架 | [`templates/`](templates/README.md) | +| 看一遍填好的中文实操走查 | [`maintainer-walkthrough.md`](maintainer-walkthrough.md) | +| frontmatter、标签分类法、生命周期 | [`SCHEMA.md`](SCHEMA.md) | +| 三个端到端的贡献示例 | [`samples/`](samples/) | +| 知识树长什么样、有哪些入口 | [`tree-map.md`](tree-map.md) | +| 这棵树从哪来、授权与同步边界 | [`../architecture/KNOWLEDGE.md`](../architecture/KNOWLEDGE.md) | + +## 两个校验器 + +它们仍然住在被校验的那棵树旁边（`knowledge/tools/`），因为它们是**代码**，不是文档： + +```bash +python knowledge/tools/check_knowledge_tree.py # 结构 / 索引 / 链接 / incident +python knowledge/tools/check_wiki_lint.py # frontmatter + 标签分类法 +``` + +两个都必须打印 0 错误（提醒需要人判断，不要求清零）。注意它们**跨树读取本目录**： +树校验器在这里检查 `CONTRIBUTING.md` 的长度门，wiki lint 从 `SCHEMA.md` 读标签分类法 +—— 搬动这两个文件时**必须同时改那两处路径**。 + +`repos/vllm-omni/` 那一片还有**第三道门**，两个校验器看不见它：owner 入口页、正文里的 +SHA pin 和 `sources:` 由 `tools/audit_vllm_omni_release.py` 按 release baseline 对账， +任何触及该切片的 PR 都会在 CI 里跑 `enforce`。见 +[同步与校验 §发版审计](contributing/validation.md#vllm-omni-页面还有一道发版审计)。 diff --git a/doc/knowledge/SCHEMA.md b/doc/knowledge/SCHEMA.md new file mode 100644 index 000000000..c12538752 --- /dev/null +++ b/doc/knowledge/SCHEMA.md @@ -0,0 +1,72 @@ +# SCHEMA — 页面元数据与生命周期 + +规定 `general/` 与 `repos/` 下**沉淀层**页面的 YAML frontmatter、标签分类法和生命周期 +规则（LLMWiki 机制）。目录归属与页面写法仍以 [贡献规范](contributing/_index.md) +为准；本文件只补充元数据机制，不重复目录规范。`repos/jianghan-roleplay-data-pipeline/` +整棵子树暂不适用（保持原样）。 + +## 分层 + +- **证据层（默认不新增）**：现存 `incidents/`、`history/`、`results/` 是旧树留下的 + 材料。PR/review 学习**一律不写**这一层：新的 raw diff、评论、thread、回放和 case 只在 + 整理临时目录存在，规则验收后必须删除。非 PR 事故也默认不新增，只有同时满足 + [复盘与规则摄取](contributing/incidents.md#incident-准入门禁) 的三项准入门禁才创建 + `incidents/` 页面；`history/`、`results/` 不再新增。 +- **沉淀层**：`rules.md`、`guides/`、`architecture.md`、`overview.md`、`_index.md` + 等 —— 携带 frontmatter，结论通过 `sources:` 引用外部 PR/代码或既有历史页。 +- **PR/review 学习产物**：只允许写最近 owner 的 `rules.md` 与必要 `_index.md`；用 + `sources:` 和段尾 PR 引用提供溯源，不保存原始证据副本。 + +## Frontmatter（沉淀层必填） + +`check_wiki_lint.py` 强制的只有前五项（`title`、`created`、`updated`、`type`、 +非空 `tags`）；其余为约定字段，写了就必须合法（`confidence` 只能取三值）。 + +```yaml +--- +title: 页面标题 +created: YYYY-MM-DD # 上游首次提交日期（来源 doc/archive/reorg-audit/baseline/dates.tsv） +updated: YYYY-MM-DD # 最近实质更新日期 +type: rule | guide | architecture | index +tags: [来自下方分类法] +sources: [] # 约定字段：incidents/... 相对路径、PR 号、file:line、上游源码路径 +confidence: high | medium | low # 可选：结论的证据强度（单一来源/有争议时标注） +contested: true # 可选：存在未解决矛盾时 +contradictions: [相对路径] # 可选：与本页冲突的页面 +--- +``` + +`repos/vllm-omni/` 的 `sources:` 另有机器用途：以 `vllm_omni/`、`tests/`、 +`benchmarks/`、`docs/`、`.buildkite/` 开头的条目会被发版审计与上游文件列表对账， +上游删除或重命名后报 `stale_knowledge_source`。见 +[同步与校验 §发版审计](contributing/validation.md#vllm-omni-页面还有一道发版审计)。 + +## 标签分类法 + +新标签必须先加入此表再使用（防止标签蔓延）： + +- 归属：`general`、`vllm-omni`、`afd-plugin` +- 工作主题：`review`、`ci`、`docs`、`git`、`debug`、`bug`、`benchmark`、`environment`、 + `remote`、`agents`、`planning`、`dev`、`rebase` +- 代码/模型轴：`components`、`models`、`diffusion`、`model-executor`、`serving`、 + `scheduler`、`distributed`、`config`、`hunyuan-image3`、`ltx2`、`qwen-omni`、 + `plugin-boundary`、`attention-runtime`、`ffn-runtime`、`connectors`、 + `model-integration`、`execution-platforms`、`compatibility` + +## 溯源标记 + +沉淀层段落引用具体事故/PR 时，段落末尾加 `^[incidents/...]` 形式的来源标记； +页面级证据在 frontmatter `sources:` 列出。每个沉淀层页面至少链接 2 个相关页面 +（相对 Markdown 链接，不用 wikilink）。 + +## 被取代页面 + +先把仍然有效的独有结论合入最近 owner 的幸存页面，再删除被取代或重复页面并修复 +入链。Git 历史负责追溯旧版本；`knowledge/` 内不再维护 `_archive/` 副本。 + +## 校验 + +```bash +python knowledge/tools/check_knowledge_tree.py # 目录/索引/链接/错题/敏感信息 +python knowledge/tools/check_wiki_lint.py # frontmatter/标签/孤页/陈旧度 +``` diff --git a/knowledge/contributing/_index.md b/doc/knowledge/contributing/_index.md similarity index 100% rename from knowledge/contributing/_index.md rename to doc/knowledge/contributing/_index.md diff --git a/knowledge/contributing/incidents.md b/doc/knowledge/contributing/incidents.md similarity index 98% rename from knowledge/contributing/incidents.md rename to doc/knowledge/contributing/incidents.md index 208b26c96..29250cd7e 100644 --- a/knowledge/contributing/incidents.md +++ b/doc/knowledge/contributing/incidents.md @@ -52,7 +52,7 @@ RULES PURITY: / DUPLICATE BODY: none | <重复正文及唯一保留位置> ``` -逐条确认新增规则脱离原事故仍可执行、下一次开工确实必须立即看到，并且只链接 guide/architecture/incident 而不复制其正文。`python tools/check_knowledge_tree.py` 只验证结构和链接，不能替代这项语义检查。 +逐条确认新增规则脱离原事故仍可执行、下一次开工确实必须立即看到，并且只链接 guide/architecture/incident 而不复制其正文。`python knowledge/tools/check_knowledge_tree.py` 只验证结构和链接，不能替代这项语义检查。 ## PR/review 学习的低成本完整摄取 diff --git a/knowledge/contributing/layout.md b/doc/knowledge/contributing/layout.md similarity index 76% rename from knowledge/contributing/layout.md rename to doc/knowledge/contributing/layout.md index 3b3742579..a8ba1c76d 100644 --- a/knowledge/contributing/layout.md +++ b/doc/knowledge/contributing/layout.md @@ -20,55 +20,43 @@ 一个问题同时影响多个入口时，只在最近 owner 保留一份正文，其他位置只链接。 -## 写入前 owner 凭证 - -选择或新建任何知识文件前，先在工作记录或 commentary 中写出下面四项： - -```text -OWNER ROUTING -- content: -- nearest owner: -- owner evidence: -- target directory: -``` - -- `nearest owner` 先回答事实归 `framework` 主题、仓库工作主题、共享源码 component、单一 model 还是 `local`；不能先选一个现有 `rules.md`，再用文件已经存在作为归属证据。 -- `owner evidence` 必须是稳定边界，例如源码目录、维护职责、测试、运行流程或明确的输入输出；“这里以前写过类似内容”和“这是一次开发工作”都不是源码 owner 证据。 -- `target directory` 必须直接位于最近 owner 下。工作主题不能包住 component/model，component/model 也不再套工作主题。 -- owner 确定后才选择载体：硬门禁进 `rules.md`，稳定职责和数据流进 `architecture.md`，展开方法进 `guides/`。载体类型不能反过来决定 owner。 - -如果删除 PR、事故或本次任务背景后，内容仍在描述某块源码的职责、字段、调用链或验证边界，它就属于对应 component/model，而不是 `dev`、`review` 或 `debug`。找不到稳定 owner 时继续核对 live 源码，不把工作主题当作 fallback。 - -父级入口只保留到唯一正文 owner 的链接和一句路由条件。不能在父级复制正文，也不能把同一结论改写一份后声称不是重复。 - ## 工作主题和代码 owner 是并列的 - `review`、`ci`、`docs`、`benchmark`、`remote`、`dev` 表示“正在做什么”。 - `components/frontend`、`components/backend`、`components/diffusion` 表示“事实属于哪块代码”。 - `models/hunyuan-image3` 表示“事实只属于哪个模型”。 -不要套娃： +不要套娃 —— 下面每一对只改了归属，其他部分刻意保持一致： ```text # 错误 -repos/acme/dev/frontend/incidents/ -repos/vllm-omni/ci/models/hunyuan-image3/ +repos/acme/dev/frontend/incidents/ # frontend 是代码 owner，被工作主题 dev 包住 +repos/vllm-omni/ci/models/hunyuan-image3/ # models/ 被工作主题 ci 包住 # 正确 -repos/acme/dev/ -repos/acme/components/frontend/ -repos/vllm-omni/ci/ +repos/acme/components/frontend/incidents/ # owner 直属仓库，incidents/ 保持不变 +repos/acme/dev/ # 前后端共用的开发流程留在工作主题 repos/vllm-omni/models/hunyuan-image3/ +repos/vllm-omni/ci/ # CI 形状的部分留在 ci/，与 model 页互相链接 ``` +**两条里只有第二条有检查器。** `check_knowledge_tree.py` 要求 `components/` 和 +`models/` 的父目录必须是仓库根，违反就报错；但没有任何检查器认得 `frontend` 是一个 +owner，第一条只能靠人。 + +理由对两条相同：工作主题记录“当时在做什么”，随任务过期；代码 owner 跟着源码目录走。 +把 owner 挂到工作主题下，同一条事实会按“谁在什么任务里发现它”分裂成多份，而知道 +owner 却不知道当时任务的读者反而找不到它。 + +`incidents/` 挂在 component/model owner 下是允许的 —— 树里就有 +`models/hunyuan-image3/incidents/`。源码 owner 下唯一被禁止的中转层是 `guides/`。 + ## 最小目录图 ```text -CLAUDE.md -README.md -CONTRIBUTING.md # 短入口 -contributing/ # 本仓库的知识树维护规范 - _index.md +AGENTS.md # agent 开工入口 +CLAUDE.md # 路由与安全短入口 + # 贡献规范不在这棵树里，见 doc/knowledge/ general/ # 换仓库仍然成立 <主题>/ @@ -179,5 +167,5 @@ rg "SSH timeout|shape mismatch" general repos -g "*.md" 2. 父级是否只有路由链接和一句触发条件？否则继续下沉，删除复制正文。 3. 同一内容是否在工作主题和源码 owner 各有一份？是则只保留最近 owner 的正文。 4. `components/`、`models/` 是否直属仓库、没有被工作主题包住？运行 - `python tools/check_knowledge_tree.py`；检查器只验证确定的目录不变量和完全相同的 + `python knowledge/tools/check_knowledge_tree.py`；检查器只验证确定的目录不变量和完全相同的 整页副本，不代替 owner 语义判断。 diff --git a/knowledge/contributing/page-rules.md b/doc/knowledge/contributing/page-rules.md similarity index 98% rename from knowledge/contributing/page-rules.md rename to doc/knowledge/contributing/page-rules.md index 94c06adf2..c86c65257 100644 --- a/knowledge/contributing/page-rules.md +++ b/doc/knowledge/contributing/page-rules.md @@ -145,13 +145,13 @@ 3. 在上一层 `_index.md` 增加入口；新增模型 owner 时由 `models/` 目录自动发现， 不修改父级 `models/_index.md`。 4. 添加真实内容，不提交空目录。 -5. 运行 `python tools/check_knowledge_tree.py`。 +5. 运行 `python knowledge/tools/check_knowledge_tree.py`。 ### 新增普通页面 1. 在最近 owner 目录新建 Markdown。 2. 在当前目录 `_index.md` 增加“遇到什么 → 查看哪里”。 3. 只有检查器提醒文件或目录达到阈值时，才按需读 [何时拆分](scaling.md) 并在同一修改中整理。 -4. 运行 `python tools/check_knowledge_tree.py`。 +4. 运行 `python knowledge/tools/check_knowledge_tree.py`。 目录名使用小写英文、数字和短横线，例如 `remote-debug`。同一层不能重名。`_index`、`local`、`components`、`models`、`incidents` 和 `guides` 是框架保留角色，只能按本文定义使用，不能拿来命名自定义主题。添加新目录不需要修改检查脚本的固定名单；上一层 `_index.md` 能找到它即可。 diff --git a/doc/knowledge/contributing/scaling.md b/doc/knowledge/contributing/scaling.md new file mode 100644 index 000000000..27fc3dd98 --- /dev/null +++ b/doc/knowledge/contributing/scaling.md @@ -0,0 +1,75 @@ +# 何时拆分 + +数字只用来阻止文件和目录无限膨胀，不表示到线后由工具机械切割。拆分必须按独立用户流程、代码 owner、输入输出或验证方式进行。 + +## 单个文件太长 + +| 大小 | 要求 | +|---|---| +| `doc/knowledge/CONTRIBUTING.md` 超过 100 个非空行或 8 KiB | 校验器报错。必须把细则继续下沉到 `contributing/`，入口只留路由和交付清单 | +| 达到 300 个非空行或 16 KiB | 校验器给提醒。检查是否已包含两个以上独立主题 | +| 达到 500 个非空行或 32 KiB | 校验器报错，必须按主题拆分。确有不拆理由时，在同目录 `_index.md` 里同时写出**文件名**和字面词 `暂不拆分`（校验器认的就是这两样），再补不拆原因和复核日期 | + +不每 300 行切一刀。一篇 350 行但只有一个完整流程，可以保留；一篇 200 行但已混了五类日常任务，应该提前拆。 + +## 一个目录页面太多 + +- 一个知识目录先直接放最多 7 个普通内容页；`_index.md`、`rules.md`、`architecture.md` 不计。 +- 出现第 8 个普通页，或已有 3 篇明显属于同一主题时，建立 `guides/`、`incidents/` 或有明确人话含义的分类目录。超过 7 页只是**提醒**，由人判断要不要分类。 +- 分类目录（`guides`、`incidents`、`history`、`references`、`results`、`rfcs` 及其子目录）不受这条 7 页提醒约束；但超过 20 篇当前有效页面时校验器**报错**，必须按稳定主题继续分类。 +- 分类最多再向下增加一层，避免重新形成很深的目录。 +- 错题先按问题主题分，不先按年份分。只有错题超过 50 篇时才增加年度历史入口。 + +## 从一个 `dev` 发展成前端和后端 + +项目早期可以只有： + +```text +repos/acme/ + _index.md + architecture.md + dev/ + _index.md +``` + +前端和后端在下列四项中至少有两项各自独立时，再拆成 component： + +- 源码目录； +- 测试命令； +- 部署方式； +- 运行流程或输入输出。 + +拆分后： + +```text +repos/acme/ + dev/ + _index.md # 前后端共用的开发流程 + components/ + _index.md + frontend/ + _index.md + architecture.md + backend/ + _index.md + architecture.md +``` + +不拆成 `dev/frontend/backend`。`dev` 是工作主题，frontend/backend 是代码 owner。仓库根部 `architecture.md` 继续保留系统总览，细节分别下沉。 + +## 模型或代码模块长大 + +- checkpoint 和尺寸增加：先补别名和差异，不新建目录。 +- 模型出现完全独立的源码、配置语义或运行流程：建并列模型目录。 +- 代码模块出现至少两个独立源码区域，而且各自有架构和测试方式：拆成并列 component。 +- `architecture.md` 很长但仍是一块代码：拆同目录专题页，不虚构新 component。 + +## 拆分必须同步完成 + +1. 新页面按主题分，不机械按行数分。 +2. 每个新页面都从最近 `_index.md` 可达。 +3. 创建或删除子目录时，更新上一层 `_index.md`。 +4. 修复全仓指向旧路径的链接。 +5. 旧文件不再继续接收新内容，重复正文已删除。 +6. `CLAUDE.md` 仍然只是短入口，不因底层页面拆分而膨胀。 +7. 运行 [同步与校验](validation.md) 中的检查。 diff --git a/doc/knowledge/contributing/validation.md b/doc/knowledge/contributing/validation.md new file mode 100644 index 000000000..f2878adc4 --- /dev/null +++ b/doc/knowledge/contributing/validation.md @@ -0,0 +1,115 @@ +# 同步与校验 + +## 修改 Markdown 时必须同步什么 + +新增、移动、重命名、拆分或删除 Markdown 时，在同一修改中： + +1. 更新当前目录 `_index.md`。 +2. 创建或删除子目录时，更新上一层 `_index.md`。 +3. 新增、删除或重命名 `general/` 一级主题时，同步 `general/_index.md` 和 + `knowledge/CLAUDE.md` 的知识地图（这棵树里已经没有 `README.md`）。 +4. 修复所有相对链接和 anchor。 +5. 移动错题时保持“编号”不变，只更新归属、路径和索引。 +6. 删除被新页面替代的重复正文，不留兼容副本。 +7. 确认本次改动没有带入机器信息、凭据、用户绝对路径或临时产物。 +8. 改到 `repos/vllm-omni/` 时，再对照下面的[发版审计](#vllm-omni-页面还有一道发版审计)： + owner 入口页、SHA pin 和 `sources:` 由机器审计对账，不在两个校验器的范围里。 + +## 运行检查 + +```powershell +python knowledge/tools/check_knowledge_tree.py +python knowledge/tools/check_wiki_lint.py +``` + +两个脚本都区分**错误**和**提醒**：只有错误让退出码非 0，提醒（接近拆分线、孤页、 +超过 365 天未更新、`confidence: low`）需要人判断，不是必须清零的门。当前这棵树本身 +就带着 2 条接近拆分线的提醒。 + +如果修改准备提交，还要查当前完整 diff 和未跟踪文件，不只看 `HEAD`。以真实 target base 为准，不硬编码 remote 或默认分支。 + +## 检查脚本负责什么 + +`tools/check_knowledge_tree.py` 负责容易明确判断的事： + +- 正式知识目录有 `_index.md`； +- 当前有效页面在最近索引中恰好有一个真实 Markdown 链接； +- 子目录的 `_index.md` 在上一层索引中恰好有一个真实 Markdown 链接 + （`models/_index.md` 声明 `` 时豁免）； +- 相对链接指向存在的文件，且不是绝对路径； +- 文件和目录大小没超过 [拆分阈值](scaling.md)； +- 错题文件名、字段、状态、编号和索引完整； +- `components/`、`models/` 直属仓库，没有被工作主题或另一个 owner 包住； +- 源码 owner 下没有 `guides/` 中转层； +- 没有两个 owner 各存一份**逐字节相同**的整页正文（≥200 字符时比较）； +- `doc/knowledge/CONTRIBUTING.md` 仍在 100 个非空行 / 8 KiB 以内； +- `local/` 没有被 Git 跟踪； +- 正式页面没有真实 IPv4（`127.` 除外）、`C:\Users\…`、远端用户 home、私钥块， + 也没有 `StrictHostKeyChecking=no`、全局 `safe.directory *`、`--gpus all`、`pkill`、 + `rm -rf`、`find … -exec rm`。 + +`tools/check_wiki_lint.py` 额外负责： + +- 沉淀层 frontmatter、类型和标签符合 `SCHEMA.md`（`repos/jianghan-roleplay-data-pipeline/` + 整棵子树豁免）； +- adapter `manifest.yaml` 的 `knowledge:` 只使用允许字段（`source`、`repo_subdir`、 + `briefing_docs`、`briefing_docs_extra`、`performance_briefing_docs`、 + `review_checklist`），briefing 和 checklist 不能指向 `incidents/history/results` + 原始证据层，`review_checklist` 指向的页面必须真实存在； +- 提醒级：没有任何入链的孤页、超过 365 天没更新的页面、`confidence: low` 或 + `contested` 的页面。 + +## vLLM-Omni 页面还有一道发版审计 + +`repos/vllm-omni/` 这一片除了两个校验器，还被 `tools/audit_vllm_omni_release.py` +按 `adapters/vllm_omni/release_baseline.yaml` 对账。它不是 lint：改错了本地两个脚本 +照样 0 错误，失败发生在 CI 的 `vLLM-Omni release drift audit` 里。 + +- **owner 入口页（`owner_documents`）**：baseline 为每个上游结构 owner 声明一个知识 + 入口，例如 `scheduler → knowledge/repos/vllm-omni/components/scheduler/rules.md`。 + 重命名、移动或删除这些页面时必须在同一改动里同步 baseline。 +- **SHA pin（`pin_documents`）**：8 个文件的正文里带 `main @ <短 SHA>` 或 + `v0.26.0rc1 @ <短 SHA>` 形式的标记，必须与 baseline 的 `audited_sha` 一致 —— + `doc/architecture/KNOWLEDGE.md`、`models/catalog.md`，以及 configuration、diffusion、 + distributed、model-executor、scheduler、serving 六个 component 的 `_index.md`。 + 人工不要手改这些 pin，它们随发版由 `imupdate` 更新。 +- **`sources:` 溯源**：frontmatter 里以 `vllm_omni/`、`tests/`、`benchmarks/`、 + `docs/`、`.buildkite/` 开头的 `sources:` 条目会与上游文件列表对账；上游删除或重命名 + 该路径后报 `stale_knowledge_source`（重命名时报告会给出新路径）。`incidents/`、 + `history/`、`results/`、`_archive` 不参与这项检查。 + +触发方式：每周一对 upstream `main` 跑 `report-only`；**任何**改到 +`knowledge/repos/vllm-omni/**`、baseline、adapter manifest、`imupdate` skill 或审计脚本 +的 PR，都会用 `previous_audited_sha → audited_sha` 跑 `enforce`。命令、模式和更新一个 +release 的步骤见[发版漂移审计](../../contributing/release-maintenance.md)。 + +## 检查脚本不会替人决定什么 + +- 两篇文章是不是重复； +- 错题最终属于哪个 owner； +- 大文件应该按哪些主题拆； +- 仓库专属页是否复制了太多通用内容； +- 规则是否真的会改变下次行为。 + +工具不会生成目录、自动移动页面、静默决定 owner，或只因行数到线就机械切文件。 + +## 当前写入规则 + +- 新的跨仓库经验只写 `general/`。 +- 新的仓库、代码模块和模型知识只写 `repos/`。 +- 稳定教训先写最近 owner 的规则，复杂事故证据才可选写错题。 +- 当前机器事实只写 ignored `local/`。 +- 不建兼容副本或第二套写入路径；历史位置通过 Git history 查询。 + +## 完成标准 + +- 不熟悉框架的人只看根入口和 `_index.md` 就能找到需要的一篇规范。 +- 日常落盘不再需要读整本贡献手册。 +- `doc/knowledge/CONTRIBUTING.md` 不超过 100 个非空行或 8 KiB（校验器跨树检查这一条）。 +- 工作主题和代码模块并列，不形成 `dev/frontend/backend` 这类套娃。 +- 复盘已经更新最近 owner 的可执行规则，确实需要的错题只有一份正文。 +- 超过阈值的文件和目录已按主题整理，或有明确的不拆原因和复核日期。 +- 每个当前有效页面可从最近索引找到，所有链接有效。 +- 第三方只需建目录、写 Markdown、更新上层索引并运行检查，不需要额外配置系统。 +- `local/` 没有被 Git 跟踪，`knowledge/CLAUDE.md` 和 `doc/knowledge/CONTRIBUTING.md` + 仍然是短入口。 diff --git a/doc/knowledge/maintainer-walkthrough.md b/doc/knowledge/maintainer-walkthrough.md new file mode 100644 index 000000000..0173754d1 --- /dev/null +++ b/doc/knowledge/maintainer-walkthrough.md @@ -0,0 +1,210 @@ +# vLLM-Omni 知识库人工维护手册 + +这份手册面向 vLLM-Omni 的模块 owner 和模型 owner。目标不是让每个人研究知识库 +框架，而是把维护者已经掌握的经验，稳定地交给人和 agent 复用。 + +社区维护者直接向 InferMatrixCopilot 提交规则即可，不需要访问私有知识源。同步流程 +必须保留目标仓库已有的模块和模型规则，不能用上游知识树整棵覆盖。 + +> **这份手册是"照着做一遍"，不是规范。** 完整的落盘位置判断、页面类型、copilot +> 消费模型和两道门禁看 +> [`writing.md`](writing.md)； +> 有约束力的规则以 [`contributing/`](contributing/_index.md) 为准； +> 页面骨架直接复制 +> [`templates/`](templates/README.md)。 + +## 只想马上加一条规则 + +不要从头写。只打开符合当前情况的一个样本： + +- [已有 component：增加规则](samples/add-component-rule.zh-CN.md) +- [已有 model：增加规则](samples/add-model-rule.zh-CN.md) +- [新模型还没有目录：创建完整 owner](samples/add-new-model-owner.zh-CN.md) + +替换尖括号里的内容，运行两个校验脚本，然后提交 PR。下面的章节用于解释为什么这样 +写，不是每次维护的必读前置。 + +## 最常见的维护方式 + +已有 owner 目录时，只需要： + +1. 打开该目录的 `rules.md`。 +2. 复制同文件最后一条规则，修改 ID、正文和来源；不要从空白开始写。 +3. 更新页面顶部的 `updated` 和 `sources`。 +4. 运行两个校验脚本。 +5. 提交 PR，由对应 owner 审核。 + +只有新增模块、模型或页面时，才需要修改 `_index.md`。 + +## 第一步：找到正确 owner + +| 经验属于什么 | 放在哪里 | +|---|---| +| 多个模型共用的源码模块 | `knowledge/repos/vllm-omni/components/<模块>/` | +| 单个模型的实现、配置或 checkpoint | `knowledge/repos/vllm-omni/models/<模型>/` | +| 整个 vLLM-Omni 都必须遵守的门禁 | `knowledge/repos/vllm-omni/rules.md` | +| 与任何仓库都无关的通用方法 | `knowledge/general/<主题>/` | + +判断标准是“哪块代码或哪个维护者对结论负责”，不是“在哪个 PR 里发现”。 + +例如： + +- deploy YAML 最终怎样进入 stage config → Configuration component。 +- HunyuanImage3 的图片数量和 prompt token → HunyuanImage3 model。 +- 所有模型共享的 scheduler 等待逻辑 → Scheduler component。 + +如果一个结论同时影响多个入口，只在最近 owner 留一份正文，其他地方只加链接。 + +## 第二步：先填一张规则卡片 + +维护者不需要先写 Markdown。先把下面六项填清楚： + +```text +规则卡片 + +- owner（模块或模型）： +- 触发：什么改动、现象或任务需要使用这条规则？ +- 必须：实现或审核时必须做什么？ +- 禁止：过去最容易犯的错误是什么？ +- 验收：什么测试、输出或代码路径能证明它满足要求？ +- 来源：相关源码路径、设计文档或 PR 链接。 +``` + +一张卡片只表达一个行为不变量。不能把一个 PR 的所有经验塞进一条规则。 + +## 第三步：写入 `rules.md` + +现有 owner 已有 `rules.md` 时，直接增加： + +```markdown +### <稳定规则 ID> — <一句话标题> + +- 触发：<适用场景> +- 必须：<必须执行的动作> +- 禁止：<不允许的做法> +- 验收：<可以检查的完成标准> +``` + +规则 ID 使用该 owner 已有前缀并继续编号，例如把样本中的 `CONF-NEXT`、`HY3-NEXT` +替换为当前未使用的下一个 ID。 +不要重排或复用旧 ID。 + +规则不保存完整 diff、review 对话、调查时间线和大段日志。GitHub PR 本身负责保存 +原始证据；知识库只保存下次可以执行的规则。 + +每条新规则必须能追溯： + +- 页面顶部 `sources` 追加 PR、源码路径或设计文档。 +- 规则最后一行追加对应来源标记，例如 `^[PR #1234]`。 +- 没有任何可核对来源时，不编造来源，也不提交规则；先请提出者补充证据。 + +### Component 示例 + +下面是 Configuration owner 的规则写法： + +```markdown +### CONF-3a — 争议以展开后的最终配置为准 + +- 触发：CLI、deploy YAML 和 stage override 对同一字段给出不同值。 +- 必须：检查合并完成后的逐 stage 配置，并跟到第一位 consumer。 +- 禁止：只看某一层 YAML 就断言最终生效值。 +- 验收：最终配置对象能读回目标值，并与第一位 consumer 收到的值一致。 +``` + +它没有保存某个 PR 的故事，而是保存 Configuration owner 下次仍会执行的判断。 + +### Model 示例 + +下面是 HunyuanImage3 owner 的规则写法： + +```markdown +### HY3-2c — 在公开入口尽早拒绝非法图片数量 + +- 触发：修改图片数量、图片输入或公开生成入口。 +- 必须：在文件读取、下载、解码、resize 和 VAE 之前校验数量。 +- 禁止：等昂贵操作开始后才报参数错误。 +- 验收：覆盖合法边界和超限输入，并证明超限请求没有进入昂贵路径。 +``` + +这条规则只属于 HunyuanImage3，不应提升到仓库根或共享 Diffusion component。 + +## 新增模块或模型 + +如果目录尚不存在： + +1. 优先复制[完整新 model owner 样本](samples/add-new-model-owner.zh-CN.md)； + 需要其他页面类型时再看 + [`templates/`](templates/README.md)。 +2. 创建 `_index.md`，写清源码路径、职责边界、测试入口和依赖关系。 +3. 有稳定架构信息时创建 `architecture.md`。 +4. 第一条真实规则出现时才创建 `rules.md`，不要提交空页面。 +5. 新增 component 时在 `components/_index.md` 增加入口；新增 model 时不用维护 + 父级长表，目录会由 `models/` 文件系统清单自动发现。新增别名或 registry key + 时才更新 `models/catalog.md`。 + +模块与模型的区别： + +- Component：被多个模型或上层功能共享，有稳定源码和输入输出边界。 +- Model：只描述某个模型自己的适配、配置、checkpoint 和行为。 + +## 人工与 agent 协作 + +推荐由 owner 提供规则卡片，agent 完成机械工作，owner 做最终语义审核。 + +不会或不想直接修改 Markdown 时，可以 +[提交中文规则建议](https://github.com/JiusiServe/InferMatrixCopilot/issues/new?template=knowledge-rule.yml)。 +表单内容就是本手册的规则卡片。知识库维护者或 agent 把它整理成 PR，再请原 owner +确认语义。 + +可以直接对 Codex 说： + +```text +Use InferMatrixCopilot to update the knowledge base. + +请根据下面的规则卡片修改最近 owner 的 rules.md： +<粘贴规则卡片> + +要求： +1. 不保存原始 PR、评论、diff 或调查过程。 +2. 不扩大到其他 component/model。 +3. 使用该 owner 现有规则 ID 前缀。 +4. 新增页面时更新最近 _index.md；只改已有 rules.md 时不要乱改索引。 +5. 运行 check_knowledge_tree.py 和 check_wiki_lint.py。 +6. 把最终 diff 交给我审核，不要自动发布评论。 +``` + +agent 可以协助查路径、补 frontmatter、更新索引和运行检查，但下面三件事必须由 +owner 最终确认： + +- owner 是否正确； +- 规则是否真的长期成立； +- 验收是否覆盖真实生产入口。 + +推荐的团队分工是： + +1. 模块/模型 owner 在重要 PR 合入或 review 结束后填写规则卡片。 +2. 知识库维护者或 agent 负责查重、生成 Markdown、更新元数据并运行校验。 +3. 原 owner 只审核自己目录中的规则语义和验收条件。 +4. 合入后，后续 review 自动从该 owner 的 `rules.md` 获取规则。 + +## PR 前检查 + +- [ ] 规则放在最近的 component/model owner，没有放进宽泛的仓库根。 +- [ ] 每条规则都有触发、必须、禁止、验收和稳定 ID。 +- [ ] 删除 PR 编号后，规则仍能独立读懂。 +- [ ] 没有保存完整 diff、review 对话、运行结果或临时路径。 +- [ ] 修改已有 `rules.md` 时更新了 `updated` 和 `sources`。 +- [ ] 新增页面或目录时更新了最近 `_index.md`。 +- [ ] 两个校验脚本均为 0 错误（提醒需要人判断，不要求清零）。 +- [ ] 没有重命名、移动或删除 owner 入口页，也没有手改正文里的 `main @ ` pin —— + 这两样由发版审计对账，改动必须同步 `adapters/vllm_omni/release_baseline.yaml`。 + 见[同步与校验 §发版审计](contributing/validation.md#vllm-omni-页面还有一道发版审计)。 + +```powershell +python knowledge/tools/check_knowledge_tree.py +python knowledge/tools/check_wiki_lint.py +``` + +更完整的目录、frontmatter 和页面规范见 +[`CONTRIBUTING.md`](CONTRIBUTING.md)。完整写作指南见 +[`writing.md`](writing.md)。 diff --git a/docs/samples/add-component-rule.zh-CN.md b/doc/knowledge/samples/add-component-rule.zh-CN.md similarity index 100% rename from docs/samples/add-component-rule.zh-CN.md rename to doc/knowledge/samples/add-component-rule.zh-CN.md diff --git a/docs/samples/add-model-rule.zh-CN.md b/doc/knowledge/samples/add-model-rule.zh-CN.md similarity index 100% rename from docs/samples/add-model-rule.zh-CN.md rename to doc/knowledge/samples/add-model-rule.zh-CN.md diff --git a/docs/samples/add-new-model-owner.zh-CN.md b/doc/knowledge/samples/add-new-model-owner.zh-CN.md similarity index 100% rename from docs/samples/add-new-model-owner.zh-CN.md rename to doc/knowledge/samples/add-new-model-owner.zh-CN.md diff --git a/doc/knowledge/templates/README.md b/doc/knowledge/templates/README.md new file mode 100644 index 000000000..55144148e --- /dev/null +++ b/doc/knowledge/templates/README.md @@ -0,0 +1,61 @@ +# 知识页模板 + +仓库根目录 `knowledge/` 那棵精选 wiki 的**可直接复制骨架**。 +工作流、决策树和门禁规则在 +[`../writing.md`](../writing.md)；**这些文件是骨架的唯一 +副本** —— 那份文档链接到这里而不是再抄一遍，因为上一版正是两份副本，最终漂移成了 +两套。 + +这些文件住在 `knowledge/` **之外**，所以不过 wiki 门禁（否则它们的占位符会直接失败）。 +把其中一个复制到正确的 owner 目录，填掉 `<...>` 占位符，删掉开头的 +`` 注释，在最近的 `_index.md` 里注册这一页，然后校验。 + +## 什么情况用哪个文件 + +| 复制这个 | 用来创建 | 放到 | +|---|---|---| +| [`_index.md`](_index.md) | 一个目录的路由表 | 任何新建的主题/组件/模型目录 | +| [`repo-index.md`](repo-index.md) | 一个**仓库**入口索引 | `knowledge/repos//_index.md` | +| [`rules.md`](rules.md) | 一个 always-on 门禁页 | 最近的 owner 目录 | +| [`architecture-component.md`](architecture-component.md) | 共享模块的架构页 | `repos//components//architecture.md` | +| [`architecture-model.md`](architecture-model.md) | 模型架构页 | `repos//models//architecture.md` | +| [`guide.md`](guide.md) | 方法 / how-to 页 | 组件或模型 owner 根目录；工作主题集合可用 `guides/` | +| [`incident.md`](incident.md) | 一次复盘记录 | `/incidents/YYYY-MM-DD-short-name.md` —— **仅**在通过三项准入门禁后 | + +## 每个模板对照的那篇实页 + +模板编码的是 vLLM-Omni 树**已经在遵守**的约定。占位符含义不清时，去打开实页 —— +它才是 ground truth，这个目录不是： + +| 模板 | 去读这篇已发布的页面 | +|---|---| +| `rules.md` | [`components/configuration/rules.md`](../../../knowledge/repos/vllm-omni/components/configuration/rules.md) · [`models/hunyuan-image3/rules.md`](../../../knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md) | +| `_index.md` | [`components/_index.md`](../../../knowledge/repos/vllm-omni/components/_index.md) | +| `repo-index.md` | [`repos/vllm-omni/_index.md`](../../../knowledge/repos/vllm-omni/_index.md) | +| `architecture-component.md` | [`components/diffusion/architecture.md`](../../../knowledge/repos/vllm-omni/components/diffusion/architecture.md) | +| `architecture-model.md` | [`models/hunyuan-image3/architecture.md`](../../../knowledge/repos/vllm-omni/models/hunyuan-image3/architecture.md) | +| `incident.md` | [`ci/incidents/`](../../../knowledge/repos/vllm-omni/ci/incidents/_index.md) | + +## 已发布的树强制的三条约定 + +1. **`rules.md` 以 Direct 代码快速入口开头。** vLLM-Omni 现有的 12 个 owner 规则页 + （5 个 component + 7 个 model）全都如此：意图 → 规则组 → *第一批 live 源码*， + 写成 producer→consumer 链。没有它，读者必须消化整页才能找到真正适用的那 3 条规则。 +2. **规则 ID 带 owner 前缀，且永不重新编号。** 一个 owner 一个前缀：vLLM-Omni 在用 + `CONF`、`SERV`、`DIFF`、`EXEC`、`SCHED`、`VOMNI-CFG`、`HY3`、`COSMOS`、`FLUX2`、 + `KREA`、`MCPMO`、`MING`、`Q3TTS`，另有 afd-plugin 的 `AFD` 和通用 `REV`。新 owner + 起新前缀，不复用别人的。评审和 incident 会引用这些 ID，所以只能 retire 一个 ID， + 不能改派。 +3. **`_index.md` 的 frontmatter 是必填，不是可选。** 全树 123 个索引页里，109 个沉淀层 + 索引全部带着它，`check_wiki_lint.py` 在沉淀层强制它。只有证据层 + （`incidents/`、`history/`、`results/`）的 14 个索引不带。 + +## 复制之后 + +```bash +python knowledge/tools/check_knowledge_tree.py # 结构 / 索引 / 链接 / incident +python knowledge/tools/check_wiki_lint.py # frontmatter + 标签分类法 +``` + +两个都必须打印 0 错误（提醒需要人判断，不是必须清零）。 +**没有被任何 `_index.md` 链接的页面会在第一个检查里失败。** diff --git a/doc/knowledge/templates/_index.md b/doc/knowledge/templates/_index.md new file mode 100644 index 000000000..684c10420 --- /dev/null +++ b/doc/knowledge/templates/_index.md @@ -0,0 +1,40 @@ + +--- +title: "" +created: +updated: +type: index +tags: [] +sources: [] +--- + +# + + + +## 什么时候查这里 + +- + +## 不放什么 + +- → `` + +## 目录内容 + +| 遇到什么 | 查看哪里 | 说明 | +|---|---|---| +| | [](.md) | | +| | [](/_index.md) | | + + diff --git a/doc/knowledge/templates/architecture-component.md b/doc/knowledge/templates/architecture-component.md new file mode 100644 index 000000000..3b2454b65 --- /dev/null +++ b/doc/knowledge/templates/architecture-component.md @@ -0,0 +1,42 @@ + +--- +title: " 共享架构" +created: +updated: +type: architecture +tags: [, components, ] +sources: [] +--- + +# 共享架构 + +## 负责什么 + +`` + +## 不负责什么 + +- 由 负责。 +- 由 负责。 + +## 当前子模块布局（ @ 复核） + + + +| 子模块 | 职责 | 主要入口 | +|---|---|---| +| `` | | `` / `` | + +## 数据怎样流动 + + → → . + +## 怎样验证 + +- diff --git a/doc/knowledge/templates/architecture-model.md b/doc/knowledge/templates/architecture-model.md new file mode 100644 index 000000000..23c672afc --- /dev/null +++ b/doc/knowledge/templates/architecture-model.md @@ -0,0 +1,45 @@ + +--- +title: " 架构" +created: +updated: +type: architecture +tags: [, models, ] +sources: [] +--- + +# 架构 + +## 模型专有部分与共享模块的边界 + + + +| 这一部分 | 归属 | 说明 | +|---|---|---| +| | **本模型** `` | | +| | 共享 [`components/`](../../components//architecture.md) | | + +## 配置、checkpoint 和兼容范围 + +- 注册键：``（``） +- checkpoint 兼容： +- 已知不兼容： + +## 从输入到输出的主要流程 + + → → → → . + + +## 怎样验证功能、精度和性能 + +- 功能： +- 精度： +- 性能： + +## 当前布局（ @ 复核） + + diff --git a/doc/knowledge/templates/guide.md b/doc/knowledge/templates/guide.md new file mode 100644 index 000000000..fc279b711 --- /dev/null +++ b/doc/knowledge/templates/guide.md @@ -0,0 +1,37 @@ + +--- +title: "" +created: +updated: +type: guide +tags: [] +sources: [, ] +--- + +# + +## 什么时候用 + +- +- 不适用： → `` + +## 步骤 / 方法 + +1. +2. + +## 怎样验证 + +- + +## 已知失败模式 + +| 现象 | 原因 | 处理 | +|---|---|---| +| | | | + + diff --git a/doc/knowledge/templates/incident.md b/doc/knowledge/templates/incident.md new file mode 100644 index 000000000..7228fcb53 --- /dev/null +++ b/doc/knowledge/templates/incident.md @@ -0,0 +1,33 @@ + +# — <人能读懂的现象标题> + +- 编号：`inc---` +- 归属：`/ | general/>` +- 状态：<待归类 | 处理中 | 已验证 | 已提炼 | 仅历史> +- 搜索词： +- 影响范围：/> + +**症状**： +**根因**： +**解法**： +**对未来的提醒**： + +## 已提炼的规则 + +- `-` in [/rules.md](../rules.md) — + + diff --git a/doc/knowledge/templates/repo-index.md b/doc/knowledge/templates/repo-index.md new file mode 100644 index 000000000..01cdeb28f --- /dev/null +++ b/doc/knowledge/templates/repo-index.md @@ -0,0 +1,53 @@ + +--- +title: " 入口" +created: +updated: +type: index +tags: [] +sources: [] +--- + +# + +上游：`/`。开始任何任务先遵守[仓库硬规则](rules.md)。 + + + +## Review 最短路径 + + + +PR title/body 先选择 owner；changed files 只校验实际范围。Direct 已返回精确 +`quick_map` 时，不再读取本页、组件总表或模型总表。 + +| PR 声明目标 | 直接 owner | +|---|---| +| | [ rules](components//rules.md) | +| 明确模型名或 registry key | 直接查看 [models 目录](models/_index.md) | + +owner 仍不明确时才看 [components 职责表](components/_index.md)。 + +## 工作主题 + +| 任务 | 入口 | +|---|---| +| PR 审查专项 | [review](review/_index.md) | +| CI 和测试配置 | [ci](ci/_index.md) | +| | [](/_index.md) | + +## 不放什么 + +- → git-ignored `local/` +- → `eval/` diff --git a/doc/knowledge/templates/rules.md b/doc/knowledge/templates/rules.md new file mode 100644 index 000000000..d4b219f64 --- /dev/null +++ b/doc/knowledge/templates/rules.md @@ -0,0 +1,57 @@ + +--- +title: " 硬门禁" +created: +updated: +type: rule +tags: [] +sources: [, , ] +--- + +# 硬门禁 + +只在<明确的触发条件：修改 X、审查 Y、调试 Z>时使用本页。<可选：第一次读先看 +[<入门 guide>](.md)；需要执行时再看 [<操作 guide>](.md)。> + +只有<加粗项目或表格第一列中的 `-数字字母`>是可审计规则 ID；章节标题只是 +分组，不计入 ID，解释性文字和链接也不计。 + +## Direct 代码快速入口 + + + +- **-0a — 先用意图选行，再用 diff 验证范围。** 只合并下表命中的规则和源码 + 入口，不在动手前手工枚举整页。命中多行时取并集，不能只选看起来最接近的一行。 + PR 描述只负责导航，冲突时以 live diff 和 consumer 为准。 +- **-0b — 命中源码后停止文档导航。** 打开第一批源码后沿 producer→consumer + 审查；只有调用链跨 owner 或具体未知量阻塞时，才多打开一个 owner 或 guide。 + +| 正在做什么 | 精确规则组 | 第一批 live 源码 | +|---|---|---| +| | ``：`-1a`–`1c` | `::{,}` → `::` | +| | ``：`-2a` | `::` | + +| 审查组 | 什么时候触发 | 规则 ID | +|---|---|---| +| `core` | 每次<该 owner 的>审查 | `-1a`, `-1b` | +| `` | | `-2a` | + +## 规则 + +### -1a + +- 触发： +- 必须： +- 禁止： +- 验收： + + diff --git a/doc/knowledge/tree-map.md b/doc/knowledge/tree-map.md new file mode 100644 index 000000000..282135d28 --- /dev/null +++ b/doc/knowledge/tree-map.md @@ -0,0 +1,58 @@ +# Knowledge + +这里存放可复用规则，不是需要整棵读取的文档站。 + +## 默认入口 + +| 任务 | 入口 | 什么时候停止 | +|---|---|---| +| PR review | Direct 根据 PR title/body 返回精确 owner/model `quick_map` | 路由返回后停止知识导航 | +| 通用方法 | [general](../../knowledge/general/_index.md) | 命中一个当前任务 guide 后停止 | +| 仓库规则和代码地图 | [repos](../../knowledge/repos/_index.md) | 命中主要 owner 后停止横向展开 | +| 写入或整理知识 | [CONTRIBUTING](CONTRIBUTING.md) | 选择最近 owner 后再写 | + +vLLM-Omni 的直接入口是 +[仓库地图](../../knowledge/repos/vllm-omni/_index.md)。已知 owner 时不要从根目录逐层点击： + +- 配置：[`components/configuration/rules.md`](../../knowledge/repos/vllm-omni/components/configuration/rules.md) +- 在线服务：[`components/serving/rules.md`](../../knowledge/repos/vllm-omni/components/serving/rules.md) +- 模型执行：[`components/model-executor/rules.md`](../../knowledge/repos/vllm-omni/components/model-executor/rules.md) +- Diffusion：[`components/diffusion/rules.md`](../../knowledge/repos/vllm-omni/components/diffusion/rules.md) +- 调度：[`components/scheduler/rules.md`](../../knowledge/repos/vllm-omni/components/scheduler/rules.md) +- 模型专属：直接查看 [`models/`](../../knowledge/repos/vllm-omni/models/_index.md) 下对应目录 + +## 目录分层 + +```text +general/ 跨仓库方法 +repos//rules.md 仓库硬规则 +repos//components// 共享代码 owner +repos//models// 模型 owner +repos/// CI、benchmark、Git 等工作主题 +contributing/ 知识维护规范 +``` + +下面这些不是默认知识入口： + +- `incidents/`、`history/`、`results/`：历史证据，只在规则指向或问题高度相似时查； +- `skills/`、`.claude/`：运行扩展； +- `tools/`：校验和维护脚本； +- 启动脚本：本地辅助材料。 + +## 找不到时 + +先查目录名，再做一次有界搜索： + +```powershell +Get-ChildItem repos/vllm-omni/models -Directory +rg -n "关键词" repos/vllm-omni/components repos/vllm-omni/models -g "*.md" +``` + +不要默认递归读取所有 `_index.md`、事故记录或同级 owner。 + +## 校验 + +```powershell +python knowledge/tools/check_knowledge_tree.py +python knowledge/tools/check_wiki_lint.py +``` diff --git a/doc/knowledge/writing.md b/doc/knowledge/writing.md new file mode 100644 index 000000000..faa6b2e1d --- /dev/null +++ b/doc/knowledge/writing.md @@ -0,0 +1,264 @@ +# 怎么写和更新知识库（`knowledge/`） + +要在仓库根目录 `knowledge/` 这棵精选 Markdown wiki 里**新增或修改一页**，**从这里 +开始**。本页是操作指南：一条事实该落在哪、由哪种页型承载、copilot 会怎么消费它、 +以及怎么过两道门禁。 + +三个surface，分清你在读哪一个： + +| Surface | 是什么 | 什么时候读 | +|---|---|---| +| **本页** | 工作流 —— 决策树、页型、消费模型、门禁 | 你正准备动笔写 | +| [`CONTRIBUTING.md`](CONTRIBUTING.md) → [`contributing/`](contributing/_index.md) | **有约束力的规范**，一个动作一篇专题 | 你需要那条硬规则；本页与它冲突时**以它为准** | +| [`templates/`](templates/) | 可复制的页面骨架，一种页型一个文件 | 你已经知道页面放哪，想直接开写 | + +中文的手把手走查（含组件与模型的填写示例）： +[`maintainer-walkthrough.md`](maintainer-walkthrough.md)。 + +`knowledge/` 在本仓库内作为普通受版本控制的文件维护；布局、消费方式与维护规则见 +[KNOWLEDGE.md](../architecture/KNOWLEDGE.md)。 + +> 同一套工作流被蒸馏成一个可检索 skill +> [`skills/knowledge-base-contribution/SKILL.md`](../../skills/knowledge-base-contribution/SKILL.md)， +> 让 copilot 自己的 agent 在复盘时也遵守它。 + +--- + +## 0. TL;DR —— 四步循环 + +1. **选定 owner 目录**（§1）。按**已验证的根因**路由，绝不按症状最先出现的地方。 +2. **写/改一页**正确页型的内容（rules / architecture / guide / incident）—— + 从 [`templates/`](templates/) 复制对应骨架（§4），并读一遍它 + 所对照的那篇 vLLM-Omni 实页（§3）。每条事实**只保留一份正文**，其他入口只链接。 +3. **在同一次修改里注册它**：在**最近的 `_index.md`** 里加一行 + `遇到什么 → 查看哪里`（或一条子目录链接）。未注册的页面过不了门禁。 +4. **校验**（两道门禁都必须 0 错误）： + + ```bash + python knowledge/tools/check_knowledge_tree.py # 结构 / 索引 / 链接 / incident + python knowledge/tools/check_wiki_lint.py # 沉淀层 frontmatter + 标签分类法 + ``` + +交付见 §7。**本页描述的是人工写入路径**；上游发版驱动的结构事实走 `imupdate` +（[发版漂移审计](../contributing/release-maintenance.md)）—— 它只更新机器已证明的 +baseline、catalog、source map、SHA pin 和 manifest，不做 owner 归属判断，也不写规则。 +运行中的 agent 只能提 candidate 由人工晋升。下面的落盘位置判断和页型选择属于人工 +路径；两个校验器三条路径都要过。 + +--- + +## 1. 该放哪？—— owner 决策树 + +### 第一层 —— general 还是仓库专属 + +| 这条经验… | 放到 | +|---|---| +| 在**任何**仓库都成立（review、ci、debug、git、planning、remote、docs、benchmark、agents、environment） | `knowledge/general//` | +| 只对**某一个仓库**成立 | `knowledge/repos//` | +| 是**当前这台机器**的事实（主机、路径、账号、cache、venv、token） | `knowledge/local/` —— 已被 git 忽略，永不入库 | + +### 第二层 —— 仓库切片内部 + +| 这条经验讲的是… | 放到 | +|---|---| +| 仓库级的工作主题（review、ci、git、benchmark、remote、rebase…） | `repos///` | +| 多个模型**共用的源码**（diffusion、scheduler、serving…） | `repos//components//` | +| **某一个模型**自己的实现 / 配置 / checkpoint | `repos//models//` | + +按**已验证**的根因路由。"前端看到 404" 不等于根因在前端 —— 页面沉到**证明了原因** +的那一层，而不是症状冒出来的那一层。 + +### 第三层 —— 选哪种页型 + +| 你手上有… | 页型 | 说明 | +|---|---|---| +| 一条必须改变**下一次**执行的规则（触发 → 必须 → 禁止 → 怎么验收） | 最近 owner 目录里的 `rules.md` | 复盘的默认产物。**always-on**（见 §2），必须写紧 | +| 一段**稳定的**数据流 / 职责 / 边界描述 | `architecture.md` | 不允许只有标题的空壳页 | +| 一套不是硬门禁的较长方法 | owner 根目录下的专题页；工作主题集合可以用 `guides/` | 组件/模型 owner 保持扁平，更深的页面一律按需拉取 | +| 规则装不下的**复杂可复现历史** | `incidents/YYYY-MM-DD-short-name.md` | 可选。只有当复现链/证据本身仍有独立查阅价值时才建 | + +**复盘的经验法则：** "复盘 / 记一条教训" 的默认产物是**规则**，不是 incident。 +只有当证据链复杂到规则本身承载不了时，才追加一个 incident。 + +--- + +## 2. copilot 怎么消费它（所以你才知道该把什么写紧） + +接线在 `src/infermatrix_copilot/engine/agent_runtime/knowledge.py` + +`src/infermatrix_copilot/config.py` + 每个 `adapters//manifest.yaml`。 + +- **always-on briefing** —— 注入**每一次** run，各自有上限： + - general 切片 `settings.knowledge_general_docs`（`general/_index.md`）； + - adapter 的 `briefing_docs` —— vllm-omni 是 `repos/vllm-omni/rules.md` + + `repos/vllm-omni/_index.md`； + - `briefing_docs_extra` —— 所有档位都加载的扩展切片，vllm-omni 目前是 + `repos/vllm-omni/review/guides/maintainer-pattern-routing.md`； + - 强模型额外的 `performance_briefing_docs`（精简版评审模式）； + - `review_checklist` —— 单页仓库评审清单，前 4k 字符注入 Strict reviewer 的 + system prompt，vllm-omni 指向 + `repos/vllm-omni/review/guides/strict-review-checklist.md`。 +- **按需** —— `doc_search` / `doc_read` 工具可递归到达 `general/` 里每一页更深的 + Markdown，**外加当前 adapter 的 `repo_subdir`**（其他仓库的切片一律拒绝；路径穿越 + 被阻断；`doc_read` 每页窗口 24k 字符，用 offset 翻页）。 +- **MCP** —— 同一套按仓库限定的 `doc_search` / `doc_read` 以只读形式暴露在 MCP 上， + 所以宿主模型不启动 run 也能查询 wiki。 + +**对作者的现实含义：** `rules.md` 和 `_index.md` 是**预算** —— 它们每个任务都会加载。 +把它们控制在触发器 + 门禁 + 导航。叙述、长复现、逐步方法推到按需拉取的专题页或 +`incidents/`，那些只有真正需要时才被取用。 + +**agent 写不进这棵树。** 运行中的 agent 经 `skill_update_candidate` 只能提出 +**candidate**；把 candidate 变成生效的 `SKILL.md`，或把一条 profile fact 写进 +`profile.yaml`，都是**人工/策展动作**（`SPEC/memory/skills.md`、`SPEC/profiles/store.md`）。 +这就是"读宽写窄"：事实随便记（RunTrace、debug memory），知识必须过门。 +所以本页的落盘规则约束的是**人**——以及那个替人准备 candidate 的 agent。 + +要注册一个**新仓库**的切片，把它的 adapter manifest 指过去 +（`knowledge.repo_subdir`、`briefing_docs`，可选 `briefing_docs_extra`、 +`performance_briefing_docs`、`review_checklist`）—— 见 §5。 + +--- + +## 3. 动笔之前先读真实的树 + +vLLM-Omni 切片是下面每一条约定的参考实现。模板里的占位符含义不清时，**实页才是 +ground truth**： + +| 要写… | 先读这篇 | +|---|---| +| owner 门禁页 | [`components/configuration/rules.md`](../../knowledge/repos/vllm-omni/components/configuration/rules.md) —— 规则组、审查组、`VOMNI-CFG-*` ID | +| 模型门禁页 | [`models/hunyuan-image3/rules.md`](../../knowledge/repos/vllm-omni/models/hunyuan-image3/rules.md) —— `HY3-*` ID，以及什么才算可审计 ID | +| 路由索引 | [`components/_index.md`](../../knowledge/repos/vllm-omni/components/_index.md) —— 一行一个 owner，以及**什么时候不要**打开 `architecture.md` | +| 仓库入口 | [`repos/vllm-omni/_index.md`](../../knowledge/repos/vllm-omni/_index.md) —— review 最短路径表 | +| 组件架构 | [`components/diffusion/architecture.md`](../../knowledge/repos/vllm-omni/components/diffusion/architecture.md) —— 负责什么 / 不负责什么 / 带 commit pin 的布局 | +| incident | [`ci/incidents/`](../../knowledge/repos/vllm-omni/ci/incidents/_index.md) —— 五个字段，以及规则是怎么从中提炼出来的 | + +那些页面共同遵守、而新手通常会漏掉的三条约定： + +1. **owner 的 `rules.md` 以 Direct 代码快速入口开头。** vLLM-Omni 现有的 12 个 + owner 规则页（5 个 component + 7 个 model）全都这么做：*意图 → 规则组 → 第一批 + live 源码*，写成 producer→consumer 链 + （`stage_config.py::{build_stage_runtime_overrides}` → `omni_config.py::…`）。 + 这一页是 always-on 预算，读者必须能在不消化其余 20 条的情况下够到命中的那 3 条。 +2. **规则 ID 带 owner 前缀，并且永不重新编号。** 一个 owner 一个前缀，新 owner 起 + 新前缀，不复用别人的：vLLM-Omni 在用 `CONF`、`SERV`、`DIFF`、`EXEC`、`SCHED`、 + `VOMNI-CFG`、`HY3`、`COSMOS`、`FLUX2`、`KREA`、`MCPMO`、`MING`、`Q3TTS`，另有 + afd-plugin 的 `AFD` 和通用 `REV`。评审和 incident 会引用这些 ID，所以只能 + **retire**，不能改派。页面上要明确写出哪些文字才是可审计 ID；章节标题只是分组。 + （数量会漂移，需要准确值时现数：ID 定义写成 `## — …` 或 + `- ** — …**`。） +3. **写清楚什么**不**在这里。** 树里每一页好文档都会点名邻居并把读者路由过去。 + 这正是"一条事实只有一处"得以成立的原因 —— 整棵 wiki 依赖的唯一性质。 + +## 4. 模板 + +**骨架在 [`templates/`](templates/)** —— 一种页型一个文件， +其 [README](templates/README.md) 给出 复制 → 填写 → 注册 → 校验 的流程。 +**它们是唯一副本**：本指南刻意不再内联重复一遍，因为上一版就是这么做的，两份副本 +最终漂移成了两套。 + +```bash +cp doc/knowledge/templates/rules.md \ + knowledge/repos//components//rules.md +``` + +只要你 (a) 替换掉占位符、(b) 在最近的 `_index.md` 里注册页面，每个模板都是过门禁的。 +frontmatter 由两条规则决定： + +- **沉淀层** —— `general/` 和 `repos/` 下的 `rule` / `guide` / `architecture` / + `index` 页 —— **必须**带 frontmatter，由 `check_wiki_lint.py` 强制：`title`、 + `created` + `updated`（都是 `YYYY-MM-DD`）、`type`（四者之一）、非空的 `tags` + （取自 [`SCHEMA.md`](SCHEMA.md) 的 `## 标签分类法`）。 + `confidence` 如果出现，必须是 `high|medium|low`；`sources` 不是必填字段。 + **这包括 `_index.md`** —— 全树 123 个索引页里，109 个沉淀层索引全部带着它， + 另外 14 个是 `incidents/`、`history/`、`results/` 的证据层索引，按规定不带。 +- **证据层** —— `incidents/`、`history/`、`results/` —— **不带** frontmatter； + incident 模板改用 `- 编号/…` 这组扁平字段，由 `check_knowledge_tree.py` 检查。 + +目录名用小写 `a-z0-9-`。`_index`、`local`、`components`、`models`、`incidents`、 +`guides` 是**保留角色名**，不要拿来当自定义主题。 + +## 5. 接入一个新仓库切片 + +1. 建出 wiki 骨架，并逐级注册上去： + + ```text + knowledge/repos//_index.md # 用 templates/repo-index.md + knowledge/repos//rules.md # 仅当确实存在仓库级门禁时才建 + # 在 knowledge/repos/_index.md 里为 加一行 + ``` + +2. 把 adapter 指向该切片（`adapters//manifest.yaml`）： + + ```yaml + knowledge: + repo_subdir: repos/ # 它在 knowledge/ 下的切片 + briefing_docs: # always-on：保持极小 + - repos//rules.md + - repos//_index.md + briefing_docs_extra: # 可选，所有档位都加载 + - repos//review/guides/.md + performance_briefing_docs: # 可选，仅强模型 + - repos//review/guides/.md + review_checklist: repos//review/guides/.md # 可选，单页 + ``` + + 这六个键之外的字段会被 `check_wiki_lint.py` 拒绝；briefing 和 checklist 都不能 + 指向 `incidents/`、`history/`、`results/`。 + +3. 如果你是从上游 fork 的 wiki，删掉不属于你的 `repos/` 切片，并相应更新 + `repos/_index.md`。 + +--- + +## 6. 门禁速查 —— 两道门到底卡什么 + +`check_knowledge_tree.py`（结构）和 `check_wiki_lint.py`（frontmatter/schema） +都在评审时运行，**两个都必须 0 错误**（提醒需要人判断，见表末一行）。 + +| 检查项 | 门 | 规则 | +|---|---|---| +| 索引存在 | tree | 任何含 Markdown 的目录都要有 `_index.md` | +| 已注册 | tree | 每个非索引页**以及**每个子目录，都要被最近的 `_index.md` **恰好链接一次** | +| 链接 | tree | 相对链接必须可解析；绝对路径一律拒绝 | +| 体量 | tree | ≥300 非空行或 16 KiB 告警；≥500 行或 32 KiB **必须拆分**（豁免要在同目录 `_index.md` 里同时出现该**文件名**和字面词 `暂不拆分`，再补复查日期） | +| 目录扇出 | tree | 非分组目录超过 7 个普通页告警；分组目录（`guides` `incidents` `history` `references` `results` `rfcs`）超过 20 个硬失败 | +| incident 格式 | tree | `YYYY-MM-DD-short-name.md` + 五个 `- 编号/归属/状态/搜索词/影响范围` 字段 + 合法状态 + 全树唯一的编号 | +| 隐私 | tree | 受版本控制的页面里不得出现真实 IPv4、Windows `C:\Users\…`、远端用户 home 或私钥块 | +| 安全 | tree | 不得出现 `StrictHostKeyChecking=no`、全局 `safe.directory *`、`--gpus all`、`pkill`、`rm -rf`、`find … -exec rm` | +| `local/` | tree | 必须保持 git 忽略（未入库） | +| 根入口 | tree | `doc/knowledge/CONTRIBUTING.md` 保持 ≤100 非空行 / ≤8 KiB（细节下沉到 `contributing/`；校验器跨树检查这一条） | +| owner 轴 | tree | `components/`、`models/` 必须直属仓库；源码 owner 下不得有 `guides/` | +| 整页重复 | tree | 两个 owner 下逐字节相同的整页正文（≥200 字符）硬失败 | +| frontmatter | lint | 沉淀层页面（`general/`+`repos/` 下的 `rule/guide/architecture/index`）需要 `title`、`created`+`updated`（`YYYY-MM-DD`）、`type`（四者之一）、非空 `tags`；可选 `confidence: high\|medium\|low` | +| 标签分类法 | lint | 每个 `tags` 取值都必须出现在 `SCHEMA.md` 的 `## 标签分类法` 里 | +| 证据层 | lint | `incidents/` `history/` `results/` **不做** frontmatter 检查（它们用扁平 incident 字段） | +| adapter briefing | lint | `manifest.yaml` 的 `knowledge:` 只允许 `source`/`repo_subdir`/`briefing_docs`/`briefing_docs_extra`/`performance_briefing_docs`/`review_checklist`；briefing 与 checklist 不得指向证据层页面，`review_checklist` 指向的页面必须存在 | +| 提醒（不失败） | 两者 | 接近拆分线、目录超过 7 个普通页、孤页、超过 365 天未更新、`confidence: low` / `contested` —— 需要人判断，不是必须清零的门 | + +`rules.md`、`architecture.md`、`_index.md` 属于"特殊页"，不计入 7 页扇出告警。 + +--- + +## 7. 交付 + +- 像改任何受版本控制的内容一样就地改 `knowledge/`；wiki 门禁在评审时运行 —— + **不要**直接推保护分支。**目标仓库不会向本仓库提 PR**：它的 owner 提 + `[Knowledge]` issue，由维护者落盘。 +- 引入任何外部页面时**有选择地**导入具体页面并只取语义增量 —— + **整树替换不是合法的更新方式**（这里没有 submodule 链接）。 +- 机器事实（主机、路径、账号、token、cache、venv）**只**放在被 git 忽略的 + `knowledge/local/`；受版本控制的页面里不得出现（门禁会拦，但提交前请自己也看一眼）。 +- 改到 `repos/vllm-omni/` 时还有**第三道门**：`owner_documents` 入口页、`pin_documents` + 里的 SHA pin 和 `sources:` 由发版审计对账，两个校验器看不见它；任何触及该切片的 PR + 都会在 CI 里跑 `enforce`。见 + [同步与校验 §发版审计](contributing/validation.md#vllm-omni-页面还有一道发版审计)。 + +--- + +## 8. 可检索 skill + +[`skills/knowledge-base-contribution/SKILL.md`](../../skills/knowledge-base-contribution/SKILL.md) +把这套工作流编码给 copilot 自己的 agent（owner 路由、页型选择、`_index.md` 注册、 +always-on 紧凑度、以及校验器）。它经 `skill_search` 被检索，在 agent 被要求记录一条 +教训时浮现。**本指南每次变更，都要在同一个 PR 里更新它。** diff --git a/docs/autonomous-workflow.md b/docs/autonomous-workflow.md deleted file mode 100644 index 7a7b0bd4a..000000000 --- a/docs/autonomous-workflow.md +++ /dev/null @@ -1,51 +0,0 @@ -# Autonomous workflow - -This page documents the optional autonomous CLI and workflow MCP. It is separate -from the default Direct MCP described in the repository README. - -The autonomous workflow runs its own model and supports longer repo-maintenance -playbooks such as review, issue handling, CI debugging, and rebase. It therefore -requires model credentials and repository configuration. - -## Install - -```bash -bash install.sh -``` - -Edit `.env`, set the required model credentials and `REPO_PATHS`, then run: - -```bash -./infermatrix-copilot doctor -``` - -## Use - -```bash -./infermatrix-copilot -./infermatrix-copilot -p "review pr 4830" --yes -./infermatrix-copilot -p "answer issue 4842, do not post" -./infermatrix-copilot -p "rebase pr 4830, then review it" -./infermatrix-copilot --resume -``` - -The autonomous MCP command is: - -```text -infermatrix-copilot-workflow-mcp -``` - -It is not registered by the default Codex installer. - -## Safety - -- Pushes require an allowing policy and are dry-run unless explicitly enabled. -- Protected branches are never direct-pushed. -- Posting requires explicit intent and configuration. -- Blocked runs write escalation artifacts instead of guessing. - -Implementation details: - -- [`../QUICKSTART.md`](../QUICKSTART.md) -- [`../doc/DESIGN.md`](../doc/DESIGN.md) -- [`../doc/IMPLEMENTATION_STATUS.md`](../doc/IMPLEMENTATION_STATUS.md) diff --git a/docs/codex/README.md b/docs/codex/README.md deleted file mode 100644 index 1a3c8d8e1..000000000 --- a/docs/codex/README.md +++ /dev/null @@ -1,189 +0,0 @@ -# Use InferMatrixCopilot in Codex - -The default MCP is a knowledge provider. Codex uses its currently selected -model to review code, so there is no API key or model configuration. - -## Install - -```text -# Windows -install.cmd - -# macOS / Linux -./install-mcp.sh -``` - -The bootstrap detects Codex automatically. It uses the same MCP descriptor and -Agent Skill as Claude, Cursor, and other compatible hosts. - -Restart Codex, then paste: - -```text -Use InferMatrixCopilot to review -https://github.com/vllm-project/vllm-omni/pull/5172. -``` - -The plugin also adds the `imreview`, `imdesign`, `imcifix`, and `imupdate` -skills, so the short review form is: - -```text -$imreview https://github.com/vllm-project/vllm-omni/pull/5172 -``` - -Codex skills use `$name`, not `/name`; run `/skills` if the skill is not shown. -For issue fixes, use: - -```text -$imcifix https://github.com/vllm-project/vllm-omni/issues/5023 -``` - -`imcifix` is a host-agent local patch workflow. It does not imply that the MCP -server exposes an `issue_fix` background tool, and it does not commit, push, -open a PR, or post comments unless explicitly asked. - -Codex calls `review`, receives the local `knowledge/AGENTS.md` path plus a -compact first-review checklist, and follows that document's routing map itself. -The MCP does not guess which owner applies and does not inject complete rule -pages. After pinning the snapshot, Codex immediately reports the pinned head SHA, -current CI, mergeability, and preliminary findings in the host conversation. -It does this before reading knowledge, searching source, or running tests and -does not wait for CI completion or resolved mergeability. Codex then calls -Direct once with the collected title, body, and changed files, uses the embedded -`quick_map` in each exact route without opening the full rule page, and runs -knowledge/source and validation tracks concurrently. It reuses one in-review -evidence packet and runs an import/version -compatibility preflight before pytest. Validation results are bound to the head -SHA and an environment fingerprint. The progress update is not an interim -GitHub comment. Before the only final review comment, Codex calls -`validate_direct_review`. A normal small fix uses `subtraction_signal="none"` -without a minimality proof. Only a diff that adds or expands a helper, class, -fallback, compatibility branch, or public behavior uses `"triggered"` and -requires subtraction evidence. - -The tool response is intentionally small: - -```json -{ - "mode": "direct", - "knowledge_entry": "C:\\...\\knowledge\\repos\\vllm-omni\\components\\serving\\rules.md", - "knowledge_routes": [ - { - "owner": "serving", - "path": "C:\\...\\knowledge\\repos\\vllm-omni\\components\\serving\\rules.md", - "reason": "title/body: endpoint, request", - "quick_map": "## Direct 代码快速入口\n...", - "read_required": false - } - ], - "navigation_policy": { - "progress_before_knowledge": true, - "use_embedded_quick_maps": true, - "open_route_file_only_for_concrete_ambiguity": true, - "max_routes": 3, - "stop_after_routes": true - }, - "execution_budget": { - "profile": "code", - "knowledge_file_reads": 0, - "validation_commands": 4, - "total_command_calls": 20, - "hard_ceiling": true, - "extension_command_calls": 4 - }, - "first_review_checklist": ["...", "Run subtraction only when the diff has a subtraction signal ..."], - "progress_update": { - "deadline_seconds": 60, - "channel": "host_conversation", - "required_fields": ["head_sha", "ci_status", "mergeability", "early_findings"], - "early_findings_status": "preliminary", - "continue_review": true, - "github_comment": false - }, - "completion_gate": { - "tool": "validate_direct_review", - "subtraction_signal": { - "none": "no subtraction evidence required", - "triggered": "require subtraction evidence" - }, - "triggered_require_one_of": [ - "subtraction[{anchor, action, risk}]", - "minimality_proof{scope_ledger, abstraction_census, why_no_safe_deletion}" - ], - "final_comment_count": 1, - "if_missing": "partial_review" - } -} -``` - -Strict uses the same installed MCP. The wheel already includes its playbooks, -adapters, and skills. Configure the local checkout during installation: - -```text -install.cmd --repo-path D:\path\to\vllm-omni -``` - -Then set either `ANTHROPIC_API_KEY` or `OPENAI_API_KEY` in -`~/.infermatrix-copilot/.env`. Each provider also has an optional -`ANTHROPIC_BASE_URL` or `OPENAI_BASE_URL` for a proxy or compatible gateway. -With one key configured the provider is selected automatically; with both, -set `LLM_PROVIDER=anthropic` or `LLM_PROVIDER=openai`. Restart Codex, and say: - -```text -Use InferMatrixCopilot in Strict mode to review -https://github.com/vllm-project/vllm-omni/pull/5172. -Do not return a review until the strict run is complete. -``` - -Strict runs the packaged `pr-review` playbook with its configured model, -progress tracking, report generation, and publishing gates. Poll -`get_review_result` until the run is terminal. - -Strict never posts implicitly. Posting still requires both an explicit -`post=true` tool argument and server-side `ALLOW_POST=1`. - -Restart Codex after installation. Use `/mcp` or `codex mcp list` to confirm that -`infermatrix-copilot` is connected, and `/skills` to confirm that `imreview`, -`imdesign`, `imcifix`, and `imupdate` are available. - -## What the default MCP exposes - -- `review(target, repo?, mode="direct", post=false, title="", body="", - changed_files=[], review_depth="", repo_path="")`: after the host progress - update, Direct uses title/body to - return at most three exact owner/model routes with compact embedded - `quick_map` excerpts. Changed files normally only validate scope, but when no - returned route matches an owner they imply they select the routes instead and - the response says so (`status: scope_fallback`). The host does not open full - rule files unless a concrete ambiguity blocks source review — any route whose - `quick_map_status` is not `ok` (`unavailable`, or `truncated` when the section - exceeds the excerpt cap) is exactly that case, and the budget grants a read for - each — and - treats the returned docs/code `execution_budget` as a hard ceiling. A single - bounded extension is reserved for one stated unresolved P1/high-risk - contract. - Strict starts the packaged workflow and accepts `review_depth` plus an - optional local checkout override through `repo_path`. -- `validate_direct_review(subtraction_signal, subtraction?, minimality_proof?, - final_comment_count=1, evidence_head_sha)`: `evidence_head_sha` must be the - frozen head commit every cited source file and validation result was read - at; evidence read from another local revision does not complete the review. - `none` completes an ordinary small fix without a - minimality proof. `triggered` requires anchored subtraction actions or - concrete evidence that the inspected scope is already minimal. -- `update_knowledge(repo?)`: keeps `repo` only for call compatibility and - returns `knowledge/CONTRIBUTING.md`; the host model follows that document and - edits the Markdown files itself. -- `get_review_result(run_id, offset?)`: polls and pages the Strict report. -- `get_review_status(run_id)`: returns the Strict run's durable progress. -- `doc_search(query, repo?)`: finds deeper model/component rules. -- `doc_read(path, repo?)`: reads a selected knowledge page. - -Direct mode does not run another model, edit knowledge, post comments, or push -code. Its deterministic router selects bounded knowledge owners from the PR -description; Codex still owns scope validation and the truth of cited code -evidence. The completion validator checks review structure. - -## Optional autonomous BYOK workflow - -The autonomous workflow has separate setup and documentation: -[`../autonomous-workflow.md`](../autonomous-workflow.md). diff --git a/docs/knowledge-maintainer.zh-CN.md b/docs/knowledge-maintainer.zh-CN.md deleted file mode 100644 index 542966bfe..000000000 --- a/docs/knowledge-maintainer.zh-CN.md +++ /dev/null @@ -1,200 +0,0 @@ -# vLLM-Omni 知识库人工维护手册 - -这份手册面向 vLLM-Omni 的模块 owner 和模型 owner。目标不是让每个人研究知识库 -框架，而是把维护者已经掌握的经验，稳定地交给人和 agent 复用。 - -社区维护者直接向 InferMatrixCopilot 提交规则即可，不需要访问私有知识源。同步流程 -必须保留目标仓库已有的模块和模型规则，不能用上游知识树整棵覆盖。 - -## 只想马上加一条规则 - -不要从头写。只打开符合当前情况的一个样本： - -- [已有 component：增加规则](samples/add-component-rule.zh-CN.md) -- [已有 model：增加规则](samples/add-model-rule.zh-CN.md) -- [新模型还没有目录：创建完整 owner](samples/add-new-model-owner.zh-CN.md) - -替换尖括号里的内容，运行两个校验脚本，然后提交 PR。下面的章节用于解释为什么这样 -写，不是每次维护的必读前置。 - -## 最常见的维护方式 - -已有 owner 目录时，只需要： - -1. 打开该目录的 `rules.md`。 -2. 复制同文件最后一条规则，修改 ID、正文和来源；不要从空白开始写。 -3. 更新页面顶部的 `updated` 和 `sources`。 -4. 运行两个校验脚本。 -5. 提交 PR，由对应 owner 审核。 - -只有新增模块、模型或页面时，才需要修改 `_index.md`。 - -## 第一步：找到正确 owner - -| 经验属于什么 | 放在哪里 | -|---|---| -| 多个模型共用的源码模块 | `knowledge/repos/vllm-omni/components/<模块>/` | -| 单个模型的实现、配置或 checkpoint | `knowledge/repos/vllm-omni/models/<模型>/` | -| 整个 vLLM-Omni 都必须遵守的门禁 | `knowledge/repos/vllm-omni/rules.md` | -| 与任何仓库都无关的通用方法 | `knowledge/general/<主题>/` | - -判断标准是“哪块代码或哪个维护者对结论负责”，不是“在哪个 PR 里发现”。 - -例如： - -- deploy YAML 最终怎样进入 stage config → Configuration component。 -- HunyuanImage3 的图片数量和 prompt token → HunyuanImage3 model。 -- 所有模型共享的 scheduler 等待逻辑 → Scheduler component。 - -如果一个结论同时影响多个入口，只在最近 owner 留一份正文，其他地方只加链接。 - -## 第二步：先填一张规则卡片 - -维护者不需要先写 Markdown。先把下面六项填清楚： - -```text -规则卡片 - -- owner（模块或模型）： -- 触发：什么改动、现象或任务需要使用这条规则？ -- 必须：实现或审核时必须做什么？ -- 禁止：过去最容易犯的错误是什么？ -- 验收：什么测试、输出或代码路径能证明它满足要求？ -- 来源：相关源码路径、设计文档或 PR 链接。 -``` - -一张卡片只表达一个行为不变量。不能把一个 PR 的所有经验塞进一条规则。 - -## 第三步：写入 `rules.md` - -现有 owner 已有 `rules.md` 时，直接增加： - -```markdown -### <稳定规则 ID> — <一句话标题> - -- 触发：<适用场景> -- 必须：<必须执行的动作> -- 禁止：<不允许的做法> -- 验收：<可以检查的完成标准> -``` - -规则 ID 使用该 owner 已有前缀并继续编号，例如把样本中的 `CONF-NEXT`、`HY3-NEXT` -替换为当前未使用的下一个 ID。 -不要重排或复用旧 ID。 - -规则不保存完整 diff、review 对话、调查时间线和大段日志。GitHub PR 本身负责保存 -原始证据；知识库只保存下次可以执行的规则。 - -每条新规则必须能追溯： - -- 页面顶部 `sources` 追加 PR、源码路径或设计文档。 -- 规则最后一行追加对应来源标记，例如 `^[PR #1234]`。 -- 没有任何可核对来源时，不编造来源，也不提交规则；先请提出者补充证据。 - -### Component 示例 - -下面是 Configuration owner 的规则写法： - -```markdown -### CONF-3a — 争议以展开后的最终配置为准 - -- 触发：CLI、deploy YAML 和 stage override 对同一字段给出不同值。 -- 必须：检查合并完成后的逐 stage 配置，并跟到第一位 consumer。 -- 禁止：只看某一层 YAML 就断言最终生效值。 -- 验收：最终配置对象能读回目标值，并与第一位 consumer 收到的值一致。 -``` - -它没有保存某个 PR 的故事，而是保存 Configuration owner 下次仍会执行的判断。 - -### Model 示例 - -下面是 HunyuanImage3 owner 的规则写法： - -```markdown -### HY3-2c — 在公开入口尽早拒绝非法图片数量 - -- 触发：修改图片数量、图片输入或公开生成入口。 -- 必须：在文件读取、下载、解码、resize 和 VAE 之前校验数量。 -- 禁止：等昂贵操作开始后才报参数错误。 -- 验收：覆盖合法边界和超限输入，并证明超限请求没有进入昂贵路径。 -``` - -这条规则只属于 HunyuanImage3，不应提升到仓库根或共享 Diffusion component。 - -## 新增模块或模型 - -如果目录尚不存在： - -1. 优先复制[完整新 model owner 样本](samples/add-new-model-owner.zh-CN.md)； - 需要其他页面类型时再看 - [`doc/knowledge-templates/`](../doc/knowledge-templates/README.md)。 -2. 创建 `_index.md`，写清源码路径、职责边界、测试入口和依赖关系。 -3. 有稳定架构信息时创建 `architecture.md`。 -4. 第一条真实规则出现时才创建 `rules.md`，不要提交空页面。 -5. 新增 component 时在 `components/_index.md` 增加入口；新增 model 时不用维护 - 父级长表，目录会由 `models/` 文件系统清单自动发现。新增别名或 registry key - 时才更新 `models/catalog.md`。 - -模块与模型的区别： - -- Component：被多个模型或上层功能共享，有稳定源码和输入输出边界。 -- Model：只描述某个模型自己的适配、配置、checkpoint 和行为。 - -## 人工与 agent 协作 - -推荐由 owner 提供规则卡片，agent 完成机械工作，owner 做最终语义审核。 - -不会或不想直接修改 Markdown 时，可以 -[提交中文规则建议](https://github.com/JiusiServe/InferMatrixCopilot/issues/new?template=knowledge-rule.yml)。 -表单内容就是本手册的规则卡片。知识库维护者或 agent 把它整理成 PR，再请原 owner -确认语义。 - -可以直接对 Codex 说： - -```text -Use InferMatrixCopilot to update the knowledge base. - -请根据下面的规则卡片修改最近 owner 的 rules.md： -<粘贴规则卡片> - -要求： -1. 不保存原始 PR、评论、diff 或调查过程。 -2. 不扩大到其他 component/model。 -3. 使用该 owner 现有规则 ID 前缀。 -4. 新增页面时更新最近 _index.md；只改已有 rules.md 时不要乱改索引。 -5. 运行 check_knowledge_tree.py 和 check_wiki_lint.py。 -6. 把最终 diff 交给我审核，不要自动发布评论。 -``` - -agent 可以协助查路径、补 frontmatter、更新索引和运行检查，但下面三件事必须由 -owner 最终确认： - -- owner 是否正确； -- 规则是否真的长期成立； -- 验收是否覆盖真实生产入口。 - -推荐的团队分工是： - -1. 模块/模型 owner 在重要 PR 合入或 review 结束后填写规则卡片。 -2. 知识库维护者或 agent 负责查重、生成 Markdown、更新元数据并运行校验。 -3. 原 owner 只审核自己目录中的规则语义和验收条件。 -4. 合入后，后续 review 自动从该 owner 的 `rules.md` 获取规则。 - -## PR 前检查 - -- [ ] 规则放在最近的 component/model owner，没有放进宽泛的仓库根。 -- [ ] 每条规则都有触发、必须、禁止、验收和稳定 ID。 -- [ ] 删除 PR 编号后，规则仍能独立读懂。 -- [ ] 没有保存完整 diff、review 对话、运行结果或临时路径。 -- [ ] 修改已有 `rules.md` 时更新了 `updated` 和 `sources`。 -- [ ] 新增页面或目录时更新了最近 `_index.md`。 -- [ ] 两个校验脚本均为 0 错误、0 提醒。 - -```powershell -python knowledge/tools/check_knowledge_tree.py -python knowledge/tools/check_wiki_lint.py -``` - -更完整的目录、frontmatter 和页面规范见 -[`knowledge/CONTRIBUTING.md`](../knowledge/CONTRIBUTING.md)。英文扩展说明见 -[`doc/EXTENDING-KNOWLEDGE.md`](../doc/EXTENDING-KNOWLEDGE.md)。 diff --git a/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_train_pr4926.md b/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_train_pr4926.md new file mode 100644 index 000000000..d97a70f1d --- /dev/null +++ b/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_train_pr4926.md @@ -0,0 +1,17 @@ +(no RUN_REPORT.md — rc=3) + +## stdout +→ task: pr_review PR #4926 on vllm-omni (tier L2) +→ plan: reuse pr-review@6 (active) steps=['pr.fetch_diff', 'pr.gate_check', 'agent.review_diff', 'pr.post_review', 'report.final_summary'] + · recalled pr-review@6 (active) +→ path: eco · agent-model=deepseek-v4-pro @ api.deepseek.com + metrics: usd≈0.00 0.1min S=1.00 (/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4926/run-20260817-105011-b1bdc1/metrics.json) + ✓ fetch: fetched PR #4926 diff (27969 chars, context 3985 chars); checkout: PR-TIME TREE (head 35aa741e1962) + ✓ gate: FAILING CHECKS (2): ['buildkite/vllm-omni-amd-ci', 'buildkite/vllm-omni-npu-ci'] — do not re-argue what CI already repor + ✗ review: unhandled error: APIStatusError: Error code: 402 - {'error': {'message': 'Insufficient Balance', 'type': 'unknown_error', 'param': None, 'code': 'invalid_request_error'}} +run run-20260817-105011-b1bdc1: blocked (/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4926/run-20260817-105011-b1bdc1) + ⚠ step 'review' (agent.review_diff): unhandled error: APIStatusError: Error code: 402 - {'error': {'message': 'Insufficient Balance', 'type': 'unknown_error', 'param': None, 'code': 'invalid_request_error'}} + see /data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4926/run-20260817-105011-b1bdc1/ESCALATION.md + + +## stderr diff --git a/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_train_pr4977.md b/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_train_pr4977.md new file mode 100644 index 000000000..c569f9b96 --- /dev/null +++ b/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_train_pr4977.md @@ -0,0 +1,17 @@ +(no RUN_REPORT.md — rc=3) + +## stdout +→ task: pr_review PR #4977 on vllm-omni (tier L2) +→ plan: reuse pr-review@6 (active) steps=['pr.fetch_diff', 'pr.gate_check', 'agent.review_diff', 'pr.post_review', 'report.final_summary'] + · recalled pr-review@6 (active) +→ path: eco · agent-model=deepseek-v4-pro @ api.deepseek.com + metrics: usd≈0.00 0.1min S=1.00 (/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4977/run-20260817-104946-ed83da/metrics.json) + ✓ fetch: fetched PR #4977 diff (4140 chars, context 1866 chars); checkout: PR-TIME TREE (head 9947f41430aa) + ✓ gate: FAILING CHECKS (1): ['buildkite/vllm-omni-npu-ci'] — do not re-argue what CI already reports; point at the gate. + ✗ review: unhandled error: APIStatusError: Error code: 402 - {'error': {'message': 'Insufficient Balance', 'type': 'unknown_error', 'param': None, 'code': 'invalid_request_error'}} +run run-20260817-104946-ed83da: blocked (/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4977/run-20260817-104946-ed83da) + ⚠ step 'review' (agent.review_diff): unhandled error: APIStatusError: Error code: 402 - {'error': {'message': 'Insufficient Balance', 'type': 'unknown_error', 'param': None, 'code': 'invalid_request_error'}} + see /data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4977/run-20260817-104946-ed83da/ESCALATION.md + + +## stderr diff --git a/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_val_pr4816.md b/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_val_pr4816.md new file mode 100644 index 000000000..93b71dfaa --- /dev/null +++ b/eval/dataset/arms/INVALID_apierror402_v17ds_stubs/copilot_v17ds_val_pr4816.md @@ -0,0 +1,17 @@ +(no RUN_REPORT.md — rc=3) + +## stdout +→ task: pr_review PR #4816 on vllm-omni (tier L2) +→ plan: reuse pr-review@6 (active) steps=['pr.fetch_diff', 'pr.gate_check', 'agent.review_diff', 'pr.post_review', 'report.final_summary'] + · recalled pr-review@6 (active) +→ path: eco · agent-model=deepseek-v4-pro @ api.deepseek.com + metrics: usd≈0.00 0.1min S=1.00 (/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_val/runs/pr4816/run-20260817-104946-e1d77c/metrics.json) + ✓ fetch: fetched PR #4816 diff (5330 chars, context 3985 chars); checkout: PR-TIME TREE (head e1dac23ac02c) + ✓ gate: FAILING CHECKS (3): ['buildkite/vllm-omni-amd-ci', 'buildkite/vllm-omni-intel-ci', 'buildkite/vllm-omni-npu-ci'] — do no + ✗ review: unhandled error: APIStatusError: Error code: 402 - {'error': {'message': 'Insufficient Balance', 'type': 'unknown_error', 'param': None, 'code': 'invalid_request_error'}} +run run-20260817-104946-e1d77c: blocked (/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_val/runs/pr4816/run-20260817-104946-e1d77c) + ⚠ step 'review' (agent.review_diff): unhandled error: APIStatusError: Error code: 402 - {'error': {'message': 'Insufficient Balance', 'type': 'unknown_error', 'param': None, 'code': 'invalid_request_error'}} + see /data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_val/runs/pr4816/run-20260817-104946-e1d77c/ESCALATION.md + + +## stderr diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/manifest.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/manifest.json new file mode 100644 index 000000000..b70019040 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_cb_grok46", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.cost.json new file mode 100644 index 000000000..d8e84abf0 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1193.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5509/run-20260814-184436-726ade", + "recorded_at": "2026-08-14T19:04:29", + "attempts": [ + { + "invocation_id": "inv-b9758385", + "outcome": "run:run-20260814-184436-726ade", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-184436-726ade", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-184436-726ade", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5451, + "minutes": 19.87, + "ci_minutes": 0, + "input_tokens": 1275375, + "output_tokens": 182505, + "tool_calls": 313, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.754, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.87, + "gate": 2.06, + "review": 1186.15, + "report": 0.06 + }, + "time_to_first_result_s": 1192.12 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 39114 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.md new file mode 100644 index 000000000..274fee61f --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] tests/diffusion/attention/test_trtllm_attn.py:315 Missing TRTLLM-GEN kernel skip: arm can occur on SM103 for int8 (docs say int8 is SM100-only); skip is justified, not a silent pass of a wrong result. +- [sweep] trtllm_attn.py:90 HAS_FLASHINFER false: _sage_kernel_available returns False; quant.enabled then raises at init (kernel-path message, not ImportError). Arm occurs. +- [sweep] trtllm_attn.py:198 quant.enabled: True only when dtype_qk is set; dense path leaves sage_kwargs empty so older FlashInfer still works. Arm occurs. +- [sweep] trtllm_attn.py:199 dtype_qk not in _QK_QUANT_DTYPES: covered by test_quant_rejects_non_sage_dtype (bfloat16 -> RuntimeError). Arm occurs; happens after AttnQuantSpec already accepted bfloat16. +- [sweep] trtllm_attn.py:204 _sage_kernel_available false: init raises rather than degrading. Arm occurs on old FlashInfer. +- [sweep] trtllm_attn.py:211 _sage_quantize_fn is None: covered by test_quant_quantize_requires_flashinfer_routine. Arm occurs. +- [sweep] trtllm_attn.py:296 forward SAGE branch: GPU test_sage_quant_matches_sdpa exercises it; skip+quant together is not forwarded in that test. +- [sweep] data.py:1415 invalid dtype: rejects int4 (test_quant_validation_rejects); does not reject TRTLLM+bfloat16 or FLASHINFER+int8 because _VALID_DTYPES is a union. + +`vllm_omni/diffusion/data.py:1447` [major] — This diff allowlists `quant` on `FLASHINFER_ATTN` (`data.py:1447`) and documents that `FLASHINFER_ATTN` reads `dtype_qk` (`float16`/`bfloat16`), `dtype_vo`, and `flashinfer_backend`, with "A dtype_qk from the wrong set raises." `FlashInferAttentionImpl.__init__` takes `**extra_impl_args` and never reads `backend_kwargs`/`quant` (`flashinfer_attn.py:55-66`), while `Attention()` always passes `spec.backend_kwargs()` (`layer.py:93`, `layer.py:109`). `{backend: FLASHINFER_ATTN, quant: {...}}` therefore constructs and runs dense FlashInfer with no error. `AttnQuantSpec._VALID_DTYPES` is a union (`data.py:1410`) so `FLASHINFER_ATTN`+`int8` also constructs (silently ignored) and `TRTLLM_ATTN`+`bfloat16` only fails later as `RuntimeError` (`trtllm_attn.py:199-202`). `test_quant_superset_fields_passed_through` only asserts `backend_kwargs()` serialization. Restrict `AttentionSpec.quant` to `TRTLLM_ATTN` until FLASHINFER consumes those fields (same pattern as `skip_softmax`), drop the FLASHINFER fields/docs/round-trip test that imply a live consumer, and reject backend-inappropriate `dtype_qk` at spec construction. (evidence: data.py:1447-1450 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` / `quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends`. data.py:1410 `_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})`. flashinfer_attn.py:55-66 `def __init__(..., **extra_impl_args) -> None:` then only `self.causal = causal; self.softmax_scale = softmax_scale` — no `backend_kwargs`/`quant` read. layer.py:93 `backend_kwargs = spec.backend_kwargs()`; layer.py:109 `backend_kwargs=backend_kwargs`. trtllm_attn.py:199-202 `if self.quant.dtype_qk not in _QK_QUANT_DTYPES: raise RuntimeError(... FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE.)`. docs/user_guide/diffusion/attention_backends.md:185-188 `The quant block is a shared AttnQuantSpec consumed by more than one backend ... FLASHINFER_ATTN reads dtype_qk (float16 / bfloat16), dtype_vo, and flashinfer_backend ... A dtype_qk from the wrong set raises.`) + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — This PR inserts the SAGE/`quant` section into `attention_backends.md` and deletes `## Known Limitations` (LTX-2 `CUDNN_ATTN`+compile crash / FA4), mixed-backends, Validation Guidance, and Reproducing the Benchmark Table; those headings have no relocated copies under `docs/` (only an unrelated Known Limitations in `video_stream_api.md`). Line 232 still says "see Known Limitations above", and lines 72–73 still claim the only typed spec field is `TRTLLM_ATTN`'s Skip-Softmax even though this PR adds typed `quant`. Restore the still-true sections (at least the LTX-2 workaround, mixed-backends example, and the validation checklist the new lossy SAGE path should link to) and update the typed-field sentence to include `quant`. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — sole remaining Known Limitations mention in this file; no `## Known Limitations` heading exists. docs/user_guide/diffusion/attention_backends.md:72-73 `A backend that needs configuration exposes it as a typed field on the spec. Today the only` / `one is \`TRTLLM_ATTN\`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):` — stale after this PR adds typed `quant`. Grep under docs/ for `Validation Guidance`, `Mixed backends across roles`, `Reproducing the Benchmark`, and `CUDNN_ATTN crashes`: no matches; `## Known Limitations` only at docs/serving/video_stream_api.md:88 (unrelated). File ends at the SAGE_ATTN_3 example (line 381) with those sections gone.) + +`vllm_omni/diffusion/attention/layer.py:?` [major] — SAGE is applied only in TrtllmAttentionImpl.forward_cuda (trtllm_attn.py:296-297: if self.quant.enabled: quantize(...) then **sage_kwargs). _run_ring_attention (layer.py:319-331) already raises for Skip-Softmax because the ring path bypasses the backend, then calls RingParallelAttention.run_attention, which uses FA/SDPA ring kernels only (ring.py:130-157) and never the TRTLLM impl. layer.py has no quant.enabled check; the ring raise at layer.py:186-192 is KV-cache dtype, not AttnQuantSpec. quant + ring_degree>1 therefore runs dense ring kernels while SAGE was requested. Extend that same Skip-Softmax guard to raise when getattr(self.attention, "quant", None) is enabled (do not add a second helper), or reject the combo at config time. (evidence: layer.py:319-326: skip = getattr(self.attention, "skip", None) / if skip is not None and getattr(skip, "configured", False): raise NotImplementedError("Skip-Softmax (TRTLLM_ATTN) is not supported with ring sequence parallelism: the ring path bypasses the backend, so the skip config would be silently ignored."). layer.py:327-331: if self.ring_runner is not None: return self.ring_runner.run_attention(...) — no quant check. trtllm_attn.py:296-297: if self.quant.enabled: q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn). ring.py:130-157: if backend_pref in {"sdpa", "torch", "torch_sdpa"}: return ring_pytorch_attn_func(...); else return ring_flash_attn_func(...) — no TRTLLM/SAGE path. layer.py:186-192 is a different feature: if dtype: if ring_degree > 1: raise ValueError("KV quantization is not compatible with ring attention").) + +`tests/diffusion/attention/test_trtllm_attn.py:50` [major] — This diff adds CPU-runnable SAGE tests (`test_quant_config_from_backend_kwargs`, `test_quant_rejects_non_sage_dtype`, `test_quant_quantize_requires_flashinfer_routine`, `test_quant_quantize_calls_routine_and_shapes_sfs`) plus `test_sage_quant_matches_sdpa` into `test_trtllm_attn.py`, which is `pytestmark`ed `cuda`. Simple · Diffusion Test runs `pytest tests/diffusion -m 'core_model and cpu'` (deselects them) and Diffusion · Other Test runs `pytest tests/diffusion/*.py` (does not collect `tests/diffusion/attention/`); `.buildkite` has zero references to that subdirectory. Those unit tests therefore never run on ready/merge lanes, `test_sage_quant_matches_sdpa` is additionally `skipif`'d for SM100+ / flashinfer>=0.6.16rc1, and no test intercepts `trtllm_ragged_attention_deepseek` to assert `sage_attn_sfs` / `num_elts_per_sage_attn_blk` (or their absence on the dense path); `test_quant_quantize_requires_flashinfer_routine` patches `_sage_kernel_available` True so the False init arm is untested. Move the GPU-free tests into `test_attention_config.py` (already cpu-marked) or mark them cpu; add a monkeypatch of `trtllm_ragged_attention_deepseek` on `_impl(quant={...}).forward_cuda` that captures those kwargs, a dense counterpart that asserts they are omitted, and coverage of `if not _sage_kernel_available()`. Verify with `pytest tests/diffusion/attention/test_attention_config.py -m 'core_model and cpu'` and `pytest tests/diffusion/attention/test_trtllm_attn.py -k 'test_quant_' -m 'core_model'`. (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` — file-level cuda mark on the new SAGE tests at :50/:60/:65/:74. tests/diffusion/attention/test_trtllm_attn.py:68 `monkeypatch.setattr(mod, "_sage_kernel_available", lambda: True)` — False init arm never hit. tests/diffusion/attention/test_trtllm_attn.py:298-300 `requires_sage = pytest.mark.skipif(not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None), reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)")`. tests/diffusion/attention/test_attention_config.py:34 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]`. .buildkite/cuda/test-ready.yml:18 `pytest -sv tests/diffusion -m 'core_model and cpu'`. .buildkite/cuda/test-ready.yml:68 `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'`. vllm_omni/diffusion/attention/backends/trtllm_attn.py:204 `if not _sage_kernel_available():` then :297-299 `sage_kwargs["sage_attn_sfs"] = sage_attn_sfs` / `sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes`. grep tests/ for `trtllm_ragged_attention_deepseek`: no matches; grep .buildkite for `tests/diffusion/attention`: no matches.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:?` [minor] — SAGE kwargs are packed in shared `TrtllmAttentionImpl.forward_cuda` with no role gate (`if self.quant.enabled:` at trtllm_attn.py:296–299). The documented enablement path puts `quant` on `AttentionConfig.default`; `get_attn_backend_for_role` then uses that spec’s `backend_kwargs()` for every unmatched role, including cross-attn (selector.py lookup: per_role → role_category → default; layer.py:92–94). Platform auto-default already selects TRTLLM_ATTN for WanPipeline/WanImageToVideoPipeline/WanVACEPipeline/WanS2VPipeline on datacenter Blackwell when `head_size==128` (model_metadata.py:39–42; layer.py:83; platform.py:204–217). Scope the quality/perf contract to the measured pipeline(s)/roles, or state that VACE/S2V, cross-attn, and non-Wan explicit-TRTLLM users are unvalidated for SAGE. (evidence: trtllm_attn.py:296-299 `if self.quant.enabled: q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn); sage_kwargs["sage_attn_sfs"] = sage_attn_sfs; sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes` — SAGE packed in shared forward, no role check. selector.py:108-111 `Lookup precedence: 1. attention_config.per_role[role] — exact match; 2. attention_config.per_role[role_category] — category fallback; 3. attention_config.default — global default; 4. Platform default`. layer.py:92-94 `if spec is not None: backend_kwargs = spec.backend_kwargs()` — default spec’s quant reaches every unmatched role’s impl. layer.py:83 `allow_trtllm_default = get_diffusion_model_metadata(model_class_name).attention_mask_free`. model_metadata.py:39-42 `"WanPipeline"/"WanImageToVideoPipeline"/"WanVACEPipeline"/"WanS2VPipeline": DiffusionModelMetadata(attention_mask_free=True)`. platform.py:204-217 `trtllm_attn_default_ok = allow_trtllm_default and compute_capability.major == 10 and head_size == 128 and trtllm_gen_available` → `Defaulting to diffusion attention backend TRTLLM_ATTN`.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:125` [minor] — QuantConfig.from_backend_kwargs uses `int(q.get("q_block_size", 1) or 1)` and the same pattern for k_block_size, so explicit 0 or None silently become 1/16. AttnQuantSpec already rejects those via `_VALID_BLOCK_SIZES = {1, 4, 16}`. The typed AttentionSpec path cannot deliver 0, but this parser is also used from raw backend_kwargs (TrtllmAttentionImpl.__init__, tests). Parse against the same allowed set and raise on 0/None instead of `or default`. (evidence: trtllm_attn.py:125 `q_block_size=int(q.get("q_block_size", 1) or 1),` — 0/None are falsy so become 1; trtllm_attn.py:126 `k_block_size=int(q.get("k_block_size", 16) or 16),` — same for 16; data.py:1411 `_VALID_BLOCK_SIZES = frozenset({1, 4, 16})`; data.py:1418-1420 `if v not in self._VALID_BLOCK_SIZES: raise ValueError(...)` so AttnQuantSpec rejects 0; trtllm_attn.py:194 `self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)` is the consumer of any raw dict, not only the typed spec.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:204` [minor] — pyproject.toml and requirements/ have no flashinfer pin (project.dependencies are dynamic from requirements/; flashinfer is treated as a vLLM hard dependency). SAGE is opt-in: TrtllmAttentionImpl.__init__ raises if quant is set but sage_attn_sfs / trtllm_sage_attention_quantize are missing (trtllm_attn.py:198-216), and forward_cuda only passes sage_kwargs when quant.enabled (295-320), so older FlashInfer still runs dense. Unpinned FlashInfer plus fail-closed SAGE is the current compatibility contract; docs already state ≥ 0.6.16rc1 as a SAGE-only requirement. (evidence: trtllm_attn.py:198 `if self.quant.enabled:` — SAGE only when quant is set. trtllm_attn.py:204 `if not _sage_kernel_available():` then raise RuntimeError that the build lacks sage_attn_sfs. trtllm_attn.py:211-214 `if self._sage_quantize_fn is None:` / `raise RuntimeError(` / `"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build lacks "` / `"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "` — missing APIs fail at init. trtllm_attn.py:295-296 `sage_kwargs: dict = {}` / `if self.quant.enabled:` — dense path omits SAGE kwargs. pyproject.toml:11 `dynamic = ["version", "dependencies"]`; pyproject.toml:33-34 `# Dependencies are now managed dynamically via setup.py ... See requirements/ directory`; grep of pyproject.toml and requirements/ for `flashinfer` returned no matches. recipes/Wan-AI/Wan2.2-T2V.md:106 `and `flashinfer` (already a hard dependency of vLLM — no extra install).` docs/user_guide/diffusion/attention_backends.md:208 `**Requirements.** Needs FlashInfer ≥ 0.6.16rc1.` is a SAGE-only doc requirement, not a package pin.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:143` [minor] — QuantConfig.quantize hardcodes FlashInfer SAGE packing as sage_attn_sfs=(q_sfs, k_sfs, None, v_sfs) and num_elts_per_sage_attn_blk=(q_block_size, k_block_size, 0, 1) (trtllm_attn.py:143-144). The only explicit layout assert is test_quant_quantize_calls_routine_and_shapes_sfs against a fake quantize_fn, not FlashInfer's C API. test_sage_quant_matches_sdpa only checks SDPA rel err < 0.1 and can pytest.skip on "Missing TRTLLM-GEN kernel", so a wrong slot order is not pinned to the vendor ABI. (evidence: trtllm_attn.py:143 `sage_attn_sfs = (q_sfs, k_sfs, None, v_sfs)` — trtllm_attn.py:144 `num_elts_per_sage_attn_blk = (self.q_block_size, self.k_block_size, 0, 1)` — tests/diffusion/attention/test_trtllm_attn.py:79-84 fake_quantize returns dummy sfs and `assert (q_q, k_q, v_q, sfs, blk) == ("qq", "kq", "vq", ("qsfs", "ksfs", None, "vsfs"), (1, 16, 0, 1))` — test_sage_quant_matches_sdpa:316-323 skips on `"Missing TRTLLM-GEN kernel"` and only asserts `rel < 0.1`.) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/attention/test_attention_config.py:75` [minor] `test_quant_and_skip_softmax_coexist` only asserts AttentionSpec.backend_kwargs() serialization (`target_sparsity` + `quant.dtype_qk`). `test_sage_quant_matches_sdpa` builds `_impl(quant={...})` with no skip kwargs; skip +- `recipes/Wan-AI/Wan2.2-T2V.md:186` [minor] This PR adds SAGE (`quant.dtype_qk` int8/fp8_e4m3) on `TRTLLM_ATTN` and shows it with `vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers` in `docs/user_guide/diffusion/attention_backends.md:191`, but `recipes/Wan-AI/Wan2. + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.trace.json.gz new file mode 100644 index 000000000..609d97218 Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.cost.json new file mode 100644 index 000000000..d21ca9da9 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1152.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5550/run-20260814-184437-21e6d3", + "recorded_at": "2026-08-14T19:03:47", + "attempts": [ + { + "invocation_id": "inv-4a1de5e1", + "outcome": "run:run-20260814-184437-21e6d3", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-184437-21e6d3", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-184437-21e6d3", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4968, + "minutes": 19.18, + "ci_minutes": 0, + "input_tokens": 1173290, + "output_tokens": 163627, + "tool_calls": 372, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7217, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.8, + "gate": 1.64, + "review": 1145.08, + "report": 0.07 + }, + "time_to_first_result_s": 1150.56 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 34149 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.md new file mode 100644 index 000000000..ef7860ddf --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] indexed test_metadata.py:37-38,68,109: constructor fixtures insert one sequence/context then read [0]; safe for those tests, not a production contract. +- [sweep] indexed test_worker_contract.py:221 calls[0][1][0].prepared_layout: follows assert calls == one 3-tuple RPC; [0] is the only recorded call. +- [sweep] branch diffusion_engine.py:199 scheduler_to_close is not None: True when initialize() fails after assignment (tested); False if _init_scheduler raises before self.scheduler is set (constructor path). +- [sweep] branch multiproc_executor.py:545 metadata is not None: both arms exist; 3-tuple dense RPC and 4-tuple metadata RPC are tested via execute_request. +- [sweep] branches interface.py:184/192: envelope vs forwarded and metadata vs forwarded mismatches are tested; both arms reachable. +- [sweep] branches diffusion_model_runner.py:172/175: dense+metadata and paged identity mismatch are tested; paged+None is allowed (W0). +- [sweep] branches diffusion_worker.py:463/1372: kwargs omit metadata when None (dense signature test) and include it when present (wrapper test). +- [validated] correctness execute_request: identity is validated before RPC so mismatches cannot become DiffusionOutput(error=) at multiproc_executor.py:459-460 vs 574-580. + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — This diff adds `and not has_diffusion_kv_metadata` to the DLO+AllGather multi-request condition in `execute_request`, so a metadata-bearing wave skips the list RPC `args=(reqs_list, od_config, kv_prefetch_job)` and falls through to the per-request 4-tuple `execute_model` loop. `make_executor()` hard-codes `enable_distributed_layerwise_offload=False`, and existing `test_dlo_dp_*` / metadata RPC cases therefore never evaluate this conjunct—deleting the gate would still pass those tests, while a later paged+DLO wave would either drop block IDs on the 3-tuple AllGather RPC or silently lose DP multi-concurrency on every `execute_request` inheritor (including MiniMax-H3 DLO). Add a worker-contract test that enables DLO+AllGather, schedules two `NewRequestData` with per-request metadata, and asserts two `(req, od_config, prefetch, metadata)` RPCs rather than one list RPC. Also cover non-DLO `execute_batch`: the `execute_model_batch` RPC must still carry `scheduled_new_reqs[*].diffusion_kv_metadata` inside `scheduler_output`. Run: `pytest -sv tests/diffusion/diffusion_kv/test_worker_contract.py -k 'dlo or batch' -m 'core_model and cpu'`. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:471 `and not has_diffusion_kv_metadata` — conjunct on the DLO+AllGather branch. :509 `args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job),` — DLO list RPC has no metadata slot. :544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None:` / `args += (new_req.diffusion_kv_metadata,)` — metadata-bearing fallthrough is per-request 4-tuple. :615-617 `result = self.collective_rpc(` / `"execute_model_batch",` / `args=(scheduler_output, self.od_config),` — non-DLO execute_batch forwards the whole scheduler_output. tests/diffusion/diffusion_kv/test_worker_contract.py:63 `enable_distributed_layerwise_offload=False`. tests/diffusion/test_multiproc_engine_concurrency.py:200-207 `_make_sched_output` builds `NewRequestData(request_id=rid, req=...)` with no metadata; :251-267 `test_dlo_dp_routes_multiple_requests_without_pipeline_batching` enables DLO and still uses that helper.) + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [minor] — ARDiffusionModelRunner.execute_model (runner.py:237) still takes only `(req, kv_prefetch_job=None)` and both `super()` calls (243, 255) omit metadata. DiffusionWorker.execute_model (462-465) and WorkerWrapperBase.execute_model (1371-1374) inject `diffusion_kv_metadata=` into runner kwargs only when the snapshot is not None, so a metadata-bearing `engine_backend=ar_diffusion` request TypeErrors at the override before DiffusionModelRunner._validate_diffusion_kv_metadata. `test_dense_worker_call_does_not_extend_model_runner_signature` only calls with metadata=None; `test_execute_model_accepts_kv_prefetch_job_kwarg` only asserts `kv_prefetch_job`. Accept and forward `diffusion_kv_metadata` on both super() paths, and require it in that inspect.signature test for both runner classes. (evidence: vllm_omni/experimental/ar_diffusion/runner.py:41 `class ARDiffusionModelRunner(DiffusionModelRunner):`; runner.py:237-243 `def execute_model(\n self,\n req: OmniDiffusionRequest,\n kv_prefetch_job: KVPrefetchJob | None = None,\n ) -> DiffusionOutput:\n if self.kv_cache is None:\n return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)`; runner.py:255 `output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` — no diffusion_kv_metadata parameter or forward. vllm_omni/diffusion/worker/diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `output = self.model_runner.execute_model(req, **kwargs)`. diffusion_worker.py:1371-1374 same conditional kwargs into `self.worker.execute_model`. vllm_omni/experimental/ar_diffusion/engine.py:40 `default_diffusion_model_runner_cls = AR_DIFFUSION_MODEL_RUNNER_CLS`. tests/diffusion/ar_diffusion/test_engine_routing.py:150-152 `for cls in (DiffusionModelRunner, ARDiffusionModelRunner):` / `params = inspect.signature(cls.execute_model).parameters` / `assert "kv_prefetch_job" in params, cls` — does not require diffusion_kv_metadata. tests/diffusion/diffusion_kv/test_worker_contract.py:266-282 DenseModelRunner.execute_model(self, req, kv_prefetch_job=None) then `worker.execute_model(req, SimpleNamespace())` with no metadata.) + +`vllm_omni/diffusion/sched/base_scheduler.py:114` [minor] — W0 producer remains unwired: BaseScheduler.schedule still calls NewRequestData.from_state(state) with no diffusion_kv_metadata, so the new field stays None on the only production path (RequestScheduler/StepScheduler do not override schedule()). Runner validation explicitly allows a missing snapshot; wiring from_state is outside this PR’s W0 boundary, and Hunyuan paged_scheduler plus MiniMax DLO therefore keep traveling with diffusion_kv_metadata=None. (evidence: base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — sole production from_state call, no metadata argument. interface.py:168-172 `def from_state(cls, state: SchedulerRequestState, *, diffusion_kv_metadata: DiffusionKVMetadata | None = None)` — omitted kwarg stays None. diffusion_model_runner.py:170 `# W0 does not wire the Scheduler allocator or Worker installer yet, so` — missing snapshot is the stated W0 remainder.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:460` [minor] — Identity checks in execute_request run before either collective_rpc and outside both try/except wrappers that turn failures into DiffusionOutput(error=). Envelope mismatches therefore raise ValueError from validate_new_request_data_identity instead of being forwarded or converted into worker error outputs. Default backend mp means every request-mode engine sharing MultiprocDiffusionExecutor inherits that raise. This is locked only by CPU test test_request_rpc_rejects_envelope_request_identity_mismatch; there is no model e2e. Please confirm that raise-vs-error-output is the intended shared W0 contract. (evidence: multiproc_executor.py:454-460 `# Validate every envelope before selecting a dispatch path. In / particular, keep identity errors outside the RPC error wrapper so an / invalid request cannot be forwarded or converted into a worker output.` then `for new_req in new_reqs: / validate_new_request_data_identity(new_req)` — this is before DLO RPC at 506 and per-request RPC at 547. interface.py:184-186 `if new_req.request_id != forwarded_request_id: / raise ValueError( / "Diffusion request identity mismatch: "`. multiproc_executor.py:574-580 `except Exception as exc: / runner_outputs.append( / RunnerOutput( / ... / result=DiffusionOutput(error=str(exc)),` — the wrapper that identity errors are kept outside of (DLO equivalent at 536). abstract.py:27-42 `if distributed_executor_backend is None: / distributed_executor_backend = "mp"` ... `executor_class = MultiprocDiffusionExecutor`. test_worker_contract.py:138-153 `with pytest.raises(ValueError, match="request identity mismatch"): / executor.execute_request(make_scheduler_output(new_req))` and `assert calls == []`; grep of `request identity mismatch` hits only interface.py and this cpu test file.) + +`tests/diffusion/diffusion_kv/test_metadata.py:15` [minor] — test_diffusion_kv_metadata_uses_native_cache_group_block_ids (tests/diffusion/diffusion_kv/test_metadata.py:15) only round-trips literal `block_ids=([7], [11])` and `([1, 2], [5, 6])` on the new dataclasses (lines 16–38). This file imports only those DTOs; it never calls KVCacheConfig, KVCacheGroupSpec, get_block_ids(), NewRequestData.from_state, or RPC. metadata.py:6–8 defers cache geometry and block-table validation to native builders / the Worker path (W1), so the name overclaims W0 native-geometry proof. The CFG share/branch tests in the same file are the same construction-only pattern. Rename the native-geometry test, or assert at a real boundary using native block IDs. Run: `pytest -sv tests/diffusion/diffusion_kv/test_metadata.py -m 'core_model and cpu'`. (evidence: tests/diffusion/diffusion_kv/test_metadata.py:15 `def test_diffusion_kv_metadata_uses_native_cache_group_block_ids() -> None:` +tests/diffusion/diffusion_kv/test_metadata.py:16-20 `context = DiffusionKVContextMetadata(\n context_id="text",\n cache_role="cross_attention",\n num_tokens=3,\n block_ids=([7], [11]),` +tests/diffusion/diffusion_kv/test_metadata.py:27 `block_ids=([1, 2], [5, 6]),` +tests/diffusion/diffusion_kv/test_metadata.py:37-38 `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])` / `assert metadata.contexts[0].block_ids == ([7], [11])` +vllm_omni/diffusion/diffusion_kv/metadata.py:6-8 `Like native vLLM's SchedulerOutput DTOs, these classes carry trusted internal Scheduler results. Cache geometry and block-table validation stay with the native cache builders and Worker installation path.` +tests/diffusion/diffusion_kv/test_metadata.py:7-11 imports only DiffusionKVContextMetadata / DiffusionKVMetadata / DiffusionKVSequenceMetadata +tests/diffusion/diffusion_kv/test_request.py:7-12 `from vllm.v1.core.kv_cache_manager import KVCacheManager` / `KVCacheConfig, KVCacheGroupSpec` — native geometry tests exist on the request side, not in test_metadata.py) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:259` [minor] — This diff’s `test_worker_wrapper_only_extends_paged_request_rpc` invokes `WorkerWrapperBase.execute_model` with a keyword (`tests/diffusion/diffusion_kv/test_worker_contract.py:259`), and `test_dense_worker_call_does_not_extend_model_runner_signature` calls `DiffusionWorker.execute_model(req, SimpleNamespace())` with no metadata (`:288`). `test_request_rpc_sends_attached_metadata_for_only_that_request` only asserts the mocked `collective_rpc` 4-tuple (`:238-241`) and never enters the worker. The executor packs a positional 4-tuple `(req, od_config, kv_prefetch_job, metadata)` (`multiproc_executor.py:544-546`); RPC unpacks with `execute_method(method, *args, **kwargs)` then `func(*args, **kwargs)` onto `self.worker` (`diffusion_worker.py:1036`, `:1442-1443`), which is `DiffusionWorker.execute_model`, not the wrapper. No test drives `DiffusionWorker.execute_model(req, od_config, None, metadata)` and asserts the model runner received `diffusion_kv_metadata=metadata`, so the Worker→runner kwargs handoff (`:462-465`) is unlocked. Add that positional call. Run: `pytest -sv tests/diffusion/diffusion_kv/test_worker_contract.py -k 'worker_wrapper or dense_worker or request_rpc_sends' -m 'core_model and cpu'`. (evidence: tests/diffusion/diffusion_kv/test_worker_contract.py:259 `wrapper.execute_model(req, od_config, diffusion_kv_metadata=metadata)` — keyword-only wrapper call; :288 `worker.execute_model(req, SimpleNamespace())` — DiffusionWorker with no metadata; :238-241 asserts mocked collective_rpc 4-tuples only. vllm_omni/diffusion/executor/multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` then `args += (new_req.diffusion_kv_metadata,)`. vllm_omni/diffusion/worker/diffusion_worker.py:1036 `result = self.worker.execute_method(method, *args, **kwargs)`; :1442-1443 `func = getattr(self.worker, method)` / `return func(*args, **kwargs)` — RPC positional unpack onto DiffusionWorker, skipping WorkerWrapperBase.execute_model; :425-426 4th param `diffusion_kv_metadata: DiffusionKVMetadata | None = None`; :462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `output = self.model_runner.execute_model(req, **kwargs)` — Worker→runner handoff never asserted with a positional metadata arg.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.trace.json.gz new file mode 100644 index 000000000..33abf8044 Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.cost.json new file mode 100644 index 000000000..af479be69 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1086.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5610/run-20260814-184436-fe6445", + "recorded_at": "2026-08-14T19:02:42", + "attempts": [ + { + "invocation_id": "inv-8a7c003f", + "outcome": "run:run-20260814-184436-fe6445", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-184436-fe6445", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-184436-fe6445", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4039, + "minutes": 18.1, + "ci_minutes": 0, + "input_tokens": 1102518, + "output_tokens": 96576, + "tool_calls": 228, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6622, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.07, + "gate": 1.54, + "review": 1080.47, + "report": 0.06 + }, + "time_to_first_result_s": 1086.12 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 25848 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.md new file mode 100644 index 000000000..1cc6ec42f --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.md @@ -0,0 +1,43 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] sample_tokens overlap path: use_async_omni_output snapshots then returns OmniAsyncGPUModelRunnerOutput with background builder; get_output joins and re-raises (gpu_ar_model_runner.py:2077-2153, 223-230). +- [validated] Connector drain after streaming payload: _build_omni_model_runner_output_from_snapshot calls get_omni_connector_output after building multimodal_outputs (gpu_ar_model_runner.py:1874-1900); mixin documents get_omni_connector_output as sole drain of _finished_load_reqs (omni_connector_model_runner_mixin.py:1080-1082). +- [validated] should_accumulate_full_payload_output returns False when async_chunk is set (omni_connector_model_runner_mixin.py:60-61), so the self.requests accumulation branch is unreachable on the feature path as documented. +- [validated] Qwen3-Omni Thinker sets use_async_omni_output=True with has_postprocess=False; Talker sets use_async_omni_output, eager_omni_postprocess_before_async_output, omni_pooler_payload_include_hidden=False (qwen3_omni.py:109-164). Qwen3-TTS Talker matches Talker flags (qwen3_tts_talker.py:351-353). +- [validated] Code2Wav is StageExecutionType.LLM_GENERATION in both pipelines (qwen3_omni/pipeline.py:63-66, qwen3_tts/pipeline.py:36-39), so it is not GPUARModelRunner. +- [validated] NPUARModelRunner builds OmniModelRunnerOutput, drains get_omni_connector_output, then wraps AsyncGPUModelRunnerOutput (npu_ar_model_runner.py:1273-1328). +- [validated] XPUARModelRunner subclasses GPUARModelRunner (xpu_ar_model_runner.py:10); MUSA get_omni_ar_worker_cls returns GPUARWorker (musa/platform.py:27). +- [validated] Default Qwen3-Omni YAML is qwen3_omni_moe.yaml with async_chunk: true and AR stages omitting async_scheduling (defaults True via engine_args.get(..., True) at stage_config.py:941-944); Code2Wav sets async_scheduling: false. qwen3_tts.yaml matches the Talker excerpt (async_chunk true, async_scheduling true, enable_prefix_caching false). + +`docs/design/feature/prefix_caching.md:44` [minor] — Note 3 (`prefix_caching.md:44`) still says prefix caching can be enabled/disabled per-stage via `enable_prefix_caching` and never mentions async Omni output. The new page's Prefix cache compatibility note (`omni_async_output_materialization.md:326`) documents that `_should_use_async_omni_output()` returns False when `omni_prefix_cache` is present and that the fallback does not prevent serving. Cross-link that note from Note 3 so operators who enable prefix cache from this page see they will silently lose async Omni output materialization. (evidence: prefix_caching.md:44 `Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. It can be enabled/disabled per-stage via the `enable_prefix_caching` parameter in the model's stage config.` — file has zero matches for `async`. omni_async_output_materialization.md:321-331 `!!! warning` / `Do not pass `--no-async-chunk` or enable prefix caching when you want this optimization. Either change causes the runner to fall back to synchronous output construction.` then `!!! note "Prefix cache compatibility"` / `Async Omni output materialization and Omni prefix caching cannot currently run together. `_should_use_async_omni_output()` returns `False` whenever `self.omni_prefix_cache` is present, so prefix caching selects synchronous output materialization.` Compatibility table also says an unsupported combination `does not prevent serving; it only falls back to synchronous output construction`.) + +`docs/design/feature/omni_async_output_materialization.md:252` [minor] — Example 1's copy-paste `vllm serve` at line 252 is unscoped, while `_should_use_async_omni_output()` has no CUDA/ROCm guard, `XPUARModelRunner` inherits `GPUARModelRunner`, MUSA selects `GPUARWorker`, and `qwen3_omni_moe.yaml` already has `async_chunk: true` plus `enable_prefix_caching: false`. Operators copying only this command on XPU/MUSA can treat it as a validated enablement path. Scope or annotate the example so the copy-paste path matches the CUDA/ROCm validation claim. (evidence: docs/design/feature/omni_async_output_materialization.md:251-254 ```bash\nvllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \\n --omni \\n --port 8091``` — Example 1 command has no CUDA/ROCm qualifier. vllm_omni/platforms/xpu/worker/xpu_ar_model_runner.py:10 `class XPUARModelRunner(GPUARModelRunner):`. vllm_omni/platforms/xpu/worker/xpu_ar_worker.py:22 `self.model_runner: XPUARModelRunner = XPUARModelRunner(self.vllm_config, self.device)`. vllm_omni/platforms/musa/platform.py:26-27 `def get_omni_ar_worker_cls(cls) -> str:` / `return "vllm_omni.worker.gpu_ar_worker.GPUARWorker"`. vllm_omni/worker/gpu_ar_model_runner.py:1595-1619 `_should_use_async_omni_output` checks async scheduling, prefix cache, speculative, async_chunk, routed experts, and model opt-in, then `return True` with no CUDA/ROCm/platform test. vllm_omni/deploy/qwen3_omni_moe.yaml:15 `async_chunk: true`; yaml:31 and yaml:42 `enable_prefix_caching: false` on Thinker/Talker.) + +`docs/design/feature/omni_async_output_materialization.md:352` [minor] — The Compatibility section (lines 351-354) says XPU inherits GPUARModelRunner and MUSA selects GPUARWorker, so they may take the async path when the listed guards pass, with the only caveat being lack of validation. That overstates equivalence: `_clone_cuda_tensor_payload` only tracks `device.type == "cuda"` tensors, `_copy_tensor_payload_to_cpu` returns non-cuda tensors unchanged, and `_snapshot_tensor_payload_to_cpu_async` returns early with no copy-stream event when `cuda_sources` is empty. `_should_use_async_omni_output` has no platform/device-type check, so those backends can still select the feature path. Add one sentence that the copy/clone path is CUDA-device-type gated, not merely unbenchmarked on XPU/MUSA. (evidence: docs/design/feature/omni_async_output_materialization.md:351-354 `CUDA and ROCm are the validated platforms. XPU inherits \`GPUARModelRunner\`, and MUSA selects \`GPUARWorker\`, so they may select the async path when these checks pass, but this optimization is not yet validated on either backend.` | vllm_omni/worker/gpu_ar_model_runner.py:73 `if value.device.type == "cuda":` (clone tracked in sources) vs :77 `return value.detach().clone()` (non-cuda, not appended) | gpu_ar_model_runner.py:89-90 `if value.device.type != "cuda":` / `return value` (no pinned async D2H) | gpu_ar_model_runner.py:132-133 `if not cuda_sources:` / `return _AsyncCPUPayloadSnapshot(cloned, None, cuda_sources)` | gpu_ar_model_runner.py:1595-1619 `_should_use_async_omni_output` checks scheduling/prefix-cache/spec/async_chunk/routed-experts/model flags only — no device.type or platform guard | vllm_omni/platforms/xpu/worker/xpu_ar_model_runner.py:10 `class XPUARModelRunner(GPUARModelRunner):` | vllm_omni/platforms/xpu/utils.py:9-12 remaps `torch.cuda.Stream`/`current_stream` only | vllm_omni/platforms/musa/platform.py:27 `return "vllm_omni.worker.gpu_ar_worker.GPUARWorker"`) + +`docs/design/feature/omni_async_output_materialization.md:73` [nit] — Figure 1 embeds a raw HTML `src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"` at line 73. From `docs/design/feature/`, three `../` resolve to repo-root `{repo}/source/architecture/`, which does not exist; this PR adds the SVG at `docs/source/architecture/`. MkDocs can still resolve the three-`../` URL from `/design/feature//` to site-root `/source/...` (`theme.logo` is `source/logos/...`), so the published site works while GitHub blob/PR preview breaks. Sibling `async_chunk.md` uses a `raw.githubusercontent.com` URL for an already-published asset. Use a markdown image `![](../../source/architecture/qwen3-omni-async-output-step-gap.svg)` so MkDocs rewrites it and GitHub preview points at the file this PR actually adds. (evidence: docs/design/feature/omni_async_output_materialization.md:73 `src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"` — from docs/design/feature/, ../../../source/ is repo-root source/ (directory does not exist). Asset this PR adds is docs/source/architecture/qwen3-omni-async-output-step-gap.svg (two ../, not three). mkdocs.yml:16 `logo: source/logos/vllm-logo-only-light.ico`. docs/design/feature/async_chunk.md:86 `Async Chunk Pipeline Architecture`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.trace.json.gz new file mode 100644 index 000000000..821877c7c Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.cost.json new file mode 100644 index 000000000..7c6dca9b6 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1027.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5703/run-20260814-190243-6dccc1", + "recorded_at": "2026-08-14T19:19:50", + "attempts": [ + { + "invocation_id": "inv-dbeaa83e", + "outcome": "run:run-20260814-190243-6dccc1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-190243-6dccc1", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-190243-6dccc1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4953, + "minutes": 17.11, + "ci_minutes": 0, + "input_tokens": 1176944, + "output_tokens": 161418, + "tool_calls": 274, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6836, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.71, + "gate": 1.63, + "review": 1021.36, + "report": 0.06 + }, + "time_to_first_result_s": 1026.79 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 27975 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.md new file mode 100644 index 000000000..c932f1f24 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] vae.py:173/217 still seeds MINIMAX_H3_KEYFRAME_ENCODE_SEED (42) on torch.default_generator inside fork_rng. +- [validated] pipeline_minimax_h3.py:545 MiniMaxH3VideoVAE(..., device=self.device) with self.device = get_local_device(); encode_image rank0 at :963; encode_video at :1066. +- [validated] CPU encode still uses devices=[] so existing FakeModel encode_video test should not AttributeError on missing device_module. +- [validated] recipes/README.md adds the MUSA row; CUDA recipe still has a distinct Ref2VA MODEL export. +- [upstream-verify] Direct serving route truncated/unreadable (path outside slices); diffusion DIFF-1b RNG is the real owner. No gh_pr_view MCP; existing PR discussion not fetched. +- [sweep] Correctness: vae.py:170 and :214 now put every non-CPU device into fork_rng; stock torch.random.fork_rng (random.py:145) defaults device_type="cuda" and cuda/random.py:41 indexes torch.cuda.default_generators by device.index, so MUSA/NPU RNG is not the module that is saved/restored. +- [sweep] Simplifiability: encode_image and encode_video duplicate the same RNG block; autoencoder_kl_wan.py:37-38 already threads first_param.device.type into a platform helper — extract one encode RNG helper rather than documenting two copies. +- [sweep] Breaking behavior: unchanged consumers pipeline_minimax_h3.py:963 encode_image and :1066 encode_video inherit the new devices list. NPU platform npu/platform.py:163 returns torch.device("npu"); old `== "cuda"` skipped NPU, new `!= "cpu"` includes it. Other models’ encode_image/encode_video (dreamzero, wan2_2, lance) are different classes and do not inherit this hunk. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [blocker] — This hunk replaces the CUDA-only gate `parameter.device.type == "cuda"` with `parameter.device.type != "cpu"` in `encode_image` (duplicated in `encode_video` at vae.py:214), then seeds via `self.device_module.device` / `manual_seed` while `fork_rng(devices=devices)` is invoked with no `device_type`. `self.device_module` is the no-arg `torch.get_device_module()` cached in `__init__`. NPU MiniMax H3 (`NPUOmniPlatform.get_torch_device` → `torch.device("npu")`; recipe: CPU-only aarch64 PyTorch + torch_npu, no CUDA) previously took `devices=[]` and still encoded; it now enters this path, so `fork_rng` can treat the NPU index as a CUDA generator or leak seed 42 into later denoise. Gate with `parameter.device.type in ("cuda", "musa")` (same allowlist as `gpu_ar_worker.py:33`), pass `device_type=parameter.device.type` into `fork_rng`, and resolve `torch.get_device_module(parameter.device)` at encode time in one helper shared with `encode_video`. (evidence: vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:172 `with torch.random.fork_rng(devices=devices):`; vae.py:120 `self.device_module = torch.get_device_module()`; vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:25 `MINIMAX_H3_KEYFRAME_ENCODE_SEED = 42`; npu/platform.py:161-163 `def get_torch_device(...)` / `return torch.device("npu")` / `return torch.device("npu", local_rank)`; MiniMax-H3-NPU.md:18 `- Runs on a CPU-only (aarch64) PyTorch build with `torch_npu`; no CUDA`; pipeline_minimax_h3.py:963 `rows = self.video_vae.encode_image(image) if rank == 0 else None`; pipeline_minimax_h3.py:1066 `self.video_vae.encode_video(load_video_frames(item["prepared_path"])) for item in prepared_videos`; gpu_ar_worker.py:33 `if self.device_config.device_type in ("cuda", "musa"):`) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — This new recipe’s only serve snippet is introduced as the validated Ref2VA TP4 MTT S5000 configuration (lines 72–74) but then `export MODEL="${MODEL_ROOT}/FL2VA"` (line 76). The same file says FL2VA is t2va/fl2va and Ref2VA is ref2va (lines 17–19, 69–70). CUDA MiniMax-H3.md:168 and NPU MiniMax-H3-NPU.md:117 switch to `${MODEL_ROOT}/Ref2VA` for Ref2VA. Copying this block loads the T2VA/FL2VA partition for the documented Ref2VA TP4 (638.6s) profile. Change the export to `${MODEL_ROOT}/Ref2VA`, or split FL2VA vs Ref2VA commands like the CUDA/NPU recipes. (evidence: MiniMax-H3-MUSA.md:72-76 `The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs and offloads inactive model components to CPU:` then `export MODEL="${MODEL_ROOT}/FL2VA"`. MiniMax-H3-MUSA.md:17-19 ``FL2VA` serves text-to-video+audio (`t2va`) and first-frame-to-video+audio (`fl2va`).` / ``Ref2VA` serves image/audio or video-reference generation (`ref2va`).` MiniMax-H3-MUSA.md:69-70 `Set `MODEL` to `FL2VA` for T2VA and FL2VA, or to `Ref2VA` for Ref2VA.` MiniMax-H3.md:168 `export MODEL="${MODEL_ROOT}/Ref2VA"`. MiniMax-H3-NPU.md:117 `export MODEL="${MODEL_ROOT}/Ref2VA"`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — This diff adds `self.device_module` (vae.py:120) and a non-CPU `fork_rng`/`manual_seed` loop in `encode_image` (vae.py:170, 175-176) and `encode_video` (vae.py:214, 219-220), but ships no test of that path. `test_video_vae_encode_uses_configured_parallel_tiling` builds `MiniMaxH3VideoVAE` via `object.__new__` (test_minimax_h3_contract.py:344) with `torch.nn.Parameter(torch.ones(1))` on CPU (line 335), so `devices=[]` and the new loop never runs; `encode_image` has no test under tests/diffusion/models/minimax_h3. `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py` is not in the tree. Add a CPU unit test of the device-type predicate (include cuda/musa, exclude cpu/npu) that would fail on today’s `!= "cpu"` gate, keep a CPU case that skips device seeding, and drop any Validation citation of the missing vae test file. Guard with `pytest -q -c /dev/null tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py -k video_vae`. (evidence: vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; vae.py:214 and vae.py:219-220 duplicate the same block in encode_video; test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))`; test_minimax_h3_contract.py:344 `video_vae = object.__new__(MiniMaxH3VideoVAE)` — CPU Parameter so devices=[] and the new loop never runs; tests/diffusion/models/minimax_h3 has no test_minimax_h3_vae.py and no encode_image / device_module / fork_rng coverage) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — MiniMaxH3VideoVAE.__init__ now calls torch.get_device_module() unconditionally (vae.py:120). Repo-wide that is the only use; pyproject.toml dependencies are dynamic and requirements/*.txt have no torch>= pin (torch is inherited from vLLM). test_video_vae_keeps_reference_fp32_weights constructs MiniMaxH3VideoVAE on CPU, so a wheel without this API fails that CPU test at construction. Please pin or document the minimum torch that provides get_device_module, or guard the call. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — unconditional in MiniMaxH3VideoVAE.__init__. tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:319-321 `video_vae = vae_module.MiniMaxH3VideoVAE(\n "unused",\n device=torch.device("cpu"),` — CPU construction now executes that __init__ call. Repo-wide grep of get_device_module hits only vae.py:120. pyproject.toml:11 `dynamic = ["version", "dependencies"]` with no torch pin; requirements/ grep for `torch>=` returns no matches.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [nit] — Head `buildkite/vllm-omni-amd-ci` is red and logs were not opened. One note is that this MiniMax H3 VAE RNG + MUSA recipe diff does not plausibly overlap that AMD lane; another is that the red lane overlaps the unpinned `torch.get_device_module` risk. Can you confirm from the logs that this failure is independent of this change before merge? (evidence: buildkite/vllm-omni-amd-ci is red; logs were not opened.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72` [nit] — The PR body cites TP8 402.3s, but this recipe’s only serve profile is the validated Ref2VA TP4 (638.6s) snippet. Please add the TP8 command and timing here, or drop TP8 402.3s from the PR body so the capacity claim matches the documented, validated profile. (evidence: PR body TP8 402.3s is absent from the recipe (TP4 only).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.trace.json.gz new file mode 100644 index 000000000..8c316c90a Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.cost.json new file mode 100644 index 000000000..c47af00fc --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 356.4, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5715/run-20260814-190348-a26f75", + "recorded_at": "2026-08-14T19:09:44", + "attempts": [ + { + "invocation_id": "inv-f0aa3409", + "outcome": "run:run-20260814-190348-a26f75", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-190348-a26f75", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-190348-a26f75", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0561, + "minutes": 5.93, + "ci_minutes": 0, + "input_tokens": 117321, + "output_tokens": 22202, + "tool_calls": 47, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2307, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.22, + "gate": 1.58, + "review": 350.89, + "report": 0.06 + }, + "time_to_first_result_s": 355.73 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 11991 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.md new file mode 100644 index 000000000..bb382dc17 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.md @@ -0,0 +1,39 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] README.md:19 0.26.0 news: MiniMax H3 joint video/audio, MiniCPM-o 4.5 experimental full-duplex, distributed layerwise offload, even-minor cadence consolidation, skills entry removed. +- [sweep] README.md:55 and docs/README.md:55 add full-duplex last in the feature list and mark it experimental; MiniCPM-o 4.5 added under omni, MiniMax H3 under diffusion. +- [sweep] docs/configuration/README.md:5 retargets the vLLM engine options link from v0.16.0 to https://docs.vllm.ai/en/v0.26.0/configuration/index.html (only remaining docs.vllm.ai/en/v0.* URL in-repo). +- [sweep] docs/getting_started/installation/README.md:3-4 adds 0.26.x↔0.26.x compatibility important note covering the platform index (CUDA/ROCm/XPU/MUSA/NPU). +- [sweep] cuda.inc.md:8 matching-major/minor wording; cuda.inc.md:127 docker --build-arg BASE_IMAGE=vllm/vllm-openai:v0.26.0. +- [sweep] npu.inc.md:56-67 from-main pins vLLM v0.26.0, vllm-ascend releases/v0.26.0rc, omni unpinned main; cd vllm-omni replaces the old absolute /vllm-workspace/vllm-omni after git clone. +- [validated] README.md:55 full-duplex bullet is last after OpenAI-compatible API server and marked experimental, matching the PR purpose. +- [validated] docker/Dockerfile.cuda:1 ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0 matches cuda.inc.md:127. + +`README.md:?` [minor] — This diff's 0.26.0 news bullet advertises an experimental full-duplex realtime runtime for MiniCPM-o 4.5 and hyperlinks that model name to recipes/OpenBMB/MiniCPM-o-4_5.md. That recipe is omni chat/completions serving and never mentions duplex or realtime, so a reader following the new headline never reaches the experimental runtime. Point the runtime claim at examples/online_serving/minicpmo/README.md (and/or vllm_omni/experimental/fullduplex/DESIGN.md), or add a duplex subsection plus those links in the recipe; the same discovery gap exists on the new feature bullets in README.md and docs/README.md. (evidence: README.md:19: 'an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md)'. recipes/OpenBMB/MiniCPM-o-4_5.md:1-4: '# MiniCPM-o 4.5' / 'Online serving and offline inference for omni multimodal chat' (grep of that file for 'duplex' returned no matches). examples/online_serving/minicpmo/README.md:6-8: 'For the experimental native duplex runtime architecture, lifecycle invariants, capability boundary, and validation scope, see [`vllm_omni/experimental/fullduplex/DESIGN.md`](...)'. examples/online_serving/minicpmo/README.md:43: '`minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay.') + +`docs/getting_started/installation/README.md:?` [minor] — This diff adds an installation-index important note that vLLM-Omni 0.26.x requires vLLM 0.26.x. That index lists MUSA as a supported platform, and this PR left the MUSA instructions on vLLM-MUSA `v0.18.0-dev`, so the new global sentence contradicts the MUSA user path. Keep MUSA unchanged, but scope the note to upstream vLLM platforms (CUDA/ROCm/XPU/NPU) or explicitly exclude the vLLM-MUSA fork. (evidence: docs/getting_started/installation/README.md:3-4: '!!! important' / 'vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.26.x requires vLLM 0.26.x; the stable v0.26.0 instructions pin vLLM 0.26.0.' docs/getting_started/installation/README.md:8-12 still lists MUSA under GPU. docs/getting_started/installation/gpu/musa.inc.md:11: 'MUSA platform requires vLLM-MUSA to be installed first.' musa.inc.md:29-31: 'git clone https://github.com/MooreThreads/vllm-musa.git' / 'cd vllm-musa' / 'git checkout v0.18.0-dev'.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.trace.json.gz new file mode 100644 index 000000000..7183940ca Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.cost.json new file mode 100644 index 000000000..687785ad8 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1812.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5840/run-20260814-190429-62f3bf", + "recorded_at": "2026-08-14T19:34:41", + "attempts": [ + { + "invocation_id": "inv-d81d7ddf", + "outcome": "run:run-20260814-190429-62f3bf", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-190429-62f3bf", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-190429-62f3bf", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7545, + "minutes": 30.2, + "ci_minutes": 0, + "input_tokens": 1933999, + "output_tokens": 211174, + "tool_calls": 449, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9959, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.16, + "gate": 2.79, + "review": 1804.92, + "report": 0.06 + }, + "time_to_first_result_s": 1811.93 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 38631 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.md new file mode 100644 index 000000000..d1a2177cc --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] extractors.py:1273-1386 vs minimax_h3_transformer.py:1085-1196 uncached path is a structural copy; test_extractor_matches_native_forward asserts numerical parity. +- [validated] backend.py:91 apply_teacache_hook(pipeline.transformer) and test asserts transformer is not transformers_ref; pipeline_minimax_h3.py:736-739 _transformer_for_task('ref2va') uses transformers_ref so combined Ref2VA is truly uncached. +- [validated] test_teacache_reuses_h3_block_residual: identical second forward with thresh=0.3 skips blocks (block_calls==1). +- [validated] Generic default still 0.2: test_enable_uses_generic_default_threshold. +- [validated] _assert_joint_output 107 frames matches ulysses smoke and duration=4.0 T2VA contract; not an accidental shape loosen. +- [upstream-verify] OmniBase.__init__ constructs AsyncOmniEngine (omni_base.py:189), so offline Omni() hits the same cache_config normalization as vllm serve. +- [sweep] correctness: enable_minimax_h3_teacache rejects ref2va, accepts fl2va/combined, hooks only pipeline.transformer; _transformer_for_task uses transformers_ref for ref2va (pipeline_minimax_h3.py:736-739) so combined Ref2VA is actually uncached. Extractor mirrors MiniMaxH3DiTModel.forward including unexpected-kwarg, x[1,S,C], token_tags/inverse_indices length, skip_mask_out_condition, and update_audio_mask. +- [validated] extractors.py:1312-1317/1341-1384 indexed accesses match native forward at minimax_h3_transformer.py:1124-1195; blocks[0] is after the empty-blocks raise and is the TeaCache similarity probe, not a silent first-element contract. + +`vllm_omni/diffusion/cache/teacache/config.py:98` [major] — This PR registers MiniMaxH3DiTModel→0.17 in `_MODEL_DEFAULT_REL_L1_THRESH` and makes `TeaCacheConfig.rel_l1_thresh` / `DiffusionCacheConfig.rel_l1_thresh` default to None so `__post_init__` can apply that lookup. That None never arrives through Omni / `vllm serve --cache-backend tea_cache`: unchanged `AsyncOmniEngine._get_default_cache_config` still returns `{"rel_l1_thresh": 0.2}` and `_normalize_cache_config` injects it whenever `cache_config` is omitted, so `enable_minimax_h3_teacache` passes 0.2 into `TeaCacheConfig` and the 0.17 lookup is skipped. `test_minimax_h3_only_enables_fl2va_teacache` only builds `DiffusionCacheConfig()` and cannot catch this. Omit `rel_l1_thresh` from the tea_cache default dict (return `{}`) and add a test through `_normalize_cache_config` that omitted `cache_config` yields 0.17 for MiniMaxH3Pipeline. (evidence: config.py:97-98 `_MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, }`; config.py:122 `rel_l1_thresh: float | None = None`; config.py:129-130 `if threshold is None: threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)`; async_omni_engine.py:853 `if cache_backend == "tea_cache":`; async_omni_engine.py:855 `"rel_l1_thresh": 0.2,`; async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"): cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`; backend.py:86-88 `teacache_config = TeaCacheConfig(transformer_type="MiniMaxH3DiTModel", rel_l1_thresh=config.rel_l1_thresh, ...)`; omni_base.py:189 `self.engine = AsyncOmniEngine(`) + +`vllm_omni/diffusion/cache/teacache/config.py:92` [major] — This diff stores MiniMax-H3 coefficients whose constant term is `2.173782527946167e-01` (config.py:87-92) and sets `_MODEL_DEFAULT_REL_L1_THRESH["MiniMaxH3DiTModel"]=0.17` (config.py:98). `TeaCacheHook` does `self.rescale_func = np.poly1d(config.coefficients)` then `accumulated_rel_l1_distance += abs(rescale_func(rel_distance))` and skips only when that accumulator is strictly below the threshold (hook.py:63, 230-236). numpy poly1d is highest-degree-first, so `|p(0)|≈0.217` already exceeds 0.17 — identical consecutive modulated inputs cannot hit at the advertised default, which is why `test_teacache_reuses_h3_block_residual` must pass `rel_l1_thresh=0.3` to get `block_calls == 1` (test_teacache_extractors.py:686-688). Do not force `p(0)≈0`. Add a hit test at 0.17 using consecutive inputs in the calibrated L1 range and report skip counts from the 50-step 107-frame A/B at 0.17; recipes advertise 0.17 but the only in-tree skip proof uses 0.3. (evidence: config.py:87-92 `"MiniMaxH3DiTModel": [ 2.283704065852778e03, -7.775977277886368e02, 9.408414741359490e01, -4.232669906169421e00, 2.173782527946167e-01, ]`; config.py:97-98 `_MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, }`; hook.py:63 `self.rescale_func = np.poly1d(config.coefficients)`; hook.py:211-213 `if state.cnt == 0: state.accumulated_rel_l1_distance = 0.0; return True`; hook.py:230-236 `rescaled_distance = float(self.rescale_func(rel_distance)); state.accumulated_rel_l1_distance += abs(rescaled_distance); if state.accumulated_rel_l1_distance < rel_l1_thresh: return False # Use cache`; test_teacache_extractors.py:682-688 `hook = TeaCacheHook(TeaCacheConfig(transformer_type="MiniMaxH3DiTModel", rel_l1_thresh=0.3))` then two identical `hook.new_forward(...)` and `assert block_calls == 1`. |p(0)|≈0.217 is not < 0.17, so the advertised default cannot skip on identical consecutive modulated inputs.) + +`tests/diffusion/cache/test_teacache_extractors.py:696` [minor] — test_teacache_reuses_h3_block_residual (tests/diffusion/cache/test_teacache_extractors.py:670) calls hook.new_forward twice on identical packed kwargs (693–694) and only asserts `block_calls == 1` plus video/audio row counts (696–698). Cache hit does `ctx.hidden_states = ctx.hidden_states + state.previous_residual` (hook.py:147) then `return ctx.postprocess(output)` (hook.py:189); a wrong residual (broadcast/dtype/video-vs-audio mixup) would still satisfy those shape checks. The uncached path already `assert_close`s at 642–643. Capture the first-call logits and `torch.testing.assert_close` them to the second-call video/audio logits so a skip that reconstructs the wrong hidden state fails. (evidence: tests/diffusion/cache/test_teacache_extractors.py:693-698 `hook.new_forward(minimax_h3_module, **sample_inputs)` then `video_logits, audio_logits = hook.new_forward(minimax_h3_module, **sample_inputs)` then `assert block_calls == 1` / `assert video_logits.shape[0] == sample_inputs["img_pos_info"]["position_ids"].shape[0]` / `assert audio_logits.shape[0] == sample_inputs["audio_pos_info"]["position_ids"].shape[0]`. Contrast 642-643 `torch.testing.assert_close(actual_video, expected_video)` / `torch.testing.assert_close(actual_audio, expected_audio)`. hook.py:147 `ctx.hidden_states = ctx.hidden_states + state.previous_residual`; hook.py:189 `return ctx.postprocess(output)`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:170` [minor] — test_minimax_h3_t2va_teacache_single_gpu_smoke sets cache_backend="tea_cache" and cache_config={"rel_l1_thresh": 0.17} (test_minimax_h3_e2e.py:155-156) but uses num_inference_steps=2 (line 170) and only _assert_joint_output (line 186), which checks frames.shape==(107, 256, 448, 3) and audio metadata (lines 19-27) — not DiT-block reuse. TeaCacheHook._should_compute_full_transformer always returns True when state.cnt==0 (hook.py:211-213), so a 2-step run has one skip chance and can pass with zero cache hits. Keep the L4 smoke; assert an actual skip (block counter as in test_teacache_reuses_h3_block_residual) or record skip count on a longer run. (evidence: test_minimax_h3_e2e.py:155-156 `cache_backend="tea_cache", cache_config={"rel_l1_thresh": 0.17}`; test_minimax_h3_e2e.py:170 `num_inference_steps=2,`; test_minimax_h3_e2e.py:186 `_assert_joint_output(outputs)`; test_minimax_h3_e2e.py:22 `assert frames.shape == (107, 256, 448, 3)`; hook.py:211-213 `if state.cnt == 0: state.accumulated_rel_l1_distance = 0.0; return True` (comment: First timestep: always compute); hook.py:183 `state.cnt += 1` after the compute/cache decision; test_teacache_extractors.py:670 `def test_teacache_reuses_h3_block_residual` asserts `block_calls == 1` after two identical hook.new_forward calls.) + +`docs/user_guide/diffusion_features.md:162` [minor] — This diff only flips the MiniMax-H3 TeaCache cell to `✅ (FL2VA)`; the Cache-DiT cell was already `✅`. The changeset adds TeaCache only (e2e `cache_backend="tea_cache"`), and `get_cache_backend` still selects a single backend string, matching the recipe: TeaCache and Cache-DiT are mutually exclusive. Attach a this-head Cache-DiT validation artifact (command + result JSON) or drop Cache-DiT from the PR title so the merge vehicle matches the diff. (evidence: docs/user_guide/diffusion_features.md:162 `| **MiniMax-H3** | ✅ (FL2VA) | ✅ | ✅ | ❌ | ✅ (DiT/TE) | ❌ | ✅ | ✅ | ✅ (tile) | ✅ (DiT) | ❌ |` — TeaCache is FL2VA-only while Cache-DiT stays a bare ✅. recipes/MiniMaxAI/MiniMax-H3.md:760 `TeaCache and Cache-DiT are mutually exclusive; pick one cache backend per server.` vllm_omni/diffusion/cache/selector.py:37-40 `if cache_backend == "cache_dit":` / `return CacheDiTBackend(cache_config)` / `elif cache_backend == "tea_cache":` / `return TeaCacheBackend(cache_config)`. tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:155 `cache_backend="tea_cache",` — the only H3 e2e cache backend added in this PR.) + +`tests/diffusion/cache/test_cache_backends.py:?` [nit] — This diff adds MiniMax-H3 TeaCache cases to `test_cache_backends.py` (`test_enable_uses_generic_default_threshold`, `test_minimax_h3_only_enables_fl2va_teacache`, `test_minimax_h3_rejects_ref2va_teacache`), and that file’s pytestmark is `core_model`+`cpu`. AMD L2 runs `pytest tests/diffusion -m 'core_model and cpu'` while ignoring `test_teacache_extractors.py`. Gate currently reports `buildkite/vllm-omni-amd-ci` red — is that failure pre-existing on main, or from these new tests? (evidence: tests/diffusion/cache/test_cache_backends.py:32: pytestmark = [pytest.mark.core_model, pytest.mark.cpu]. tests/diffusion/cache/test_cache_backends.py:490-532: new test_minimax_h3_only_enables_fl2va_teacache and test_minimax_h3_rejects_ref2va_teacache. .buildkite/amd/test-amd-ready.yml:19: timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py. gate_report: FAILING CHECKS (1): ['buildkite/vllm-omni-amd-ci'].) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.trace.json.gz new file mode 100644 index 000000000..4563ed9aa Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5863.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5863.cost.json new file mode 100644 index 000000000..221a1f838 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5863.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 476.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5863/run-20260814-190944-190616", + "recorded_at": "2026-08-14T19:17:41", + "attempts": [ + { + "invocation_id": "inv-64bc57e5", + "outcome": "run:run-20260814-190944-190616", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-190944-190616", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-190944-190616", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0746, + "minutes": 7.94, + "ci_minutes": 0, + "input_tokens": 163213, + "output_tokens": 27782, + "tool_calls": 68, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2929, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.74, + "gate": 1.72, + "review": 470.59, + "report": 0.06 + }, + "time_to_first_result_s": 476.1 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 15057 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5863.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5863.md new file mode 100644 index 000000000..9a53ffa77 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5863.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:83-100 MODEL=.../FL2VA without --task-type: pipeline_minimax_h3.py:130-151 auto-detects partition from a FL2VA/Ref2VA directory with model_index.json and uses path.parent as model root. +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:46 `--cfg-parallel-size` must remain 1: pipeline_minimax_h3.py:551-552 raises if cfg_parallel_size != 1. +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:218-223 SM120 vs TRTLLM_ATTN: platform.py:72-73 names sm_120 as RTX Pro 6000; TRTLLM default requires compute_capability.major == 10 (platform.py:207-218); SM120 falls through to CUDNN_ATTN (platform.py:222-227). +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:301-305 t2va requires explicit aspect_ratio: pipeline_minimax_h3.py:346-349 raises OmniClientError; Ref2VA None defaults to 16:9 at line 349. +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:228-231 `--quantization fp8` cannot combine with layerwise offload: MiniMax-H3.md known limitations and vllm_omni/diffusion/data.py:969 appends layerwise offload to incompatible_features. +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:253 X-Stage-Durations exists at api_server.py:3510. + +`recipes/README.md:?` [minor] — This PR adds recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md with validated 2/4/8-GPU no-offload serve commands, but does not add a catalog row next to the existing MiniMax-H3 entries. recipes/README.md:48-49 list MiniMax-H3.md (1x offload / 4x B300) and MiniMax-H3-MUSA.md only, and MiniMax-H3.md has zero MiniMax-H3-* hardware-recipe links, so a 96 GiB 6000 user following the indexed main recipe still lands on the DLO 5090/4090 profile. Add a README row for the 6000 file (2x/4x/8x RTX PRO 6000, no offload) and a one-liner under MiniMax-H3.md “Start a server” pointing at it. (evidence: Diff adds only recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md. recipes/README.md:48: `| [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |`. recipes/README.md:49: `| [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |`. Grep MiniMax-H3- in MiniMax-H3.md: no matches. New recipe line 7: `CPU offload and distributed layerwise offload are not required in any of these configurations.`) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [minor] — This diff’s two-server layout tells readers to pass `--cpunodebind=0 --membind=0` alongside `--port 8091`, which looks like vllm serve flags. `--cpunodebind` / `--membind` are numactl options (an earlier draft of this recipe correctly used `numactl --cpunodebind=0 --membind=0 \` before `vllm serve`); vllm_omni/entrypoints/cli/serve.py has no such flags. Copying the sentence onto the four-GPU command will fail argument parsing. Rewrite as `numactl --cpunodebind=0 --membind=0 vllm serve ... --port 8091` (and the node-1 twin). (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201-204: `Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and `--port 8091` for the FL2VA server. Use `CUDA_VISIBLE_DEVICES=4,6,5,7`, `--cpunodebind=1 --membind=1`, and `--port 8092` for the Ref2VA server.` Grep `cpunodebind` in vllm_omni/entrypoints/cli/serve.py: no matches. Prior review hunk on the same file used `numactl --cpunodebind=0 --membind=0 \ ++vllm serve "${MODEL}" \`.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340` [minor] — This diff adds a Ref2VA curl that posts `audio_reference= float:\n if self._uses_native_speed_control():` +serving_speech.py:517-519 `def _uses_native_speed_control(self) -> bool:\n adapter = self._get_tts_adapter()\n return bool(adapter is not None and adapter.native_speed_control)` +serving_speech.py:508-514 `adapter_cls = resolve_adapter(self._tts_model_type)` then `if self._adapter is None or type(self._adapter) is not adapter_cls:\n ctx = SpeechServingContext(server=self, engine_client=self.engine_client)` +serving_speech.py:411-416 `instance = cls.__new__(cls)` … `instance._tts_model_type = "omnivoice"` — no `_adapter` or `engine_client` assignment (only `_init_speaker_storage()`). +tts_adapters/omnivoice.py:20-23 `@register_tts_adapter\nclass OmniVoiceAdapter(ARTTSAdapter):` … `name = "omnivoice"` +serving_speech.py:3612-3614 `except Exception as e:\n logger.exception("Diffusion speech generation failed: %s", e)\n return self._diffusion_error_response(f"Speech generation failed: {e}")` — default HTTP 500. +test_serving_speech.py:707-710 `await server._create_diffusion_speech(req)` then `mock_engine.generate.assert_called_once()` — response status unasserted.) + +`vllm_omni/entrypoints/openai/protocol/audio.py:?` [major] — This diff deletes the protocol-level `elif self.speed != 1.0: raise ValueError(...)` in `OpenAICreateSpeechRequest.validate_streaming_constraints`, so a streaming request may carry `speed` in `[0.25, 4.0]`. The replacement gate (`_validate_speech_streaming_request` at serving_speech.py:3642) is only called from the AR streaming branches (3700, 3721). `create_speech` returns `_create_diffusion_speech` first when `_diffusion_mode` (3676-3677), and that dispatcher never checks streaming+speed; it still does `CreateAudio(..., speed=self._audio_encode_speed(request))` (3600). Restore a serving-layer streaming+speed check on the diffusion dispatcher, or keep protocol rejection except when the resolved adapter sets `native_speed_control`. (evidence: vllm_omni/entrypoints/openai/protocol/audio.py:319-321 `if self.speed is None:\n self.speed = 1.0\n return self` — streaming no longer raises on speed != 1.0. vllm_omni/entrypoints/openai/serving_speech.py:3642 `if request.speed is not None and request.speed != 1.0 and not self._uses_native_speed_control():` — replacement gate. vllm_omni/entrypoints/openai/serving_speech.py:3676-3677 `if self._diffusion_mode:\n return await self._create_diffusion_speech(request)` — returns before that gate. vllm_omni/entrypoints/openai/serving_speech.py:3596-3600 `audio_obj = CreateAudio(\n audio_tensor=audio_tensor,\n sample_rate=sample_rate,\n response_format=request.response_format or "wav",\n speed=self._audio_encode_speed(request),` — diffusion applies request speed with no streaming check. tests/entrypoints/openai_api/test_serving_speech.py:2226-2227 `request = OpenAICreateSpeechRequest(input="Hello", stream=True, response_format="pcm", speed=2.0)\n assert request.speed == 2.0` — protocol now accepts the combo.) + +`docs/serving/speech_api.md:?` [major] — This PR deletes the schema-level HTTP streaming speed ban (`OpenAICreateSpeechRequest.validate_streaming_constraints` at protocol/audio.py:319-321 only defaults a missing speed) and replaces it with serving_speech.py:3642 `if request.speed is not None and request.speed != 1.0 and not self._uses_native_speed_control()`. That helper is True only for adapters with `native_speed_control` (base.py:119 default False; indextts2.py:276 True on IndexTTS25Adapter only). docs/serving/speech_api.md:150 still says every stream requires speed=1.0, and Supported Models (line 693) never mentions IndexTTS-2.5. Rewrite that contract to: (1) HTTP stream speed≠1.0 is allowed only for native-speed adapters (today IndexTTS 2.5), (2) every other TTS model still 4xx, (3) WebSocket `stream_audio=true` still requires speed=1.0 (protocol/audio.py:558-561), (4) list IndexTTS-2.5. Leave user_guide/.../text_to_speech.md:422 — Qwen3TTSAdapter does not set `native_speed_control`. (evidence: docs/serving/speech_api.md:150 `response_format="pcm"` or `"wav"`, and `speed` must be `1.0` (or omitted). docs/serving/speech_api.md:693-695 `## Supported Models` / `### Qwen3-TTS` — grep IndexTTS in this file: no matches. protocol/audio.py:319-321 `if self.speed is None:` / `self.speed = 1.0` / `return self` — no speed≠1.0 raise on HTTP OpenAICreateSpeechRequest. serving_speech.py:3642-3645 `if request.speed is not None and request.speed != 1.0 and not self._uses_native_speed_control():` / `return response_format, self.create_error_response(` / `f"{mode_label} is not supported with speed adjustment. "`. tts_adapters/base.py:119 `native_speed_control: ClassVar[bool] = False`. tts_adapters/indextts2.py:276 `native_speed_control: ClassVar[bool] = True` on IndexTTS25Adapter only. protocol/audio.py:558-561 `elif self.speed != 1.0:` / `raise ValueError(` / `"WebSocket stream_audio=true currently requires speed=1.0; "` / `"native speed control is only available through HTTP streaming."`. docs/user_guide/examples/online_serving/text_to_speech.md:422 ``speed` is not supported when streaming.`) + +`recipes/IndexTeam/IndexTTS-2_5.md:40` [minor] — The recipe heading (`### 1x NVIDIA H20 96GB`) and recipes/README.md hardware column both advertise H20 96GB, but Notes say the standard recipe was exercised on one NVIDIA H200. Align heading, README row, and Notes to the GPU that was actually used so the support matrix is not a different SKU from the evidence. (evidence: recipes/IndexTeam/IndexTTS-2_5.md:40 `### 1x NVIDIA H20 96GB` — recipes/IndexTeam/IndexTTS-2_5.md:290 `- Hardware scope: the standard recipe has been exercised on one NVIDIA H200.` — recipes/README.md:53 `| [`IndexTeam/IndexTTS-2_5.md`](./IndexTeam/IndexTTS-2_5.md) | Online + offline multilingual voice-cloned TTS with native speed and emotion control | 1x NVIDIA H20 96GB |`) + +`vllm_omni/entrypoints/openai/serving_chat.py:87` [minor] — This PR's Test Plan still lists vLLM 0.26.0 E2E, but XPU CI now pins v0.27.0 (`.buildkite/intel/pipeline-intel.yml:13`, `docker/Dockerfile.xpu:91`) while the same change adds 0.26/0.27 shims (`serving_chat.py:87-91` `MistralToolCall` fallback; `request.py:44-47` `num_stale_output_tokens`; `api_server.py:1001-1003` optional `OnlineRenderer.warmup`) and ungated renderer APIs (`stage_init_utils.py:23` `BaseRenderer`, `stage_init_utils.py:1141-1144` `InputProcessor(..., renderer=)`). State the oldest supported vLLM and align the Test Plan and XPU pin to it. (evidence: PR #5957 Test Plan: `**vLLM Version:** 0.26.0`. serving_chat.py:87-91 `try:\n from vllm.parser.mistral import MistralToolCall\nexcept ImportError:\n # Accelerator images can lag the core vLLM release by one version.\n from vllm.tool_parsers.mistral_tool_parser import MistralToolCall`. .buildkite/intel/pipeline-intel.yml:13 `VLLM_VERSION: "v0.27.0"`. docker/Dockerfile.xpu:91 `ARG VLLM_VERSION=v0.27.0`. stage_init_utils.py:23 `from vllm.renderers import BaseRenderer`. stage_init_utils.py:1141-1144 `input_processor = InputProcessor(\n vllm_config=stage_vllm_config,\n renderer=_build_token_only_renderer(stage_vllm_config),\n )`. request.py:44-47 `# vLLM 0.27 owns this counter; accelerator images still based on 0.26\n # do not. Keep the Omni scheduler's stale-output drain compatible with\n # both request layouts until those images move to 0.27.`) + +`gpu_ar_model_runner.py:1723` [minor] — The new IndexTTS 2.5 `omni_payload_at_request_end` defer-D2H path still force-sends retained `_pending_full_payload_send` entries when `rid` is in `finished_req_ids` or `rid not in self.requests` (`gpu_ar_model_runner.py:1058-1062` → `flush_full_payload_outputs` → `send_full_payload_outputs`). Abort is not given a drop path; it still rides that pre-existing finished/not-in-requests keying. Confirm whether abort should skip this force-send for the retain-until-end path. (evidence: gpu_ar_model_runner.py:1723-1728 `def _should_defer_full_payload_d2h(self) -> bool:` / `return (` / `getattr(self, "omni_prefix_cache", None) is None` / `and self._runner_model_omni_flag("omni_payload_at_request_end")` / `and self._should_accumulate_full_payload_output()` — defer-D2H is the new retain path. indextts2_talker.py:171 `self.omni_payload_at_request_end = self.config.model_type == "indextts2_5"` — only IndexTTS 2.5 sets the flag. gpu_ar_model_runner.py:1058-1062 `if self._pending_full_payload_send:` / `flush_ids = set(getattr(scheduler_output, "finished_req_ids", set()))` / `flush_ids.update({rid for rid in self._pending_full_payload_send if rid not in self.requests})` / `if flush_ids:` / `self.flush_full_payload_outputs(flush_ids)` — abort/finish still force-flushes. omni_connector_model_runner_mixin.py:964-970 `for req_id in finished_req_ids:` / `entry = self._pending_full_payload_send.pop(req_id, None)` / `if entry is not None:` / `to_send[req_id] = self._materialize_full_payload_entry(entry)` / `if to_send:` / `self.send_full_payload_outputs(scheduler_output=None, outputs=to_send)` — pop then send, not record-without-post. gpu_model_runner.py:481-482 `for req_id in scheduler_output.finished_req_ids:` / `self.requests.pop(req_id, None)` and 496-498 `There could be an edge case where finished_req_ids and scheduled_req_ids overlap. This happens when a request is aborted` — abort is keyed off finished/not-in-requests.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:237` [minor] `_normalize_with_official_backend` treats missing `nemo_text_processing` as a silent Spanish no-op (`return text` at text_processing_v2_5.py:237), while missing Japanese `fugashi` raises with `pip install 'vllm-omni[inde +- `vllm_omni/deploy/qwen2_5_omni.yaml:90` [minor] This IndexTTS-2.5 PR also changes every default Qwen2.5-Omni XPU deploy: `vllm_omni/deploy/qwen2_5_omni.yaml:90-93` flips stages 0/1 to `enforce_eager: true` (replacing the previous XPU cudagraph overlay). `pipeline.py:2 +- `vllm_omni/model_executor/models/common/qwen3_code_predictor.py:577` [minor] This PR adds `current_omni_platform.is_xpu()` to shared `CodePredictorWrapper._setup_compile` (`qwen3_code_predictor.py:577`), so Qwen3-Omni MTP (`Qwen3OmniMoeTalkerCodePredictor`) also stays eager on XPU. The new test o +- `vllm_omni/model_executor/models/indextts2/s2mel/modules/gpt_fast/model.py:276` [minor] This PR adds a packed varlen FlashAttention path in gpt_fast Attention gated on `q.device.type == "cuda"`, non-fp32 dtype, and `_forward_varlen_packed`. The new CPU test `test_indextts_attention_keeps_padding_metadata_mo + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5957.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5957.trace.json.gz new file mode 100644 index 000000000..d99cb850c Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.cost.json new file mode 100644 index 000000000..b8be77016 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1379.9, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5976/run-20260814-193442-9e26dc", + "recorded_at": "2026-08-14T19:57:41", + "attempts": [ + { + "invocation_id": "inv-e0013cc2", + "outcome": "run:run-20260814-193442-9e26dc", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-193442-9e26dc", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-193442-9e26dc", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7628, + "minutes": 22.99, + "ci_minutes": 0, + "input_tokens": 1910262, + "output_tokens": 224531, + "tool_calls": 444, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8922, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.6, + "gate": 2.48, + "review": 1372.13, + "report": 0.07 + }, + "time_to_first_result_s": 1379.26 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 66665 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.md new file mode 100644 index 000000000..637cf36a3 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] 1 correctness: AR _update_request_as_session += in_flight (omni_ar_scheduler.py:627-629) then may call mixin _replace_streaming_session which += the same in_flight (omni_scheduler_mixin.py:105); AR segment-stop also += (omni_ar_scheduler.py:490-491). Drain subtracts per-frame num_tokens_scheduled (336-339) so leftover stale swallows new-segment frames. Generation scheduler only seeds via mixin (omni_generation_scheduler.py:604) — no double-add there. +- [sweep] 2 simplifiability: three += sites for one in-flight set; DELETE the AR pre-mixin increment (and do not += again at session replace after segment-stop). Assign or seed once. async_tokens_to_discard is written (479, 634) and never read in omni after this rebase — dead, not posted. +- [sweep] 3 breaking: ModelRegistry.register_model now overwrites every omni arch (arg_utils.py:115-116). Qwen3 thinker forward returns a bare tensor without capture (qwen3_omni_moe_thinker.py:1516-1521); combined make_omni_output accepts both shapes (qwen3_omni.py:511-518). Qwen2.5 combined already branched on tuple vs tensor (qwen2_5_omni.py:320-323). FusedMoE DiT still calls vllm_omni.diffusion.layers.fused_moe.FusedMoE, which now constructs FusedMoEFactory (fused_moe.py:78-80). +- [sweep] 4 rebase damage: no dropped FusedMoEFactory consumer in hunyuan_image3.py / ming_flash_omni; hunyuan_image3_transformer.py uses the omni wrapper. is_interleaved and copy_missing_attrs are re-homes of removed upstream helpers. +- [sweep] 5 tests: test_omni_ar_scheduler_stale_drain.py sets stage_id=0 and calls _update_request_as_session without replace_streaming_prompt (71, 84), so mixin replace and stop-then-replace never run. Qwen3 forward-contract tests bind real thinker.forward / make_omni_output. replica cache tests hit scoped_spawn_device_env. SSIM 0.96 fallback is documented for SDPA-only images. +- [sweep] 6 docs: inductor patch, replica cache, and Qwen3 is_staged_run comments match the new code. Generation drain comment claims segment-stop seeds; generation stop path (461-470) does not seed — stale comment only, not posted. +- [sweep] 7 invariants: drain assumes seed units == num_tokens_scheduled; that holds only if each in-flight token is counted once. Overlapping += breaks it. replica_id 0 is skipped via `if not replica_id` (stage_engine_startup.py:936). +- [sweep] 8 scope: PR discloses seed-TTS WER still open after this drain fix (build 2957 0.3916). Overlapping += on Qwen2.5/Qwen3 talker (LLM_AR, pipeline.py:40 / qwen3 pipeline.py:50) is a remaining mechanism for that hole, not a separate omitted bug list. + +`vllm_omni/core/sched/omni_ar_scheduler.py:629` [major] — OmniARScheduler._update_request_as_session seeds `session.num_stale_output_tokens += in_flight_tokens` (omni_ar_scheduler.py:627-629) before `if replace_streaming_prompt: self._replace_streaming_session(...)` (669-671). The mixin then does `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` (omni_scheduler_mixin.py:105) without clearing `num_in_flight_tokens`, so in-flight=1 becomes stale=2. MiniCPM-o 4.5 Talker is LLM_AR (minicpmo_4_5/pipeline.py:55-58) and sets `handoff_meta["replace_streaming_prompt"] = True` on turn/handoff (minicpmo_4_5_omni.py:953-954). Drain then marks leftover stale and `continue`s (omni_ar_scheduler.py:336-353), swallowing the new turn's first valid talker frame — the codec hole this PR is fixing. `test_omni_ar_scheduler_stale_drain.py` uses `stage_id=0` and a bare StreamingUpdate (no replace flag), so it only hits the stage-0 extend path. Seed in one place (mixin or AR, not both +=) and add a case with `replace_streaming_prompt=True` and `in_flight>0` asserting stale==in-flight once and the new-segment frame is delivered. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0:` / `session.num_stale_output_tokens += in_flight_tokens` — unconditional seed at the start of `_update_request_as_session`. vllm_omni/core/sched/omni_ar_scheduler.py:669-671 `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)` / `return` — same call then invokes the mixin without clearing in-flight. vllm_omni/core/sched/omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` — second += of the same counter. vllm_omni/core/sched/omni_ar_scheduler.py:336-339,348-353 `if request is not None and request.num_stale_output_tokens > 0:` / `output_is_stale = True` / `request.num_stale_output_tokens -= num_tokens_scheduled` then `if output_is_stale: continue` — leftover stale swallows the next frame. vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py:55-58 Talker `stage_id=1` / `execution_type=StageExecutionType.LLM_AR`. vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:953-954 `if not is_native_duplex_handoff or native_turn_start:` / `handoff_meta["replace_streaming_prompt"] = True`. tests/core/sched/test_omni_ar_scheduler_stale_drain.py:73 `sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=0))` — tests never set replace_streaming_prompt.) + +`vllm_omni/worker/gpu_generation_model_runner.py:580` [major] — GPU generation `_dummy_run` now accepts `randomize_inputs` for vLLM 0.27 `kernel_warmup` compat (`gpu_generation_model_runner.py:580`) but still calls `maybe_randomize_inputs` without forwarding it (`:847`) — intentional for codec dummy ids. The NPU overrides this PR already touched do not accept the kwarg: `OmniNPUModelRunner._dummy_run` (`npu_model_runner.py:112-127`) and NPU generation `_dummy_run` (`npu_generation_model_runner.py:641-656`) both end at `profile_cpp` with no `randomize_inputs` and no `**kwargs` (grep of `vllm_omni/platforms/npu` is empty). The same keyword call that TypeError'd CUDA in build 2952 will TypeError on NPU. Accept-and-ignore `randomize_inputs` on both NPU `_dummy_run`s (do not forward into `maybe_randomize_inputs`); `tests/worker` has no `randomize_inputs` coverage. (evidence: gpu_generation_model_runner.py:580 ` randomize_inputs: bool = False,`; gpu_generation_model_runner.py:847 ` self.maybe_randomize_inputs(input_ids, inputs_embeds),`; npu_model_runner.py:112-127 `def _dummy_run(self, num_tokens: int, ... profile_seq_lens: int | None = None, profile_cpp: bool = False,)` — no `randomize_inputs`, no `**kwargs`; npu_generation_model_runner.py:641 ` def _dummy_run(` and :656 ` profile_cpp: bool = False,`; grep `vllm_omni/platforms/npu` for `randomize_inputs`: no matches; grep `tests/worker` for `randomize_inputs`: no matches.) + +`vllm_omni/engine/arg_utils.py:116` [major] — This hunk drops the `if arch not in supported_archs` guard and unconditionally `ModelRegistry.register_model`s every `_OMNI_MODELS` entry on the `vllm.general_plugins` path (`pyproject.toml:148-149` → `arg_utils.py:107-116`). Installing vllm-omni now changes stock `vllm serve` for overlapping upstream archs: `Qwen2_5OmniForConditionalGeneration` is in `_OMNI_MODELS` (`registry.py:9-12`) and constructs the omni wrapper with `model_stage` defaulting to `"thinker"` and `self.talker = None` (`qwen2_5_omni.py:86-97`), so audio is gone on that entrypoint. Other staged constructors still read bare `model_config.model_stage` and AttributeError on a plain ModelConfig — `MiniCPMO45OmniForConditionalGeneration` (`minicpmo_4_5_omni.py:94`) then raises `ValueError` unless the stage is `'llm'` or `'tts'` (`:123`). Overwrite only the 0.27 archs that need omni features (the Qwen3OmniMoe* realtime gap), or give every remaining `.model_stage` reader a documented getattr default that actually builds a valid submodule. (evidence: pyproject.toml:148-149 `[project.entry-points."vllm.general_plugins"]` / `vllm_omni_register_models = "vllm_omni.engine.arg_utils:register_omni_models_to_vllm"`. arg_utils.py:107-116 `# Unconditionally (re)register every omni arch into the upstream global ModelRegistry` then `for arch, (mod_folder, mod_relname, cls_name) in _OMNI_MODELS.items():` / `ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")`. registry.py:9-12 `_OMNI_MODELS = { "Qwen2_5OmniForConditionalGeneration": ( "qwen2_5_omni", "qwen2_5_omni", "Qwen2_5OmniForConditionalGeneration", ),`. qwen2_5_omni.py:86-97 `self.model_stage = getattr(vllm_config.model_config, "model_stage", None) or "thinker"` / `if self.model_stage == "thinker":` / `self.talker = None`. minicpmo_4_5_omni.py:94 `self.model_stage = vllm_config.model_config.model_stage` ; :123 `raise ValueError(f"Invalid model stage: {self.model_stage}. Must be one of: 'llm', 'tts'")`.) + +`test_quantization_quality.py:525` [minor] — Quality tests now pass `enforce_eager=True` on both Omni arms (`test_quantization_quality.py:525` and `:537`). Comments at 518–523 make compile explicitly out of scope for LPIPS, so the quantization-quality job that originally hit inductor CantSplit on FLUX fp8 no longer compile-proves `_patch_inductor_factorable_divisibility`. CPU tests in `tests/diffusion/test_inductor_divisibility_patch.py` cover the exact SizeVarAllocator expression; GPU compile-path coverage of FLUX fp8 is no longer in this job. (evidence: tests/diffusion/quantization/test_quantization_quality.py:518-525 `# Run both arms eager: LPIPS must measure quantization only. With compile / on, a compiler failure in one arm (e.g. inductor's CantSplit on fp8 FLUX) / silently drops that arm to eager while the other stays compiled` then `bl_kwargs: dict = {"model": config.baseline_ref(), "enforce_eager": True}`; :537 `qt_kwargs: dict = {"model": config.quantized_ref(), "enforce_eager": True}` — both quality arms skip compile. tests/diffusion/test_inductor_divisibility_patch.py:59 `assert method(sv, 15360 * _S1 + 15360 * _S2, _S1 + _S2) is True` — replacement coverage is the CPU SizeVarAllocator unit test, not a compiled FLUX fp8 run.) + +`vllm_omni/core/sched/omni_generation_scheduler.py:385` [minor] — OmniGenerationScheduler.update_from_output drains `num_stale_output_tokens` (`:385–387`) but this file never seeds it. AR seeds at resumable segment-stop (`omni_ar_scheduler.py:490–491`: `request.num_stale_output_tokens += request.num_in_flight_tokens`). Generation session-replace only calls mixin `_replace_streaming_session` (`omni_scheduler_mixin.py:105`). Generation's resumable stop (`_handle_stopped_request` re-enqueues chunk receivers; the `if not finished` arm only `segment_finished_requests.discard`) does not seed. MiniCPM-o Code2Wav is `LLM_GENERATION` (`minicpmo_4_5/pipeline.py:70`). A resumable segment stop with leftover in-flight tokens then has a drain with nothing to drain and will append the late frame. Mirror the AR segment-stop seed, or document that generation never has in-flight at stop and assert it. (evidence: vllm_omni/core/sched/omni_generation_scheduler.py:385-387 `if request is not None and request.num_stale_output_tokens > 0:` / `output_is_stale = True` / `request.num_stale_output_tokens -= num_tokens_scheduled` — drain only; no `num_stale_output_tokens +=` in this file. vllm_omni/core/sched/omni_ar_scheduler.py:490-491 `if request.num_in_flight_tokens > 0:` / `request.num_stale_output_tokens += request.num_in_flight_tokens` — AR segment-stop seed. OmniGenerationScheduler._update_request_as_session body is only `self._replace_streaming_session(session, update)`. vllm_omni/core/sched/omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`. Generation `_handle_stopped_request`: `request.resumable and not request.streaming_queue and self.chunk_transfer_adapter is not None and self.chunk_transfer_adapter.receives_chunks` → `request.status = RequestStatus.WAITING` / `_enqueue_waiting_request(request)` / `return False` with no stale seed. vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py:70 `execution_type=StageExecutionType.LLM_GENERATION`.) + +`vllm_omni/platforms/xpu/platform.py:139` [minor] — This hunk changes `XPUOmniPlatform.get_default_ir_op_priority` from `xpu_kernels` to `vllm_c` for every non-inductor XPU config (`platform.py:129–139`); there is no Hunyuan/Qwen (or other model) gate. CUDA already prefers `vllm_c` (`cuda/platform.py:314–315`), but this PR adds no XPU IR test: `tests/platforms/test_cuda_ir_op_priority.py` is CUDA-only (module-level skip) and `tests/platforms` has no XPU IR file (`XPUOmniPlatform` has zero test matches). Scope the default to the families you measured, or add an XPU unit test mirroring the CUDA IR assertions. (evidence: vllm_omni/platforms/xpu/platform.py:129-139 `def get_default_ir_op_priority(cls, vllm_config: VllmConfig) -> IrOpPriorityConfig:` / `default = ["native"] if using_inductor else ["vllm_c", "native"]` — inductor vs native only, no model-family gate. vllm_omni/platforms/cuda/platform.py:314-315 `"""Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops."""` / `default = ["vllm_c", "native"]`. tests/platforms/test_cuda_ir_op_priority.py:13-14 `if not current_platform.is_cuda():` / `pytest.skip("CUDA-only IR op priority tests", allow_module_level=True)`. tests/platforms/test_cuda_ir_op_priority.py:32-35 `def test_cuda_default_ir_op_priority_prefers_vllm_c_when_inductor_backend` / `priority = CudaOmniPlatform.get_default_ir_op_priority(_vllm_config(mode=mode))` / `assert priority.rms_norm == ["vllm_c", "native"]`. tests/platforms listing: npu/, test_cuda_ir_op_priority.py, test_record_device_event.py — no XPU IR test; grep tests for `XPUOmniPlatform` has no matches.) + +`vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py:312` [minor] — `Qwen2_5OmniVideoProcessorItems.get_processor_data` now builds items via `self.get(index)` (`qwen2_5_omni_thinker.py:312`) so MediaWithBytes unwraps before the HF processor. `tests/` has zero `Qwen2_5OmniVideoProcessorItems` or `get_processor_data` hits; the only qwen2_5_omni test change is `mm_hasher_algorithm="sha256"` on a hasher fake in `test_per_video_audio_mask.py:294`. Add a CPU unit test that plants a wrapper in the items, implements `get()` to return `(frames, metadata)`, and fails if `convert_pil_frames_to_video` would see the wrapper. (evidence: qwen2_5_omni_thinker.py:296 `def get_processor_data(self) -> Mapping[str, object]:`; qwen2_5_omni_thinker.py:312 `items = [self.get(index) for index in range(self.get_count())]`. tests/ grep for Qwen2_5OmniVideoProcessorItems and get_processor_data: no matches. tests/model_executor/models/qwen2_5_omni/ contains only test_audio_length.py, test_per_video_audio_mask.py, test_qwen2_5_omni_embed.py, test_weight_loading.py. test_per_video_audio_mask.py:294 `mm_hasher_algorithm="sha256"`.) + +`vllm_omni/platforms/xpu/platform.py:139` [nit] — intel-ci is red on this head, and gate_report also lists buildkite/vllm-omni-amd-ci, npu-ci, and rebase as red, with attribution vs main unknown. Which of these red checks are this PR versus inherited from main? (evidence: [sweep 9 blast] XPU get_default_ir_op_priority default flipped xpu_kernels -> vllm_c for every non-inductor XPU model (xpu/platform.py:139). ... intel-ci is red. | gate_report lists buildkite/vllm-omni-amd-ci, intel-ci, npu-ci, rebase red; attribution vs main is unknown.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.trace.json.gz new file mode 100644 index 000000000..76fba3193 Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.cost.json b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.cost.json new file mode 100644 index 000000000..1f1d8c49c --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1330.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5509/run-20260814-224628-d8debb", + "recorded_at": "2026-08-14T23:08:37", + "attempts": [ + { + "invocation_id": "inv-ec25b59c", + "outcome": "run:run-20260814-224628-d8debb", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-224628-d8debb", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-224628-d8debb", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6827, + "minutes": 22.15, + "ci_minutes": 0, + "input_tokens": 1849593, + "output_tokens": 166619, + "tool_calls": 309, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8479, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.7, + "gate": 1.79, + "review": 1323.57, + "report": 0.07 + }, + "time_to_first_result_s": 1329.1 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 37935 +} \ No newline at end of file diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.md b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.md new file mode 100644 index 000000000..5adcb90b6 --- /dev/null +++ b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.md @@ -0,0 +1,57 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] test_trtllm_attn.py:315 Missing TRTLLM-GEN kernel skip: reachable on SM103 for int8; string-gated and re-raises other RuntimeErrors. +- [sweep] trtllm_attn.py:90 HAS_FLASHINFER False → _sage_kernel_available False → init raises if quant.enabled; dense path unchanged. +- [sweep] trtllm_attn.py:198 quant.enabled is dtype_qk is not None; empty quant stays dense. +- [sweep] trtllm_attn.py:199 dtype not in {int8,fp8_e4m3} raises RuntimeError; covered by test_quant_rejects_non_sage_dtype. +- [sweep] trtllm_attn.py:204 kernel signature missing sage_attn_sfs raises; no dedicated unit test (only quantize-fn None is mocked). +- [sweep] trtllm_attn.py:211 missing trtllm_sage_attention_quantize raises; covered by test_quant_quantize_requires_flashinfer_routine. +- [sweep] trtllm_attn.py:296 forward SAGE kwargs only when quant.enabled after init resolved the fn; dense path omits unknown kwargs (compat with older FlashInfer). +- [sweep] data.py:1415/1418 AttnQuantSpec rejects int4 and k_block_size=8; covered by test_quant_validation_rejects. + +`vllm_omni/diffusion/data.py:1447` [major] — This diff allowlists `quant` on FLASHINFER_ATTN as well as TRTLLM_ATTN (`data.py:1447`) and the new docs say FLASHINFER_ATTN reads `dtype_qk` (float16/bfloat16), `dtype_vo`, and `flashinfer_backend` and that a wrong-set dtype raises (`attention_backends.md:185-188`); `test_quant_superset_fields_passed_through` only round-trips that dict. FlashInferAttentionImpl never consumes it (`flashinfer_attn.py:55-66` stores only causal/softmax_scale via `**extra_impl_args`; `forward_cuda` always calls dense `single_prefill_with_kv_cache` at line 178), while `layer.py:109` still passes `backend_kwargs=` into every impl, so FLASHINFER_ATTN+quant accepts, serializes, and silently runs dense — unlike skip_softmax (fail-loud on non-TRTLLM at `data.py:1442`) and SDPAImpl (warns on ignored kwargs at `sdpa.py:89-90`). There is no FLASHINFER quant section despite “see its section”, and `AttnQuantSpec._VALID_DTYPES` is a union (`data.py:1410`) with no per-backend reject. Restrict `quant` to TRTLLM_ATTN, drop `dtype_vo`/`flashinfer_backend`/float16/bfloat16 from AttnQuantSpec (replace the superset round-trip with a rejection test), or implement the consumer and reject leftover fields per backend at AttentionSpec.__post_init__. (evidence: data.py:1447 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):`; data.py:1410 `_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})`; attention_backends.md:185-188 `The quant block is a shared AttnQuantSpec consumed by more than one backend; each reads the fields that apply and rejects the rest. ... FLASHINFER_ATTN reads dtype_qk (float16 / bfloat16), dtype_vo, and flashinfer_backend (see its section). A dtype_qk from the wrong set raises.`; flashinfer_attn.py:55-66 `__init__(..., prefix: str = "", **extra_impl_args)` then only `self.causal = causal` / `self.softmax_scale = softmax_scale` — no backend_kwargs/quant use; flashinfer_attn.py:178 `out = single_prefill_with_kv_cache(query[b], key[b], value[b], **kwargs)` with kwargs only sm_scale/causal/return_lse/custom_mask; layer.py:109 `backend_kwargs=backend_kwargs,`; sdpa.py:89-90 `if backend_kwargs: logger.warning("SDPAImpl ignoring backend_kwargs: %s", list(backend_kwargs.keys()))`; data.py:1442 `if self.skip_softmax is not None and self.backend.upper() != "TRTLLM_ATTN":` fail-loud contrast.) + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — This SAGE insert also deletes `## Known Limitations` (LTX-2 CUDNN_ATTN+torch.compile crash / #3121, FA4 note), mixed-backend, Validation Guidance, and `python benchmarks/diffusion/bench_attention_backends.py --preset hv15` reproduction. Line 232 still says “see Known Limitations above”, and that bench command still exists in the script. Restore those unrelated sections, or rewrite the dangling xref and keep the LTX-2 workaround; SAGE docs should not ship by dropping them. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — this file’s remaining headings are Overview, Backend Options, Configuration, Platform Defaults, TRTLLM_ATTN Backend and Skip-Softmax, TRTLLM_ATTN SAGE Quantization, End-to-End Benchmark, Choosing a Backend Manually, SageAttention Installation, SageAttention3 Installation, HuggingFace Kernels Hub Backends, Usage Examples (no `## Known Limitations`). benchmarks/diffusion/bench_attention_backends.py:15 `python benchmarks/diffusion/bench_attention_backends.py --preset hv15`) + +`tests/diffusion/attention/test_trtllm_attn.py:50` [minor] — These four new tests (`test_quant_config_from_backend_kwargs` L50, `test_quant_rejects_non_sage_dtype` L60, `test_quant_quantize_requires_flashinfer_routine` L65, `test_quant_quantize_calls_routine_and_shapes_sfs` L74) are CPU-pure QuantConfig / TrtllmAttentionImpl.__init__ error-path checks. The file is module-marked cuda (`pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` at L13) under tests/diffusion/attention/. L2 CPU (`pytest tests/diffusion -m 'core_model and cpu'` in .buildkite/cuda/test-ready.yml:18) collects then deselects the whole file; L2 CUDA (`pytest tests/diffusion/*.py -m 'core_model and cuda'` at test-ready.yml:68) is a non-recursive glob and never collects this subdirectory. `.buildkite` has no `tests/diffusion/attention` job. `test_attention_config.py` is already cpu-marked (L34) and only covers AttentionSpec.quant serialization, not QuantConfig.from_backend_kwargs / impl init. Move the four tests there so L2 actually executes those contracts; a green ready pipeline is not SAGE kernel coverage (`test_sage_quant_matches_sdpa` L306). (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]`. tests/diffusion/attention/test_trtllm_attn.py:50 `def test_quant_config_from_backend_kwargs():`. tests/diffusion/attention/test_trtllm_attn.py:60 `def test_quant_rejects_non_sage_dtype():`. tests/diffusion/attention/test_trtllm_attn.py:65 `def test_quant_quantize_requires_flashinfer_routine(monkeypatch):`. tests/diffusion/attention/test_trtllm_attn.py:74 `def test_quant_quantize_calls_routine_and_shapes_sfs():`. tests/diffusion/attention/test_attention_config.py:34 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]`. .buildkite/cuda/test-ready.yml:18 `timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' ...`. .buildkite/cuda/test-ready.yml:68 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"`. Grep of .buildkite for `tests/diffusion/attention` returns no matches. vllm_omni/diffusion/attention/backends/trtllm_attn.py:76-79 flashinfer import is try/except (`HAS_FLASHINFER = True` / `False`); QuantConfig.from_backend_kwargs and TrtllmAttentionImpl.__init__ quant checks do not touch CUDA.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:211` [minor] — vllm-omni does not pin flashinfer (absent from pyproject.toml and requirements/{common,cuda}.txt). Docs require FlashInfer >=0.6.16rc1 for SAGE, but a missing trtllm_sage_attention_quantize only raises in TrtllmAttentionImpl.__init__. Dense TRTLLM stays compatible because sage kwargs are passed only when quant is enabled. Consider declaring that lower bound (or an extra) so SAGE enablement is not discoverable only as an init RuntimeError. (evidence: docs/user_guide/diffusion/attention_backends.md:208 `**Requirements.** Needs FlashInfer ≥ 0.6.16rc1.` — docs floor this repo does not install-pin. requirements/cuda.txt:1-3 `-r common.txt` / `onnxruntime>=1.23.2` / `fa3-fwd==0.0.3` and requirements/common.txt:12-36 (`transformers >= 5.5.3` … `gguf>=0.10.0`) contain no flashinfer entry; pyproject.toml:[project.optional-dependencies] likewise has none. trtllm_attn.py:211-216 `if self._sage_quantize_fn is None: raise RuntimeError("TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build lacks trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade FlashInfer, or remove the quant config.")` — SAGE-only init failure. trtllm_attn.py:292-296 `sage_kwargs: dict = {}` / `if self.quant.enabled:` — dense/oldest FlashInfer path never passes the new kwargs.) + +`tests/diffusion/attention/test_trtllm_attn.py:321` [minor] — The only in-repo SAGE quality gate is skip-gated kernel test `test_sage_quant_matches_sdpa` (`rel < 0.1` vs dense `rel < 0.01`). There is no `pytest.mark.L4` / e2e / `QUALITY_CONFIGS` LPIPS case for `TRTLLM_ATTN` quant, and SAGE+skip_softmax is kwargs-only (`test_quant_and_skip_softmax_coexist`). If PR-body LPIPS is the intended bar for this lossy path, add an L4 e2e case to the quality grid. (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` — no L4 marker. tests/diffusion/attention/test_trtllm_attn.py:295 `assert rel < 0.01, f"BF16 dense rel err {rel:.4f} too high"`. tests/diffusion/attention/test_trtllm_attn.py:298-301 `requires_sage = pytest.mark.skipif(\n not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None),\n reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)",\n)`. tests/diffusion/attention/test_trtllm_attn.py:321 `assert rel < 0.1, f"SAGE({dtype_qk}) rel err {rel:.4f} too high"`. tests/diffusion/attention/test_attention_config.py:75-79 `def test_quant_and_skip_softmax_coexist(self):` only asserts `bk["target_sparsity"] == 0.5 and bk["quant"]["dtype_qk"] == "int8"` (kwargs round-trip, no forward). tests/diffusion/quantization/test_quantization_quality.py QUALITY_CONFIGS has no TRTLLM/SAGE/dtype_qk entry; tests/e2e has zero matches for TRTLLM_ATTN or dtype_qk.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:319` [minor] — forward_cuda always passes skip_softmax_threshold_scale_factor=_skip_factor into the same trtllm_ragged_attention_deepseek call that adds sage_attn_sfs when quant is enabled (trtllm_attn.py:290-320). AttentionSpec.backend_kwargs serializes skip_softmax and quant independently with no mutual exclusion (data.py:1464-1483); test_quant_and_skip_softmax_coexist only asserts kwargs. No GPU forward exercises both: test_sage_quant_matches_sdpa uses only quant, test_skip_* use only skip_softmax_threshold. Add a GPU case matching test_sage_quant_matches_sdpa that constructs `_impl(quant={"dtype_qk": "fp8_e4m3"}, skip_softmax_threshold=1e-9)` and `_impl(quant={"dtype_qk": "fp8_e4m3"}, skip_softmax_threshold=2.0)` so a kernel that rejects the combo fails CI. (evidence: trtllm_attn.py:290 `_skip_factor = self._resolve_skip_factor(kv_len)`; trtllm_attn.py:296-299 `if self.quant.enabled:\n q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)\n sage_kwargs["sage_attn_sfs"] = sage_attn_sfs\n sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes`; trtllm_attn.py:319-320 `skip_softmax_threshold_scale_factor=_skip_factor,\n **sage_kwargs,`; data.py:1464-1483 independently serializes skip_softmax keys and `kw["quant"] = quant_kw` with no combo guard; test_attention_config.py:75-79 `def test_quant_and_skip_softmax_coexist(self):` asserts only `bk["target_sparsity"] == 0.5 and bk["quant"]["dtype_qk"] == "int8"`; test_trtllm_attn.py:311 `impl = _impl(quant={"dtype_qk": dtype_qk, "q_block_size": 1, "k_block_size": 16})` — SAGE GPU forward has no skip_softmax_threshold.) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — Configuration intro still claims Skip-Softmax is the only typed AttentionSpec field (attention_backends.md:72-78), but this PR adds `quant: AttnQuantSpec` (data.py:1434) and a SAGE how-to (attention_backends.md:174). Update the "Today the only one" sentence and the skip_softmax-only dotted-flag example to include `quant`. (evidence: docs/user_guide/diffusion/attention_backends.md:72-73 `A backend that needs configuration exposes it as a typed field on the spec. Today the only` / `one is \`TRTLLM_ATTN\`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):`. Same file:76-77 `--diffusion-attention-config.default.backend TRTLLM_ATTN \` / `--diffusion-attention-config.default.skip_softmax.target_sparsity 0.5`. Same file:174 `Enable it through the typed \`quant\` block on the attention spec:`. vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None`.) + +`recipes/Wan-AI/Wan2.2-T2V.md:?` [minor] — This PR adds opt-in SAGE on TRTLLM_ATTN (int8/fp8_e4m3 QK plus per-channel fp8 V) and documents that as the attention-level quant path, using Wan-AI/Wan2.2-T2V-A14B-Diffusers as the example. The Wan2.2 T2V recipe — which already teaches TRTLLM_ATTN + Skip-Softmax for this model — still says at recipes/Wan-AI/Wan2.2-T2V.md:186 that `TRTLLM_ATTN` is BF16-only (no attention-level quantization). Update that limitation and point at the new `quant` block so the recipe matches the new public contract. (evidence: recipes/Wan-AI/Wan2.2-T2V.md:186 ` - \`TRTLLM_ATTN\` is BF16-only (no attention-level quantization).` — stale known-limitation. vllm_omni/diffusion/attention/backends/trtllm_attn.py:108-111 `_QK_QUANT_DTYPES = { "int8": torch.int8, "fp8_e4m3": torch.float8_e4m3fn }` and :198-202 `if self.quant.enabled: / if self.quant.dtype_qk not in _QK_QUANT_DTYPES: / raise RuntimeError( f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got "` — SAGE is a real opt-in path. docs/user_guide/diffusion/attention_backends.md:172-174 `On top of dense/Skip-Softmax, \`TRTLLM_ATTN\` can run **SAGE** ... Enable it through the typed \`quant\` block on the attention spec:` and :191-193 `vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \ / --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN", / "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}'` — same model family now documented as supporting SAGE.) + +**Additional observations (beyond the comment budget):** +- `docs/user_guide/diffusion/attention_backends.md:191` [minor] The SAGE snippet sets `attention_config.default` on `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, so `AttentionConfig.resolve_with_source` applies SAGE to every role with no per_role override — including Wan2.2 `role="cross"`. All +- `tests/diffusion/attention/test_trtllm_attn.py:321` [minor] This diff’s only SAGE forward test asserts `rel < 0.1` vs SDPA (dense TRTLLM uses `rel < 0.01`) and pytest.skips on `Missing TRTLLM-GEN kernel`, so a no-op SAGE path (output ≈ dense, rel ~0.01) still passes. Assert SAGE + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.trace.json.gz b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.trace.json.gz new file mode 100644 index 000000000..a4e46b201 Binary files /dev/null and b/eval/dataset/arms/INVALID_skillleak_copilot_cb_grok46_r3/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/manifest.json b/eval/dataset/arms/copilot_cb_composer25/manifest.json new file mode 100644 index 000000000..5f1af16e8 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_cb_composer25", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5509.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5509.cost.json new file mode 100644 index 000000000..7c3277b93 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5509.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 414.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5509/run-20260814-181729-ff9540", + "recorded_at": "2026-08-14T18:24:22", + "attempts": [ + { + "invocation_id": "inv-2b7d46e1", + "outcome": "run:run-20260814-181729-ff9540", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-181729-ff9540", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-181729-ff9540", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2966, + "minutes": 6.89, + "ci_minutes": 0, + "input_tokens": 768612, + "output_tokens": 80983, + "tool_calls": 280, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.366, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.98, + "gate": 1.74, + "review": 407.32, + "report": 0.06 + }, + "time_to_first_result_s": 413.07 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 31567 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5509.md b/eval/dataset/arms/copilot_cb_composer25/pr5509.md new file mode 100644 index 000000000..a5007f11a --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5509.md @@ -0,0 +1,56 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] vllm_omni/diffusion/attention/backends/trtllm_attn.py:296-320 — `sage_kwargs` are attached only when `self.quant.enabled`, keeping the dense `trtllm_ragged_attention_deepseek` call compatible with FlashInfer builds lacking SAGE parameters. +- [validated] vllm_omni/diffusion/attention/backends/trtllm_attn.py:194-216 — init resolves `_sage_quantize_fn` once and fails fast when `dtype_qk` is unsupported, `sage_attn_sfs` is absent from the kernel signature, or `trtllm_sage_attention_quantize` is missing; covered by `test_quant_rejects_non_sage_dtype` and `test_quant_quantize_requires_flashinfer_routine`. +- [validated] vllm_omni/diffusion/data.py:1472-1483 — `backend_kwargs()` emits the `quant` block with defaults (`q_block_size=1`, `k_block_size=16`) when enabled; `test_quant_serialized_with_defaults_and_overrides` and coexistence with `skip_softmax` pass. +- [validated] vllm_omni/diffusion/data.py:1447-1450 — non-TRTLLM/FLASHINFER backends reject `quant` at parse time (`test_quant_validation_rejects` for TORCH_SDPA). +- [validated] tests/diffusion/attention/test_trtllm_attn.py:315-316 — int8 SAGE skips cleanly when the runtime reports `Missing TRTLLM-GEN kernel` (SM100-only int8 vs SM103 fp8 matrix documented). +- [validated] tests/diffusion/attention/test_trtllm_attn.py:13 — `pytestmark` includes `core_model`, `diffusion`, and `cuda`; new tests are not silently deselected by marker filters. +- [sweep] vllm_omni/diffusion/attention/backends/trtllm_attn.py:90-95 — `_sage_kernel_available()` returns False when `HAS_FLASHINFER` is false or `sage_attn_sfs` is absent from the kernel signature; paired with init-time RuntimeError at lines 204-208. +- [sweep] vllm_omni/diffusion/data.py:1415-1420 — invalid `dtype_qk`/`dtype_vo` and block sizes raise ValueError before backend init (`test_quant_validation_rejects`). + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — This diff deletes `## Known Limitations` (LTX-2 `CUDNN_ATTN` + `torch.compile` workaround, FA4 status) and related guidance, but line 232 still says “see Known Limitations above.” Restore those sections or fix/remove that bullet so operators retain the LTX-2 escape hatch (`cudnn_attn.py` still documents the symbolic head_dim failure). (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — only file occurrence of "Known Limitations"; no `## Known Limitations` heading exists (headings jump from `## TRTLLM_ATTN SAGE Quantization` at :170 to `## End-to-End Benchmark` at :212 then `## Choosing a Backend Manually` at :226). vllm_omni/diffusion/attention/backends/cudnn_attn.py:75-76 `# e.g. under torch.compile where Dynamo sees a symbolic head_dim and` / `# cuDNN's kernel selection fails (observed in LTX-2 audio attention).`) + +`vllm_omni/diffusion/data.py:1447` [major] — `quant` is accepted and serialized for `FLASHINFER_ATTN` (data.py:1447-1483), but `FlashInferAttentionImpl` (flashinfer_attn.py:55-66) has no `backend_kwargs` handling—`layer.py:109` passes kwargs that are silently dropped via `**extra_impl_args`, unlike `SDPAImpl` (sdpa.py:88-89) and `TRTLLM_ATTN` (trtllm_attn.py:194). Restrict `quant` to `TRTLLM_ATTN` or implement consumption/rejection in `flashinfer_attn.py`; tests at test_attention_config.py:65-73 only check serialization. (evidence: vllm_omni/diffusion/data.py:1447 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):`; vllm_omni/diffusion/attention/layer.py:109 `backend_kwargs=backend_kwargs,`; vllm_omni/diffusion/attention/backends/flashinfer_attn.py:63 `**extra_impl_args,` with only `self.causal = causal` and `self.softmax_scale = softmax_scale` in __init__; vllm_omni/diffusion/attention/backends/sdpa.py:88-89 `if backend_kwargs: logger.warning("SDPAImpl ignoring backend_kwargs: %s", list(backend_kwargs.keys()))`) + +`tests/diffusion/attention/test_trtllm_attn.py:?` [minor] — This diff adds CPU-safe quant plumbing tests (`test_quant_config_from_backend_kwargs`, `test_quant_rejects_non_sage_dtype`, `test_quant_quantize_requires_flashinfer_routine`, `test_quant_quantize_calls_routine_and_shapes_sfs`) to `test_trtllm_attn.py`, which is module-marked `pytest.mark.cuda` and lives under `tests/diffusion/attention/`. test-ready.yml's CPU job (`tests/diffusion -m 'core_model and cpu'`) skips cuda-marked tests, and its CUDA job (`tests/diffusion/*.py`) does not recurse into `attention/`, so these backend guards never run in CI. Mark them `@pytest.mark.cpu` or move them to `test_attention_config.py` (config-layer quant tests there already run in CPU CI). (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]`; .buildkite/cuda/test-ready.yml:18 `timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu'`; .buildkite/cuda/test-ready.yml:68 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda'`; tests/diffusion/attention/test_trtllm_attn.py:50-74 four new `test_quant_*` functions with no `@pytest.mark.cpu` override.) + +`vllm_omni/diffusion/model_metadata.py:42` [minor] — SAGE plugs into the shared TRTLLM_ATTN backend, which auto-defaults for all mask-free Wan pipelines (WanPipeline/T2V, WanImageToVideoPipeline/I2V, WanVACEPipeline, WanS2VPipeline — model_metadata.py:39–42). The PR diff only adds synthetic kernel tests (test_sage_quant_matches_sdpa on random tensors); can you confirm SAGE quality/perf for other Wan TRTLLM call sites or document that validation scope is intentionally kernel-level / Wan2.2-only? (evidence: vllm_omni/diffusion/model_metadata.py:39 `"WanPipeline": DiffusionModelMetadata(attention_mask_free=True),` — vllm_omni/diffusion/model_metadata.py:40 `"WanImageToVideoPipeline": DiffusionModelMetadata(attention_mask_free=True),` — vllm_omni/diffusion/model_metadata.py:41 `"WanVACEPipeline": DiffusionModelMetadata(attention_mask_free=True),` — vllm_omni/diffusion/model_metadata.py:42 `"WanS2VPipeline": DiffusionModelMetadata(attention_mask_free=True),` — vllm_omni/diffusion/attention/layer.py:83 `allow_trtllm_default = get_diffusion_model_metadata(model_class_name).attention_mask_free` — tests/diffusion/attention/test_trtllm_attn.py:308 `B, S, H, D = 2, 256, 8, 128` — tests/diffusion/attention/test_trtllm_attn.py:310 `q, k, v = (torch.randn(B, S, H, D, device="cuda", dtype=torch.bfloat16) for _ in range(3))`) + +`docs/design/module/dit_module.md:543` [minor] — Design doc §5.1 still says skip_softmax is the only AttentionSpec typed field serialized by backend_kwargs(); this PR adds quant (AttnQuantSpec) for TRTLLM_ATTN SAGE and FLASHINFER_ATTN. Consider a follow-up to align dit_module.md with attention_backends.md. (evidence: docs/design/module/dit_module.md:543 `Today the only such field is \`skip_softmax\` (TRTLLM_ATTN), so e.g. \`target_sparsity\` can be set without changing model code.` — stale; vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None` and vllm_omni/diffusion/data.py:1472-1483 `if self.quant is not None and self.quant.enabled: ... kw["quant"] = quant_kw` — second typed backend_kwargs field added in PR.) + +`docs/user_guide/diffusion/attention_backends.md:188` [minor] — Line 188 says FLASHINFER_ATTN quant fields are documented in 'its section', but the file has no FLASHINFER_ATTN quant section (only TRTLLM_ATTN SAGE Quantization), and flashinfer_attn.py does not consume quant kwargs. Please add the missing section, drop the cross-reference, or mark FLASHINFER quant as unsupported until the backend implements it. (evidence: docs/user_guide/diffusion/attention_backends.md:188 `dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.` — no `## FLASHINFER` quant heading exists (nearest quant section is line 170 `## TRTLLM_ATTN SAGE Quantization`); vllm_omni/diffusion/attention/backends/flashinfer_attn.py:55-66 `def __init__(self, num_heads: int, head_size: int, softmax_scale: float, causal: bool = False, num_kv_heads: int | None = None, prefix: str = "", **extra_impl_args,) -> None:` with no quant/backend_kwargs handling) + +`vllm_omni/diffusion/data.py:?` [minor] — AttnQuantSpec.enabled is `dtype_qk is not None or dtype_vo is not None` (data.py:1424-1425), so `AttentionSpec(backend='TRTLLM_ATTN', quant={'dtype_vo': 'fp8_e4m3'})` passes __post_init__ and backend_kwargs() emits `quant` with `dtype_qk: None` (data.py:1472-1483). TRTLLM QuantConfig.enabled only checks `dtype_qk is not None` (trtllm_attn.py:129-131), so SAGE init/forward paths are skipped and dense runs with no error — dtype_vo is FLASHINFER-only on this shared spec. Reject dtype_vo-only quant for TRTLLM_ATTN in AttentionSpec.__post_init__ or require dtype_qk when backend is TRTLLM_ATTN. (evidence: data.py:1424-1425 `return self.dtype_qk is not None or self.dtype_vo is not None`; data.py:1472-1480 `if self.quant is not None and self.quant.enabled:` / `quant_kw: dict[str, Any] = {"dtype_qk": q.dtype_qk, ...}` / `if q.dtype_vo is not None: quant_kw["dtype_vo"] = q.dtype_vo`; trtllm_attn.py:129-131 `def enabled(self) -> bool: return self.dtype_qk is not None`; trtllm_attn.py:197 `if self.quant.enabled:` (SAGE validation and sage_kwargs only when dtype_qk is set)) + +`vllm_omni/diffusion/data.py:?` [minor] — Shared AttnQuantSpec accepts bfloat16 (and float16) in _VALID_DTYPES, so AttentionSpec(backend="TRTLLM_ATTN", quant={"dtype_qk": "bfloat16"}) passes parse-time validation even though SAGE only supports int8/fp8_e4m3 and TrtllmAttentionImpl raises RuntimeError at init. Add backend-specific dtype_qk checks in AttentionSpec.__post_init__ so TRTLLM_ATTN misconfig raises ValueError at parse time, matching attention_backends.md:188. (evidence: vllm_omni/diffusion/data.py:1410 `_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})`; vllm_omni/diffusion/data.py:1447-1451 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` (no per-backend dtype_qk check); vllm_omni/diffusion/attention/backends/trtllm_attn.py:108-111 `_QK_QUANT_DTYPES = {"int8": torch.int8, "fp8_e4m3": torch.float8_e4m3fn}`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:199-203 `if self.quant.dtype_qk not in _QK_QUANT_DTYPES: raise RuntimeError(...)`) + +**Additional observations (beyond the comment budget):** +- `docs/user_guide/diffusion/attention_backends.md:72` [minor] The Configuration intro at attention_backends.md:72-73 still says skip_softmax is the only typed backend field on TRTLLM_ATTN, but this PR adds a typed quant block (see TRTLLM_ATTN SAGE Quantization at line 170). Update +- `tests/diffusion/attention/test_attention_config.py:65` [minor] This diff adds `test_quant_superset_fields_passed_through` (`test_attention_config.py:65-73`), which only asserts `backend_kwargs()["quant"]` round-trips for FLASHINFER_ATTN fields. `FlashInferAttentionImpl` (`flashinfer +- `tests/diffusion/attention/test_trtllm_attn.py:306` [minor] This diff's only forward-path SAGE fidelity check is `test_sage_quant_matches_sdpa` (`test_trtllm_attn.py:306`), gated on Blackwell + FlashInfer≥0.6.16rc1 via `@requires_sage` and `_has_trtllm_attn()` (L4/sm_89 CI GPUs s + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5509.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5509.trace.json.gz new file mode 100644 index 000000000..c33a8bb10 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5550.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5550.cost.json new file mode 100644 index 000000000..d1e9d7623 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5550.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 373.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5550/run-20260814-181729-e8c67f", + "recorded_at": "2026-08-14T18:23:41", + "attempts": [ + { + "invocation_id": "inv-24a8fd7d", + "outcome": "run:run-20260814-181729-e8c67f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-181729-e8c67f", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-181729-e8c67f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1967, + "minutes": 6.19, + "ci_minutes": 0, + "input_tokens": 483993, + "output_tokens": 60020, + "tool_calls": 230, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3037, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.75, + "gate": 1.68, + "review": 366.08, + "report": 0.06 + }, + "time_to_first_result_s": 371.56 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 25700 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5550.md b/eval/dataset/arms/copilot_cb_composer25/pr5550.md new file mode 100644 index 000000000..734e085ef --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5550.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] tests/diffusion/diffusion_kv/test_metadata.py:37-38 — indexed sequences[0]/contexts[0] accesses are test-constructed single-element tuples; safe by construction. +- [validated] tests/diffusion/diffusion_kv/test_worker_contract.py:221 — calls[0][1][0].prepared_layout assertion covers dense RPC preserving req object identity through executor collective_rpc args. +- [sweep] vllm_omni/diffusion/diffusion_engine.py:199 — scheduler_to_close branch is reachable because _init_scheduler assigns self.scheduler before initialize() (line 247-252). +- [sweep] vllm_omni/diffusion/executor/multiproc_executor.py:545 — metadata fourth positional arg branch only taken when diffusion_kv_metadata is not None; dense legacy keeps 3-tuple shape per test_worker_contract.py:214-220. +- [sweep] vllm_omni/diffusion/sched/interface.py:184,192 — both identity mismatch branches are covered by test_worker_contract.py envelope/metadata mismatch tests. +- [sweep] vllm_omni/diffusion/worker/diffusion_model_runner.py:172,175 — dense rejection and metadata request_id mismatch covered by test_dense_request_rejects_metadata and test_paged_request_rejects_mismatched_metadata_identity. +- [sweep] vllm_omni/diffusion/worker/diffusion_worker.py:463,1372 — kwargs extension only when metadata non-None; WorkerWrapperBase and dense-worker tests confirm signature compatibility. +- [validated] vllm_omni/diffusion/executor/multiproc_executor.py:459-460 — validate_new_request_data_identity runs before dispatch; identity errors never enter per-request RPC error wrapper (test_request_rpc_rejects_envelope_request_identity_mismatch asserts calls==[]). + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — This PR adds `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)` and gates the DLO+AllGather fused multi-request `execute_model` list-RPC (`len(new_reqs) > 1` branch at line 471) on `not has_diffusion_kv_metadata`, falling back to per-request RPCs that append metadata when set (lines 544-546). That is the correct W0 safety choice because the fused path passes only `(reqs_list, od_config, kv_prefetch_job)` (lines 506-509) with no metadata slot. Consider adding a unit test in `test_multiproc_engine_concurrency.py` that schedules two `NewRequestData` entries with `diffusion_kv_metadata` under DLO fixtures and asserts per-request `collective_rpc` (args include metadata) instead of a single list-RPC, plus a short inline TODO for W1 per-DP-rank metadata in the fused path. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)`; vllm_omni/diffusion/executor/multiproc_executor.py:471 `and not has_diffusion_kv_metadata`; vllm_omni/diffusion/executor/multiproc_executor.py:509 `args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job)`; vllm_omni/diffusion/executor/multiproc_executor.py:545-546 `if new_req.diffusion_kv_metadata is not None: args += (new_req.diffusion_kv_metadata,)`) + +`vllm_omni/experimental/ar_diffusion/runner.py:237` [minor] — ARDiffusionModelRunner.execute_model (runner.py:237) still omits diffusion_kv_metadata and super().execute_model (line 255) only passes kv_prefetch_job. Safe while W0 never attaches metadata; W1 paged_scheduler+AR will need signature forwarding or an explicit out-of-scope note. (evidence: vllm_omni/experimental/ar_diffusion/runner.py:237-241 `def execute_model( + self, + req: OmniDiffusionRequest, + kv_prefetch_job: KVPrefetchJob | None = None, + ) -> DiffusionOutput:` — no diffusion_kv_metadata parameter; vllm_omni/experimental/ar_diffusion/runner.py:255 `output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` — metadata not forwarded) + +`vllm_omni/diffusion/diffusion_engine.py:193` [minor] — DiffusionEngine.__init__ (diffusion_engine.py:191-208) shares one except path for _init_scheduler, _init_runtime_state, _init_execute_fn, and _log_execution_mode, calling scheduler.close() and executor.shutdown() on any failure. test_scheduler_initialization_failure_closes_scheduler_and_executor only covers scheduler.initialize() raising; add targeted tests (or document why unnecessary) for failures in _init_runtime_state or _init_execute_fn so partial-init cleanup stays verified if init ordering changes. (evidence: diffusion_engine.py:191-208 `try: self._init_scheduler(...); self._init_runtime_state(); self._init_execute_fn(); ... except Exception: ... scheduler_to_close.close(); self.executor.shutdown(); raise` — tests/diffusion/test_diffusion_engine_cleanup.py:126-164 `initialize=Mock(side_effect=initialization_error)` with asserts on close/shutdown; repo-wide tests grep has zero matches for _init_runtime_state or _init_execute_fn failure tests) + +`vllm_omni/diffusion/executor/multiproc_executor.py:?` [minor] — `execute_request` validates envelopes before RPC (454-460), but fused `execute_batch` (615-617) and `execute_step` (651-653) call `collective_rpc` without that loop; identity checks run only in `DiffusionModelRunner.execute_model_batch`/`execute_stepwise` (637, 751) after RPC, so mismatches become `RuntimeError` instead of the pre-RPC `ValueError`. Mirror the pre-loop in both executor paths and add executor-level tests. (evidence: multiproc_executor.py:454-460 `# Validate every envelope before selecting a dispatch path.` / `for new_req in new_reqs:` / `validate_new_request_data_identity(new_req)`; multiproc_executor.py:615-617 `result = self.collective_rpc(` / `"execute_model_batch",` / `args=(scheduler_output, self.od_config),`; multiproc_executor.py:651-653 `result = self.collective_rpc(` / `"execute_stepwise",` / `args=(scheduler_output,),`; diffusion_model_runner.py:636-637 `for new_req in scheduler_output.scheduled_new_reqs:` / `validate_new_request_data_identity(new_req)`; multiproc_executor.py:207-211 `if isinstance(response, dict) and response.get("status") == "error":` / `raise RuntimeError(`) + +`tests/diffusion/diffusion_kv/test_metadata.py:37` [minor] — test_metadata.py adds three tests that construct DiffusionKVMetadata dataclasses and assert constructor inputs round-trip (e.g. line 37: `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])` mirrors line 27 input). metadata.py is plain @dataclass with no __post_init__ or validation, so these cannot fail unless dataclass storage breaks and do not guard the W0 transport contract. test_worker_contract.py already covers from_state wiring (77-85), runner rejection (103-126), and executor RPC identity checks (129-141); fold CFG/context scenarios there or drive metadata through those boundaries. (evidence: tests/diffusion/diffusion_kv/test_metadata.py:27 `block_ids=([1, 2], [5, 6]),` then :37 `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])`; vllm_omni/diffusion/diffusion_kv/metadata.py:16-57 plain `@dataclass` fields with no `__post_init__` or validation; tests/diffusion/diffusion_kv/test_worker_contract.py:82 `new_req = NewRequestData.from_state(state, diffusion_kv_metadata=metadata)` and :107-113 `with pytest.raises(ValueError, match="dense_legacy.*must not carry"): runner.execute_model(..., diffusion_kv_metadata=make_metadata())` with `runner._execute_request_list.assert_not_called()`) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:129` [minor] — This PR adds `validate_new_request_data_identity` with a second check when `metadata.request_id != forwarded_request_id` (interface.py:191-196), and `test_request_rpc_rejects_envelope_request_identity_mismatch` (test_worker_contract.py:129-141) covers envelope/request_id mismatch at the executor, but no executor test covers matching envelope/request_id with stale `diffusion_kv_metadata.request_id` (`test_paged_request_rejects_mismatched_metadata_identity` at lines 116-126 exercises only the runner). Add `test_request_rpc_rejects_metadata_request_identity_mismatch` through `executor.execute_request`, expecting `ValueError` and `calls == []`. (evidence: vllm_omni/diffusion/sched/interface.py:192 `if metadata is not None and metadata.request_id != forwarded_request_id:`; vllm_omni/diffusion/executor/multiproc_executor.py:460 `validate_new_request_data_identity(new_req)`; tests/diffusion/diffusion_kv/test_worker_contract.py:132-135 `request_id="envelope-id", req=req, diffusion_kv_metadata=make_metadata("envelope-id")` with `req.request_id="actual-request-id"`; tests/diffusion/diffusion_kv/test_worker_contract.py:116-124 `runner.execute_model(..., diffusion_kv_metadata=make_metadata("stale"))` — no `executor.execute_request` metadata-mismatch test in tests/diffusion/diffusion_kv/) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5550.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5550.trace.json.gz new file mode 100644 index 000000000..7bddf444e Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5610.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5610.cost.json new file mode 100644 index 000000000..e57fad012 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5610.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 417.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5610/run-20260814-181728-27f39a", + "recorded_at": "2026-08-14T18:24:25", + "attempts": [ + { + "invocation_id": "inv-9a21e829", + "outcome": "run:run-20260814-181728-27f39a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-181728-27f39a", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-181728-27f39a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1114, + "minutes": 6.95, + "ci_minutes": 0, + "input_tokens": 290779, + "output_tokens": 29898, + "tool_calls": 155, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2854, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.63, + "gate": 1.7, + "review": 411.29, + "report": 0.06 + }, + "time_to_first_result_s": 416.67 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 19723 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5610.md b/eval/dataset/arms/copilot_cb_composer25/pr5610.md new file mode 100644 index 000000000..9d26dc4f9 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5610.md @@ -0,0 +1,37 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness: read `gpu_ar_model_runner.py:1595-1619` — `_should_use_async_omni_output()` guards match the Compatibility table (async scheduling, `async_chunk`, model opt-in, `omni_prefix_cache`, speculative decoding, routed experts, eager postprocess). +- [validated] `should_accumulate_full_payload_output()` at `omni_connector_model_runner_mixin.py:60-61` returns False when `async_chunk` is enabled, matching the doc claim that the full-payload accumulation branch is unreachable on this path. +- [validated] Qwen3-Omni thinker/talker opt-in at `qwen3_omni.py:133-164` matches stage behavior table (talker sets `eager_omni_postprocess_before_async_output=True`, `omni_pooler_payload_include_hidden=False`). +- [validated] Qwen3-TTS talker opt-in at `qwen3_tts_talker.py:351-353` matches doc; moss_tts explicitly sets `use_async_omni_output=False` at `modeling_moss_tts_talker.py:1326`. +- [validated] NPU path at `npu_ar_model_runner.py:1289-1328` builds `OmniModelRunnerOutput`, calls `get_omni_connector_output()` synchronously, then wraps with `AsyncGPUModelRunnerOutput` — matches Ascend description. +- [validated] Deploy YAML snippets match `qwen3_omni_moe.yaml:15,31,43,60` and `qwen3_tts.yaml:14,53,81`; omitted `async_scheduling` defaults to True per `stage_config.py:941`. +- [validated] `enable_prefix_caching` creates `omni_prefix_cache` at `gpu_model_runner.py:170-176`, which disables async output — warning at doc lines 321-328 is accurate. +- [validated] Background builder lifecycle at `gpu_ar_model_runner.py:201-239` matches doc (daemon thread, `get_output()` join + exception propagation, connector drain in builder at `:1899-1900`). + +`docs/design/feature/omni_async_output_materialization.md:73` [major] — Figure 1 in `docs/design/feature/omni_async_output_materialization.md` uses `src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"` (line 73). From `docs/design/feature/`, three `../` reach the repo root, so the resolved path is `source/architecture/qwen3-omni-async-output-step-gap.svg` outside `docs/` where no file exists. The SVG is at `docs/source/architecture/qwen3-omni-async-output-step-gap.svg`; use `../../source/architecture/qwen3-omni-async-output-step-gap.svg` or the raw GitHub URL pattern from `async_chunk.md`. (evidence: docs/design/feature/omni_async_output_materialization.md:73 `src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"`; docs/source/architecture/ contains `qwen3-omni-async-output-step-gap.svg`; repo-root `source/` directory does not exist.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5610.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5610.trace.json.gz new file mode 100644 index 000000000..b7391c76f Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5703.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5703.cost.json new file mode 100644 index 000000000..f8e22e220 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5703.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 432.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5703/run-20260814-182342-01a22a", + "recorded_at": "2026-08-14T18:30:53", + "attempts": [ + { + "invocation_id": "inv-92bbddca", + "outcome": "run:run-20260814-182342-01a22a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-182342-01a22a", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-182342-01a22a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2031, + "minutes": 7.19, + "ci_minutes": 0, + "input_tokens": 456271, + "output_tokens": 72603, + "tool_calls": 271, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3345, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.25, + "gate": 1.76, + "review": 425.38, + "report": 0.06 + }, + "time_to_first_result_s": 431.44 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 23787 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5703.md b/eval/dataset/arms/copilot_cb_composer25/pr5703.md new file mode 100644 index 000000000..1436d0b9d --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5703.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] vllm_omni/diffusion/models/minimax_h3/vae.py:170,214 — devices list widened from parameter.device.type == "cuda" to != "cpu", so MUSA (and other non-CPU) parameters now enter fork_rng and device_module.manual_seed; CUDA behavior preserved. +- [validated] vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:963,1066 — sole in-repo callers of encode_image/encode_video; signatures and return contracts unchanged. +- [sweep] tests/diffusion/models/minimax_h3/ — no RNG/determinism assertions for encode_image/encode_video; PR validation command references test_minimax_h3_vae.py which is not in the tree; diff touches zero test files. +- [sweep] recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76 — prose says "validated Ref2VA configuration" and summary line 11 cites "4x MTT S5000 for the validated Ref2VA profile", but server example exports MODEL="${MODEL_ROOT}/FL2VA". +- [sweep] PR description claims added coverage for torchaudio/soundfile/ffmpeg audio-loading paths, but diff contains no changes to vllm_omni/diffusion/models/minimax_h3/reference_video.py or related tests. +- [validated] vllm_omni/diffusion/models/minimax_h3/vae.py:120 — torch.get_device_module() called without device; pipeline passes device=self.device from get_local_device() at pipeline_minimax_h3.py:481,547, so worker init likely aligns accelerator with weights. +- [sweep] Blast radius: encode_image/encode_video now seed RNG for any non-CPU device.type (mps/npu/xpu/etc.), broader than PR-stated CUDA+MUSA allowlist; only MUSA is evidenced. +- [sweep] gate_report: buildkite/vllm-omni-amd-ci failing; this diff is MiniMax H3 VAE RNG + docs only — likely unrelated but unattributed from this checkout. + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — The TP4 serve block under "The validated Ref2VA configuration" (lines 72–76) still sets `export MODEL="${MODEL_ROOT}/FL2VA"` even though the section documents Ref2VA validation and the file says Ref2VA serving must use the Ref2VA partition. Operators copying this block will load FL2VA for Ref2VA requests. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72 `The validated Ref2VA configuration uses tensor parallelism across four MTT`; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76 `export MODEL="${MODEL_ROOT}/FL2VA"`; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:69-70 `Set \`MODEL\` to \`FL2VA\` for T2VA and FL2VA, or to \`Ref2VA\` for Ref2VA.`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — This diff widens `encode_image`/`encode_video` RNG gating from CUDA-only to any non-CPU device (`vae.py:170`, `:214`), while the PR documents MUSA VAE RNG validation and an existing NPU recipe (`MiniMax-H3-NPU.md`) does not. On NPU, `MiniMaxH3VideoVAE` is still built on `get_local_device()` (`pipeline_minimax_h3.py:481`, `:545-547`), so Ref2VA `encode_image`/`encode_video` would now run `fork_rng` and `device_module.manual_seed` on `npu` without cited validation. Either restrict seeding to validated backends (e.g. `("cuda", "musa")`) or document and smoke-test the NPU behavior change. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:481 `self.device = get_local_device()`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:545-547 `self.video_vae = MiniMaxH3VideoVAE(..., device=self.device,)`; vllm_omni/platforms/npu/platform.py:163 `return torch.device("npu")`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — MiniMaxH3VideoVAE.__init__ takes device and calls .to(device=...) before caching self.device_module = torch.get_device_module() without that device. encode_image/encode_video seed accelerator RNG via the cached module while building devices from parameter.device; use torch.get_device_module(device) (or resolve from parameter.device in the encode helpers) so MUSA/CUDA seeding targets the backend that owns the weights. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:104 `device: torch.device,` — constructor accepts device; vllm_omni/diffusion/models/minimax_h3/vae.py:115 `self.remote.eval().to(device=device, dtype=torch.float32)` — weights placed on device; vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — no device passed; vllm_omni/diffusion/models/minimax_h3/vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` — encode_image uses init-cached module; vllm_omni/diffusion/models/minimax_h3/vae.py:219-220 same pattern in encode_video.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — This diff switches encode_image/encode_video RNG seeding from torch.cuda.* to self.device_module.* (vae.py:170-176, :214-220) but adds no tests. The only encode_video contract test builds the VAE with object.__new__, omits device_module, and uses CPU weights so devices stays empty and the new seeding loop is never hit; there are also zero encode_image tests under tests/diffusion/models/minimax_h3/. Consider adding a contract test that constructs MiniMaxH3VideoVAE through __init__, places parameters on a non-CPU device (cuda/musa mock), calls encode_image/encode_video twice with a fixed seed, and asserts identical latents plus restored CPU/accelerator RNG state. (evidence: vae.py:120 `self.device_module = torch.get_device_module()`; vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; test_minimax_h3_contract.py:344 `video_vae = object.__new__(MiniMaxH3VideoVAE)`; test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))`) + +`.buildkite/cuda/test-ready.yml:18` [minor] — Diffusion Simple/L1-L2 CI is `pytest -sv tests/diffusion -m 'core_model and cpu'` (`.buildkite/cuda/test-ready.yml:18`), and there is no MUSA Buildkite lane under `.buildkite/`. Should merge wait on automated MUSA coverage, or is CPU L2 plus the cited manual Ref2VA smoke the intended gate for the new `device_module.manual_seed` encode path? (evidence: .buildkite/cuda/test-ready.yml:18 — `- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; grep under `.buildkite/` for `musa|MUSA` returns no matches) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:545` [minor] — `MiniMaxH3VideoVAE` is only constructed at pipeline_minimax_h3.py:545, so every H3 partition shares one video_vae whose encode_image/encode_video paths widen RNG seeding to all non-CPU backends (`parameter.device.type != "cpu"` + `torch.get_device_module()`). FL2VA/Ref2VA conditioned requests hit those encode paths; pure T2VA may only decode. The PR documents MUSA validation only—should ROCm/XPU/NPU (or other non-CPU backends) be explicitly scoped out, documented, or smoke-tested before merge? (evidence: pipeline_minimax_h3.py:545 `self.video_vae = MiniMaxH3VideoVAE(` — sole production construction site; vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` — widened non-CPU seeding gate in encode_image; vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` — generic accelerator RNG seeding; pipeline_minimax_h3.py:963 `rows = self.video_vae.encode_image(image) if rank == 0 else None` — conditioned encode path used by FL2VA/Ref2VA visual references) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:?` [nit] — PR Purpose says this change "adds coverage for the torchaudio, soundfile, and ffmpeg audio-loading paths", but the diff only adds recipe validation prose (e.g. the soundfile-fallback bullet here) and vae.py RNG edits—no reference_video.py changes and no tests under tests/diffusion/models/minimax_h3/ for load_audio/soundfile/ffmpeg. Trim that Purpose bullet or land the promised loader tests separately. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:111 `- the soundfile fallback preserves stereo channel layout, FP32 samples, and` — manual validation prose only, not automated coverage; vllm_omni/diffusion/models/minimax_h3/reference_video.py:477 `def load_audio_file(path: str) -> tuple[torch.Tensor, int]:` with torchaudio/soundfile/ffmpeg fallback at :486-514 — unchanged in this diff (only vae.py, recipes/MiniMax-H3-MUSA.md, recipes/README.md touched); tests/diffusion/models/minimax_h3/ — grep load_audio|soundfile|ffmpeg returns no matches) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5703.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5703.trace.json.gz new file mode 100644 index 000000000..b8ba3b7a8 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5715.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5715.cost.json new file mode 100644 index 000000000..1415436e4 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5715.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 97.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5715/run-20260814-182423-98d514", + "recorded_at": "2026-08-14T18:26:00", + "attempts": [ + { + "invocation_id": "inv-bb006dc8", + "outcome": "run:run-20260814-182423-98d514", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-182423-98d514", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-182423-98d514", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0179, + "minutes": 1.62, + "ci_minutes": 0, + "input_tokens": 42982, + "output_tokens": 5746, + "tool_calls": 30, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.0734, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.26, + "gate": 1.78, + "review": 92.0, + "report": 0.06 + }, + "time_to_first_result_s": 97.07 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 10200 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5715.md b/eval/dataset/arms/copilot_cb_composer25/pr5715.md new file mode 100644 index 000000000..4732960f8 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5715.md @@ -0,0 +1,37 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] README.md:19-21 — v0.26.0 announcement, consolidated even-minor cadence bullet, and recipe links (recipes/MiniMaxAI/MiniMax-H3.md, recipes/OpenBMB/MiniCPM-o-4_5.md) exist in repo +- [validated] README.md:54-60 and docs/README.md:55-61 — full-duplex (experimental) feature and model example updates are mirrored between root and docs README +- [validated] docs/configuration/README.md:5 — vLLM 0.26 configuration URL https://docs.vllm.ai/en/v0.26.0/configuration/index.html resolves; no stale v0.16.0 links remain under docs/ +- [validated] docs/getting_started/installation/gpu/cuda.inc.md:8,127 — compatibility text and Docker BASE_IMAGE example match docker/Dockerfile.cuda:1 (vllm/vllm-openai:v0.26.0) +- [validated] docs/getting_started/installation/npu/npu.inc.md:56-65 — vLLM v0.26.0 tag and vllm-ascend releases/v0.26.0rc branch exist upstream; vllm-ascend has no v0.26.0 tag (404), so RC branch pin is correct +- [upstream-verify] GitHub API confirms vllm-project/vllm tag v0.26.0 and vllm-project/vllm-ascend branch releases/v0.26.0rc +- [sweep] Breaking-change consumer grep — no in-repo docs still link to docs.vllm.ai/en/v0.16|0.18|0.22; cuda/rocm/npu install inc files already at 0.26.0 +- [sweep] Scope — PR description deliverables (news, models, experimental duplex, cadence consolidation, skills removal, compatibility note, CUDA/NPU/config pins) all present in diff; MUSA/quickstart intentionally unchanged + +`docs/getting_started/installation/README.md:4` [minor] — This PR adds an unconditional 0.26.x ↔ vLLM 0.26.x compatibility callout at the top of the installation index, but the MUSA install path in the same section (docs/getting_started/installation/gpu/musa.inc.md:31) still documents `git checkout v0.18.0-dev` for vLLM-MUSA and was deliberately left unchanged. Scope the admonition (e.g., "except MUSA, which follows vLLM-MUSA pinning") or add a cross-link/caveat so MUSA readers are not told to install vLLM 0.26.x when their platform docs still target 0.18.0-dev. (evidence: docs/getting_started/installation/README.md:4 states "vLLM-Omni 0.26.x requires vLLM 0.26.x"; docs/getting_started/installation/gpu/musa.inc.md:31 still has "git checkout v0.18.0-dev" with no 0.26.x mention.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5715.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5715.trace.json.gz new file mode 100644 index 000000000..e3fb2fbef Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5840.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5840.cost.json new file mode 100644 index 000000000..87f5e1914 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5840.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 565.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5840/run-20260814-182426-5f9fec", + "recorded_at": "2026-08-14T18:33:51", + "attempts": [ + { + "invocation_id": "inv-f42c8646", + "outcome": "run:run-20260814-182426-5f9fec", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-182426-5f9fec", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-182426-5f9fec", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3263, + "minutes": 9.42, + "ci_minutes": 0, + "input_tokens": 856363, + "output_tokens": 86413, + "tool_calls": 372, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4462, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.81, + "gate": 2.03, + "review": 559.05, + "report": 0.06 + }, + "time_to_first_result_s": 564.94 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 32122 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5840.md b/eval/dataset/arms/copilot_cb_composer25/pr5840.md new file mode 100644 index 000000000..b745f3098 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5840.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness: read minimax_h3_transformer.py forward vs extract_minimax_h3_context — preprocessing, blocks loop, final_layer/masks align; test_extractor_matches_native_forward asserts parity. +- [validated] backend.py:76-101 — ref2va raises ValueError; combined warns; hook applied only to pipeline.transformer; _transformer_for_task routes ref2va to transformers_ref. +- [validated] config.py:96-133 — TeaCacheConfig resolves None to MiniMaxH3DiTModel=0.17 else 0.2; other custom enablers pass None through and still resolve to 0.2 for non-H3 types. +- [sweep] Breaking defaults: grep rel_l1_thresh consumers — HunyuanImage3 uses tea_cache_config after TeaCacheConfig __post_init__; generic models still effective 0.2 via TeaCacheConfig. +- [validated] test_cache_backends.py:474-516 — tests TeaCacheBackend(DiffusionCacheConfig()) → 0.17 for H3 but does not cover AsyncOmniEngine normalization. +- [validated] test_teacache_extractors.py TestMiniMaxH3Extractor — parity, packed layout, cache-hit skip; pytest.mark.cpu + core_model. +- [sweep] E2E: test_minimax_h3_e2e.py frame assert 107 + teacache smoke uses explicit cache_config 0.17; markers local_model/cuda (L4, not L2 CI). +- [sweep] sp_prepare is currently no-op (minimax_h3_transformer.py:790-799); modulated_input before sp_prepare matches today; future SP sharding would need extractor alignment. + +`vllm_omni/engine/async_omni_engine.py:?` [major] — PR defers rel_l1_thresh to TeaCacheConfig model defaults (MiniMaxH3DiTModel→0.17), but AsyncOmniEngine._get_default_cache_config still injects rel_l1_thresh=0.2 when cache_config is omitted. Omni(..., cache_backend="tea_cache") and vllm serve --cache-backend tea_cache without --cache-config therefore bypass the calibrated 0.17. Return {} for tea_cache (or stop injecting rel_l1_thresh) and add an AsyncOmniEngine-path regression test. (evidence: async_omni_engine.py:853-856 `return {"rel_l1_thresh": 0.2}`; async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"): cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`; config.py:97-98 `"MiniMaxH3DiTModel": 0.17`; backend.py:86-90 `teacache_config = TeaCacheConfig(transformer_type="MiniMaxH3DiTModel", rel_l1_thresh=config.rel_l1_thresh, ...)`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:?` [minor] — This PR’s e2e TeaCache smoke hardcodes cache_config={"rel_l1_thresh": 0.17}, so it never exercises the omitted-cache_config Omni path. AsyncOmniEngine._normalize_cache_config still injects {"rel_l1_thresh": 0.2} for tea_cache (lines 853–856, 881–882), which bypasses the PR’s MiniMax-H3 model default of 0.17. test_minimax_h3_only_enables_fl2va_teacache covers None→0.17 at TeaCacheBackend only. Add a sibling test on AsyncOmniEngine normalization (or fix the 0.2 injection) so the Omni/serve path is covered. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:156 `cache_config={"rel_l1_thresh": 0.17},`; vllm_omni/engine/async_omni_engine.py:853-856 `if cache_backend == "tea_cache": return {"rel_l1_thresh": 0.2,}`; vllm_omni/engine/async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"): cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`) + +`vllm_omni/diffusion/cache/teacache/backend.py:?` [minor] — This PR adds enable_minimax_h3_teacache, which hooks only pipeline.transformer and logs that combined serving leaves Ref2VA uncached; test_minimax_h3_only_enables_fl2va_teacache mocks enable() but no test drives a combined-pipeline Ref2VA request through _transformer_for_task to prove transformers_ref denoise never hits a TeaCache hook. Add a CPU contract or integration test that issues task=ref2va on a combined pipeline and asserts hook absence on transformers_ref (or zero block-skip count). (evidence: backend.py:91-97 hooks only `pipeline.transformer`; pipeline_minimax_h3.py:736-739 `_transformer_for_task` returns `transformers_ref` for ref2va; test_cache_backends.py:490-516 mocks partition but no combined Ref2VA generation test.) + +`tests/diffusion/cache/test_teacache_extractors.py:670` [minor] — test_teacache_reuses_h3_block_residual verifies cache skip via block_calls==1 and output shapes only; consider also assert_close against an uncached baseline so incorrect residual reuse cannot pass. (evidence: tests/diffusion/cache/test_teacache_extractors.py:693-698 `hook.new_forward(minimax_h3_module, **sample_inputs)` then `video_logits, audio_logits = hook.new_forward(minimax_h3_module, **sample_inputs)`; `assert block_calls == 1`; shape asserts on video_logits/audio_logits — no torch.testing.assert_close vs uncached outputs) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:790` [minor] — MiniMaxH3SPPrepare (sp_prepare) is currently a no-op (minimax_h3_transformer.py:798-799). extract_minimax_h3_context computes modulated_input from decoder_input before sp_prepare (extractors.py:1341-1352), which matches today's behavior because sp_prepare does not transform tensors. If SP sharding lands later, extractor modulated_input placement may need a tracked alignment follow-up. (evidence: minimax_h3_transformer.py:798-799 `return hidden_states, rope_freqs, combined_indices` — sp_prepare is a no-op; extractors.py:1341-1349 modulated_input built from decoder_input via blocks[0] adaln/norm1 before extractors.py:1352 `hidden, block_rope, block_combined = module.sp_prepare(decoder_input, rope_freqs, combined_indices)` inside run_transformer_blocks) + +`docs/user_guide/diffusion/cache_acceleration/teacache.md:?` [minor] — This PR changes `DiffusionCacheConfig.rel_l1_thresh` to `None` and resolves model-specific defaults in `TeaCacheConfig` (`MiniMaxH3DiTModel` → 0.17, 0.2 fallback), but `docs/user_guide/diffusion/cache_acceleration/teacache.md` still documents a single global default of `0.2` at line 122. Update the parameter table to state that unset `rel_l1_thresh` defers to TeaCache model defaults and note H3's 0.17. (evidence: docs/user_guide/diffusion/cache_acceleration/teacache.md:122 `| \`rel_l1_thresh\` | float | \`0.2\` | Similarity threshold for cache reuse.`; vllm_omni/diffusion/data.py:422 `rel_l1_thresh: float | None = None`; vllm_omni/diffusion/cache/teacache/config.py:94-98 `_DEFAULT_REL_L1_THRESH = 0.2` and `_MODEL_DEFAULT_REL_L1_THRESH = {"MiniMaxH3DiTModel": 0.17}`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143` [minor] — This PR adds test_minimax_h3_t2va_teacache_single_gpu_smoke (inherits module pytestmark local_model and skipif on VLLM_TEST_MINIMAX_H3_FL2VA_MODEL), but no .buildkite job selects local_model or tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py—nightly only runs test_minimax_h3_quantization_quality.py with -m 'full_model and cuda and H100'. Wire the TeaCache smoke into a nightly/merge lane or drop it from the PR test plan as CI evidence. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:9-16 `pytestmark = [pytest.mark.local_model, ...]`; `MODEL_ENV = "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL"`; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:139-142 `@pytest.mark.skipif(not os.environ.get(MODEL_ENV), ...)` then `def test_minimax_h3_t2va_teacache_single_gpu_smoke():`; .buildkite/cuda/test-nightly.yml:510 `- pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"`; repo-wide .buildkite grep for `local_model` and `test_minimax_h3_e2e` returns no matches.) + +`tests/diffusion/cache/test_cache_backends.py:32` [nit] — gate_report shows buildkite/vllm-omni-amd-ci failing on this PR head. The new TeaCache backend tests here are CPU-only (core_model+cpu); can you confirm whether the AMD failure is pre-existing/unrelated or needs resolution before merge? (evidence: [sweep] AMD CI: gate_report lists buildkite/vllm-omni-amd-ci failing; diff is CUDA TeaCache/cache code, likely unrelated—confirm attribution.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5840.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5840.trace.json.gz new file mode 100644 index 000000000..1bc3d6a1d Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5863.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5863.cost.json new file mode 100644 index 000000000..ad082eae8 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5863.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 112.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5863/run-20260814-182600-834123", + "recorded_at": "2026-08-14T18:27:53", + "attempts": [ + { + "invocation_id": "inv-1afe740a", + "outcome": "run:run-20260814-182600-834123", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-182600-834123", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-182600-834123", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0238, + "minutes": 1.87, + "ci_minutes": 0, + "input_tokens": 56635, + "output_tokens": 7722, + "tool_calls": 37, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.0854, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.74, + "gate": 1.75, + "review": 106.58, + "report": 0.06 + }, + "time_to_first_result_s": 112.11 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 13194 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5863.md b/eval/dataset/arms/copilot_cb_composer25/pr5863.md new file mode 100644 index 000000000..d9eaac41c --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5863.md @@ -0,0 +1,43 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] Correctness/logic: read recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md — parallel configs satisfy tp×usp×ring=num_gpus (2×1, 2×2, 2×4, 4×2); text_encoder_tp_size ≤ dit world size per pipeline_minimax_h3.py:642-644 +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:94-100,123-130,152-159 — --usp, --ring, --text-encoder-tp-size, --vae-patch-parallel-size, --diffusion-attention-backend are real serve.py flags (serve.py:461,719,538) +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:306-307 — t2va requires explicit aspect_ratio matches pipeline_minimax_h3.py:347-348 +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:332-333 — ref2va optional aspect_ratio defaults to 16:9 per pipeline_minimax_h3.py:338-349 +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:220-224 — SM120 uses CUDNN_ATTN not TRTLLM_ATTN; platform.py:190-196 rejects TRTLLM on non-10.x, defaults CUDNN on Blackwell sm_120 +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:253-258,325-328 — X-Stage-Durations and X-Peak-Memory-MB headers exist in api_server.py:3510-3511 with --enable-diffusion-pipeline-profiler (serve.py:784) +- [sweep] Breaking changes: no code/API defaults changed; docs-only diff +- [sweep] Rebase/merge damage: single new file, no symbol drift + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340` [minor] — This diff adds a Ref2VA curl example with hardcoded `/root/` paths and a curl `"}` — `decode_audio_url` only accepts http(s) or data URLs (video_api_utils.py:297-298), so readers need the file contents spelled out, not an opaque path. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:510 uses `-F "audio_reference={\"audio_url\":\"${AUDIO_URL}\"}"`; vllm_omni/entrypoints/openai/video_api_utils.py:297-298: `raise InvalidInputReferenceError("Invalid audio_reference.audio_url: must be an http(s) URL or data URL.")`) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [minor] — This hunk lists `--cpunodebind` and `--membind` alongside vllm `--port`, but those are not vllm serve flags (repo-wide grep finds them only here). Wrap the server launch in `numactl --cpunodebind=0 --membind=0` (and node 1 for the second server) so copy-paste does not fail argparse. (evidence: grep cpunodebind/membind across repo returns only recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201-203; vllm_omni/entrypoints/cli/serve.py has no such options) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:4` [nit] — The intro compares against an "RTX PRO 5000 profile" and a four-GPU minimum, but no RTX PRO 5000 recipe exists under recipes/MiniMaxAI/. Add a link to the baseline doc or one sentence stating what that profile is (e.g., datacenter four-GPU no-offload from MiniMax-H3.md) so the comparison is actionable. (evidence: grep 'RTX PRO 5000' under recipes/ returns only this file; recipes/MiniMaxAI/MiniMax-H3.md:210-241 documents the four-GPU datacenter profile without RTX PRO 5000 naming) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:274` [nit] — The scaling prose rounds all text-encode times to 0.04 s, but the table above reports 0.035 s for eight GPUs (line 245). Change to "~0.04 s" or cite the table range so the narrative matches the measured values. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:245 `| Text encode | 0.040 s | 0.044 s | 0.035 s |` vs line 274 fixed 0.04 s claim) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5863.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5863.trace.json.gz new file mode 100644 index 000000000..88c55a027 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5863.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5884.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5884.cost.json new file mode 100644 index 000000000..5177885c7 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5884.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 418.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5884/run-20260814-182753-e9a649", + "recorded_at": "2026-08-14T18:34:51", + "attempts": [ + { + "invocation_id": "inv-2e35140b", + "outcome": "run:run-20260814-182753-e9a649", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-182753-e9a649", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-182753-e9a649", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.197, + "minutes": 6.97, + "ci_minutes": 0, + "input_tokens": 449503, + "output_tokens": 68719, + "tool_calls": 256, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3256, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.48, + "gate": 1.86, + "review": 412.56, + "report": 0.06 + }, + "time_to_first_result_s": 417.95 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5884.trace.json.gz", + "trace_bytes": 16739 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5884.md b/eval/dataset/arms/copilot_cb_composer25/pr5884.md new file mode 100644 index 000000000..5ce3d74ce --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5884.md @@ -0,0 +1,45 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5884, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5dc24e7f6a0c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] backend.py:43-53 — `_dit_module_names` now resolves dotted paths via `attrgetter`, catching `AttributeError` and skipping `None`; matches BagelPipeline `_dit_modules=['language_model.model']` (pipeline_bagel.py:159) and ModuleDiscovery pattern (module_collector.py:102-110). +- [sweep] backend.py:45 `if not isinstance(name, str)` — unchanged semantics; non-string entries in `_dit_modules` are still skipped. +- [sweep] backend.py:51 `if module is not None` — reachable when a dotted path resolves to an existing attribute set to `None`; entry is correctly omitted from enable/summary lists. +- [sweep] backend.py:61 `if not BlockAdapter.is_cached(transformer)` — verified against cache_dit `BlockAdapter.is_cached` (checks `getattr(module, '_is_cached', False)` for `nn.Module`); Cosmos3 custom enabler only caches outer `transformer` (model_specific.py:781-804), so skipping `transformer.language_model` in summary is intentional. +- [validated] tests/diffusion/cache/test_cache_backends.py:139-161 — `test_enable_and_refresh_nested_declared_dit` drives real `CacheDiTBackend.enable/refresh` and `cache_summary` with `_dit_modules=['language_model.model']`, asserting `enable_cache`/`refresh_context`/`summary` target the nested transformer. +- [validated] tests/diffusion/cache/test_cache_backends.py:121-135 — `test_summary_skips_uncached_nested_dit` calls unpatched `cache_summary` and confirms only the `_is_cached=True` module is summarized. +- [validated] tests/diffusion/cache/test_cache_backends.py:32 — `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` ensures new tests are not marker-deselected in CI. +- [upstream-verify] tests/diffusion/offloader/test_module_collector.py:119-130,266 — `ResidentPipeline` already validates `language_model.model` dotted discovery for offload; Cache-DiT path now consistent. + +`tests/diffusion/cache/test_cache_backends.py:121` [minor] — This diff adds the BlockAdapter.is_cached gate in cache_summary for Cosmos3-style nested _dit_modules, but test_summary_skips_uncached_nested_dit only exercises Mock _is_cached flags and never the post-enable Cosmos3 path (enable_cache_for_cosmos3). Extend test_cosmos3_cache_dit_wraps_gen_layers or add a sibling test that calls cache_summary after enable_cache_for_cosmos3 and asserts cache_dit.summary runs for outer transformer while transformer.language_model is skipped—otherwise a regression that leaves the outer module uncached would silence all Cosmos3 summaries. (evidence: backend.py:61-62 `if not BlockAdapter.is_cached(transformer):` / `continue`; pipeline_cosmos3.py:816 `_dit_modules: ClassVar[list[str]] = ["transformer.language_model", "transformer"]`; test_cache_backends.py:123-126 `language_model._is_cached = False` / `transformer._is_cached = True`; test_cache_dit.py:167-180 `def test_cosmos3_cache_dit_wraps_gen_layers(...)` through `assert adapter_kwargs["check_forward_pattern"] is False` with no cache_summary call) + +`tests/diffusion/cache/test_cache_backends.py:32` [minor] — gate_report shows `buildkite/vllm-omni-amd-ci` red; AMD ready runs `pytest -sv tests/diffusion -m 'core_model and cpu'` (`.buildkite/amd/test-amd-ready.yml:19`), which collects this file via `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` at line 32. Has the AMD failure been triaged for attribution to this PR's new cache tests before merge? (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — entire module (including new tests) is in AMD ready CPU diffusion scope; .buildkite/amd/test-amd-ready.yml:19 `timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py` — `test_cache_backends.py` is not ignored) + +`diffusion_model_runner.py:153` [minor] — This PR fixes dotted `_dit_modules` resolution in Cache-DiT via `attrgetter`, but `vllm_omni/diffusion/worker/diffusion_model_runner.py:_compile_transformer` (line 153) and `vllm_omni/diffusion/lora/manager.py` (lines 386-388) still resolve `_dit_modules` with top-level `getattr`/`hasattr`. Bagel (`_dit_modules = ["language_model.model"]`) compile/LoRA paths remain unfixed by this PR. Should the PR body or #5879 explicitly call out that remainder as out of scope? (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:153 `model = getattr(self.pipeline, attr_name, None)` — top-level getattr, not dotted-path resolution; vllm_omni/diffusion/lora/manager.py:386-388 `if not hasattr(self.pipeline, component_name):` / `component = getattr(self.pipeline, component_name)` — same top-level resolution; vllm_omni/diffusion/models/bagel/pipeline_bagel.py:159 `_dit_modules: ClassVar[list[str]] = ["language_model.model"]` — nested path affected by the gap) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [minor] — attrgetter in shared _dit_module_names / CacheDiTBackend.enable (backend.py:47-48, 239-246) fixes dotted _dit_modules for all generic-path pipelines, not only Bagel. SoulXSingerPipeline also declares a dotted path (`cfm_decoder.model.diff_estimator`, pipeline_soulx_singer_base.py:63) and has no CUSTOM_DIT_ENABLERS entry. Consider a SoulXSinger enable/refresh smoke or document Bagel-only test scope. (evidence: pipeline_soulx_singer_base.py:63 `_dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]`; pipeline_bagel.py:159 `_dit_modules: ClassVar[list[str]] = ["language_model.model"]`; backend.py:47-48 `module = attrgetter(name)(pipeline)`; backend.py:239 `for name in _dit_module_names(pipeline):`; model_specific.py:846-856 CUSTOM_DIT_ENABLERS lists Wan22/Cosmos3/Krea2 only — no BagelPipeline or SoulXSingerPipeline) + +`vllm_omni/diffusion/cache/cachedit/backend.py:57` [nit] — Update the `cache_summary` docstring (backend.py:57): it still says "Log Cache-DiT statistics for every transformer on the pipeline," but lines 61–62 skip modules where `BlockAdapter.is_cached(transformer)` is false. State that only DiT modules with an active Cache-DiT context are summarized (e.g., nested paths like `transformer.language_model` are skipped when uncached). (evidence: backend.py:57 """Log Cache-DiT statistics for every transformer on the pipeline.""" — docstring unchanged; backend.py:61 `if not BlockAdapter.is_cached(transformer):` — summary gated to cached modules only; backend.py:62 `continue` — uncached transformers skipped before `cache_dit.summary`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5884.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5884.trace.json.gz new file mode 100644 index 000000000..e3adcbde6 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5884.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5957.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5957.cost.json new file mode 100644 index 000000000..3e70a6a90 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5957.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 505.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5957/run-20260814-183054-fd114b", + "recorded_at": "2026-08-14T18:39:18", + "attempts": [ + { + "invocation_id": "inv-1beda600", + "outcome": "run:run-20260814-183054-fd114b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-183054-fd114b", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-183054-fd114b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4984, + "minutes": 8.41, + "ci_minutes": 0, + "input_tokens": 1568433, + "output_tokens": 68086, + "tool_calls": 291, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4872, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.42, + "gate": 1.94, + "review": 498.15, + "report": 0.06 + }, + "time_to_first_result_s": 504.57 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5957.trace.json.gz", + "trace_bytes": 117420 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5957.md b/eval/dataset/arms/copilot_cb_composer25/pr5957.md new file mode 100644 index 000000000..405fe4018 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5957.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5957, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 19a6e513f28e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] Read `vllm_omni/entrypoints/openai/tts_adapters/indextts2.py` and `serving_speech.py:517-524,3642-3646`: IndexTTS25Adapter sets `duration_factor=1.0/speed` and serving skips generic waveform speed adjustment via `_audio_encode_speed` returning 1.0 when `native_speed_control` is true. +- [validated] Read `indextts2_talker.py:166-171,453-486` and `stage_input_processors/indextts2.py`: v2.5 code-only path defers mel codes to `request.output_token_ids` at request end; `meta.use_gpt_latent` is always emitted; latent/crop mismatches now raise instead of silently truncating. +- [validated] Read `configuration_indextts2.py:158-160,169-203`: `IndexTTS25Config` sets `vocab_size` from `number_mel_codes` for vLLM 0.27 warmup when `skip_tokenizer_init=true`. +- [validated] Read `engine/arg_utils.py:245-275` and `tests/engine/test_arg_utils.py`: `_patch_empty_hf_config` is scoped to `indextts2_5` bundles missing `config.json`; malformed configs still surface through vLLM's loader. +- [sweep] Correctness: reviewed duration_factor bounds in `indextts2_s2mel_decoder.py:670-678`, CFM seeded noise batch-order tests, and stage-0 tokenizer-only renderer in `stage_init_utils.py` — no unguarded None/empty paths found in production hot paths. +- [sweep] Breaking behavior: `preprocess_utils.py:265-282` `compute_fbank` now raises if torchaudio Kaldi fbank is unavailable (removed zero-fbank fallback); affects IndexTTS 2 and 2.5 callers such as `indextts2_talker.py:634`. +- [sweep] Tests: `test_configuration_and_pipeline.py`, `test_enhanced_codec.py`, and `test_recipe_defaults.py` have no `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` unlike sibling files; `.buildkite/amd/test-amd-ready.yml:19` and `.buildkite/cuda/test-ready.yml:13` select `-m 'core_model and cpu'`, so these tests are deselected. +- [sweep] Docs: `recipes/IndexTeam/IndexTTS-2_5.md:40` titles H20 96GB but line 290 Notes say exercised on H200; `tokenizer_v2_5.py` supports `es`/`ar` but user READMEs only list zh/en/zhen/ja/yue. + +`tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:16` [major] — Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` to `test_configuration_and_pipeline.py`, `test_enhanced_codec.py`, and `test_recipe_defaults.py` (same as `test_s2mel_v2_5.py:26`). PR L2 runs `pytest -sv tests/model_executor -m 'core_model and cpu'` (`.buildkite/cuda/test-ready.yml:13`, `.buildkite/amd/test-amd-ready.yml:19`), so these modules are currently deselected even though they are the only guards for `OMNI_PIPELINES['indextts2_5']` (`test_configuration_and_pipeline.py:55`), real `EnhancedCodec` decode/layout (`test_enhanced_codec.py`), and `s2mel_cfm_batch_size` in `indextts2_5.yaml` (`test_recipe_defaults.py:14`). (evidence: test_configuration_and_pipeline.py:1-16 — file imports yaml/OMNI_PIPELINES then defines `def test_indextts25_defaults_are_distinct_from_v2():` with no `pytestmark`; test_s2mel_v2_5.py:26 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; .buildkite/cuda/test-ready.yml:13 `"timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' ..."`; test_configuration_and_pipeline.py:55 `assert OMNI_PIPELINES["indextts2_5"] is INDEXTTS25_PIPELINE`; test_recipe_defaults.py:14 `assert stage1["hf_overrides"]["s2mel_cfm_batch_size"] == 4`) + +`vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:224` [major] — Spanish (`lang=="es"`) is accepted, but `_normalize_with_official_backend` silently returns raw text when `nemo_text_processing` is missing (lines 234-237). The `[indextts2]` extra installs only fugashi/unidic-lite/wetext, and the recipe documents zh/en/zhen/ja/yue. With default `text_normalization=true`, Spanish requests get clean+uppercase preprocessing only, not official NeMo TN. Add NeMo to optional deps and error when TN is unavailable, or narrow language claims. (evidence: text_processing_v2_5.py:234-237 `except ModuleNotFoundError as exc: if exc.name != "nemo_text_processing": raise; return text`; pyproject.toml:87-92 `indextts2 = ["fugashi>=1.2.0", "unidic-lite>=1.0.0", "wetext>=0.0.9; sys_platform != 'linux'", "WeTextProcessing; sys_platform == 'linux'"]`; text_processing_v2_5.py:282-288 `if text_normalization: text = normalize(text, lang)` then `elif lang == "es": text = text.upper()`; recipes/IndexTeam/IndexTTS-2_5.md:298-299 `Languages: common codes include zh, en, zhen, ja, and yue`) + +`.buildkite/amd/test-amd-ready.yml:?` [minor] — AMD L2 gate `buildkite/vllm-omni-amd-ci` is red on this PR. The blocking step in `.buildkite/amd/test-amd-ready.yml` (`Simple · Diffusion & Model Executor Test`, line 19) runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`, which now includes seven new `tests/model_executor/models/indextts2/*` suites marked `core_model` and `cpu`. Please paste the failing Buildkite step/log from build #10484 and confirm whether the failure is PR-specific or already present on main. (evidence: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; tests/model_executor/models/indextts2/test_s2mel_v2_5.py:26 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +`vllm_omni/model_executor/models/indextts2/preprocess_utils.py:281` [minor] — The diff makes `compute_fbank` fail fast when torchaudio Kaldi fbank is unavailable (`preprocess_utils.py:281-282`), replacing the old zero-fbank fallback. The shared talker prefill path calls it on speaker-cache miss (`indextts2_talker.py:633-636`), affecting both IndexTTS 2.0 and 2.5. Document this breaking behavior change in release notes for deployments that previously relied on silent zero conditioning. (evidence: preprocess_utils.py:281-282 `except (ImportError, RuntimeError, OSError) as exc: raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc`; indextts2_talker.py:633-636 `campplus = load_campplus(self.model_path, device)` / `fbank = compute_fbank(wav_16k, device)` / `style = campplus(fbank)` on the shared cache-miss prefill path used by both model types via `IndexTTS2TalkerForConditionalGeneration`) + +`vllm_omni/deploy/qwen2_5_omni.yaml:90` [minor] — The XPU platform block in `qwen2_5_omni.yaml` sets `enforce_eager: true` on AR stages 0 and 1 for every Qwen2.5-Omni XPU launch (collateral to IndexTTS). Please confirm an XPU/Qwen2.5-Omni run or Intel CI covered this change and note any perf impact in the PR. (evidence: vllm_omni/deploy/qwen2_5_omni.yaml:88-93 `- stage_id: 0` / `enforce_eager: true` and `- stage_id: 1` / `enforce_eager: true` under `platforms.xpu` — disables cudagraph on both Qwen2.5-Omni AR stages for all XPU launches, not IndexTTS-only.) + +`docker/Dockerfile.xpu:91` [minor] — PR Test Plan still cites vLLM 0.26.0, but `docker/Dockerfile.xpu` (line 91) and `.buildkite/intel/pipeline-intel.yml` (line 13) pin v0.27.0. Please align the Test Plan vLLM Version with the baseline this PR actually builds against. (evidence: docker/Dockerfile.xpu:91 `ARG VLLM_VERSION=v0.27.0`; .buildkite/intel/pipeline-intel.yml:13 `VLLM_VERSION: "v0.27.0"`; PR #5957 Test Plan `**vLLM Version:** 0.26.0` (frozen pr_context snapshot).) + +`tests/e2e/test_indextts2_expansion.py:1` [minor] — IndexTTS 2.0 has e2e expansion tests at tests/e2e/online_serving/test_indextts2_expansion.py and tests/e2e/offline_inference/test_indextts2_expansion.py, but this PR adds no IndexTTS 2.5 e2e counterpart despite new pipeline/codec paths (coverage is mostly CPU/unit under tests/model_executor/models/indextts2/). Is 2.5 e2e intentionally deferred, or should a minimal online/offline e2e be added before claiming recipe/H200 acceptance? (evidence: tests/e2e/online_serving/test_indextts2_expansion.py:24 `MODEL = "IndexTeam/IndexTTS-2"`; tests/e2e/online_serving/test_indextts2_expansion.py:32 `stage_config_path=get_deploy_config_path("indextts2.yaml")`; tests/e2e/offline_inference/test_indextts2_expansion.py:24 `MODEL_NAME = "IndexTeam/IndexTTS-2"`; repo-wide grep under tests/e2e/ for indextts2_5|IndexTTS25|IndexTTS-2.5 returned 0 files) + +`vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:?` [minor] — `_normalize_with_official_backend` special-cases only zh/en/zhen/es; with default `text_normalization=True`, `lang=ar` gets shared cleaning/annotation but no language-specific normalization. The es/ar tokenizer test sets `text_normalization=False`, and the recipe documents zh/en/zhen/ja/yue only. If Arabic preprocessing is an acceptance target, implement or document the route; otherwise narrow the test name/claims. (evidence: text_processing_v2_5.py:221-238 `def _normalize_with_official_backend(text: str, lang: str) -> str:` … `if lang in {"zh", "en", "zhen"}: return _normalize_zh_or_en(text)` … `if lang == "es":` … `return text`; test_tokenizer_v2_5.py:236-240 `prepare_indextts25_text(..., text_normalization=False,)` for `lang="ar"`; recipes/IndexTeam/IndexTTS-2_5.md:166-167 lists `zh`, `en`, `zhen`, `ja`, and `yue` only.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/deploy/qwen2_5_omni.yaml:90` [minor] Collateral change: `qwen2_5_omni.yaml` XPU overrides now set stages 0/1 `enforce_eager: true` (PyTorch 2.13 XPU Dynamo duplicate-handler comment). Only `test_qwen2_5_omni_xpu_uses_eager_ar_stages` covers it, and Intel CI + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5957.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5957.trace.json.gz new file mode 100644 index 000000000..919f36302 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5976.cost.json b/eval/dataset/arms/copilot_cb_composer25/pr5976.cost.json new file mode 100644 index 000000000..36474fc3c --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5976.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 633.4, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_composer25/runs/pr5976/run-20260814-183352-6d3fc9", + "recorded_at": "2026-08-14T18:44:24", + "attempts": [ + { + "invocation_id": "inv-4e578aa0", + "outcome": "run:run-20260814-183352-6d3fc9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-183352-6d3fc9", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-183352-6d3fc9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4631, + "minutes": 10.55, + "ci_minutes": 0, + "input_tokens": 1273137, + "output_tokens": 108496, + "tool_calls": 417, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5297, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.73, + "gate": 2.0, + "review": 625.97, + "report": 0.07 + }, + "time_to_first_result_s": 632.76 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 60178 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5976.md b/eval/dataset/arms/copilot_cb_composer25/pr5976.md new file mode 100644 index 000000000..cb74dbb2b --- /dev/null +++ b/eval/dataset/arms/copilot_cb_composer25/pr5976.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness: read omni_ar_scheduler.py:327-353,490-491,627-670 — stale drain subtracts num_tokens_scheduled commensurate with in_flight seeding; segment-stop and stage-0 extend paths seed once. +- [validated] omni_generation_scheduler.py:598-604 calls only mixin _replace_streaming_session (single seed); generation path not double-seeded. +- [sweep] Generation segment-stop gap: omni_generation_scheduler.py:461-470 lacks AR segment-stop stale seeding at omni_ar_scheduler.py:490-491; only _replace_streaming_session seeds — unverified whether async in-flight can arrive between segment stop and session replace. +- [sweep] Dead field: omni_ar_scheduler.py:634,479 set async_tokens_to_discard but no vllm_omni consumer reads it post-0.27; suppression relies solely on num_stale_output_tokens. +- [validated] Qwen3 forward contract: qwen3_omni.py:144-148,409-422 gates capture on is_staged_run; qwen3_omni_moe_thinker.py:1516-1521 returns bare tensor without capture; test_qwen3_omni_forward_contract.py covers both shapes. +- [validated] Unconditional registry overwrite: engine/arg_utils.py:115-116 re-registers all _OMNI_MODELS over upstream 0.27 entries — intentional per comment. +- [validated] Replica compile cache: stage_engine_startup.py:936-951 isolates VLLM_CACHE_ROOT for replica_id>0; launch_stage_replica:1147 passes stage_id/replica_id; stage_runtime.py:580-590 delegates spawn to launch_stage_replica. +- [validated] EP all2all: parallel_state.py:537-538 version-guards use_all2all kwarg; initialize_model_parallel:969 passes use_all2all=True for EP. + +`vllm_omni/core/sched/omni_ar_scheduler.py:628` [major] — OmniARScheduler._update_request_as_session seeds num_stale_output_tokens from num_in_flight_tokens (lines 627–629) before the replace_streaming_prompt check; when that flag is set (lines 669–670) it calls mixin _replace_streaming_session, which adds num_in_flight_tokens again at line 105 without clearing the counter. With async lookahead (e.g. num_in_flight_tokens=2) the stale drain becomes 4 instead of 2 and can swallow the first valid new-segment frame—the same codec-hole failure this PR fixes for placeholder-based seeding. Seed in one place only: skip the pre-call seed when delegating to _replace_streaming_session (generation scheduler already uses mixin-only seeding at line 604), and add a replace_streaming_prompt test with num_in_flight_tokens>0 asserting num_stale_output_tokens equals the in-flight count (not 2×) and that the first post-replace frame is delivered after exactly that many stale drains. (evidence: omni_ar_scheduler.py:628 `if in_flight_tokens > 0:`; omni_ar_scheduler.py:629 `session.num_stale_output_tokens += in_flight_tokens`; omni_ar_scheduler.py:669 `if replace_streaming_prompt:`; omni_ar_scheduler.py:670 `self._replace_streaming_session(session, update)`; omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`; omni_generation_scheduler.py:604 `self._replace_streaming_session(session, update)` — generation path has no prior stale seed.) + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [minor] — After migrating discard draining to `num_stale_output_tokens` in `update_from_output` (lines 336–353), `async_tokens_to_discard` is still written at segment stop (line 479) and stage-0 streaming replacement (line 634) but nothing reads it in production code. Remove those dead assignments and update `test_stage0_streaming_update_discards_outstanding_async_placeholder_token` to seed `num_in_flight_tokens=1` and assert `update_from_output` suppression via `num_stale_output_tokens` (mirror `test_omni_ar_scheduler_stale_drain.py`). (evidence: omni_ar_scheduler.py:336-353 `if request is not None and request.num_stale_output_tokens > 0: output_is_stale = True; request.num_stale_output_tokens -= num_tokens_scheduled` — only stale drain path in update_from_output; omni_ar_scheduler.py:479 `request.async_tokens_to_discard = outstanding_async_tokens`; omni_ar_scheduler.py:634 `session.async_tokens_to_discard = 1`; repo grep shows no production reads of async_tokens_to_discard outside those two writes; tests/core/sched/test_omni_ar_scheduler_streaming.py:221 `assert session.async_tokens_to_discard == 1` with no num_in_flight_tokens or update_from_output assertion) + +`vllm_omni/core/sched/omni_generation_scheduler.py:?` [minor] — AR segment-stop continuation seeds `num_stale_output_tokens` from `num_in_flight_tokens` (`omni_ar_scheduler.py:490-491`), but the generation branch at `omni_generation_scheduler.py:467-470` only discards `chunk_transfer_adapter` state. Async-chunk LLM_GENERATION stages (code2wav, moss codec, etc.) resume via `_handle_stopped_request` → WAITING without `_replace_streaming_session`, so mixin stale seeding (`omni_scheduler_mixin.py:105`) never runs. If segment-stop can occur with `num_in_flight_tokens > 0`, late frames can still append because `update_from_output` only drops when `num_stale_output_tokens > 0`. Mirror AR seeding here or document why generation cannot hit that combination; add a generation-scheduler test with `num_in_flight_tokens=2` at segment boundary and a follow-on frame. (evidence: omni_generation_scheduler.py:467-470 `if not finished: + # for streaming input request only + if self.chunk_transfer_adapter: + self.chunk_transfer_adapter.segment_finished_requests.discard(req_id)` — no `num_stale_output_tokens` seeding; omni_ar_scheduler.py:490-491 `if request.num_in_flight_tokens > 0: + request.num_stale_output_tokens += request.num_in_flight_tokens`; omni_generation_scheduler.py:385-401 stale drain runs only when `request.num_stale_output_tokens > 0` before `if output_is_stale: continue`) + +`tests/e2e/accuracy/test_qwen_image.py:262` [minor] — SSIM is gated at 0.97 on the FA3-hub path but 0.96 when `kernels-community/flash-attn3` is unavailable (line 262); both sides use matched SDPA on that branch. Inline comments cite build 2954 at 0.9687 — is 0.96 the right permanent floor, or should it stay env/marker-gated as torch variants evolve? (evidence: tests/e2e/accuracy/test_qwen_image.py:32 `SSIM_THRESHOLD = 0.97`; tests/e2e/accuracy/test_qwen_image.py:38 `SSIM_THRESHOLD_SDPA_FALLBACK = 0.96`; tests/e2e/accuracy/test_qwen_image.py:262 `ssim_threshold=SSIM_THRESHOLD if _flash_attn3_hub_available() else SSIM_THRESHOLD_SDPA_FALLBACK`) + +`vllm_omni/core/sched/omni_ar_scheduler.py:627` [minor] — The stale-drain fix seeds `num_stale_output_tokens` from `num_in_flight_tokens`, but on the `replace_streaming_prompt` path it is applied twice: `_update_request_as_session` increments at :627-629 before branching, then `_replace_streaming_session` increments again at omni_scheduler_mixin.py:105. That over-seeds the counter drained at :336 and can still swallow valid new-segment frames. If nightly seed-TTS WER (~0.39) remains open for this reason, should merge explicitly track a follow-up (dedupe + regression test) versus what this PR is expected to close? (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0: session.num_stale_output_tokens += in_flight_tokens`; vllm_omni/core/sched/omni_ar_scheduler.py:669-670 `if replace_streaming_prompt: self._replace_streaming_session(session, update)`; vllm_omni/core/sched/omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`; vllm_omni/core/sched/omni_ar_scheduler.py:336-338 `if request is not None and request.num_stale_output_tokens > 0: output_is_stale = True` / `request.num_stale_output_tokens -= num_tokens_scheduled`) + +`tests/core/sched/test_omni_ar_scheduler_streaming.py:210` [minor] — `test_stage0_streaming_update_discards_outstanding_async_placeholder_token` (lines 210–221) sets `num_output_placeholders=1` but not `num_in_flight_tokens`, calls only `_update_request_as_session`, and asserts `async_tokens_to_discard == 1` without calling `update_from_output` or checking `num_stale_output_tokens`. With default `num_in_flight_tokens=0`, `_update_request_as_session` never seeds the stale counter (omni_ar_scheduler.py:627–629), so this case does not prove late-output suppression via `num_stale_output_tokens`; extend it or rely on `test_omni_ar_scheduler_stale_drain.py`. (evidence: tests/core/sched/test_omni_ar_scheduler_streaming.py:216 `session.num_output_placeholders = 1`; tests/core/sched/test_omni_ar_scheduler_streaming.py:219 `sched._update_request_as_session(session, _make_update([10, 20]))`; tests/core/sched/test_omni_ar_scheduler_streaming.py:221 `assert session.async_tokens_to_discard == 1`; vllm_omni/core/sched/omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0: session.num_stale_output_tokens += in_flight_tokens`) + +`vllm_omni/platforms/xpu/platform.py:139` [minor] — This diff changes the default XPU diffusion IR-op priority from `["xpu_kernels", "native"]` to `["vllm_c", "native"]` when inductor is inactive (`vllm_omni/platforms/xpu/platform.py:139`). `DiffusionWorker._resolve_ir_op_priority` reads the platform default for every model, so the change is platform-wide unless a model registry override applies. Either scope behind a model/platform override or cite an XPU diffusion smoke/benchmark proving parity before merge. (evidence: ['vllm_omni/platforms/xpu/platform.py:139 `default = ["native"] if using_inductor else ["vllm_c", "native"]`', 'vllm_omni/diffusion/worker/diffusion_worker.py:184 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`', 'vllm_omni/platforms/rocm/platform.py:210 `default = ["native"] if using_inductor else ["vllm_c", "native"]`']) + +`vllm_omni/engine/arg_utils.py:115` [minor] — This diff unconditionally overwrites every `_OMNI_MODELS` entry in the global vLLM `ModelRegistry` (arg_utils.py:115-116), so plain `vllm serve` paths always resolve to omni classes. Only Qwen2.5/Qwen3 received `getattr(..., 'model_stage', None) or 'thinker'` hardening in this PR; MiniCPM-o (minicpmo_4_5_omni.py:94), voxtral_tts (voxtral_tts.py:112), step_audio2 (step_audio2.py:57), ming_flash_omni (ming_flash_omni.py:98), and at least six others still read `vllm_config.model_config.model_stage` directly. Enumerate which archs vLLM 0.27 also registers and confirm each either has a non-staged default or cannot be reached via stock serve. (evidence: vllm_omni/engine/arg_utils.py:115-116 `for arch, (mod_folder, mod_relname, cls_name) in _OMNI_MODELS.items():` / `ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")`; vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py:144 `self.model_stage = getattr(vllm_config.model_config, "model_stage", None) or "thinker"`; vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py:94 `self.model_stage = vllm_config.model_config.model_stage`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_composer25/pr5976.trace.json.gz b/eval/dataset/arms/copilot_cb_composer25/pr5976.trace.json.gz new file mode 100644 index 000000000..091568b73 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_composer25/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.cost.json b/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.cost.json new file mode 100644 index 000000000..b66b0d082 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 578.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5840/run-20260814-201248-3ede23", + "recorded_at": "2026-08-14T20:22:26", + "attempts": [ + { + "invocation_id": "inv-f9f3b134", + "outcome": "run:run-20260814-201248-3ede23", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-201248-3ede23", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-201248-3ede23", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4015, + "minutes": 9.63, + "ci_minutes": 0, + "input_tokens": 1097046, + "output_tokens": 95743, + "tool_calls": 372, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4824, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.54, + "gate": 1.95, + "review": 571.99, + "report": 0.06 + }, + "time_to_first_result_s": 577.53 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 36085 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.md b/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.md new file mode 100644 index 000000000..4de3c42ad --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] vllm_omni/diffusion/cache/teacache/extractors.py:1312 batch x.shape[0]==1 matches recipe single-request batch; raise on mismatch. +- [validated] extractors.py:1341-1342 blocks[0] for modulated_input is intentional TeaCache decision input; residual is gathered full-seq (run_transformer_blocks + sp_gather) so USP fast-path addback is consistent with slow path. +- [validated] backend.py:79-91 ref2va raises; fl2va/combined hook only pipeline.transformer; combined warns; pipeline_minimax_h3.py:736-738 Ref2VA uses transformers_ref (uncached). +- [validated] test_cache_backends.py:490-532 exercises fl2va/combined hook target, None→0.17 / 0.2 override, and ref2va reject with enabled=False. +- [validated] OmniBase→AsyncOmniEngine._create_default_diffusion_stage_cfg calls _normalize_cache_config (async_omni_engine.py:954-958); omitted cache_config → None → inject 0.2. +- [sweep] Blast radius of DiffusionCacheConfig.rel_l1_thresh 0.2→None: TeaCacheConfig.__post_init__ still resolves None→0.2 for non-H3 types and →0.17 for MiniMaxH3DiTModel; MagCache/CacheDiT projection paths do not read rel_l1_thresh. +- [validated] Non-H3 TeaCache enable path still gets effective 0.2 when DiffusionCacheConfig() is empty: backend.py passes config.rel_l1_thresh into TeaCacheConfig which fills the default. +- [validated] MiniMax combined partition hooks only pipeline.transformer (FL2VA); transformers_ref is a separate MiniMaxH3DiTModel (pipeline_minimax_h3.py:634/639) and Ref2VA routes via _transformer_for_task — FL2VA-only enable is structurally correct. + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:?` [major] — This PR registers MiniMaxH3Pipeline→enable_minimax_h3_teacache and documents server-level TeaCache↔Cache-DiT mutual exclusivity in MiniMax-H3.md, but request-scoped quality="high" still builds a CacheDiTRequestSpec and pipeline_minimax_h3.py always calls _cache_dit_runtime.prepare(...), ignoring tea_cache. On a --cache-backend tea_cache server, quality=high can wrap DiTs TeaCache already hooked—the dual-cache state the recipe forbids. Gate quality=high (and force-refresh) when the configured backend is tea_cache (or tear down TeaCache before Cache-DiT), add a unit/e2e test, and rewrite the recipe quality / Known-limitations text so it no longer invites combining tea_cache serve with quality=high. (evidence: quality_policy.py:149-151 `if quality == "high":` / `base_key = MINIMAX_H3_HIGH_CACHE_KEY` / `base_config = _high_quality_cache_config()` — no tea_cache check; quality_policy.py:152 `elif self._configured_backend == "cache_dit" and quality is None:` — only omitted quality special-cases cache_dit; quality_policy.py:164-169 returns `MiniMaxH3QualityPlan(cache_dit=CacheDiTRequestSpec(...))` for high; pipeline_minimax_h3.py:1761-1766 `quality_plan = self._quality_policy.resolve(...)` then `self._cache_dit_runtime.prepare(quality_plan.cache_dit)`; runtime.py:71-74 `backend = CacheDiTBackend(spec.cache_config)` / `backend.enable(self._pipeline)` with no tea_cache gate; backend.py:91 `apply_teacache_hook(pipeline.transformer, teacache_config)`; MiniMax-H3.md TeaCache section `TeaCache and Cache-DiT are mutually exclusive`; MiniMax-H3.md:684 `-F 'quality=high' # H3's conservative Cache-DiT profile.` plus `No Cache-DiT startup option is required`) + +`vllm_omni/diffusion/cache/teacache/config.py:?` [major] — This PR sets DiffusionCacheConfig.rel_l1_thresh to None and registers MiniMaxH3DiTModel→0.17 in TeaCacheConfig, and the recipe calls 0.17 the model-specific default, but AsyncOmniEngine._get_default_cache_config still returns {"rel_l1_thresh": 0.2} and _normalize_cache_config injects it whenever cache_config is None (Omni / `vllm serve --cache-backend tea_cache`; arg_utils default None). Tests that build DiffusionCacheConfig() then TeaCacheBackend.enable never hit that omit path; recipe/e2e smoke pass explicit 0.17 and also miss it. Change the tea_cache default to {} (omit the key) so TeaCacheConfig can apply 0.17 for MiniMax (non-H3 still fall back to 0.2), and add a regression for _normalize_cache_config/Omni omit→0.17. (evidence: vllm_omni/diffusion/cache/teacache/config.py:97-99 `_MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, }`; vllm_omni/diffusion/data.py:422 `rel_l1_thresh: float | None = None`; vllm_omni/engine/async_omni_engine.py:853-856 `if cache_backend == "tea_cache": return { "rel_l1_thresh": 0.2, }`; vllm_omni/engine/async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"): cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`; vllm_omni/engine/arg_utils.py:489 `cache_config: str | None = None`; recipes/MiniMaxAI/MiniMax-H3.md:762 `The model-specific default and examples use \`rel_l1_thresh=0.17\``; tests/.../test_minimax_h3_e2e.py:156 `cache_config={"rel_l1_thresh": 0.17}`) + +`docs/user_guide/diffusion/cache_acceleration/teacache.md:?` [minor] — This PR changes DiffusionCacheConfig.rel_l1_thresh from float=0.2 to float|None=None and resolves MiniMaxH3DiTModel→0.17 (else 0.2) inside TeaCacheConfig.__post_init__, but docs/user_guide/diffusion/cache_acceleration/teacache.md:122 still documents type float and Default `0.2` only. Update the Configuration table (and omit-cache_config Quick Start) to describe deferred/model-specific defaults, including MiniMax-H3 FL2VA’s 0.17—same way coefficients already point at _MODEL_COEFFICIENTS. (evidence: data.py:421-422 `# None defers to the model-specific TeaCache default (0.2 fallback).` / `rel_l1_thresh: float | None = None`; config.py:96-99 `_DEFAULT_REL_L1_THRESH = 0.2` / `_MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, }`; config.py:122 `rel_l1_thresh: float | None = None`; config.py:128-130 `threshold = self.rel_l1_thresh` / `if threshold is None:` / `threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)`; teacache.md:122 `| \`rel_l1_thresh\` | float | \`0.2\` | Similarity threshold for cache reuse. Lower values prioritize quality (less caching), higher values prioritize speed (more caching). Suggested range: 0.1-0.8 |`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:?` [minor] — test_minimax_h3_t2va_teacache_single_gpu_smoke enables cache_backend="tea_cache" with rel_l1_thresh=0.17 but uses num_inference_steps=2 and only calls _assert_joint_output. The TeaCache guide lists short runs (<20 steps) under "Not for", and hook.py always fully computes cnt==0, so a 2-step smoke can pass without proving a cache hit. The only MiniMax-H3 hit assertion is CPU test_teacache_reuses_h3_block_residual (block_calls == 1). Raise steps and/or assert TeaCache hit/refresh telemetry if this should guard TeaCache behavior; otherwise treat/rename it as plumbing-only. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:153-154 `cache_backend="tea_cache",` / `cache_config={"rel_l1_thresh": 0.17},`; :170 `num_inference_steps=2,`; :186 `_assert_joint_output(outputs)`; docs/user_guide/diffusion/cache_acceleration/teacache.md:158 `- Very short inference runs (< 20 steps) where caching overhead may outweigh benefits`; vllm_omni/diffusion/cache/teacache/hook.py:227-230 `# First timestep: always compute` / `if state.cnt == 0:` / `return True`; tests/diffusion/cache/test_teacache_extractors.py:696 `assert block_calls == 1`) + +`tests/diffusion/cache/test_cache_backends.py:490` [minor] — gate_report shows buildkite/vllm-omni-amd-ci failing on this head, and this PR touches AMD-selected test_cache_backends.py. Can you confirm whether that red is on main already or introduced here (live log / main baseline), and green or explain before merge? (evidence: [gate] buildkite/vllm-omni-amd-ci failing per gate_report. / [sweep] gate_report: buildkite/vllm-omni-amd-ci failing; diff touches AMD-selected test_cache_backends.py — attribution unknown without main comparison. / BLOCKERS: buildkite/vllm-omni-amd-ci is failing per gate_report; cannot attribute to this PR vs main without live log diff.) + +`tests/diffusion/cache/test_teacache_extractors.py:570` [minor] — AMD ready/merge workflows explicitly `--ignore=tests/diffusion/cache/test_teacache_extractors.py`, so new `TestMiniMaxH3Extractor` coverage never runs on AMD; `test_cache_backends.py` (`pytestmark` core_model+cpu) is not ignored and still runs. Intentional for this PR, or should AMD select these CPU extractor tests? (evidence: .buildkite/amd/test-amd-ready.yml:19 `timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml`; .buildkite/amd/test-amd-merge.yml:21 `timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml`; tests/diffusion/cache/test_teacache_extractors.py:570 `class TestMiniMaxH3Extractor(BaseExtractorTest):`; tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +`recipes/MiniMaxAI/MiniMax-H3.md:753` [nit] — Title `[MiniMax-H3] TeaCache support and Cache-DiT validation` does not match the net diff: production changes are TeaCache-only (`teacache/{backend,config,extractors,hook}.py`, `data.py`, recipe TeaCache section). `quality_policy.py`’s Cache-DiT path is untouched and no Cache-DiT code lands. Drop Cache-DiT from the title, or link a durable Cache-DiT validation artifact/issue so the scope claim matches the tree. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:753 `## TeaCache acceleration` — new PR docs surface is TeaCache-only; vllm_omni/diffusion/cache/teacache/backend.py:127 `"MiniMaxH3Pipeline": enable_minimax_h3_teacache,` — MiniMax registers TeaCache only; vllm_omni/diffusion/models/minimax_h3/quality_policy.py:124 `class MiniMaxH3QualityPolicy:` / `:126 "Resolve H3 request quality into a declarative Cache-DiT target."` — Cache-DiT path exists but is not in this PR’s 10 changed files (teacache + docs/tests/data only); GitHub PR #5840 title `[MiniMax-H3] TeaCache support and Cache-DiT validation`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.trace.json.gz new file mode 100644 index 000000000..84e9a5f95 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/QUARANTINE_skillleak_pr5840/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/manifest.json b/eval/dataset/arms/copilot_cb_grok45/manifest.json new file mode 100644 index 000000000..ad52b321b --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_cb_grok45", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5509.cost.json b/eval/dataset/arms/copilot_cb_grok45/pr5509.cost.json new file mode 100644 index 000000000..06b55b016 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5509.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 424.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5509/run-20260814-200226-44a566", + "recorded_at": "2026-08-14T20:09:29", + "attempts": [ + { + "invocation_id": "inv-adc05c51", + "outcome": "run:run-20260814-200226-44a566", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-200226-44a566", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-200226-44a566", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2966, + "minutes": 7.06, + "ci_minutes": 0, + "input_tokens": 838601, + "output_tokens": 63827, + "tool_calls": 212, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3708, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.85, + "gate": 1.54, + "review": 417.81, + "report": 0.06 + }, + "time_to_first_result_s": 423.25 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 33146 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5509.md b/eval/dataset/arms/copilot_cb_grok45/pr5509.md new file mode 100644 index 000000000..9a0e7a767 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5509.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] Correctness None/empty: QuantConfig.from_backend_kwargs(None|{}) → enabled=False; AttnQuantSpec.enabled requires dtype_qk or dtype_vo; backend_kwargs omits quant when not enabled — OK for TRTLLM. +- [sweep] trtllm_attn.py:198/296 quant.enabled arms reachable and co-consistent; sage kwargs passed only when enabled (compat with older FlashInfer) — validated. +- [sweep] trtllm_attn.py:199 wrong dtype_qk raises RuntimeError at init; 204/211 fail-fast if sage_attn_sfs or trtllm_sage_attention_quantize missing — validated. +- [sweep] trtllm_attn.py:90 HAS_FLASHINFER short-circuit in _sage_kernel_available — reachable; avoids unbound name when import failed — OK. +- [sweep] data.py:1415/1418 AttnQuantSpec dtype/block validation arms reachable via tests — OK. +- [sweep] data.py:1447 allows TRTLLM_ATTN and FLASHINFER_ATTN for quant — FLASHINFER arm is live config but has no consumer (see review_comments). +- [sweep] data.py:1455/1464–1481 coerce and backend_kwargs skip_softmax+quant serialization — exercised by test_quant_* — OK. +- [sweep] test_trtllm_attn.py:315 Missing TRTLLM-GEN kernel skip arm is intentional for int8 SM100-only; can occur — OK. + +`vllm_omni/diffusion/data.py:?` [major] — This diff widens `AttentionSpec.__post_init__` so `quant` is legal on `FLASHINFER_ATTN` as well as `TRTLLM_ATTN`, and the new docs/tests treat `dtype_vo` / `flashinfer_backend` / float16|bfloat16 as live FLASHINFER fields (`AttnQuantSpec` serializes them into `backend_kwargs`). `FlashInferAttentionImpl.__init__` (flashinfer_attn.py:55–66) only keeps `**extra_impl_args` and never reads `backend_kwargs`/`quant`, while `layer.py:109` always passes `backend_kwargs=spec.backend_kwargs()` — unlike SDPA’s unused-kwargs warning — so configs like `{"backend":"FLASHINFER_ATTN","quant":{"dtype_qk":"bfloat16","dtype_vo":"fp8_e4m3"}}` are accepted then silently run dense; wrong `dtype_qk` never raises on FLASHINFER. Restrict the allowlist to `TRTLLM_ATTN`, drop the multi-backend docs claim and `test_quant_superset_fields_passed_through` as product behavior, or implement/reject quant in `flashinfer_attn.py` the way TRTLLM does. (evidence: data.py:1447 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):`; data.py:1479–1482 `if q.dtype_vo is not None: quant_kw["dtype_vo"] = q.dtype_vo` / `if q.flashinfer_backend is not None: quant_kw["flashinfer_backend"] = q.flashinfer_backend`; layer.py:109 `backend_kwargs=backend_kwargs,`; flashinfer_attn.py:55–66 `__init__(..., **extra_impl_args)` then only `self.causal = causal` / `self.softmax_scale = softmax_scale` (no `backend_kwargs` read); sdpa.py:89–90 `if backend_kwargs: logger.warning("SDPAImpl ignoring backend_kwargs: %s", list(backend_kwargs.keys()))`; docs attention_backends.md:185–188 ``FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`), `dtype_vo`, and `flashinfer_backend``; test_attention_config.py:65–73 `test_quant_superset_fields_passed_through` round-trips those fields with no FlashInfer forward assert.) + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — This diff deletes the entire `## Known Limitations` block (LTX-2.0 `CUDNN_ATTN`+compile workaround and FA4 note) plus Validation Guidance / mixed-backend / benchmark-reproduction sections, while `## Choosing a Backend Manually` still has `- **Workaround for known issues**: see Known Limitations above.` at docs/user_guide/diffusion/attention_backends.md:232. Restore that section (or delete/retarget this bullet and decide explicitly whether the LTX workaround still belongs in-tree) so the SAGE doc landing does not ship a dead cross-reference and drop unrelated user guidance. (evidence: attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — file-wide grep finds only this `Known Limitations` mention; no `## Known Limitations` heading remains.) + +`recipes/Wan-AI/Wan2.2-T2V.md:?` [major] — This PR adds typed SAGE `quant` (`int8`/`fp8_e4m3`) for `TRTLLM_ATTN` and documents enablement for Wan-AI/Wan2.2-T2V-A14B-Diffusers, but `recipes/Wan-AI/Wan2.2-T2V.md:186` still says `TRTLLM_ATTN` is BF16-only with no attention-level quantization. Update or remove that limitation bullet and point at the new `quant` / SAGE section so recipe consumers are not left on the old contract. (evidence: recipes/Wan-AI/Wan2.2-T2V.md:186 ` - `TRTLLM_ATTN` is BF16-only (no attention-level quantization).`; vllm_omni/diffusion/data.py:1434 ` quant: AttnQuantSpec | None = None`; vllm_omni/diffusion/data.py:1447-1450 ` if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` / ` raise ValueError(` / ` f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but "`; docs/user_guide/diffusion/attention_backends.md:170 `## TRTLLM_ATTN SAGE Quantization`; docs/user_guide/diffusion/attention_backends.md:191-193 `vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \` / ` --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",` / ` "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}'`) + +`tests/diffusion/attention/test_trtllm_attn.py:?` [major] — This diff’s only SAGE numeric check (`test_sage_quant_matches_sdpa` under `@requires_sage`) can soft-skip on `Missing TRTLLM-GEN kernel`, and ready CUDA selects `pytest … tests/diffusion/*.py` (does not recurse into `tests/diffusion/attention/`), so merge-ready green does not execute this regression. Paste local B200 logs for `pytest -sv tests/diffusion/attention/test_trtllm_attn.py -k test_sage_quant_matches_sdpa` showing both `fp8_e4m3` and `int8` arms asserted (not skipped), or wire an SM100 lane that runs that file without swallowing the int8 arm. (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]`; :315-316 `if "Missing TRTLLM-GEN kernel" in str(e):` / `pytest.skip(...)`; .buildkite/cuda/test-ready.yml:18 `pytest -sv tests/diffusion -m 'core_model and cpu'`; :68 `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` with :69 `mirror_hardwares: l4_1`; docs/user_guide/diffusion/attention_backends.md:209-210 ``int8` QK kernels are compiled for **SM100 only**.`) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — This PR adds typed `quant` / `AttnQuantSpec` on `AttentionSpec` and a full TRTLLM SAGE section, but the Configuration intro still says the only typed backend field is TRTLLM Skip-Softmax. Update that sentence (and its example) at lines 72–78 to also name `quant` so readers do not treat SAGE as env-only or undocumented. (evidence: docs/user_guide/diffusion/attention_backends.md:72-73 `A backend that needs configuration exposes it as a typed field on the spec. Today the only` / `one is TRTLLM_ATTN's Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):` — vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None` — docs/user_guide/diffusion/attention_backends.md:174 `Enable it through the typed quant block on the attention spec:`) + +`vllm_omni/diffusion/data.py:?` [minor] — AttnQuantSpec._VALID_DTYPES is a shared union (`float16`/`bfloat16`/`int8`/`fp8_e4m3`) with no backend gate, and AttentionSpec.__post_init__ only checks that quant is allowed on TRTLLM_ATTN/FLASHINFER_ATTN. So `AttentionSpec(backend="TRTLLM_ATTN", quant={"dtype_qk":"bfloat16"})` validates at construction and only fails later in `TrtllmAttentionImpl.__init__` (`dtype_qk not in _QK_QUANT_DTYPES`). Tighten per-backend dtype validation in `AttentionSpec.__post_init__` (or split specs) so the public contract matches the docs’ “A dtype_qk from the wrong set raises” at config time. (evidence: data.py:1410 `_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})` — data.py:1447-1450 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"): raise ValueError(...)` (backend presence only) — trtllm_attn.py:199-202 `if self.quant.dtype_qk not in _QK_QUANT_DTYPES: raise RuntimeError(... FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE.)` — attention_backends.md:188 `A dtype_qk from the wrong set raises.` — test_trtllm_attn.py:60-62 `test_quant_rejects_non_sage_dtype` / `_impl(quant={"dtype_qk": "bfloat16"})`) + +`docs/user_guide/diffusion/attention_backends.md:208` [minor] — SAGE docs (attention_backends.md:208) and init fail-close (trtllm_attn.py:210-216) require FlashInfer ≥0.6.16rc1 (`trtllm_sage_attention_quantize`), but this repo has no `flashinfer` floor in pyproject/requirements (transitive via vLLM). Add an extras/optional pin or an explicit “unpinned transitive; fail-closed at init / upgrade before `quant` works” note beside the SAGE Requirements so install docs match packaging. (evidence: docs/user_guide/diffusion/attention_backends.md:208 `**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:210-216 `self._sage_quantize_fn = _sage_quantize_fn()` / `if self._sage_quantize_fn is None:` / `raise RuntimeError(` / `"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build lacks "` / `"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "`; pyproject.toml/requirements*: no `flashinfer` match.) + +`vllm_omni/diffusion/data.py:?` [nit] — This diff adds `quant` to `spec_keys` in `_flatten_per_role_entry` (data.py:1553), but the mixed-entry error at :1559 still says only `backend/skip_softmax`. Mention `quant` in that message so a nested/quant misconfig is diagnosable. (evidence: data.py:1553 `spec_keys = {"backend", "skip_softmax", "quant"}` — data.py:1559 `f"Invalid per_role entry for role {role!r}: cannot mix backend/skip_softmax with nested role keys."`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5509.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/pr5509.trace.json.gz new file mode 100644 index 000000000..7921be798 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5550.cost.json b/eval/dataset/arms/copilot_cb_grok45/pr5550.cost.json new file mode 100644 index 000000000..34d75898b --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5550.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 623.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5550/run-20260814-200226-4aed43", + "recorded_at": "2026-08-14T20:12:48", + "attempts": [ + { + "invocation_id": "inv-fa41bcb5", + "outcome": "run:run-20260814-200226-4aed43", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-200226-4aed43", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-200226-4aed43", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3823, + "minutes": 10.36, + "ci_minutes": 0, + "input_tokens": 1086667, + "output_tokens": 80811, + "tool_calls": 268, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4929, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.86, + "gate": 1.73, + "review": 616.2, + "report": 0.06 + }, + "time_to_first_result_s": 621.83 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 32384 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5550.md b/eval/dataset/arms/copilot_cb_grok45/pr5550.md new file mode 100644 index 000000000..5aa403913 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5550.md @@ -0,0 +1,60 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] Correctness: execute_request validates identity before RPC (multiproc_executor.py:459-460); execute_batch (586+) and execute_step (646+) do not — invalid envelopes still cross the MQ and surface as RuntimeError via async/error wrappers (worker_worker.py:1110-1127; multiproc_executor.py:852-853). +- [sweep] Indexed accesses in test_metadata.py:37-38/68/109 are on tuples the test just built — safe; test_worker_contract.py:221 prepared_layout identity is preserved through 3-arg dense RPC — [validated]. +- [sweep] New branches: dense rejects metadata (model_runner.py:172-173) [validated by test]; paged mismatch (175-178) [validated]; optional 4th arg (executor:545-546) [validated]; DLO skip when any metadata (462/471) can occur but has no dedicated test; engine scheduler_to_close None arm (199) reachable if assign fails before self.scheduler — defensive OK. +- [sweep] Breaking/consumers: NewRequestData.from_state still called without metadata from base_scheduler.py:114 (intentional W0); SimpleNamespace fixtures updated except _update_states test (line 230) which does not read diffusion_kv_metadata — no stale AttributeError; OmniNewRequestData is a different AR type — no collision. +- [sweep] Rebase/merge: no duplicated symbols; metadata.py new; imports resolve; head matches checkout claim 20ca4b93. +- [sweep] Tests: markers pytest.mark.diffusion/core_model/cpu present so L2 selects files; metadata tests are tautological constructors; worker_contract covers request RPC + runner gates but not executor batch/step preflight or DLO+metadata fallback. +- [sweep] Docs: module docstring claims trusted Scheduler snapshot; sibling DiffusionKVRequest/Context freeze hashes — mutable list[int] block_ids undercut snapshot wording. +- [sweep] Undocumented invariants: context_ids may dangle; empty sequences allowed; W0 explicitly defers geometry validation — OK if W1 owns checks. + +`vllm_omni/diffusion/executor/multiproc_executor.py:?` [major] — This diff adds `has_diffusion_kv_metadata = any(...)` and gates the DLO+AllGather multi-request wave with `and not has_diffusion_kv_metadata` (multiproc_executor.py:462–474), so any attached metadata falls through to the serial per-request RPC loop (`args=(req, od_config, ...[, metadata])`, `unique_reply_rank=0` at :544–551) instead of the DP wave (`args=(reqs_list, ...)`, `unique_reply_rank=None` at :504–511; worker `dp_rank % len(req)` at diffusion_worker.py:449). That silently drops DP multi-concurrency when W1 attaches metadata under DLO+allgather, and the list RPC still cannot carry per-request metadata. No test covers the arm (`make_executor` forces `enable_distributed_layerwise_offload=False` at test_worker_contract.py:62–65). Either raise ValueError for multi-req DLO+metadata until a per-request metadata wave RPC exists, or add a CPU contract test that locks the intentional serial fallthrough (two `collective_rpc` calls with the 4-arg shape) and documents that DP concurrency is disabled when metadata is present. (evidence: multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)`; multiproc_executor.py:469-474 `if ( len(new_reqs) > 1 and not has_diffusion_kv_metadata and getattr(self.od_config, "enable_distributed_layerwise_offload", False) and getattr(self.od_config, "dlo_use_allgather", True) ):`; multiproc_executor.py:504-511 `reqs_list = [nr.req for nr in new_reqs]` then `args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job), unique_reply_rank=None`; multiproc_executor.py:544-551 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None: args += (new_req.diffusion_kv_metadata,)` / `unique_reply_rank=0`; diffusion_worker.py:449 `idx = dp_rank % len(req)`; test_worker_contract.py:62-65 `enable_distributed_layerwise_offload=False`) + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [major] — This diff extends `DiffusionWorker.execute_model` to optionally forward `diffusion_kv_metadata` into `model_runner.execute_model(req, **kwargs)` when metadata is present (`diffusion_worker.py:462-465`), but the only `DiffusionModelRunner` subclass—`ARDiffusionModelRunner.execute_model` (`ar_diffusion/runner.py:237-241`)—still declares only `(req, kv_prefetch_job=None)` and calls `super().execute_model(req, kv_prefetch_job=...)` at both sites (`:243`, `:255`) without metadata. The first Worker call that attaches metadata on an AR backend therefore TypeErrors at the subclass boundary before parent dense/paged validation. Accept `diffusion_kv_metadata: DiffusionKVMetadata | None = None` and pass it through to `super()` at both call sites. (evidence: diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `output = self.model_runner.execute_model(req, **kwargs)`. ar_diffusion/runner.py:237-241 `def execute_model(self, req: OmniDiffusionRequest, kv_prefetch_job: KVPrefetchJob | None = None) -> DiffusionOutput:`. ar_diffusion/runner.py:243 `return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)`. ar_diffusion/runner.py:255 `output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)`. Grep: only `class ARDiffusionModelRunner(DiffusionModelRunner)` subclasses DiffusionModelRunner.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:?` [minor] — This PR adds pre-RPC `validate_new_request_data_identity` in `execute_request` (multiproc_executor.py:454–460) so identity errors stay outside the RPC error wrapper and invalid envelopes are not forwarded. The fused multi-request `execute_batch` path and `execute_step` still `collective_rpc` the raw `scheduler_output` with no executor-side check (615–620 / 651–655); identity and `_validate_diffusion_kv_metadata` run only after the call has crossed the RPC boundary in `DiffusionModelRunner.execute_model_batch`/`execute_stepwise` (637–641 / 750–755). Apply the same pre-RPC validate loop used in `execute_request` to those entrypoints, and add a contract test asserting zero RPC enqueue on envelope/request/metadata identity mismatch (today only `test_request_rpc_rejects_envelope_request_identity_mismatch` covers the executor). (evidence: multiproc_executor.py:454-460 `# Validate every envelope before selecting a dispatch path...` / `for new_req in new_reqs: validate_new_request_data_identity(new_req)` — pre-RPC in execute_request. multiproc_executor.py:615-620 `result = self.collective_rpc("execute_model_batch", args=(scheduler_output, self.od_config), ...)` — no validate before enqueue. multiproc_executor.py:651-655 `result = self.collective_rpc("execute_stepwise", args=(scheduler_output,), ...)` — likewise. diffusion_model_runner.py:637-641 / 750-755 `validate_new_request_data_identity(new_req)` only inside post-RPC runner methods. diffusion_worker.py:1122-1126 `AsyncDiffusionOutput(..., error=str(e))`; multiproc_executor.py:852-853 `fut.set_exception(RuntimeError(msg.error))` — worker-side identity failures become RPC-wrapped RuntimeError.) + +`base_scheduler.py:114` [minor] — W0 still schedules via `NewRequestData.from_state(state)` with no metadata (`base_scheduler.py:114`). Runner comment and metadata module both defer allocator/installer and cache geometry/block-table checks. Please confirm that production, installation, consumption, and geometry invariants (dangling `context_ids`, empty `sequences`) are intentionally owned by W1, not an accidental omission. (evidence: vllm_omni/diffusion/sched/base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — no diffusion_kv_metadata. | vllm_omni/diffusion/worker/diffusion_model_runner.py:170-171 `# W0 does not wire the Scheduler allocator or Worker installer yet, so` / `# validate an allocation snapshot when present without requiring one.` | vllm_omni/diffusion/diffusion_kv/metadata.py:7-8 `Cache geometry and block-table validation stay with the` / `native cache builders and Worker installation path.` | vllm_omni/diffusion/diffusion_kv/metadata.py:43 `context_ids: tuple[str, ...] = ()` — no dangling-id check; sequences tuple has no empty-sequence guard.) + +`diffusion_model_runner.py:636` [minor] — On execute_model_batch/execute_stepwise (and execute_model), metadata is validated then not forwarded into _execute_request_list, which has no metadata parameter. This matches the explicit W0 boundary (validate snapshot when present; allocator/installer not wired yet)—intentional non-consumption, not a missing forward. (evidence: diffusion_model_runner.py:170-171 `# W0 does not wire the Scheduler allocator or Worker installer yet, so` / `# validate an allocation snapshot when present without requiring one.` | diffusion_model_runner.py:636-649 `for new_req in scheduler_output.scheduled_new_reqs:` … `self._validate_diffusion_kv_metadata(... metadata=new_req.diffusion_kv_metadata)` then `reqs = [nr.req for nr in scheduler_output.scheduled_new_reqs]` / `return self._execute_request_list(reqs, ...)` with no metadata | diffusion_model_runner.py:750-755 same validate-then-continue without forwarding metadata | diffusion_model_runner.py:506-515 `_execute_request_list(... kv_prefetch_job=... record_name=...)` has no `diffusion_kv_metadata` parameter) + +`diffusion_model_runner.py:172` [minor] — Metadata gates land on shared MultiprocDiffusionExecutor / DiffusionModelRunner / DiffusionWorker paths used by all diffusion models. Default `diffusion_kv_mode` is `DENSE_LEGACY` (`data.py`); `_validate_diffusion_kv_metadata` rejects unexpected metadata whenever mode is not `PAGED_SCHEDULER`. Please confirm that cross-model blast radius is intended. (evidence: diffusion_model_runner.py:169 `cache_mode = getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)` +diffusion_model_runner.py:172 `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:` +diffusion_model_runner.py:173 `raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")` +data.py:666 `diffusion_kv_mode: DiffusionKVCacheMode = DiffusionKVCacheMode.DENSE_LEGACY` +multiproc_executor.py:459-460 `for new_req in new_reqs:` / `validate_new_request_data_identity(new_req)` +diffusion_worker.py:463-465 `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `output = self.model_runner.execute_model(req, **kwargs)`) + +`vllm_omni/diffusion/diffusion_kv/metadata.py:?` [minor] — This PR introduces `DiffusionKVContextMetadata.block_ids` and `DiffusionKVSequenceMetadata.block_ids` as `tuple[list[int], ...]` (metadata.py:27, :42) while documenting them as trusted one-shot Scheduler allocation snapshots. Mutable `list[int]` cells let a Worker/installer mutate the snapshot in place before W1 BlockTable installation; sibling Scheduler-owned `DiffusionKVContext` already freezes `block_hashes` in `__post_init__`. Freeze to `tuple[tuple[int, ...], ...]` (or copy-on-init). (evidence: metadata.py:27 `block_ids: tuple[list[int], ...]`; metadata.py:42 `block_ids: tuple[list[int], ...]`; metadata.py:4-8 docstring "carry trusted internal Scheduler results"; diffusion_kv/request.py:13 `@dataclass(frozen=True)` on DiffusionKVContext; diffusion_kv/request.py:35 `object.__setattr__(self, "block_hashes", tuple(self.block_hashes))` — metadata dataclasses have no freeze/copy of block_ids.) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:244` [minor] — This PR sends request-mode metadata as a positional 4th `collective_rpc` arg (`args=(req, od_config, kv_prefetch_job[, metadata])` at multiproc_executor.py:544–549). Production WorkerProc invokes `execute_method` → `getattr(self.worker, method)` → `DiffusionWorker.execute_model(*args)` (diffusion_worker.py:1036, 1442–1443), which bypasses `WorkerWrapperBase.execute_model`. `test_worker_wrapper_only_extends_paged_request_rpc` only calls `wrapper.execute_model(..., diffusion_kv_metadata=...)` kwargs. Add a `DiffusionWorker.execute_model(..., diffusion_kv_metadata=...)` and/or `execute_method("execute_model", req, od_config, None, metadata)` assertion so the production RPC binding and Worker→ModelRunner kwargs gate are covered. (evidence: multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None:` / `args += (new_req.diffusion_kv_metadata,)`; diffusion_worker.py:1036 `result = self.worker.execute_method(method, *args, **kwargs)`; diffusion_worker.py:1442-1443 `func = getattr(self.worker, method)` / `return func(*args, **kwargs)`; test_worker_contract.py:258-259 `wrapper.execute_model(req, od_config)` / `wrapper.execute_model(req, od_config, diffusion_kv_metadata=metadata)`) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/diffusion_kv/test_metadata.py:37` [minor] These new tests in `test_metadata.py` only construct the dataclasses and assert the same literals just passed in (e.g. `block_ids=([1, 2], [5, 6])` then `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])` at lin +- `tests/diffusion/diffusion_kv/test_worker_contract.py:144` [minor] This PR wires `_validate_diffusion_kv_metadata` into `execute_model_batch`/`execute_stepwise` (diffusion_model_runner.py:636-640 / 750-754: non-paged must not carry metadata; matching request_id allowed on paged), but `t + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5550.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/pr5550.trace.json.gz new file mode 100644 index 000000000..7357d7f8b Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5610.cost.json b/eval/dataset/arms/copilot_cb_grok45/pr5610.cost.json new file mode 100644 index 000000000..5952d5801 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5610.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 614.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5610/run-20260814-200225-5c4756", + "recorded_at": "2026-08-14T20:12:39", + "attempts": [ + { + "invocation_id": "inv-49ab9f02", + "outcome": "run:run-20260814-200225-5c4756", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-200225-5c4756", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-200225-5c4756", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2112, + "minutes": 10.23, + "ci_minutes": 0, + "input_tokens": 583215, + "output_tokens": 48816, + "tool_calls": 170, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4147, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.42, + "gate": 1.71, + "review": 608.67, + "report": 0.06 + }, + "time_to_first_result_s": 613.85 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 23290 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5610.md b/eval/dataset/arms/copilot_cb_grok45/pr5610.md new file mode 100644 index 000000000..2b510e65e --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5610.md @@ -0,0 +1,41 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness (docs vs code): read `_should_use_async_omni_output` (gpu_ar_model_runner.py:1595-1619), OmniAsyncGPUModelRunnerOutput lifecycle, NPU sample_tokens wrap (npu_ar_model_runner.py:1273-1323), and model opt-ins; activation/fallback claims match code. +- [validated] Guard table matches live checks: use_async_scheduling, omni_prefix_cache is not None, speculative_config, async_chunk, enable_return_routed_experts, use_async_omni_output, and has_postprocess⇒eager_omni_postprocess_before_async_output (gpu_ar_model_runner.py:1595-1617). +- [validated] Qwen3-Omni Thinker sets use_async_omni_output=True with has_postprocess=False; Talker sets use_async_omni_output=True, eager_omni_postprocess_before_async_output=True, omni_pooler_payload_include_hidden=False (qwen3_omni.py:109-164). +- [validated] Qwen3-TTS Talker opts in the same way (qwen3_tts_talker.py:351-353); only these two model families set use_async_omni_output=True (moss_tts explicitly False). +- [validated] Deploy examples: qwen3_omni_moe.yaml has async_chunk:true, AR stages omit async_scheduling (defaults true via stage_config.py:941), Code2Wav async_scheduling:false; qwen3_tts.yaml Stage0 async_scheduling:true and enable_prefix_caching:false; pipelines declare those default deploy names. +- [validated] should_accumulate_full_payload_output returns False when async_chunk (omni_connector_model_runner_mixin.py:60-61); accumulation branch reading self.requests is gated (gpu_ar_model_runner.py:1867-1872). +- [validated] Snapshot path clones CUDA tensors then async D2H on a dedicated stream with wait() (gpu_ar_model_runner.py:124-141, 2096-2098); connector drain is after wire payload build (1899-1900). +- [validated] XPUARModelRunner subclasses GPUARModelRunner; MUSA platform returns GPUARWorker; NPU builds OmniModelRunnerOutput + get_omni_connector_output before AsyncGPUModelRunnerOutput — platform note is accurate. + +`docs/design/feature/omni_async_output_materialization.md:?` [minor] — Overview states absolute `It does not change model computation or generated output.` while the following platform note and Performance section only substantiate CUDA/ROCm validation plus the Qwen3-Omni hash match (PR #4476), and explicitly leave XPU/MUSA unvalidated with no separate TTS hash. Qualify that sentence to the validated CUDA/ROCm Qwen3-Omni evidence, or cite proof that covers TTS/XPU/MUSA. (evidence: docs/design/feature/omni_async_output_materialization.md:58 `It does not change model computation or generated output.` | docs/design/feature/omni_async_output_materialization.md:61-65 `The performance and correctness validation in this document covers CUDA and ROCm.` / `XPU and MUSA have not been validated for this optimization.` | docs/design/feature/omni_async_output_materialization.md:122-124 `The text and audio output hashes matched, confirming that overlap does not change the generated result.` (Qwen3-Omni Performance only)) + +`docs/design/feature/omni_async_output_materialization.md:241` [minor] — Enabling claims the bundled profile turns the feature on for supported Qwen3-Omni and Qwen3-TTS pipelines, but alternate deploy `qwen3_omni_moe_thinking.yaml` sets `enable_prefix_caching: true` and `async_scheduling: false`, both of which make `_should_use_async_omni_output()` return False. Narrow the sentence to the default registry profiles in the examples (`qwen3_omni_moe.yaml`, `qwen3_tts.yaml`), or note that other deploy configs can remain on the sync path. (evidence: docs/design/feature/omni_async_output_materialization.md:240-242 `using the bundled deployment profile is enough to turn it on for the supported Qwen3-Omni and Qwen3-TTS pipelines.` | vllm_omni/deploy/qwen3_omni_moe_thinking.yaml:5 `enable_prefix_caching: true` | vllm_omni/deploy/qwen3_omni_moe_thinking.yaml:13 `async_scheduling: false` | vllm_omni/worker/gpu_ar_model_runner.py:1596-1599 `if not self.use_async_scheduling: return False` / `if self.omni_prefix_cache is not None: return False` | vllm_omni/worker/gpu_model_runner.py:170-171 `if self.cache_config.enable_prefix_caching: self.omni_prefix_cache = OmniTensorPrefixCache(`) + +`docs/.nav.yml:122` [nit] — Feature Design nav lists `design/feature/vae_parallel.md` twice (line 115 after tensor_parallel, and again at line 122 after the new `omni_async_output_materialization.md`). Drop the second entry so the path stays unique. (evidence: docs/.nav.yml:115 ` - design/feature/vae_parallel.md`; docs/.nav.yml:121-122 ` - design/feature/omni_async_output_materialization.md` then ` - design/feature/vae_parallel.md` — same path appears twice in Feature Design nav.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5610.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/pr5610.trace.json.gz new file mode 100644 index 000000000..0ce252efd Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5703.cost.json b/eval/dataset/arms/copilot_cb_grok45/pr5703.cost.json new file mode 100644 index 000000000..a54db1ea0 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5703.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 483.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5703/run-20260814-200930-378d02", + "recorded_at": "2026-08-14T20:17:33", + "attempts": [ + { + "invocation_id": "inv-f48ea491", + "outcome": "run:run-20260814-200930-378d02", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-200930-378d02", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-200930-378d02", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2863, + "minutes": 8.05, + "ci_minutes": 0, + "input_tokens": 726500, + "output_tokens": 81995, + "tool_calls": 260, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3939, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.13, + "gate": 1.72, + "review": 477.09, + "report": 0.06 + }, + "time_to_first_result_s": 482.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 25795 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5703.md b/eval/dataset/arms/copilot_cb_grok45/pr5703.md new file mode 100644 index 000000000..223159ea3 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5703.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [validated][sweep:1] Mechanical CUDA→device_module substitution is coherent for cuda/musa when parameter.device is that accelerator: vae.py:120,170-176,214-220. +- [validated][sweep:6 docs] recipes/README.md index row for MiniMax-H3-MUSA.md is consistent with the new file. +- [validated] Old CUDA path still seeds via device_module when device.type==cuda; MUSA recipe documents FLASH_ATTN/MATE and VAE tiling for the claimed profile. +- [validated] MiniMaxH3VideoVAE.__init__ now always assigns self.device_module = torch.get_device_module() (vae.py:120). +- [validated] NPUOmniPlatform.get_torch_device returns torch.device('npu'[, rank]) so NPU params have type 'npu' and satisfy != 'cpu'. +- [validated] CUDA path still qualifies as non-cpu, so prior cuda seeding still enters the device branch; callers outside MiniMax H3 are unaffected. +- [validated] `test_video_vae_encode_uses_configured_parallel_tiling` in test_minimax_h3_contract.py builds a CPU FakeModel so `devices=[]` and never touches `device_module` — existing suite does not cover the new branch. +- [validated] recipes/README.md index row for MiniMax-H3-MUSA.md is present and consistent with the new recipe filename. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This hunk replaces `parameter.device.type == "cuda"` with `!= "cpu"` in `encode_image`/`encode_video` (vae.py:170 and :214), so every non-CPU accelerator—including Ascend `npu`—now enters `fork_rng(devices=...)` and `self.device_module.manual_seed`. That expands beyond CUDA+MUSA and changes the shared NPU path (`recipes/MiniMaxAI/MiniMax-H3-NPU.md`; `pipeline_minimax_h3.py:963` calls `encode_image`), which previously stayed on the CPU-only seed path. Gate with `parameter.device.type in ("cuda", "musa")` (cf. `gpu_ar_worker.py:33`), or add explicit NPU evidence before merging. (evidence: vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:172-174 `with torch.random.fork_rng(devices=devices):` / `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; vae.py:214 same `!= "cpu"` gate in `encode_video`; pipeline_minimax_h3.py:963 `rows = self.video_vae.encode_image(image) if rank == 0 else None`; gpu_ar_worker.py:33 `if self.device_config.device_type in ("cuda", "musa"):`) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — This new recipe labels the serve block as “The validated Ref2VA configuration” (TP4 + CPU offload + VAE tiling) but sets `export MODEL="${MODEL_ROOT}/FL2VA"` (MiniMax-H3-MUSA.md:76). FL2VA and Ref2VA are separately served partitions; copying that command loads the wrong checkpoint for the Ref2VA results claimed later in the same file. Change the export to `${MODEL_ROOT}/Ref2VA` for that profile (or split FL2VA vs Ref2VA blocks as MiniMax-H3.md / MiniMax-H3-NPU.md do). (evidence: MiniMax-H3-MUSA.md:73-76 `The validated Ref2VA configuration uses tensor parallelism across four MTT\nS5000 GPUs and offloads inactive model components to CPU:\n\n```bash\nexport MODEL="${MODEL_ROOT}/FL2VA"` — labeled Ref2VA profile exports FL2VA; MiniMax-H3.md:168 `export MODEL="${MODEL_ROOT}/Ref2VA"`; MiniMax-H3-NPU.md:117 `export MODEL="${MODEL_ROOT}/Ref2VA"`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This diff adds `device_module`-based non-CPU seeding in `encode_image`/`encode_video` but adds no test coverage of that path. `test_video_vae_encode_uses_configured_parallel_tiling` (test_minimax_h3_contract.py:327-358) builds a CPU `Parameter(torch.ones(1))`, never sets `video_vae.device_module`, so `devices=[]` and the new `device_module.manual_seed` branch is never exercised; there is no `test_minimax_h3_vae*` and no KEYFRAME/`device_module` assertions under tests/diffusion/models/minimax_h3. Add a CPU-safe mock that sets `device_module`, uses a fake non-cpu Parameter, and asserts `manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` on both encode paths plus CPU skip; drop undelivered `test_minimax_h3_vae.py`/audio-loading coverage claims from the PR body. (evidence: vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:174-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; vae.py:214-220 same pattern in `encode_video`; test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))`; test_minimax_h3_contract.py:344-353 constructs `video_vae` without assigning `device_module` then calls `encode_video("frames")`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — Caches `self.device_module = torch.get_device_module()` with no device argument at `__init__` (vae.py:120) even though `device` is available and weights are placed with `.to(device=device)`. `encode_image`/`encode_video` later seed via that cached module against live `parameter.device` (vae.py:166-176, 210-220). Resolve the module from allowlisted `parameter.device.type` on the encode path (e.g. `torch.get_device_module(parameter.device.type)`) so the seeded DeviceModule always matches the tensor device. (evidence: vae.py:104 `device: torch.device,`; vae.py:115 `self.remote.eval().to(device=device, dtype=torch.float32)`; vae.py:120 `self.device_module = torch.get_device_module()`; vae.py:166 `parameter = next(self.parameters())`; vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; vae.py:210/214/219-220 same pattern in `encode_video`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:170` [minor] — vae.py:170 (and :214) now gates keyframe RNG with `device.type != "cpu"` and seeds via `self.device_module` from `torch.get_device_module()` instead of hard-coded `torch.cuda`. For devices that already had `type == "cuda"`, membership in the forked RNG device list is unchanged, so this reads as MUSA enablement rather than an AMD/ROCm behavioral break. gate_report shows buildkite/vllm-omni-amd-ci red on this head — before merge, please confirm whether that failure is pre-existing on main vs introduced here (attribution ask, not a claimed regression from this diff). (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` — cuda-typed devices still enter the list (old gate was `== "cuda"`); vllm_omni/diffusion/models/minimax_h3/vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` — seeding goes through get_device_module(), not a ROCm-excluding path.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — This is the sole in-repo call to `torch.get_device_module()` (vae.py:120). requirements/common.txt and requirements/musa.txt do not lower-bound `torch` (API since ~PyTorch 2.5; MUSA smoke cites 2.9.1+musa). Please confirm the supported torch floor for MiniMax CUDA/MUSA/NPU images, or explicitly scope support to platform builds that already ship this API. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — sole repo match for get_device_module; requirements/common.txt has `torchsde>=0.2.6` but no `torch` pin; requirements/musa.txt pins `torchada>=0.1.52` / `mate>=0.2.0` only; grep `torch>=` under requirements/ returned no matches) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:172` [minor] — encode_image/encode_video allowlist any non-CPU `parameter.device` into `devices` then call `torch.random.fork_rng(devices=devices)` with no `device_type=`, while seeding uses `self.device_module` from `torch.get_device_module()` at init. Pass `device_type=parameter.device.type` (or the allowlisted type) so fork/restore matches the parameter device rather than an implicit accelerator default—OK on single-accelerator MUSA hosts per recipe smoke, fragile if types diverge. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:117 `self.device_module = torch.get_device_module()`; vae.py:169 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:172 `with torch.random.fork_rng(devices=devices):`; vae.py:211 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:216 `with torch.random.fork_rng(devices=devices):`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5703.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/pr5703.trace.json.gz new file mode 100644 index 000000000..adbfc78e3 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5715.cost.json b/eval/dataset/arms/copilot_cb_grok45/pr5715.cost.json new file mode 100644 index 000000000..5201a8b08 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5715.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 142.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5715/run-20260814-201239-8f9872", + "recorded_at": "2026-08-14T20:15:01", + "attempts": [ + { + "invocation_id": "inv-3345b24f", + "outcome": "run:run-20260814-201239-8f9872", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-201239-8f9872", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-201239-8f9872", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0476, + "minutes": 2.36, + "ci_minutes": 0, + "input_tokens": 138908, + "output_tokens": 9151, + "tool_calls": 40, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.114, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.51, + "gate": 1.61, + "review": 136.25, + "report": 0.06 + }, + "time_to_first_result_s": 141.42 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 11499 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5715.md b/eval/dataset/arms/copilot_cb_grok45/pr5715.md new file mode 100644 index 000000000..04df1480c --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5715.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] README.md:19–21 / docs/README.md:54–55: full-duplex listed last and marked experimental; `vllm_omni/experimental/fullduplex/` present (MiniCPM-o 4.5 + JoyVL). +- [validated] Recipe links resolve: recipes/MiniMaxAI/MiniMax-H3.md, recipes/OpenBMB/MiniCPM-o-4_5.md, recipes/cosmos3/Cosmos3-Nano.md. +- [validated] docs/getting_started/installation/gpu/cuda.inc.md:127 BASE_IMAGE=vllm/vllm-openai:v0.26.0 matches docker/Dockerfile.cuda:1 default ARG. +- [validated] cuda.inc.md / rocm.inc.md / npu.inc.md release-path pins already on 0.26.0; this PR updates the previously stale NPU main-branch clone block (v0.26.0 + releases/v0.26.0rc) and fixes `cd vllm-omni`. +- [validated] docs/configuration/README.md:5 now points at docs.vllm.ai/en/v0.26.0/configuration/index.html (was v0.16.0). +- [validated] vllm-omni-skills news fully removed (repo grep: no matches). +- [upstream-verify] Could not live-check remote existence of docs.vllm.ai/en/v0.26.0 or vllm-ascend branch releases/v0.26.0rc (no network/gh tools in this step); PR body asserts both verified. + +`docs/getting_started/installation/README.md:?` [minor] — This diff adds a page-level !!! important that all vLLM-Omni 0.26.x installs require upstream vLLM 0.26.x, but the same Installation page routes MUSA users through gpu.md → musa.inc.md, which still pins `vllm-musa` at `v0.18.0-dev` (intentionally unchanged in this PR). Narrow the note to platforms that track upstream vLLM (CUDA/ROCm/XPU/NPU), or add an explicit MUSA exception pointing at the MUSA section, so the new compatibility contract does not contradict the published MUSA path. (evidence: docs/getting_started/installation/README.md:3-4: `!!! important` / `vLLM-Omni 0.26.x requires vLLM 0.26.x`; README.md:12 lists MThreads MUSA; docs/getting_started/installation/gpu.md:25-27 includes the MUSA tab; docs/getting_started/installation/gpu/musa.inc.md:29-31: `git clone https://github.com/MooreThreads/vllm-musa.git` / `git checkout v0.18.0-dev`.) + +`README.md:?` [nit] — This hunk collapses the old per-release news into one `[2026/06]` bullet that starts with 0.14.0, but the removed bullets dated 0.14.0/0.16.0 as 2026/02, 0.18.0 as 2026/03, and 0.20.0 as 2026/05—only 0.22.0 was June. Relabel as a range (e.g. `[2026/02–06]`) or date the cadence intro without implying all of 0.14–0.22 shipped in June. (evidence: Removed lines in the PR diff: `[2026/02] We released [0.14.0]`, `[2026/02] We released [0.16.0]`, `[2026/03] We released [0.18.0]`, `[2026/05] We released [0.20.0]`, `[2026/06] We released [0.22.0]`; new README.md:21 opens with `[2026/06] Starting with [0.14.0]`.) + +`docs/getting_started/installation/npu/npu.inc.md:56` [nit] — This hunk retitles the main-branch pin as the “v0.26 release line” while the Ascend clone uses branch `releases/v0.26.0rc` (not a `v0.26.0` tag like vLLM on the next line). Add a short clarifying clause that Ascend tracks the `releases/v0.26.0rc` branch for this line so readers do not treat the two refs as the same tag style. (evidence: docs/getting_started/installation/npu/npu.inc.md:56-62: comment `Pin vLLM and vLLM-Ascend to the v0.26 release line`; `git clone -b v0.26.0 https://github.com/vllm-project/vllm.git`; `git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5715.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/pr5715.trace.json.gz new file mode 100644 index 000000000..aedadf19f Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5840.cost.json b/eval/dataset/arms/copilot_cb_grok45/pr5840.cost.json new file mode 100644 index 000000000..131560101 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5840.cost.json @@ -0,0 +1,110 @@ +{ + "wall_s": 486.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5840/run-20260814-203741-d273e6", + "recorded_at": "2026-08-14T20:45:46", + "attempts": [ + { + "invocation_id": "inv-35f33b67", + "outcome": "run:run-20260814-203741-d273e6", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-201248-3ede23", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-203543-d1bc38", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-203741-d273e6", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-203741-d273e6", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4428, + "minutes": 8.09, + "ci_minutes": 0, + "input_tokens": 1288610, + "output_tokens": 86292, + "tool_calls": 304, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4578, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.19, + "gate": 1.77, + "review": 479.58, + "report": 0.06 + }, + "time_to_first_result_s": 485.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 104424 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5840.md b/eval/dataset/arms/copilot_cb_grok45/pr5840.md new file mode 100644 index 000000000..b81e8b0ad --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5840.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] Extractor uncached path parity: tests/diffusion/cache/test_teacache_extractors.py::test_extractor_matches_native_forward vs minimax_h3_transformer.py:1077-1196. +- [validated] FL2VA-only hook: backend.py:91 apply_teacache_hook(pipeline.transformer); test_minimax_h3_only_enables_fl2va_teacache asserts transformer is not transformers_ref. +- [sweep] Correctness/None: DiffusionCacheConfig.rel_l1_thresh 0.2→None; TeaCacheConfig.__post_init__ resolves None via _MODEL_DEFAULT_REL_L1_THRESH/MiniMax=0.17 else 0.2 (config.py:96-133); enable paths pass None through and rely on that resolution (backend.py:84-90,176-180). +- [validated] Blast radius for non-H3: test_enable_uses_generic_default_threshold asserts Qwen empty DiffusionCacheConfig() → TeaCacheConfig.rel_l1_thresh==0.2 (test_cache_backends.py:475-482). +- [validated] MiniMax partition gate: enable_minimax_h3_teacache rejects ref2va, allows fl2va/combined, hooks only pipeline.transformer not transformers_ref (backend.py:76-93); pipeline.partition is always lowercase fl2va|ref2va|combined (pipeline_minimax_h3.py:130-144). +- [validated] Extractor parity vs MiniMaxH3DiTModel.forward: same kwargsarg contract, _embed, sp_prepare/blocks/sp_gather, final_layer+masks (extractors.py:1249-1395 vs minimax_h3_transformer.py:1077-1196); unit test_extractor_matches_native_forward covers uncached path. +- [validated] Indexed blocks[0] for modulated_input matches TeaCache pattern; residual reuse test forces identical second forward and asserts block_calls==1 (test_teacache_extractors.py:675-698). +- [sweep] Simplifiability: combined branch only warns; ref2va raise is live; threshold None arm is live — no dead arms found. + +`docs/user_guide/diffusion/cache_acceleration/teacache.md:?` [major] — This PR changes `DiffusionCacheConfig.rel_l1_thresh` from `0.2` to `float | None = None` (deferring to TeaCache model defaults; MiniMax-H3 → 0.17) and documents empty-config deferral in `data.py`, but `docs/user_guide/diffusion/cache_acceleration/teacache.md:122` still lists type float and Default `0.2` with no model-specific note. Update that parameter table (and nearby omit-`cache_config` guidance) to state None/omitted → model default (generic 0.2; MiniMax-H3 FL2VA 0.17). (evidence: docs/user_guide/diffusion/cache_acceleration/teacache.md:122 `| `rel_l1_thresh` | float | `0.2` | Similarity threshold for cache reuse. Lower values prioritize quality (less caching), higher values prioritize speed (more caching). Suggested range: 0.1-0.8 |`; vllm_omni/diffusion/data.py:417 `>>> print(default_config.rel_l1_thresh) # None`; vllm_omni/diffusion/data.py:422 `rel_l1_thresh: float | None = None`; vllm_omni/diffusion/cache/teacache/config.py:94-97 `_DEFAULT_REL_L1_THRESH = 0.2` / `_MODEL_DEFAULT_REL_L1_THRESH = {` / `"MiniMaxH3DiTModel": 0.17,`; vllm_omni/diffusion/cache/teacache/config.py:119 `rel_l1_thresh: float | None = None`) + +`vllm_omni/engine/async_omni_engine.py:?` [major] — This PR sets `DiffusionCacheConfig.rel_l1_thresh` to `None` and registers `_MODEL_DEFAULT_REL_L1_THRESH["MiniMaxH3DiTModel"]=0.17`, but `AsyncOmniEngine._get_default_cache_config("tea_cache")` still returns `{"rel_l1_thresh": 0.2}` and `_normalize_cache_config` applies that when `cache_config is None` (OmniBase → AsyncOmniEngine → `_create_default_diffusion_stage_cfg`). So `Omni(..., cache_backend="tea_cache")` / `vllm serve ... --cache-backend tea_cache` never reach the H3 default that `test_minimax_h3_only_enables_fl2va_teacache` asserts via bare `DiffusionCacheConfig()`. Change the tea_cache factory to `{}` and add a unit test that `_normalize_cache_config("tea_cache", None)` leaves the threshold unset. (evidence: async_omni_engine.py:853-856 `if cache_backend == "tea_cache": return { "rel_l1_thresh": 0.2, }`; async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"): cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`; async_omni_engine.py:954-958 `cache_backend = normalized_kwargs.get("cache_backend", "none"); cache_config = AsyncOmniEngine._normalize_cache_config(cache_backend, normalized_kwargs.get("cache_config", None),)`; diffusion/data.py:422 `rel_l1_thresh: float | None = None`; teacache/config.py:96-98 `_MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, }`; teacache/backend.py:86-89 `TeaCacheConfig(transformer_type="MiniMaxH3DiTModel", rel_l1_thresh=config.rel_l1_thresh, ...)`; test_cache_backends.py:485-516 parametrize `(None, 0.17)` with `DiffusionCacheConfig()` → `assert teacache_config.rel_l1_thresh == expected_threshold`; omni_base.py:189-196 `self.engine = AsyncOmniEngine(..., **kwargs)`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143` [minor] — This PR adds `test_minimax_h3_t2va_teacache_single_gpu_smoke` with `cache_backend="tea_cache"`, `rel_l1_thresh=0.17`, and `num_inference_steps=2`, but only asserts joint output shape via `_assert_joint_output`. TeaCache always fully computes the first step (`TeaCacheHook._should_compute_full_transformer` when `state.cnt == 0`), so a short run can pass with a silently inactive TeaCache. Either assert enablement after `Omni(...)` (e.g. transformer teacache hook present) or label this as crash-only smoke so CI green is not read as cache-hit coverage (CPU `test_teacache_reuses_h3_block_residual` already covers hits). (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:153-156 `cache_backend="tea_cache"` / `cache_config={"rel_l1_thresh": 0.17}`; :170 `num_inference_steps=2`; :187 `_assert_joint_output(outputs)`; :19-22 `_assert_joint_output` only `assert frames.shape == (107, 256, 448, 3)`; vllm_omni/diffusion/cache/teacache/hook.py:312-315 `# First timestep: always compute` / `if state.cnt == 0:` / `return True`; tests/diffusion/cache/test_teacache_extractors.py:670-701 `test_teacache_reuses_h3_block_residual` asserts `block_calls == 1` on guaranteed hit.) + +`recipes/MiniMaxAI/MiniMax-H3.md:760` [minor] — PR title claims TeaCache and Cache-DiT validation, but this diff only adds MiniMax-H3 FL2VA TeaCache (teacache backend/extractor/config/tests + feature table). The only Cache-DiT edits are recipe mutual-exclusivity lines; H3 quality_policy/pipeline already owns Cache-DiT. Confirm validation is intentionally out of code scope, or retitle/narrow to FL2VA TeaCache. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:760 `TeaCache and Cache-DiT are mutually exclusive; pick one cache backend per server.` — recipes/MiniMaxAI/MiniMax-H3.md:838 `- TeaCache and Cache-DiT cannot be enabled on the same server.` — vllm_omni/diffusion/models/minimax_h3/quality_policy.py:126-133 `class MiniMaxH3QualityPolicy:` / `Resolve H3 request quality into a declarative Cache-DiT target.` / `The policy therefore owns whether a request installs Cache-DiT; the startup backend only controls the omitted-quality default.` — vllm_omni/diffusion/cache/teacache/backend.py:76-77 `def enable_minimax_h3_teacache(pipeline: Any, config: DiffusionCacheConfig) -> None:` / `"""Enable TeaCache for the calibrated MiniMax-H3 FL2VA partition."""` — GitHub PR #5840 title: `[MiniMax-H3] TeaCache support and Cache-DiT validation`; changed paths are TeaCache/docs/recipe/tests only (no `vllm_omni/diffusion/cache/cachedit/` edits).) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143` [minor] — `test_minimax_h3_t2va_teacache_single_gpu_smoke` inherits module `pytestmark` `local_model`+`cuda`+`diffusion`+`parallel` (not `core_model`) and is skip-gated on `VLLM_TEST_MINIMAX_H3_FL2VA_MODEL`; ready CI runs `tests/diffusion/models/` with `-m 'core_model and cuda'`, so this smoke is opt-in/local-only unless markers (or CI selection) are changed. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:9-16 `pytestmark = [pytest.mark.local_model, pytest.mark.cuda, pytest.mark.diffusion, pytest.mark.parallel]` and `MODEL_ENV = "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL"`; :139-143 `@pytest.mark.skipif(not os.environ.get(MODEL_ENV), ...)` then `def test_minimax_h3_t2va_teacache_single_gpu_smoke():`; .buildkite/cuda/test-ready.yml:65 `pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"`) + +`.buildkite/amd/test-amd-ready.yml:19` [minor] — AMD ready’s Simple Diffusion job explicitly `--ignore`s `tests/diffusion/cache/test_teacache_extractors.py` (comment + pytest flag at .buildkite/amd/test-amd-ready.yml:17-19), so the new `TestMiniMaxH3Extractor` coverage (`pytestmark` core_model + class `@pytest.mark.cpu`) never runs on that AMD path, while `test_cache_backends.py` (`pytestmark` core_model+cpu) still does. Confirm the ignore remains intentional for this PR’s MiniMax TeaCache extractor work. (evidence: .buildkite/amd/test-amd-ready.yml:17-19 `# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM` / `# that is not supported on CPU` / `pytest ... -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py ...`; tests/diffusion/cache/test_teacache_extractors.py:41 `pytestmark = [pytest.mark.core_model]`; tests/diffusion/cache/test_teacache_extractors.py:569-570 `@pytest.mark.cpu` / `class TestMiniMaxH3Extractor(BaseExtractorTest):`; tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +`recipes/MiniMaxAI/MiniMax-H3.md:762` [minor] — This PR’s recipe and `_MODEL_DEFAULT_REL_L1_THRESH` ship MiniMax TeaCache at `rel_l1_thresh=0.17`, but the PR body’s online smoke used `--cache-config '{"rel_l1_thresh":0.2}'` and logged `TeaCache applied with rel_l1_thresh=0.2`. Re-run (or cite) the online smoke at 0.17 — or with omitted config after the engine default-injection fix — so acceptance evidence matches the threshold users are told to run. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:762-763 / :783 `cache_config={"rel_l1_thresh": 0.17}`; PR online smoke evidence: `--cache-config '{"rel_l1_thresh":0.2}'` and `TeaCache applied with rel_l1_thresh=0.2, transformer_class=MiniMaxH3DiTModel`.) + +`vllm_omni/diffusion/cache/teacache/config.py:?` [nit] — This PR registers MiniMax-H3 FL2VA TeaCache coefficients in `_MODEL_DEFAULT_COEFFICIENTS` while the PR calibration report lists Polynomial R-squared ≈ 0.0949 (RMSE 0.0397). Disclose that weak-fit limitation next to the coefficients/default in `recipes/MiniMaxAI/MiniMax-H3.md`, or justify why the fit is still safe for the claimed quality/speed tradeoff. (evidence: config.py:83-93 registers the five MiniMax coefficients; PR body Calibration statistics: `Polynomial R-squared: 0.094873` with RMSE 0.039703.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5840.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/pr5840.trace.json.gz new file mode 100644 index 000000000..4fca3e6db Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5863.cost.json b/eval/dataset/arms/copilot_cb_grok45/pr5863.cost.json new file mode 100644 index 000000000..1f0e09b92 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5863.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 197.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5863/run-20260814-201501-1b72fd", + "recorded_at": "2026-08-14T20:18:18", + "attempts": [ + { + "invocation_id": "inv-cac2dfee", + "outcome": "run:run-20260814-201501-1b72fd", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-201501-1b72fd", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-201501-1b72fd", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0482, + "minutes": 3.27, + "ci_minutes": 0, + "input_tokens": 129197, + "output_tokens": 12135, + "tool_calls": 56, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.146, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.84, + "gate": 2.95, + "review": 188.5, + "report": 0.06 + }, + "time_to_first_result_s": 196.35 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 13939 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5863.md b/eval/dataset/arms/copilot_cb_grok45/pr5863.md new file mode 100644 index 000000000..d7b628b49 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5863.md @@ -0,0 +1,41 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] Attention claim: vllm_omni/platforms/cuda/platform.py:70-72,205-226 — TRTLLM default only for compute_capability.major==10; SM120 falls through to CUDNN_ATTN when cudnn>=9.5; explicit --diffusion-attention-backend CUDNN_ATTN is valid pinning. +- [validated] Profiler headers: serve.py:784 --enable-diffusion-pipeline-profiler; api_server.py:3510-3511 emit X-Stage-Durations and X-Peak-Memory-MB. +- [validated] T2VA aspect_ratio: MiniMax-H3.md:588 and pipeline aspect logic — T2VA requires named ratio; Ref2VA defaults 16:9. +- [validated] Perf table internal arithmetic (per-step, speedups, headroom, Ulysses memory fit) matches stated numbers within rounding. +- [validated] FP8 note aligns with MiniMax-H3.md:365-388 and docs/user_guide/quantization/fp8.md MiniMax row (incompatible with layerwise offload). +- [validated] Omitting VLLM_WORKER_MULTIPROC_METHOD is safe: stage_init_utils.py:509-511 auto-sets spawn (5090 recipe also omits it). + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:?` [minor] — This new two-server section lists `--cpunodebind` / `--membind` next to `vllm serve`'s `--port`, which makes them look like serve CLI flags. They are `numactl` options and do not appear anywhere else as vLLM args (`rg cpunodebind|membind` only hits this recipe). Rewrite as an explicit prefix, e.g. `numactl --cpunodebind=0 --membind=0 env CUDA_VISIBLE_DEVICES=... vllm serve ... --port 8091`, and the same for the Ref2VA server. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201-203: `Use CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and `--port 8091`...` / `--cpunodebind=1 --membind=1`, and `--port 8092`. Repo-wide: only this file uses those flag names; docker/Dockerfile.xpu:25 installs `numactl` as a package, not a serve argument.) + +`recipes/README.md:?` [minor] — This PR adds `recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md` as a hardware-Y answer, but `recipes/README.md` still only indexes `MiniMax-H3.md` and `MiniMax-H3-MUSA.md`. Add a table row for the PRO 6000 recipe (2/4/8×96 GiB) so the directory's stated purpose — "How do I run model X on hardware Y" — actually surfaces it; optionally also link it from `MiniMax-H3.md` the way MUSA/NPU point back to the main guide. (evidence: Diff adds only MiniMax-H3-RTX-PRO-6000.md. recipes/README.md:48-49 list MiniMax-H3.md and MiniMax-H3-MUSA.md only. recipes/MiniMaxAI/MiniMax-H3-MUSA.md:143-144 already cross-links the main guide; the new PRO 6000 file has no Additional resources section.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340` [minor] — The new Ref2VA example sends `audio_reference= str:\n if lang in {"zh", "en", "zhen"}:\n return _normalize_zh_or_en(text)\n if lang == "es":\n ...\n return text\n return text` — no `ar` branch; Arabic falls through unchanged. PR Purpose: `Official Chinese, English, Japanese, Spanish, and Arabic language preprocessing`.) + +`Dockerfile.xpu:91` [minor] — PR Test Plan still declares `**vLLM Version:** 0.26.0` while this PR pins XPU/Intel CI to `VLLM_VERSION=v0.27.0` (`docker/Dockerfile.xpu:91`, `.buildkite/intel/pipeline-intel.yml`). Please update the Test Plan evidence contract to 0.27 (or explicitly state which vLLM the reported IndexTTS-2.5 results were gathered against) so reviewers can judge dependency compatibility. (evidence: PR #5957 Test Plan: `**vLLM Version:** 0.26.0` — docker/Dockerfile.xpu:91 `ARG VLLM_VERSION=v0.27.0` — .buildkite/intel/pipeline-intel.yml:13 `VLLM_VERSION: "v0.27.0"`) + +`vllm_omni/model_executor/models/registry.py:?` [minor] — IndexTTS-2.5 is registered onto the shared IndexTTS2Talker/IndexTTS2S2MelDecoder implementations, and INDEXTTS2_PIPELINE uses the same talker2s2mel_full_payload hook plus shared Stage-1 CFM path that this PR rewrote. IndexTTS-2 therefore inherits those changes; add an IndexTTS-2 audio smoke (offline end2end.py --model-version 2.0 with seed 0, or online speech_client default) proving Stage-0→1 still produces audio, or explicitly track residual IndexTTS-2 risk if deferring that validation. (evidence: registry.py:373-381 `"IndexTTS25TalkerForConditionalGeneration": ("indextts2", "indextts2_talker", "IndexTTS2TalkerForConditionalGeneration"),` / `"IndexTTS25S2MelDecoder": ("indextts2", "indextts2_s2mel_decoder", "IndexTTS2S2MelDecoder"),` — 2.5 maps onto shared IndexTTS-2 classes; pipeline.py:30 `custom_process_next_stage_input_func=f"{_PROC}.talker2s2mel_full_payload"` on INDEXTTS2_PIPELINE and pipeline.py:66 same hook on INDEXTTS25_PIPELINE; stage_input_processors/indextts2.py:217-219 `if use_gpt_latent_value is None: raise ValueError(f"IndexTTS payload is missing meta.use_gpt_latent for req={rid}")` — shared fail-fast; indextts2_s2mel_decoder.py:229 `self.s2mel_cfm_batch_size: int = int(getattr(self.config, "s2mel_cfm_batch_size", 1))` — IndexTTS-2 default CFM batch path.) + +`vllm_omni/engine/stage_init_utils.py:?` [minor] — This shared stage-0 path injects `_TokenOnlyRenderer` whenever `skip_tokenizer_init` is true (`stage_init_utils.py:1140-1144`). That flag is set on stage 0 in `indextts2.yaml`, `indextts2_5.yaml`, `indextts2_low_latency.yaml`, and `personaplex.yaml`. The new unit test only stubs `_build_token_only_renderer`; either run a PersonaPlex (or other skip_tokenizer) stage-0 init smoke, or note in the PR that those deploy configs inherit this renderer and were not re-validated beyond the unit test. (evidence: stage_init_utils.py:1140-1144 `if bool(getattr(stage_vllm_config.model_config, "skip_tokenizer_init", False)):\n input_processor = InputProcessor(\n vllm_config=stage_vllm_config,\n renderer=_build_token_only_renderer(stage_vllm_config),\n )` — unconditional for any skip_tokenizer stage; indextts2_low_latency.yaml:29 `skip_tokenizer_init: true`; personaplex.yaml:42 `skip_tokenizer_init: true`; test_async_omni_engine_stage_init.py:852-865 monkeypatches `_build_token_only_renderer` and sets `skip_tokenizer_init=True` only — no PersonaPlex/low-latency smoke.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/indextts2/preprocess_utils.py:281` [minor] This diff changes shared IndexTTS preprocess fail-fast behavior: `compute_fbank` now raises `RuntimeError("IndexTTS requires torchaudio Kaldi fbank support")` instead of returning zeros (`preprocess_utils.py:281-282`), a +- `pyproject.toml:91` [minor] The new `indextts2` extra pins `wetext>=0.0.9` off-Linux but leaves Linux `WeTextProcessing` unpinned (pyproject.toml:90-91). `_load_wetext_normalizer` on Linux constructs `tn.chinese.normalizer.Normalizer(remove_interje + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5957.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/pr5957.trace.json.gz new file mode 100644 index 000000000..fa7616087 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5976.cost.json b/eval/dataset/arms/copilot_cb_grok45/pr5976.cost.json new file mode 100644 index 000000000..c41cfda2a --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5976.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 634.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok45/runs/pr5976/run-20260814-202227-787d8c", + "recorded_at": "2026-08-14T20:33:01", + "attempts": [ + { + "invocation_id": "inv-90357c55", + "outcome": "run:run-20260814-202227-787d8c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-202227-787d8c", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-202227-787d8c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5117, + "minutes": 10.57, + "ci_minutes": 0, + "input_tokens": 1521713, + "output_tokens": 91670, + "tool_calls": 339, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5489, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 5.49, + "gate": 2.16, + "review": 626.38, + "report": 0.06 + }, + "time_to_first_result_s": 634.08 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 62333 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5976.md b/eval/dataset/arms/copilot_cb_grok45/pr5976.md new file mode 100644 index 000000000..1c6d1798a --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok45/pr5976.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness stale-drain: read omni_ar_scheduler.py:611-671 and omni_scheduler_mixin.py:90-105 — AR seeds at 627-629 then calls mixin which seeds again at 105 when replace_streaming_prompt is True; in_flight is not cleared between seeds → 2× counter. +- [sweep] replace_streaming_prompt producers: grep hit minicpmo_4_5_omni.py:954 (live duplex/turn handoff) and tests/core/sched/test_omni_ar_scheduler_streaming.py:252-313 (exercises replace but never asserts num_stale_output_tokens). +- [sweep] Stale-drain tests: test_omni_ar_scheduler_stale_drain.py helper calls _update_request_as_session without replace_streaming_prompt meta → takes super() path; asserts seed==in_flight once; does not cover mixin double-seed. +- [sweep] Generation scheduler: update_from_output drains stale; _update_request_as_session only calls mixin once — single seed OK on generation path. +- [validated] Segment-stop seed at omni_ar_scheduler.py:490-491 uses num_in_flight_tokens (same units as drain) — correct for non-replace stop path. +- [validated] Qwen3 thinker bare-tensor: qwen3_omni.py:144-148/407-417 and qwen3_omni_moe_thinker.py:1515-1521 return tensor when not capturing; make_omni_output accepts both shapes; covered by test_qwen3_omni_forward_contract.py. +- [validated] gpu_generation_model_runner.py:840-847 accepts randomize_inputs but calls maybe_randomize_inputs without forwarding it — intentional for code2wav. +- [validated] Replica cache: launch_stage_replica/stage_engine_startup pass stage_id+replica_id; StageRuntime._scoped_spawn_device_env omits ids but only wraps acquire_device_locks, not engine spawn — Cubin race path still isolated. + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [blocker] — This hunk adds `session.num_stale_output_tokens += in_flight_tokens` at the top of every `_update_request_as_session` (omni_ar_scheduler.py:627-629) without clearing `num_in_flight_tokens`, then on `replace_streaming_prompt` immediately calls `_replace_streaming_session`, which seeds again from the same counter (omni_scheduler_mixin.py:105) — MiniCPM-o sets that flag on turn handoff (minicpmo_4_5_omni.py:954). Net effect is 2× stale on the real streaming-replace path, so leftover drain can swallow new-segment talker frames (the failure mode this PR claims to fix). The same unconditional pre-seed also runs before any stage_id!=0 resume-only early return and can re-add units on a later StreamingUpdate while in-flight is still non-zero. Seed in exactly one owner per discard (skip AR pre-seed when calling replace, or seed only in the mixin for that path), clear/snapshot in-flight so a second update cannot re-add, and extend `test_omni_ar_scheduler_stale_drain.py` with `replace_streaming_prompt=True` + `num_in_flight_tokens>0` asserting stale==in_flight (not 2×) before drain. (evidence: omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0:` / `session.num_stale_output_tokens += in_flight_tokens` — no clear of num_in_flight_tokens; omni_ar_scheduler.py:669-670 `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)`; omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` — second seed from same uncleared counter; minicpmo_4_5_omni.py:954 `handoff_meta["replace_streaming_prompt"] = True`) + +`vllm_omni/platforms/xpu/platform.py:?` [minor] — This hunk flips the shared XPU non-inductor IR default from `xpu_kernels` to `vllm_c` in `XPUOmniPlatform.get_default_ir_op_priority` (platform.py:139). `diffusion_worker._resolve_ir_op_priority` applies that platform default for every model (diffusion_worker.py:184) before any optional `get_diffusion_ir_op_priority_func` override. Scope via registry override for the intended models, or attach XPU same-seed accuracy/latency evidence for affected diffusion paths before treating the rename as merge-safe. (evidence: vllm_omni/platforms/xpu/platform.py:133 `otherwise prefer vllm_c where available.`; vllm_omni/platforms/xpu/platform.py:139 `default = ["native"] if using_inductor else ["vllm_c", "native"]`; vllm_omni/diffusion/worker/diffusion_worker.py:184 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)` — platform default applied before optional per-model `ir_op_priority_func`.) + +`adapter.py:235` [minor] — Async-chunk replace via `construct_next_stage_streaming_input_prompt` (adapter.py:235-247) still does not seed `num_stale_output_tokens`, unlike `omni_scheduler_mixin._replace_streaming_session`. MiniCPM-o deploy defaults `async_chunk: true` and sets `replace_streaming_prompt` at minicpmo_4_5_omni.py:954 — accept as unfixed residual in this PR. (evidence: adapter.py:235-247 `if replace_prompt and isinstance(next_stage_prompt_len, int) and next_stage_prompt_len > 0:` … `request._output_token_ids.clear()` … `request.num_computed_tokens = 0` … `request.update_block_hashes()` / `return` — no `num_stale_output_tokens` seed (adapter.py has zero matches for that symbol). Contrast omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`. Producer: minicpmo_4_5_omni.py:954 `handoff_meta["replace_streaming_prompt"] = True`. Deploy: minicpmo_4_5.yaml:3 `async_chunk: true`.) + +`vllm_omni/benchmarks/patch/patch.py:?` [minor] — probe_task is started at patch.py:1712-1714 when probe_request_rate > 0, but probe_stop/await only run after asyncio.gather(*tasks) succeeds (1779-1783). A gather failure leaves the probe issuing requests with no cancel/finally. probe_outputs is appended in probe_loop (1708) and never folded into the result dict. Mirror upstream serve.py teardown (cancel + await in finally) or gate the feature until lifecycle matches. (evidence: vllm_omni/benchmarks/patch/patch.py:1712-1714 `if probe_request_rate > 0:` / `probe_task = asyncio.create_task(probe_loop())`; :1779-1783 `outputs: list[MixRequestFuncOutput] = await asyncio.gather(*tasks)` then `if probe_task is not None: probe_stop.set(); await probe_task` with no finally; :1694/:1708 `probe_outputs: list[...] = []` and `probe_outputs.append(...)` with no later read of probe_outputs in this file.) + +`vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py:?` [minor] — This hunk changes `Qwen2_5OmniVideoProcessorItems.get_processor_data` to iterate `self.get()` instead of `self.data` so `MediaWithBytes` unwraps before HF video batching (`qwen2_5_omni_thinker.py:312-317`), claimed to fix IndexError/400 on every video request. Repo grep of `tests/` for `MediaWithBytes`/`get_processor_data`/`Qwen2_5OmniVideoProcessorItems`/`VideoProcessorItems` is empty, so CI cannot catch a regression to `self.data`. Add a CPU unit test wrapping frames in `MediaWithBytes` (or equivalent) asserting `get_processor_data()["videos"]` yields unwrapped frame sequences. (evidence: qwen2_5_omni_thinker.py:312-317 `items = [self.get(index) for index in range(self.get_count())]` / `videos = [frames for frames, _ in items]` / `return {"videos": videos, "video_metadata": video_metadata}` — comment at :299-309 documents MediaWithBytes unwrap vs `self.data` causing IndexError/400; tests/ grep MediaWithBytes|get_processor_data|Qwen2_5OmniVideoProcessorItems|VideoProcessorItems → no matches.) + +`vllm_omni/worker/gpu_generation_model_runner.py:?` [minor] — Generation `_dummy_run` adds `randomize_inputs: bool = False` for API compat but intentionally does not forward it: `self.maybe_randomize_inputs(input_ids, inputs_embeds)` (gpu_generation_model_runner.py:580, :848). Thinker runner does forward it (gpu_model_runner.py:1134). Comment cites code2wav CUDA asserts if randomized. `tests/` has no `randomize_inputs` coverage. Add a unit test that calls generation `_dummy_run(..., randomize_inputs=True)` with `maybe_randomize_inputs` mocked and asserts it is not invoked with `randomize_inputs=True`. (evidence: gpu_generation_model_runner.py:580 `randomize_inputs: bool = False,` — param accepted; :848 `self.maybe_randomize_inputs(input_ids, inputs_embeds),` — kwarg not forwarded (comment :840-:847 explains deliberate non-forward). Contrast gpu_model_runner.py:1134 `self.maybe_randomize_inputs(input_ids, inputs_embeds, randomize_inputs=randomize_inputs),`. Grep tests/ for `randomize_inputs` → no matches.) + +`vllm_omni/core/sched/omni_generation_scheduler.py:?` [minor] — This hunk mirrors AR stale-output drain into `OmniGenerationScheduler.update_from_output` (omni_generation_scheduler.py:385-401) while generation seeds only via mixin `_replace_streaming_session` (called from `_update_request_as_session` at :598-604), but only AR gained `test_omni_ar_scheduler_stale_drain.py` (imports/`OmniARScheduler.update_from_output` only). Add a generation counterpart or parametrize the drain helper over `OmniGenerationScheduler.update_from_output` so a future edit cannot drop the generation `continue` while AR stays green. (evidence: omni_generation_scheduler.py:385 `if request is not None and request.num_stale_output_tokens > 0:` — drain; :396-401 `if output_is_stale: ... continue`. omni_generation_scheduler.py:598-604 `_update_request_as_session` only `self._replace_streaming_session(session, update)`. omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`. test_omni_ar_scheduler_stale_drain.py:42 `from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler`; :132 `OmniARScheduler.update_from_output(...)` — no OmniGenerationScheduler coverage.) + +`tests/e2e/accuracy/test_qwen_image.py:38` [nit] — This hunk adds `SSIM_THRESHOLD_SDPA_FALLBACK = 0.96` and selects it when `_flash_attn3_hub_available()` is false (`tests/e2e/accuracy/test_qwen_image.py:38`, `:262`), weakening the SSIM gate from 0.97 on the common post-0.27 missing-FA3 path. Keep the split if intentional, but note in PR acceptance that CI may run the looser path while FA3-available jobs still bind 0.97. (evidence: tests/e2e/accuracy/test_qwen_image.py:38 `SSIM_THRESHOLD_SDPA_FALLBACK = 0.96`; tests/e2e/accuracy/test_qwen_image.py:262 `ssim_threshold=SSIM_THRESHOLD if _flash_attn3_hub_available() else SSIM_THRESHOLD_SDPA_FALLBACK` — FA3-missing path uses 0.96 instead of `SSIM_THRESHOLD = 0.97`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok45/pr5976.trace.json.gz b/eval/dataset/arms/copilot_cb_grok45/pr5976.trace.json.gz new file mode 100644 index 000000000..2a537b373 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok45/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.cost.json b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.cost.json new file mode 100644 index 000000000..c3527bc39 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1136.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5550/run-20260814-203330-5a76e3", + "recorded_at": "2026-08-14T20:52:25", + "attempts": [ + { + "invocation_id": "inv-36de51f8", + "outcome": "run:run-20260814-203330-5a76e3", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-203330-5a76e3", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-203330-5a76e3", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5772, + "minutes": 18.92, + "ci_minutes": 0, + "input_tokens": 1471910, + "output_tokens": 163479, + "tool_calls": 364, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7504, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.94, + "gate": 2.08, + "review": 1129.02, + "report": 0.06 + }, + "time_to_first_result_s": 1135.08 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 36158 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.md b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.md new file mode 100644 index 000000000..11d82a318 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] indexed test_metadata.py:37-38 sequences[0]/contexts[0].block_ids: fixtures construct exactly one sequence and one context; [0] is that element, not a production first-group invariant. +- [sweep] indexed test_metadata.py:68 and :109 sequences[0].context_ids: CFG tests build two sequences then assert [0] and [1]; [0] is sequence_id 0 from range(2)/tuple order, guaranteed by construction. +- [sweep] indexed test_worker_contract.py:221 calls[0][1][0].prepared_layout: precedes assert calls == one 3-tuple; [0] is that RPC, [1][0] is req. Safe. +- [sweep] branch diffusion_engine.py:199 scheduler_to_close is not None: _init_scheduler assigns self.scheduler before initialize(); the None arm is only if getattr misses. test_scheduler_initialization_failure_closes_scheduler_and_executor hits the taken arm (close+shutdown, original exception re-raised). +- [sweep] branch multiproc_executor.py:545 metadata is not None: live for attached snapshots; covered by test_request_rpc_sends_attached_metadata_for_only_that_request. Dense 3-tuple arm covered by test_dense_request_rpc_keeps_legacy_positional_shape. +- [sweep] branch interface.py:184 envelope != forwarded: can occur; test_request_rpc_rejects_envelope_request_identity_mismatch and batch/step twins. Raises before collective_rpc (assert calls == []). +- [sweep] branch interface.py:192 metadata.request_id != forwarded: can occur when envelope matches req but snapshot is stale; same helper. Runner twin test_paged_request_rejects_mismatched_metadata_identity. +- [sweep] branch diffusion_model_runner.py:172 non-paged + metadata: can occur; test_dense_request_rejects_metadata_before_request_forward. PAGED_WORKER_LOCAL is rejected at parse_diffusion_kv_cache_mode before startup. + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — This diff adds `has_diffusion_kv_metadata = any(...)` and `and not has_diffusion_kv_metadata` to the DLO+AllGather gate in `MultiprocDiffusionExecutor.execute_request` (462–473), so any envelope with a snapshot skips the list RPC `args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job)` (`unique_reply_rank=None`, 506–511) and falls through to serial per-request 4-tuples (`unique_reply_rank=0`, 541–552). `execute_batch` re-enters `execute_request` for DLO DP (605–613), so HunyuanImage3 (`supports_request_batch = False`; engine already requires DLO for `max_num_seqs>1` on non-batch pipelines) and MiniMax-H3 DLO inherit a silent switch from `req[dp_rank % len]` to every rank running each request one-by-one once W1 attaches metadata. The list RPC stays a 3-tuple, so an inverted or dropped gate would omit snapshots entirely. `DiffusionWorker.execute_model` still accepts `req: list` plus one `diffusion_kv_metadata` and only then picks `req[idx]` (444–450, 463–465), so a mismatched `metadata.request_id` would raise on some ranks (`_validate_diffusion_kv_metadata` 175–178) and desync AllGather. `test_worker_contract.make_executor` hardcodes `enable_distributed_layerwise_offload=False` and its `collective_rpc` mock does not accept `timeout=`; `test_multiproc_engine_concurrency.py` builds `NewRequestData` without metadata. Add a CPU test with DLO+AllGather, two `NewRequestData`s that include metadata, `timeout=` allowed on the mock, and assert two per-request 4-tuples rather than one `reqs_list` 3-tuple; fail-fast `isinstance(req, list) and diffusion_kv_metadata is not None` on `DiffusionWorker.execute_model` (WorkerProc `_execute_rpc` → `execute_method` target) or plumb per-request snapshots through the list RPC — do not leave the mapping change implicit. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)` +vllm_omni/diffusion/executor/multiproc_executor.py:469-474 `if (\n len(new_reqs) > 1\n and not has_diffusion_kv_metadata\n and getattr(self.od_config, "enable_distributed_layerwise_offload", False)\n and getattr(self.od_config, "dlo_use_allgather", True)\n ):` +vllm_omni/diffusion/executor/multiproc_executor.py:506-511 `results = self.collective_rpc(\n "execute_model",\n timeout=_DLO_DP_WAVE_TIMEOUT_S,\n args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job),\n unique_reply_rank=None,\n exec_all_ranks=True,\n )` +vllm_omni/diffusion/executor/multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)\n if new_req.diffusion_kv_metadata is not None:\n args += (new_req.diffusion_kv_metadata,)` +vllm_omni/diffusion/executor/multiproc_executor.py:605-613 `if (\n dp_size > 1\n and getattr(self.od_config, "enable_distributed_layerwise_offload", False)\n and getattr(self.od_config, "dlo_use_allgather", True)\n ):\n ...\n return self.execute_request(scheduler_output)` +vllm_omni/diffusion/worker/diffusion_worker.py:444-450 `is_batch = isinstance(req, list)\n if is_batch:\n ...\n idx = dp_rank % len(req)\n req = req[idx]` +vllm_omni/diffusion/worker/diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}\n if diffusion_kv_metadata is not None:\n kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` +vllm_omni/diffusion/worker/diffusion_model_runner.py:175-178 `if metadata is not None and metadata.request_id != request_id:\n raise ValueError(\n f"Diffusion KV metadata request mismatch: expected={request_id!r}, got={metadata.request_id!r}"\n )` +vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py:323 `supports_request_batch = False` +vllm_omni/diffusion/diffusion_engine.py:229 `if not self.supports_request_batch and _max_num_seqs(od_config) > 1 and not _uses_dlo_dp_concurrency(od_config):` +tests/diffusion/diffusion_kv/test_worker_contract.py:63 `enable_distributed_layerwise_offload=False,` +tests/diffusion/diffusion_kv/test_worker_contract.py:69 `def collective_rpc(method, *, args, unique_reply_rank, exec_all_ranks):` +tests/diffusion/test_multiproc_engine_concurrency.py:200-207 `NewRequestData(\n request_id=rid,\n req=OmniDiffusionRequest(\n prompt=f"prompt_{rid}",\n sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),\n request_id=rid,\n ),\n )`) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:?` [minor] — This diff packs an optional fourth positional `execute_model` argument in `MultiprocDiffusionExecutor.execute_request` (`args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` then `args += (new_req.diffusion_kv_metadata,)` at `multiproc_executor.py:544-546`). Production unpack is `WorkerProc._execute_rpc` → `self.worker.execute_method(method, *args)` → `getattr(self.worker, method)(*args)` onto `DiffusionWorker.execute_model(req, od_config, kv_prefetch_job=None, diffusion_kv_metadata=None)`, not `WorkerWrapperBase.execute_model`. `test_worker_wrapper_only_extends_paged_request_rpc` keyword-calls the wrapper; `test_request_rpc_sends_attached_metadata_for_only_that_request` only asserts the packed 4-tuple on a mocked `collective_rpc`; `test_dense_worker_call_does_not_extend_model_runner_signature` and `test_paged_request_without_metadata_reaches_request_forward` never attach a matching snapshot, so a regression that rejects valid paged metadata after the identity check, or that breaks positional 4-tuple binding, still passes. Add `worker.execute_model(req, od_config, None, metadata)` and/or `wrapper.execute_method("execute_model", req, od_config, None, metadata)` asserting the runner received `diffusion_kv_metadata`, and `runner.execute_model(req, diffusion_kv_metadata=make_metadata())` asserting `_execute_request_list` is called; retarget or drop the wrapper-only test so it cannot pass while the live `*args` binding is broken. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None:` / `args += (new_req.diffusion_kv_metadata,)`. vllm_omni/diffusion/worker/diffusion_worker.py:1036 `result = self.worker.execute_method(method, *args, **kwargs)`. diffusion_worker.py:1442-1443 `func = getattr(self.worker, method)` / `return func(*args, **kwargs)` — lookup is the inner worker, so RPC does not call WorkerWrapperBase.execute_model. diffusion_worker.py:421-426 `def execute_model(self, req: OmniDiffusionRequest | list[OmniDiffusionRequest], od_config: OmniDiffusionConfig, kv_prefetch_job: KVPrefetchJob | None = None, diffusion_kv_metadata: DiffusionKVMetadata | None = None)`. tests/diffusion/diffusion_kv/test_worker_contract.py:259 `wrapper.execute_model(req, od_config, diffusion_kv_metadata=metadata)` — keyword wrapper call. test_worker_contract.py:98 `assert runner.execute_model(req) is expected_output` — paged success path has no metadata. test_worker_contract.py:288 `worker.execute_model(req, SimpleNamespace())` — DiffusionWorker called without metadata. Repo grep found no `execute_method("execute_model", ...)` or positional `execute_model(..., None, metadata)` call in this test file.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:574` [minor] — Identity mismatches are rejected before RPC so they raise instead of becoming a finished output (multiproc_executor.py:454-460). Dense+metadata rejection lives in the worker (`_validate_diffusion_kv_metadata`), so execute_request still catches that exception at 574-580 and returns RunnerOutput(finished=True, result=DiffusionOutput(error=...)). If invalid KV envelopes must not be forwarded or converted into worker outputs, reject dense+metadata on the executor before collective_rpc the same way as identity; otherwise this wrapping is the existing worker-error contract. (evidence: multiproc_executor.py:454-460 `keep identity errors outside the RPC error wrapper so an invalid request cannot be forwarded or converted into a worker output` then `validate_new_request_data_identity(new_req)` before RPC. multiproc_executor.py:545-546 `if new_req.diffusion_kv_metadata is not None: args += (new_req.diffusion_kv_metadata,)` forwards metadata into collective_rpc. multiproc_executor.py:574-580 `except Exception as exc:` `result=DiffusionOutput(error=str(exc))` with `finished=True`. diffusion_model_runner.py:172-173 `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None: raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")` — this runs inside worker execute_model (609-612), not before the executor RPC.) + +`interface.py:615` [minor] — execute_request validates identity before dispatch (multiproc_executor.py:454-460) so those ValueErrors never hit the except that emits finished DiffusionOutput(error=) (574-581). Serial execute_batch (len<=1 or DLO DP) already delegates to that path. The residual is fused execute_batch (615-620) and execute_step (651-656): they RPC first and only validate on the runner after RPC (diffusion_model_runner.py:636-641 / 750-755); those executor paths raise rather than wrapping into fake outputs. In scope for this W0, or defer because they do not synthesize finished error outputs? (evidence: multiproc_executor.py:454-460 `# Validate every envelope before selecting a dispatch path. In particular, keep identity errors outside the RPC error wrapper so an invalid request cannot be forwarded or converted into a worker output.` then `for new_req in new_reqs: validate_new_request_data_identity(new_req)`. multiproc_executor.py:574-581 `except Exception as exc: runner_outputs.append(RunnerOutput(..., finished=True, result=DiffusionOutput(error=str(exc))))`. multiproc_executor.py:615-620 fused execute_batch `result = self.collective_rpc("execute_model_batch", args=(scheduler_output, self.od_config), ...)` with no validate; 642-644 `raise RuntimeError(...)` not DiffusionOutput(error=). multiproc_executor.py:651-656 execute_step `result = self.collective_rpc("execute_stepwise", args=(scheduler_output,), ...)` with no validate; 658-660 `raise RuntimeError(...)`. diffusion_model_runner.py:636-637 / 750-751 `for new_req in scheduler_output.scheduled_new_reqs: validate_new_request_data_identity(new_req)` only after RPC.) + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [minor] — DiffusionWorker.execute_model only adds diffusion_kv_metadata to runner kwargs when it is not None (diffusion_worker.py:462–465). ARDiffusionModelRunner.execute_model (runner.py:237–255) still takes (req, kv_prefetch_job=None) and calls super().execute_model(req, kv_prefetch_job=kv_prefetch_job), so a present snapshot is TypeError unexpected keyword at the worker→runner boundary (DreamZero request mode). Accept and forward diffusion_kv_metadata the same way kv_prefetch_job was added, and extend tests/diffusion/ar_diffusion/test_engine_routing.py::test_execute_model_accepts_kv_prefetch_job_kwarg (141–152) to require that parameter on both DiffusionModelRunner and ARDiffusionModelRunner. W0 has no producer so this does not crash today; the first W1 envelope that attaches metadata on an AR request will. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `output = self.model_runner.execute_model(req, **kwargs)`. vllm_omni/experimental/ar_diffusion/runner.py:237-255 `def execute_model(` / `self,` / `req: OmniDiffusionRequest,` / `kv_prefetch_job: KVPrefetchJob | None = None,` / `) -> DiffusionOutput:` ... `return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` ... `output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` — no diffusion_kv_metadata param or forward. tests/diffusion/ar_diffusion/test_engine_routing.py:150-152 `for cls in (DiffusionModelRunner, ARDiffusionModelRunner):` / `params = inspect.signature(cls.execute_model).parameters` / `assert "kv_prefetch_job" in params, cls`.) + +`tests/diffusion/diffusion_kv/test_metadata.py:?` [nit] — Rename `test_diffusion_kv_metadata_uses_native_cache_group_block_ids` — it is a constructor round-trip of plain dataclasses, not native cache-group coverage. `DiffusionKVContextMetadata` / `DiffusionKVSequenceMetadata` / `DiffusionKVMetadata` (metadata.py:16–57) have no `__post_init__` or transform; the test assigns sequence `block_ids=([1, 2], [5, 6])` at test_metadata.py:27 and asserts the same tuple at :37 (context `block_ids=([7], [11])` at :20/:38). Native `KVCacheConfig` / cache-group order is exercised in test_request.py, not here. Keep the CFG shared- vs branch-context tests as DTO schema docs; do not treat a green run of this file as the native cache-group contract the name claims. (evidence: tests/diffusion/diffusion_kv/test_metadata.py:15 `def test_diffusion_kv_metadata_uses_native_cache_group_block_ids() -> None:` — name claims native cache-group contract. :27 `block_ids=([1, 2], [5, 6]),` injects the value; :37 `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])` asserts it back. vllm_omni/diffusion/diffusion_kv/metadata.py:16 `@dataclass` / :30 `@dataclass` / :46 `@dataclass` with no `__post_init__`; :42 `block_ids: tuple[list[int], ...]` stored as given. Native groups are elsewhere: tests/diffusion/diffusion_kv/test_request.py:123 `config = KVCacheConfig(`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.trace.json.gz new file mode 100644 index 000000000..220557c3a Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.cost.json b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.cost.json new file mode 100644 index 000000000..c25213901 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1122.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5703/run-20260814-204837-4aa9fd", + "recorded_at": "2026-08-14T21:07:18", + "attempts": [ + { + "invocation_id": "inv-6dba7e68", + "outcome": "run:run-20260814-204837-4aa9fd", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-204837-4aa9fd", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-204837-4aa9fd", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5726, + "minutes": 18.69, + "ci_minutes": 0, + "input_tokens": 1477139, + "output_tokens": 157972, + "tool_calls": 341, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7444, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.79, + "gate": 1.98, + "review": 1115.76, + "report": 0.07 + }, + "time_to_first_result_s": 1121.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 29209 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.md b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.md new file mode 100644 index 000000000..a5e5f7138 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] 1. Correctness: vae.py encode_image:170 and encode_video:214 flip `== "cuda"` to `!= "cpu"`; on NPU (`torch.device("npu")` in platforms/npu/platform.py:163) this newly enters fork_rng + device_module.manual_seed. PR text claimed a CUDA/MUSA allowlist that is not in the code. +- [sweep] 2. Simplifiability: encode_image:172-176 and encode_video:216-220 duplicate the same RNG block; in-repo allowlist is already `device_type in ("cuda", "musa")` at gpu_generation_worker.py:32. Extract one helper gated on that set rather than `!= "cpu"`. +- [sweep] 3. Breaking behavior: callers of MiniMaxH3VideoVAE.encode_image/encode_video are pipeline_minimax_h3.py:963 and :1066 (plus object.__new__ tests). DreamZero/Wan2.2 encode_image hits are different classes. NPU MiniMax-H3-NPU.md is the stale inheritor: previously CUDA-only seeding skipped NPU; this PR does not update that recipe or gate it out. +- [sweep] 4. Rebase/merge damage: single squashed commit 1c6ee28f; no dropped hunks. test_minimax_h3_vae.py named in the PR body does not exist in this tree (grep: 0 files). +- [sweep] 5. Tests: DIFF HUNK vae.py:117/167/211 has TEST FILES TOUCHED: NONE. test_video_vae_encode_uses_configured_parallel_tiling (test_minimax_h3_contract.py:341-346) uses a CPU Parameter so devices=[] and never touches device_module. test_video_vae_keeps_reference_fp32_weights constructs MiniMaxH3VideoVAE on CPU and now executes get_device_module() at vae.py:120. Claimed torchaudio/soundfile/ffmpeg coverage is not in this diff. +- [validated] pipeline_minimax_h3.py:545-548 constructs MiniMaxH3VideoVAE(device=self.device); FL2VA/Ref2VA encode goes through _encode_visual_condition:963 and _encode_video_conditions:1066. +- [validated] MUSA device type is "musa" (platforms/musa/platform.py:133-134); NPU device type is "npu" (platforms/npu/platform.py:163-164). +- [sweep] 6. Docs: MiniMax-H3-MUSA.md:72-76 labels the only serve block as the validated Ref2VA profile but exports MODEL_ROOT/FL2VA and never shows Ref2VA, unlike MiniMax-H3.md:168 and MiniMax-H3-NPU.md:117. recipes/README.md:48-49 adds the MUSA row and rewrites the CUDA H3 blurb (drops "full mixed-reference"). supported_models.md:47 is unchanged and has no MUSA column. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This diff replaces the CUDA-only gate `parameter.device.type == "cuda"` with `parameter.device.type != "cpu"` in both `encode_image()` and `encode_video()` (vae.py:170 and :214) and caches `self.device_module = torch.get_device_module()` with no argument in `__init__` (vae.py:120). Every non-CPU device now enters `torch.random.fork_rng(devices=...)` plus `self.device_module.manual_seed`. MiniMax H3 already ships an Ascend path: `NPUOmniPlatform.get_torch_device` returns `torch.device("npu")`, `recipes/MiniMaxAI/MiniMax-H3-NPU.md` exists, and `pipeline_minimax_h3.py` constructs this VAE with `device=self.device` from `get_local_device()` then calls `encode_image` on rank 0 and `encode_video` on rank 0 or distributed VAE encode. NPU FL2VA/Ref2VA therefore newly inherits this RNG path. The no-arg `get_device_module()` binds the process current accelerator, not the VAE's `parameter.device`, so CPU construction (`test_video_vae_keeps_reference_fp32_weights`) and a MUSA VAE under a different current device can resolve the wrong module. Gate both copies on `parameter.device.type in ("cuda", "musa")` (the same pair `GPUGenerationWorker.init_device` already uses), resolve `torch.get_device_module(device)` inside the encode-time loop, and leave other accelerators on the previous CPU-only seed path. (evidence: vae.py:120 `self.device_module = torch.get_device_module()` — no device argument, bound at construction. vae.py:170 and :214 `devices = [parameter.device] if parameter.device.type != "cpu" else []` — any non-CPU type, including npu. vae.py:173-175 / :217-219 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` — uses that cached module whenever devices is non-empty. pipeline_minimax_h3.py:481 `self.device = get_local_device()`; :545-547 `self.video_vae = MiniMaxH3VideoVAE(` / `device=self.device,`; :963 `rows = self.video_vae.encode_image(image) if rank == 0 else None`; :1062 `if rank == 0 or distributed_encode:`. vllm_omni/diffusion/distributed/utils.py:14 `return current_omni_platform.get_torch_device(current_omni_platform.current_device())`. platforms/npu/platform.py:163 `return torch.device("npu")`. recipes/MiniMaxAI contains MiniMax-H3-NPU.md. tests/.../test_minimax_h3_contract.py:319-322 `video_vae = vae_module.MiniMaxH3VideoVAE(` / `device=torch.device("cpu"),`. gpu_generation_worker.py:32 `if self.device_config.device_type in ("cuda", "musa"):`.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — This new recipe's only serve snippet is introduced as the validated Ref2VA TP4 configuration, then sets `export MODEL="${MODEL_ROOT}/FL2VA"` (MiniMax-H3-MUSA.md:72-76). Two lines above, the same page says to set MODEL to Ref2VA for Ref2VA; known limitations say each server loads only one partition; CUDA/NPU siblings keep the FL2VA template and add `export MODEL="${MODEL_ROOT}/Ref2VA"` as a restart. As written, a reader reproducing the claimed TP4 Ref2VA profile (638.6s, 1344x768, 50 steps) loads the FL2VA checkpoint. Change this block's MODEL to Ref2VA, or keep FL2VA only if you add the sibling restart snippet and stop labeling the command as the Ref2VA profile. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-73 `The validated Ref2VA configuration uses tensor parallelism across four MTT` / `S5000 GPUs and offloads inactive model components to CPU:`; MiniMax-H3-MUSA.md:76 `export MODEL="${MODEL_ROOT}/FL2VA"`; MiniMax-H3-MUSA.md:69-70 `Set \`MODEL\` to \`FL2VA\` for T2VA and FL2VA, or to \`Ref2VA\` for Ref2VA.`; MiniMax-H3-MUSA.md:137 `- Each server process loads only one checkpoint partition.`; recipes/MiniMaxAI/MiniMax-H3.md:168 `export MODEL="${MODEL_ROOT}/Ref2VA"`; recipes/MiniMaxAI/MiniMax-H3-NPU.md:117 `export MODEL="${MODEL_ROOT}/Ref2VA"`.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:126` [minor] — MUSA validation records the 50.6s image+audio smoke and official Ref2VA TP4 638.6s result, matching the PR body, but the PR body also publishes true TP8 on 8x MTT S5000 at 402.3s. This page's hardware summary and serve command are TP4-only (`--num-gpus 4`, `--tensor-parallel-size 4`) and never mention TP8. Add the TP8 serve/profile that produced 402.3s, or drop that number from the PR body, so every published MUSA capacity claim is reproducible from this page. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:117 `The synchronous HTTP request completed in 50.6 seconds and returned an MP4` — smoke matches PR body. recipes/MiniMaxAI/MiniMax-H3-MUSA.md:125-127 `A true TP4 run on four MTT S5000 GPUs returned HTTP 200 in 638.6 seconds and produced 124 H.264 frames at 24 FPS with AAC stereo audio at 32 kHz.` — TP4 only; no TP8 or 402.3s. recipes/MiniMaxAI/MiniMax-H3-MUSA.md:87-89 `--num-gpus 4` / `--tensor-parallel-size 4` / `--vae-patch-parallel-size 4` — only published serve profile. PR 5703 Validation also lists `true TP8 on 8 x MTT S5000: 402.3 seconds`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:?` [minor] — This diff changes production keyframe RNG in encode_image/encode_video (vae.py:170/214 `parameter.device.type != "cpu"` plus `self.device_module.device`/`manual_seed`) but touches no test files. `test_video_vae_encode_uses_configured_parallel_tiling` builds MiniMaxH3VideoVAE via `object.__new__` without setting `device_module` (test_minimax_h3_contract.py:344) and uses a CPU `Parameter(torch.ones(1))` (line 335), so `devices=[]` and the new branch cannot fail; `encode_image` has no test callers. `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py` is not in the tree. Extend the existing L2 test — `pytest -q -c /dev/null tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py -k vae` — with a stub `device_module` and a fake `cuda`/`musa` `parameter.device` asserting `fork_rng`/`manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; CPU should leave `devices` empty. Do not cite a vae test file this PR does not add. (evidence: vae.py:120 `self.device_module = torch.get_device_module()` is only assigned in `__init__`. vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []` then vae.py:219-220 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` (same pattern at vae.py:170,175-176 in encode_image). test_minimax_h3_contract.py:344 `video_vae = object.__new__(MiniMaxH3VideoVAE)` never sets `device_module`; :335 `self.weight = torch.nn.Parameter(torch.ones(1))` is CPU so `devices=[]`; :353 `rows, shape = video_vae.encode_video("frames")`. Grep under tests/diffusion/models/minimax_h3 finds no `device_module`, no `MINIMAX_H3_KEYFRAME_ENCODE_SEED`, no `.encode_image(`, and no `test_minimax_h3_vae.py`.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:111` [minor] — This PR’s Purpose/Validation claim coverage of the torchaudio, soundfile, and ffmpeg audio-loading paths via tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py, but that file is absent, tests/diffusion/models/minimax_h3 has no torchaudio/soundfile/ffmpeg matches, and tests/ has zero load_audio_file callers. The loader in vllm_omni/diffusion/models/minimax_h3/reference_video.py:477 is untouched by this diff. recipes/MiniMaxAI/MiniMax-H3-MUSA.md:111 is an MTT S5000 hardware-validation bullet under line 103, not L2. Delete those coverage claims from the PR body, or restore CPU tests that drive load_audio_file. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:103 `The following weight-independent gates have been validated on one MTT S5000:` — hardware validation, not a pytest. recipes/MiniMaxAI/MiniMax-H3-MUSA.md:111-112 `- the soundfile fallback preserves stereo channel layout, FP32 samples, and` / `the native sample rate when torchaudio/TorchCodec is unavailable.` vllm_omni/diffusion/models/minimax_h3/reference_video.py:477 `def load_audio_file(path: str) -> tuple[torch.Tensor, int]:` then :489 `return torchaudio.load(path)` / :490 `except (ImportError, RuntimeError, OSError):` / :493 `return _soundfile_to_waveform(path)` / :501 `"ffmpeg"` demux fallback. tests/ grep `test_minimax_h3_vae` and `load_audio_file`: no matches. tests/diffusion/models/minimax_h3 listing: test_minimax_h3_contract.py, test_minimax_h3_e2e.py, test_minimax_h3_packing.py, test_minimax_h3_parallel.py, test_minimax_h3_quantization.py, test_minimax_h3_quantization_quality.py — no test_minimax_h3_vae.py; grep soundfile|ffmpeg|torchaudio in that dir: no matches. Squashed diff files: MiniMax-H3-MUSA.md, recipes/README.md, vae.py only.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [nit] — gate_report shows buildkite/vllm-omni-amd-ci red on this head, but this review cannot tell whether main is also red or attribute the failure to this hunk: supported_models.md leaves MiniMax H3 AMD empty, and ROCm typically still reports device.type=="cuda" so this class would follow the old CUDA RNG path. The only coupling noted is that `__init__` now always calls no-arg `torch.get_device_module()`, including CPU construction that AMD core_model+cpu jobs would run. Can you confirm from the job logs whether this check is unique to the PR and whether that init-time `get_device_module()` call is involved? (evidence: BLOCKERS: buildkite/vllm-omni-amd-ci is red per gate_report; cannot tell from this lens whether main is also red. ASSUMPTIONS: AMD CI redness cannot be attributed from this worktree; only plausibility of MiniMaxH3VideoVAE.__init__ now calling get_device_module() is claimed. AMD CI likely includes core_model+cpu tests; attribution of buildkite/vllm-omni-amd-ci is unknown without job logs.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.trace.json.gz new file mode 100644 index 000000000..551f878c6 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.cost.json b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.cost.json new file mode 100644 index 000000000..bf3de861f --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1372.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5957/run-20260814-210832-202771", + "recorded_at": "2026-08-14T21:31:24", + "attempts": [ + { + "invocation_id": "inv-f3d314c0", + "outcome": "run:run-20260814-210832-202771", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-210832-202771", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-210832-202771", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8824, + "minutes": 22.87, + "ci_minutes": 0, + "input_tokens": 2342765, + "output_tokens": 227176, + "tool_calls": 510, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9335, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 5.32, + "gate": 1.88, + "review": 1364.89, + "report": 0.06 + }, + "time_to_first_result_s": 1372.15 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5957.trace.json.gz", + "trace_bytes": 126843 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.md b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.md new file mode 100644 index 000000000..6ed3a8a72 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5957, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 19a6e513f28e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] 1. Correctness: read talker payload-at-request-end (indextts2_talker.py:171,454-486), accumulator per-key latest replace (omni_connector_model_runner_mixin.py:920-947), serving native speed (serving_speech.py:517-524,3642-3646; adapter duration_factor=1/speed), stage handoff fail-fast (stage_input_processors/indextts2.py), HF missing-config patch (arg_utils.py:247-265). No crash/off-by-one found on the code-only 2.5 path; duration_factor from first decode is kept because later meta only replaces meta.use_gpt_latent. +- [validated] native speed is applied once: IndexTTS25Adapter sets duration_factor=[1.0/speed] (tts_adapters/indextts2.py:318-320) and serving_speech._audio_encode_speed returns 1.0 when adapter.native_speed_control (serving_speech.py:521-524); tests assert create_audio.speed==1.0 for indextts2_5. +- [validated] streaming speed schema move is scoped: OpenAICreateSpeechRequest no longer rejects stream+speed!=1.0 (protocol/audio.py:319-321); create_speech still calls _validate_speech_streaming_request for raw and SSE (serving_speech.py:3699-3726) and rejects unless _uses_native_speed_control; WebSocket still requires speed=1.0 (protocol/audio.py:556-563). +- [validated] request-end D2H defer is opted-in: omni_payload_at_request_end only when model_type==indextts2_5 (talker.py:171); _should_defer_full_payload_d2h also requires no prefix cache and full-payload accumulation (gpu_ar_model_runner.py:1723-1728); deploy yaml sets enable_prefix_caching:false. +- [validated] HF empty-config patch is model-scoped: except path requires model_type==indextts2_5 and isdir(self.model) and not lexists(config.json) (arg_utils.py:257-265); tests assert malformed/remote errors reach the parent loader. +- [validated] end2end.py stage_configs[0] is guarded by `if not stage_configs: return None` before indexing (end2end.py:96-98). +- [validated] EnhancedCodec 2D codes unsqueeze to [1,B,T] so codes.shape[0]==num_quantizers for the single-codebook path (codec/models.py:97-102); s2mel ref_mel.shape[0] is checked against ref_lengths (indextts2_s2mel_decoder.py:513-519). +- [sweep] 2. Simplifiability: 2.5 adapter/talker/stage1 duration bounds are the same [0.5,2.0] closed under reciprocal; no dead co-occurring branches found. Shared _preprocess_prefill duration check is not model_type-gated but defaults to 1.0 for v2. + +`tests/model_executor/models/indextts2/test_enhanced_codec.py:11` [major] — These three new modules have no pytestmark, so CUDA/AMD L1/L2 (`pytest -sv tests/model_executor -m 'core_model and cpu'` in .buildkite/cuda/test-ready.yml:13 and .buildkite/amd/test-amd-ready.yml:19) silently deselect them. Siblings such as test_s2mel_v2_5.py:26 and test_talker_v2_5.py:17 set `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. Unique coverage lives only here: real EnhancedCodec decode (test_enhanced_codec.py:11; marked test_s2mel_v2_5.py uses `_FakeEnhancedCodec`), IndexTTS25Config defaults (test_configuration_and_pipeline.py:16), and `s2mel_cfm_batch_size == 4` (test_recipe_defaults.py:14). A broken codec, config default, or recipe batch size can merge with a green L2 job. Add the same pytestmark to all three modules (or fold the unique asserts into a marked file). (evidence: test_enhanced_codec.py:4-11 imports EnhancedCodec then `def test_enhanced_codec_decode_restores_hidden_size_and_doubles_time():` with no pytestmark; test_configuration_and_pipeline.py:16 `def test_indextts25_defaults_are_distinct_from_v2():` (IndexTTS25Config import at :11, no pytestmark); test_recipe_defaults.py:9-14 `def test_indextts25_default_recipe_enables_validated_cfm_batching():` then `assert stage1["hf_overrides"]["s2mel_cfm_batch_size"] == 4` with no pytestmark; test_s2mel_v2_5.py:26 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` and :29 `class _FakeEnhancedCodec:`; .buildkite/cuda/test-ready.yml:13 `pytest -sv tests/model_executor -m 'core_model and cpu'`; pyproject.toml:247 `"core_model: L1&L2 tests (run in each PR)"`.) + +`vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py:251` [minor] — `load_indextts25_encoding` lazily `import tiktoken` at tokenizer_v2_5.py:251, but `tiktoken` is not listed in `pyproject.toml` (including the new `indextts2` extra) or `requirements/`. That extra already pins fugashi/unidic-lite/wetext; a missing tiktoken would raise an unguarded ImportError at encode time. Declare it on the extra or common requirements, or document that vLLM's transitive pin is the supported contract. (evidence: tokenizer_v2_5.py:251 ` import tiktoken` — lazy import inside load_indextts25_encoding with no ModuleNotFoundError handling. pyproject.toml:87-92 `indextts2 = [\n "fugashi>=1.2.0",\n "unidic-lite>=1.0.0",\n "wetext>=0.0.9; sys_platform != 'linux'",\n "WeTextProcessing; sys_platform == 'linux'",\n]` — extra declares those four packages only; repo grep of pyproject.toml and requirements/ finds no `tiktoken` pin.) + +`gpu_ar_model_runner.py:1058` [minor] — `gpu_ar_model_runner.py:1058-1062` flushes `_pending_full_payload_send` for `finished_req_ids` and rids already gone from `self.requests`. Mixin `cleanup_finished_request` (`omni_connector_model_runner_mixin.py:291-293`) force-flushes before drop, so GPU snapshots are popped. `flush_full_payload_outputs` has no abort-vs-complete gate: aborted IDs in `finished_req_ids` (`gpu_model_runner.py:496-500`) still `send_full_payload_outputs` whatever was accumulated — the pre-existing full-payload contract. Confirm whether request-end D2H deferral (`_should_defer_full_payload_d2h`) should still send that partial GPU snapshot on abort. (evidence: gpu_ar_model_runner.py:1058-1062 `if self._pending_full_payload_send:` / `flush_ids = set(getattr(scheduler_output, "finished_req_ids", set()))` / `flush_ids.update({rid for rid in self._pending_full_payload_send if rid not in self.requests})` / `self.flush_full_payload_outputs(flush_ids)`. omni_connector_model_runner_mixin.py:291-293 `if req_id in self._pending_full_payload_send:` / `self.flush_full_payload_outputs({req_id})`. omni_connector_model_runner_mixin.py:951-970 `def flush_full_payload_outputs(self, finished_req_ids: set[str]) -> None:` / `"""Send accumulated full_payload outputs for requests that just finished."""` / `entry = self._pending_full_payload_send.pop(req_id, None)` / `self.send_full_payload_outputs(scheduler_output=None, outputs=to_send)` — no abort vs complete check. gpu_model_runner.py:481-492 `for req_id in scheduler_output.finished_req_ids:` / `self.requests.pop(req_id, None)` / `cleanup_finished_request(req_id)`. gpu_model_runner.py:496-498 `There could be an edge case where finished_req_ids and scheduled_req_ids overlap. This happens when a request is aborted`.) + +`recipes/IndexTeam/IndexTTS-2_5.md:40` [minor] — The published hardware heading is `### 1x NVIDIA H20 96GB` (recipes/IndexTeam/IndexTTS-2_5.md:40) and recipes/README.md:53 lists `1x NVIDIA H20 96GB`, but the same recipe Notes say the standard recipe was exercised on one NVIDIA H200 (IndexTTS-2_5.md:290). Align the heading and README row with the GPU that was actually run, or list both and state which one was validated. (evidence: recipes/IndexTeam/IndexTTS-2_5.md:40 `### 1x NVIDIA H20 96GB`. recipes/IndexTeam/IndexTTS-2_5.md:290 `- Hardware scope: the standard recipe has been exercised on one NVIDIA H200.`. recipes/README.md:53 `| [`IndexTeam/IndexTTS-2_5.md`](./IndexTeam/IndexTTS-2_5.md) | Online + offline multilingual voice-cloned TTS with native speed and emotion control | 1x NVIDIA H20 96GB |`.) + +`vllm_omni/engine/arg_utils.py:?` [minor] — `_patch_empty_hf_config` synthesizes a missing config.json only when `model_type == "indextts2_5"` and `os.path.isdir(self.model)` with no `config.json` (vllm_omni/engine/arg_utils.py:260). `test_remote_hf_config_error_reaches_parent_loader` keeps a remote id on vLLM’s normal failure (tests/engine/test_arg_utils.py:333 `model="remote/model"`, :345 `hf_config_path is None`). The recipe Summary still lists Model: `IndexTeam/IndexTTS-2.5` and links the Hub repo, while serve commands and docs/models/supported_models.md require a local native `checkpoints/` bundle. Change that Summary so only the local bundle is documented as `--model`, or extend the missing-config patch to a Hub id / downloaded snapshot. (evidence: vllm_omni/engine/arg_utils.py:257 `# The official IndexTTS 2.5 bundle has no HuggingFace config.json.`; vllm_omni/engine/arg_utils.py:260 `if model_type != "indextts2_5" or not os.path.isdir(self.model):` then `return` — Hub ids are not directories so no synthesized config.json; tests/engine/test_arg_utils.py:333 `model="remote/model"` with :345 `assert args.hf_config_path is None`; recipes/IndexTeam/IndexTTS-2_5.md:6 `- Model: \`IndexTeam/IndexTTS-2.5\`` vs :56 `Obtain the native IndexTTS-2.5 bundle and point \`MODEL\` at its root.`; docs/models/supported_models.md:83 `native \`checkpoints/\` bundle`.) + +`vllm_omni/engine/stage_init_utils.py:?` [minor] — This PR unguardedly constructs `InputProcessor(..., renderer=_build_token_only_renderer(...))` in `build_stage0_input_processor` whenever `skip_tokenizer_init` is set (`vllm_omni/engine/stage_init_utils.py:1140-1144`) and subclasses `BaseRenderer` from `vllm.renderers`. The same series already shims vLLM 0.26 accelerator images for `MistralToolCall`, `renderer.warmup`, and `num_stale_output_tokens` (`vllm_omni/request.py:44-47`). IndexTTS-2.5 stage 0 sets `skip_tokenizer_init: true` (`vllm_omni/deploy/indextts2_5.yaml:21`); pre-existing IndexTTS-2 and PersonaPlex deploy configs do too. Guard this constructor like the other 0.26 shims, or document that `skip_tokenizer_init` stages require vLLM 0.27. (evidence: vllm_omni/engine/stage_init_utils.py:1140-1144 `if bool(getattr(stage_vllm_config.model_config, "skip_tokenizer_init", False)): input_processor = InputProcessor(vllm_config=stage_vllm_config, renderer=_build_token_only_renderer(stage_vllm_config))` — no try/except or version guard. vllm_omni/engine/stage_init_utils.py:23 `from vllm.renderers import BaseRenderer`. vllm_omni/request.py:44-47 `# vLLM 0.27 owns this counter; accelerator images still based on 0.26 do not.` then `self.num_stale_output_tokens = int(getattr(self, "num_stale_output_tokens", 0) or 0)`. vllm_omni/entrypoints/openai/api_server.py:1001 `renderer_warmup = getattr(state.online_renderer, "warmup", None)`. vllm_omni/entrypoints/openai/serving_chat.py:88 and :91 import `MistralToolCall` from `vllm.parser.mistral` and fall back to `vllm.tool_parsers.mistral_tool_parser`. vllm_omni/deploy/indextts2_5.yaml:21 `skip_tokenizer_init: true`.) + +`vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:?` [minor] — `_protect_values` and `_restore_values` build placeholders with `chr(ord('a') + index)` and never cap the unique list. `_normalize_zh_or_en` passes every `_PINYIN_TONE_PATTERN` / `_NAME_PATTERN` hit into that helper, so the 27th unique value becomes `` or ``. Those punctuation markers can be rewritten by WeText or collide with user text, so restore cannot reliably round-trip. Use a numeric marker such as ``. (evidence: vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:153-155 `for index, value in enumerate(unique_values):\n marker = f"<{prefix}_{chr(ord('a') + index)}>"` — same scheme at :167 in `_restore_values`. Callers at :194-203 pass every hit with no cap: `pinyin_values = [match.group(0) for match in _PINYIN_TONE_PATTERN.finditer(text)]` then `_protect_values(..., prefix="pinyin")`, and `_protect_values(text, _NAME_PATTERN.findall(text), prefix="n")`. Index 26 is `chr(97+26)` = `{`.) + +`vllm_omni/deploy/qwen2_5_omni.yaml:?` [minor] — This PR also flips Qwen2.5-Omni XPU AR stages from `enforce_eager: false` to `true` in `vllm_omni/deploy/qwen2_5_omni.yaml:90` and `:93` (comment cites PyTorch 2.13 XPU Dynamo duplicate-handler assertions). Any Qwen2.5-Omni XPU deploy that loads this YAML now inherits eager AR. The only coverage is `tests/config/test_config_factory.py:2279` `test_qwen2_5_omni_xpu_uses_eager_ar_stages`, which asserts the YAML projection after `_apply_platform_overrides`. This PR’s documented hardware exercise is IndexTTS-2.5 on NVIDIA H200 (`recipes/IndexTeam/IndexTTS-2_5.md:290`), with no Qwen2.5-Omni XPU smoke. Split the override into an accelerator PR, or add an XPU Qwen2.5-Omni run that shows the old cudagraph path crashes and eager still produces audio. (evidence: vllm_omni/deploy/qwen2_5_omni.yaml:85-93 `# PyTorch 2.13 XPU Dynamo raises duplicate-handler assertions while compiling both Qwen2.5 AR stages. Keep them eager until that toolchain issue is fixed; stage 2 already inherits enforce_eager: true above.` then stage 0/1 `enforce_eager: true`. tests/config/test_config_factory.py:2279-2287 `def test_qwen2_5_omni_xpu_uses_eager_ar_stages(self):` / `deploy = _apply_platform_overrides(deploy, platform="xpu")` / `assert deploy.stages[0].enforce_eager is True` / `assert deploy.stages[1].enforce_eager is True` — YAML projection only; repo grep finds no other `qwen2_5_omni` XPU smoke. recipes/IndexTeam/IndexTTS-2_5.md:290 `- Hardware scope: the standard recipe has been exercised on one NVIDIA H200.`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.trace.json.gz new file mode 100644 index 000000000..58552d62c Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r2/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.cost.json b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.cost.json new file mode 100644 index 000000000..7bbb22f16 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.cost.json @@ -0,0 +1,102 @@ +{ + "wall_s": 1193.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5550/run-20260814-215450-2bd4a2", + "recorded_at": "2026-08-14T22:14:42", + "attempts": [ + { + "invocation_id": "inv-f4c0f8b0", + "outcome": "run:run-20260814-215450-2bd4a2", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-203330-5a76e3", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-215450-2bd4a2", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-215450-2bd4a2", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5272, + "minutes": 19.87, + "ci_minutes": 0, + "input_tokens": 1300234, + "output_tokens": 160081, + "tool_calls": 334, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7466, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.42, + "gate": 1.76, + "review": 1187.13, + "report": 0.11 + }, + "time_to_first_result_s": 1192.35 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 72763 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.md b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.md new file mode 100644 index 000000000..0ee104b06 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.md @@ -0,0 +1,58 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [sweep] indexed test_metadata.py:37 sequences[0].block_ids — assumes sequences is non-empty; guaranteed only because the test itself constructed a 1-tuple; no production consumer. +- [sweep] indexed test_metadata.py:38 contexts[0].block_ids — same constructor-local guarantee. +- [sweep] indexed test_metadata.py:68 sequences[0].context_ids — same; does not prove CFG sharing is interpreted by scheduler/worker. +- [sweep] indexed test_metadata.py:109 sequences[0].context_ids — same for branch-specific contexts. +- [sweep] indexed test_worker_contract.py:221 calls[0][1][0].prepared_layout — calls[0] is guaranteed by the preceding assert calls == [3-tuple]; prepared_layout identity is a real executor-path check. +- [sweep] branch diffusion_engine.py:199 scheduler_to_close is not None — reachable when _init_runtime_state/_init_execute_fn fails after _init_scheduler assigned self.scheduler; not reachable on initialize() failure because assignment always precedes initialize() (diffusion_engine.py:246-252). +- [sweep] branch multiproc_executor.py:545 metadata is not None — reachable and tested by test_request_rpc_sends_attached_metadata_for_only_that_request. +- [sweep] branch interface.py:184 envelope vs forwarded request_id — reachable and tested (test_request_rpc_rejects_envelope_request_identity_mismatch). + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — This diff adds `and not has_diffusion_kv_metadata` so any attached snapshot disables the DLO+AllGather list RPC (`args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job)` at line 509, still no per-request snapshot slot) and falls through to serial per-request 4-tuples with no log. `any(...)` means one snapshot disables DLO for MiniMax-H3 (`examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py`) and every other DLO+AllGather user of this shared executor; OmniDiffusionConfig does not forbid combining `paged_scheduler` with `enable_distributed_layerwise_offload`. `make_executor()` hardcodes `enable_distributed_layerwise_offload=False` and its `collective_rpc` stub cannot accept the DLO `timeout=` kwarg, so the new arm is never executed; `tests/diffusion/test_multiproc_engine_concurrency.py` DLO cases build `NewRequestData` without metadata. Add a CPU test with DLO+AllGather enabled and two `NewRequestData` carrying metadata, and assert `collective_rpc` sees per-request 4-tuples rather than one list broadcast — otherwise W1 attaching metadata silently drops DP concurrency with no CI signal. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)` — vllm_omni/diffusion/executor/multiproc_executor.py:469-474 `if ( len(new_reqs) > 1 and not has_diffusion_kv_metadata and getattr(self.od_config, "enable_distributed_layerwise_offload", False) and getattr(self.od_config, "dlo_use_allgather", True) ):` — vllm_omni/diffusion/executor/multiproc_executor.py:509 `args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job),` — vllm_omni/diffusion/executor/multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None:` / `args += (new_req.diffusion_kv_metadata,)` — tests/diffusion/diffusion_kv/test_worker_contract.py:62-71 `enable_distributed_layerwise_offload=False` and `def collective_rpc(method, *, args, unique_reply_rank, exec_all_ranks):` (no `timeout=`) — vllm_omni/diffusion/data.py:941-947 forbids only `paged_scheduler` + `need_recv_cache`, not DLO — vllm_omni/diffusion/sched/base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — tests/diffusion/test_multiproc_engine_concurrency.py:200-207 builds `NewRequestData(request_id=rid, req=...)` with no metadata — examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py:83-84 `enable_distributed_layerwise_offload=True, dlo_use_allgather=True`) + +`tests/diffusion/diffusion_kv/test_metadata.py:37` [minor] — tests/diffusion/diffusion_kv/test_metadata.py:15-110 only constructs DiffusionKV*Metadata and asserts the same block_ids/context_ids just injected (line 37). The dataclasses in metadata.py have no __post_init__, and this file never calls NewRequestData, validate_new_request_data_identity, execute_request, or a native cache-group builder despite test_diffusion_kv_metadata_uses_native_cache_group_block_ids. Delete these roundtrips, or attach the shared-vs-branch context payloads to test_request_rpc_sends_attached_metadata_for_only_that_request (test_worker_contract.py:238-241 currently compares whole metadata objects from make_metadata(), which has no contexts). (evidence: tests/diffusion/diffusion_kv/test_metadata.py:15 `def test_diffusion_kv_metadata_uses_native_cache_group_block_ids() -> None:` — name claims native cache-group block_ids +tests/diffusion/diffusion_kv/test_metadata.py:6-10 imports only DiffusionKVContextMetadata/DiffusionKVMetadata/DiffusionKVSequenceMetadata +tests/diffusion/diffusion_kv/test_metadata.py:20 `block_ids=([7], [11]),` then :27 `block_ids=([1, 2], [5, 6]),` then :37-38 `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])` / `assert metadata.contexts[0].block_ids == ([7], [11])` — same values just injected +vllm_omni/diffusion/diffusion_kv/metadata.py:16 `@dataclass` / :31 `@dataclass` / :47 `@dataclass` with no __post_init__ — field storage only +tests/diffusion/diffusion_kv/test_worker_contract.py:27-40 make_metadata() builds sequences with `block_ids=([1, 2],)` and no contexts +tests/diffusion/diffusion_kv/test_worker_contract.py:238-241 `assert [call[1] for call in calls] == [(req_0, executor.od_config, None, metadata_0), (req_1, executor.od_config, None, metadata_1)]` — whole-object RPC attach, not block_ids or CFG context shape) + +`vllm_omni/diffusion/sched/base_scheduler.py:114` [minor] — `NewRequestData.from_state(state)` at `base_scheduler.py:114` still omits `diffusion_kv_metadata`, so production never attaches a snapshot. Worker `execute_model` validates then drops it (`diffusion_model_runner.py:170-171`, `:609-618`) with no installer. This is the stated W0 Worker-contract boundary; a follow-up must emit and install snapshots. (evidence: base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — only production from_state call, no diffusion_kv_metadata kwarg. interface.py:171-177 from_state accepts `diffusion_kv_metadata: DiffusionKVMetadata | None = None` and stores it, but the scheduler never passes it. diffusion_model_runner.py:170-171 `# W0 does not wire the Scheduler allocator or Worker installer yet, so / # validate an allocation snapshot when present without requiring one.` diffusion_model_runner.py:609-618 `_validate_diffusion_kv_metadata(...)` then `_execute_request_list([req], ...)` with no metadata forwarded.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:574` [minor] — `execute_request` now raises identity mismatches before the RPC `try/except` that fabricates `DiffusionOutput(error=)`, but dense+metadata is still appended onto the `execute_model` RPC args with no cache-mode check. The worker then raises in `_validate_diffusion_kv_metadata`, and that failure is caught by the same `except` and converted into a worker error output. Reject dense+metadata pre-RPC the same way identity mismatches are rejected. (evidence: multiproc_executor.py:459-460 `for new_req in new_reqs: validate_new_request_data_identity(new_req)` — identity errors are raised before any RPC. multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None: args += (new_req.diffusion_kv_metadata,)` — metadata is forwarded with no dense/paged check. diffusion_worker.py:463-465 `if diffusion_kv_metadata is not None: kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` then `self.model_runner.execute_model(req, **kwargs)` — worker delivers it to the runner. diffusion_model_runner.py:172-173 `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None: raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")` — dense+metadata is rejected only after RPC. multiproc_executor.py:574-580 `except Exception as exc: runner_outputs.append(RunnerOutput(..., result=DiffusionOutput(error=str(exc))))` — that post-RPC failure is converted into a worker error output.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:457` [minor] — execute_request gates identity before RPC (`multiproc_executor.py:454-460`) so a mismatch cannot be forwarded. `execute_batch` still `collective_rpc("execute_model_batch", args=(scheduler_output, self.od_config))` at 615-619 with no equivalent loop (only `len<=1` and DLO-DP fall back to `execute_request`). `execute_step` still `collective_rpc("execute_stepwise", args=(scheduler_output,))` at 651-655 with none. Runner checks in `execute_model_batch` (636-640) and `execute_stepwise` (750-754) run only after those RPCs. Call the same helper on `scheduled_new_reqs` before the fused/step RPCs. (evidence: multiproc_executor.py:454-460 ` # Validate every envelope before selecting a dispatch path. In\n # particular, keep identity errors outside the RPC error wrapper so an\n # invalid request cannot be forwarded or converted into a worker output.\n from vllm_omni.diffusion.sched.interface import validate_new_request_data_identity\n\n for new_req in new_reqs:\n validate_new_request_data_identity(new_req)` — only inside execute_request. multiproc_executor.py:600-601 ` if len(scheduler_output.scheduled_new_reqs) <= 1:\n return self.execute_request(scheduler_output)` — len<=1 covered. multiproc_executor.py:615-619 ` result = self.collective_rpc(\n "execute_model_batch",\n args=(scheduler_output, self.od_config),\n unique_reply_rank=0,\n exec_all_ranks=True,\n )` — fused batch RPC, no identity loop. multiproc_executor.py:651-655 ` result = self.collective_rpc(\n "execute_stepwise",\n args=(scheduler_output,),\n unique_reply_rank=0,\n exec_all_ranks=True,\n )` — step RPC, no identity loop. diffusion_model_runner.py:636-640 ` for new_req in scheduler_output.scheduled_new_reqs:\n validate_new_request_data_identity(new_req)\n self._validate_diffusion_kv_metadata(` and 750-754 same loop in execute_stepwise — runner-side only, after RPC.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:465` [minor] — This diff forwards `diffusion_kv_metadata` as a kwarg from `DiffusionWorker.execute_model` (and `WorkerWrapperBase.execute_model`) whenever it is not None (`diffusion_worker.py:462-465` and `:1371-1374`). The only in-repo `DiffusionModelRunner` subclass, `ARDiffusionModelRunner.execute_model` at `vllm_omni/experimental/ar_diffusion/runner.py:237-255`, still declares `(req, kv_prefetch_job=None)` and both `super()` calls omit the new kwarg, so the first attached snapshot is `TypeError: unexpected keyword argument 'diffusion_kv_metadata'` rather than `_validate_diffusion_kv_metadata`. `ARDiffusionEngine.default_diffusion_model_runner_cls` points here (`engine.py:13,40`); `test_dense_worker_call_does_not_extend_model_runner_signature` uses a fake `DenseModelRunner` and only covers metadata-is-None. Add the optional argument and pass it through to `super()` in both branches as part of this Worker-contract PR. (evidence: diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` then `if diffusion_kv_metadata is not None: kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` then `output = self.model_runner.execute_model(req, **kwargs)` — extra kwarg only when attached. diffusion_worker.py:1371-1374 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None: kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `return self.worker.execute_model(req, od_config, **kwargs)`. experimental/ar_diffusion/runner.py:237-243 `def execute_model(self, req: OmniDiffusionRequest, kv_prefetch_job: KVPrefetchJob | None = None) -> DiffusionOutput:` then `return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` — no `diffusion_kv_metadata`. runner.py:255 `output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` — second branch also omits it. experimental/ar_diffusion/engine.py:13 `AR_DIFFUSION_MODEL_RUNNER_CLS = "vllm_omni.experimental.ar_diffusion.runner.ARDiffusionModelRunner"` and engine.py:40 `default_diffusion_model_runner_cls = AR_DIFFUSION_MODEL_RUNNER_CLS`. tests/diffusion/diffusion_kv/test_worker_contract.py:274-278 `def test_dense_worker_call_does_not_extend_model_runner_signature()` uses `class DenseModelRunner: def execute_model(self, req, kv_prefetch_job=None):` not this override.) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:609` [minor] — This diff adds `_validate_diffusion_kv_metadata` on `execute_model` / `execute_model_batch` / `execute_stepwise` so paged mode may carry a snapshot without requiring one. New tests only cover the 'not required' half (`test_paged_request_without_metadata_reaches_request_forward` calls `execute_model(req)` with no metadata; `test_batch_path_allows_missing_metadata_before_forward` likewise) plus dense reject and identity mismatch; nothing drives matching metadata through the runner and asserts `_execute_request_list` is still called. Add `test_paged_request_with_matching_metadata_reaches_request_forward` (and the same for `execute_model_batch`) that passes `make_metadata("req-0")` on a `PAGED_SCHEDULER` runner — that is the W0 claim 'paged mode accepts metadata when supplied' and it is currently unasserted at the worker boundary. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:170-173 `# W0 does not wire the Scheduler allocator or Worker installer yet, so / validate an allocation snapshot when present without requiring one. / if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None: / raise ValueError(...)` — paged mode does not require metadata and only rejects non-paged carry. vllm_omni/diffusion/worker/diffusion_model_runner.py:609-613 `self._validate_diffusion_kv_metadata(request_id=req.request_id, metadata=diffusion_kv_metadata)` then immediately `_execute_request_list`. tests/diffusion/diffusion_kv/test_worker_contract.py:88-98 `test_paged_request_without_metadata_reaches_request_forward` — `assert runner.execute_model(req) is expected_output` with no metadata kwarg. tests/diffusion/diffusion_kv/test_worker_contract.py:116-124 `test_paged_request_rejects_mismatched_metadata_identity` — `diffusion_kv_metadata=make_metadata("stale")` raises. tests/diffusion/diffusion_kv/test_worker_contract.py:144-155 `test_batch_path_allows_missing_metadata_before_forward` — `NewRequestData(request_id="req-0", req=...)` with no metadata. No `test_paged_request_with_matching_metadata` exists; matching metadata is only forwarded by executor/wrapper tests, not asserted through the runner.) + +`vllm_omni/diffusion/diffusion_kv/metadata.py:?` [nit] — W0 snapshot DTOs type `block_ids` as `tuple[list[int], ...]` on unfrozen dataclasses with no `__post_init__` copy (`DiffusionKVContextMetadata` at metadata.py:27 and `DiffusionKVSequenceMetadata` at :42), so they store aliases to the caller's inner lists. That contradicts the one-shot trusted-snapshot claim and the rule that mutable Scheduler `DiffusionKVRequest` state must never reach a Worker; sibling `DiffusionKVContext` already tuple-izes `block_hashes` in `__post_init__`. A later allocator that passes live native `get_block_ids()` lists would let the Worker mutate Scheduler block tables. Copy the inner lists or freeze to `tuple[tuple[int, ...], ...]`. (evidence: vllm_omni/diffusion/diffusion_kv/metadata.py:16 `@dataclass` (not frozen); :27 ` block_ids: tuple[list[int], ...]`; :30 `@dataclass`; :42 ` block_ids: tuple[list[int], ...]` — neither class has `__post_init__` or a copy. Contrast vllm_omni/diffusion/diffusion_kv/request.py:13 `@dataclass(frozen=True)` and :35 `object.__setattr__(self, "block_hashes", tuple(self.block_hashes))`. vllm_omni/diffusion/sched/base_scheduler.py:290-292 `# DiffusionKVRequest objects are mutable Scheduler/native-KVCacheManager` / `# state and must never ride the normal request payload to a Worker.`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.trace.json.gz new file mode 100644 index 000000000..f95e6cc9a Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.cost.json b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.cost.json new file mode 100644 index 000000000..cdbd9aa52 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.cost.json @@ -0,0 +1,102 @@ +{ + "wall_s": 1124.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5703/run-20260814-221443-7af57a", + "recorded_at": "2026-08-14T22:33:27", + "attempts": [ + { + "invocation_id": "inv-9daea44a", + "outcome": "run:run-20260814-221443-7af57a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-204837-4aa9fd", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-221443-7af57a", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-221443-7af57a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5448, + "minutes": 18.73, + "ci_minutes": 0, + "input_tokens": 1297170, + "output_tokens": 176864, + "tool_calls": 307, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7338, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.76, + "gate": 1.81, + "review": 1118.33, + "report": 0.06 + }, + "time_to_first_result_s": 1123.96 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 56668 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.md b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.md new file mode 100644 index 000000000..56da15034 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] MINIMAX_H3_KEYFRAME_ENCODE_SEED remains 42 and CPU fork_rng still wraps the encode. CUDA devices still match the old ==cuda branch. +- [validated] pipeline encode_image/encode_video call sites do not pass extra RNG kwargs; rank-0 encode_image and VPP encode_video still call the changed methods. +- [validated] _AudioVAEDeterminismContext remains CUDA/cudnn-only; out of this PR's video-VAE RNG claim. +- [validated] CPU encode path still compiles: when parameter.device.type == "cpu", devices=[] so self.device_module is unused; the existing CPU mock test can still pass after this change. +- [validated] get_device_module() with no args: torch/__init__.py:2894-2896 uses torch._C._get_accelerator().type, not the VAE constructor device. +- [validated] CUDA get_rng_state (torch/cuda/random.py:23-42) uses device.index into torch.cuda.default_generators after _lazy_init(); it does not reject a musa/npu torch.device by type. +- [upstream-verify] recipes/MiniMaxAI/MiniMax-H3-NPU.md:18-19: NPU path has no CUDA runtime. platforms/npu/platform.py:163-164: device type is npu. +- [validated] vllm_omni/diffusion/models/minimax_h3/vae.py:170,214: CUDA still satisfies != "cpu", so CUDA still enters fork_rng; MUSA (not "cuda") is the intended newly included type. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [blocker] — This diff replaces `parameter.device.type == "cuda"` with `!= "cpu"` in `encode_image()`/`encode_video()` (vae.py:170 and :214), then calls `torch.random.fork_rng(devices=devices)` with no `device_type` and seeds via `self.device_module = torch.get_device_module()` with no device argument (vae.py:120). That is not a CUDA/MUSA allowlist: every non-CPU MiniMax H3 VAE now enters accelerator `fork_rng`, including in-tree Ascend (`NPUOmniPlatform.get_torch_device` returns `torch.device("npu", local_rank)`; MiniMaxH3Pipeline sets `self.device = get_local_device()` and constructs this VAE on `self.device`, then rank 0 calls `encode_image`/`encode_video`). Stock `fork_rng` defaults `device_type="cuda"` and `torch.cuda.get_rng_state` `_lazy_init()`s CUDA using `device.index` only, while the NPU recipe runs CPU-only PyTorch with no CUDA runtime — the old `== "cuda"` path left `devices=[]` there. `get_device_module()` at init is also unbound from the constructor `device`, so CPU construction follows the process default accelerator. Gate with `parameter.device.type in ("cuda", "musa")`, pass `device_type=parameter.device.type` into `fork_rng`, and bind `torch.get_device_module(parameter.device)` (or the constructor `device`). (evidence: vae.py:120 `self.device_module = torch.get_device_module()` — no device argument. vae.py:170/214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`. vae.py:172/216 `with torch.random.fork_rng(devices=devices):` — no device_type. pipeline_minimax_h3.py:481 `self.device = get_local_device()`; :545-547 `self.video_vae = MiniMaxH3VideoVAE(..., device=self.device)`; :963 `rows = self.video_vae.encode_image(image) if rank == 0 else None`. platforms/npu/platform.py:163-164 `return torch.device("npu")` / `return torch.device("npu", local_rank)`. recipes/MiniMaxAI/MiniMax-H3-NPU.md:18-19 `Runs on a CPU-only (aarch64) PyTorch build with `torch_npu`; no CUDA runtime is present.` torch/random.py:145 `device_type="cuda"` default; :210 `device_rng_states = [device_mod.get_rng_state(device) for device in devices]`. torch/cuda/random.py:33 `_lazy_init()` then :38-41 `idx = device.index` / `torch.cuda.default_generators[idx]`. torch/__init__.py:2894-2896 no-arg `get_device_module` uses `torch._C._get_accelerator().type`.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — This new recipe introduces the only serve block as “The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs” (MiniMax-H3-MUSA.md:72-73) then `export MODEL="${MODEL_ROOT}/FL2VA"` (line 76). FL2VA cannot serve Ref2VA: lines 69-70 say set MODEL to Ref2VA for Ref2VA; MiniMax-H3.md:164-168 restarts with `export MODEL="${MODEL_ROOT}/Ref2VA"`; pipeline_minimax_h3.py:597 rejects tasks the loaded partition does not support. Change this export to Ref2VA, or split FL2VA vs Ref2VA commands the way MiniMax-H3.md / MiniMax-H3-NPU.md do. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:69-76 `Set \`MODEL\` to \`FL2VA\` for T2VA and FL2VA, or to \`Ref2VA\` for Ref2VA.` then `The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs and offloads inactive model components to CPU:` followed by `export MODEL="${MODEL_ROOT}/FL2VA"`. recipes/MiniMaxAI/MiniMax-H3.md:164-168 `To serve Ref2VA, stop the FL2VA server and restart the same one- or four-GPU command with:` `export MODEL="${MODEL_ROOT}/Ref2VA"`. vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:597 `f"checkpoint partition {self.partition!r} supports {sorted(self.supported_tasks)}, got task={task!r}"`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This diff widens the encode RNG gate to `parameter.device.type != "cpu"` and seeds with `self.device_module.device` / `manual_seed` instead of `torch.cuda` (vae.py:170 encode_image, vae.py:214 encode_video), but no test file is in the diff. `test_video_vae_encode_uses_configured_parallel_tiling` still does `object.__new__(MiniMaxH3VideoVAE)` with `torch.nn.Parameter(torch.ones(1))` (CPU), so `devices=[]` and the new seed loop never runs; `object.__new__` also skips `__init__`, so `device_module` is unset. There is no MiniMax `encode_image` test, and `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py` is not in the tree. Extend that contract test to inject a fake non-CPU `parameter.device`, set `video_vae.device_module` to a mock, and assert `fork_rng(devices=[that device])` plus `manual_seed` for `cuda`/`musa`. Current `!= "cpu"` would also include NPU (`get_torch_device` returns `torch.device("npu")`); asserting npu stays on `devices=[]` requires narrowing the production allowlist to cuda/musa. Guard with: `pytest -q -c /dev/null tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py::test_video_vae_encode_uses_configured_parallel_tiling -m 'core_model and cpu'`. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` — encode_image gate is any non-CPU device, not cuda-only. vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` — GPU seed loop only runs when devices is non-empty. tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))` — CPU Parameter, so devices=[]. test_minimax_h3_contract.py:344 `video_vae = object.__new__(MiniMaxH3VideoVAE)` — skips __init__, never sets device_module. test_minimax_h3_contract.py:353 `rows, shape = video_vae.encode_video("frames")` — only encode_video is exercised; no MiniMax encode_image test. vllm_omni/platforms/npu/platform.py:163-164 `return torch.device("npu")` / `return torch.device("npu", local_rank)` — NPU type is not cpu, so the new gate would include it.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — vae.py:120 is the only `torch.get_device_module()` call in the tree. `pyproject.toml` marks dependencies dynamic, `setup.py` loads `requirements/{device}.txt`, and `requirements/musa.txt` (plus `common.txt`) never declare a `torch>=` floor, so the vLLM-MUSA stack is the implicit oldest bound. Please confirm that the vLLM 0.26 / vLLM-MUSA torch already includes this API; this change was not shown against an oldest-torch bound. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — repo-wide grep finds only this call. pyproject.toml:11 `dynamic = ["version", "dependencies"]`. pyproject.toml:32 `# Dependencies are now managed dynamically via setup.py based on detected hardware platform.`. setup.py:226 `requirements_file = requirements_dir / f"{device}.txt"`. requirements/musa.txt is only `-r common.txt`, `torchada>=0.1.52`, `onnxruntime>=1.23.2`, `mate>=0.2.0`, `flash_attn_3>=0.1.4` (no torch pin). requirements/common.txt:21 `torchsde>=0.2.6` — still no `torch>=` floor.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:170` [nit] — buildkite/vllm-omni-amd-ci is red on this head. Existing MiniMax H3 tests are cpu-marked and skip device_module on CPU parameters, and AMD typically keeps device.type "cuda", so this hunk is probably behavior-preserving and was not attributed to this PR — but the lane was not logged. Can you confirm the red AMD check is pre-existing (or post a log) before treating this head as mergeable? (evidence: [gate] buildkite/vllm-omni-amd-ci is red. Existing MiniMax H3 tests are cpu-marked and skip device_module on CPU parameters, so this hunk is unlikely to be the AMD failure; not attributing it to this PR.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72` [nit] — The new recipe’s validated serve profile is TP4 (four MTT S5000 GPUs), while the PR body cites a TP8 402.3s figure that is not in the recipe (docs gap only; extra evidence, not missing code). Should that TP8 number be added to MiniMax-H3-MUSA.md, or is it out of scope so the supported MUSA claim stays the TP4 serve block? (evidence: [sweep 8 scope] TP8 402.3s figure in the PR body is not in the recipe (docs gap only).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.trace.json.gz new file mode 100644 index 000000000..bd6efb609 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.cost.json b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.cost.json new file mode 100644 index 000000000..23198ea9c --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.cost.json @@ -0,0 +1,102 @@ +{ + "wall_s": 1439.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5957/run-20260814-223327-5dee94", + "recorded_at": "2026-08-14T22:57:27", + "attempts": [ + { + "invocation_id": "inv-9ebc292e", + "outcome": "run:run-20260814-223327-5dee94", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-210832-202771", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-223327-5dee94", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-223327-5dee94", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8979, + "minutes": 23.98, + "ci_minutes": 0, + "input_tokens": 2463344, + "output_tokens": 211650, + "tool_calls": 441, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9574, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.65, + "gate": 1.81, + "review": 1432.41, + "report": 0.06 + }, + "time_to_first_result_s": 1438.93 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5957.trace.json.gz", + "trace_bytes": 200208 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.md b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.md new file mode 100644 index 000000000..9ec37c38e --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.md @@ -0,0 +1,56 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5957, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 19a6e513f28e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] correctness: native speed is opt-in via IndexTTS25Adapter.native_speed_control and serving_speech._audio_encode_speed returns 1.0 so generic waveform speed is not applied twice; HTTP streaming speed!=1.0 is re-gated in _validate_speech_streaming_request; WebSocket still requires speed=1.0 (existing thread, fixed at head). +- [validated] vllm_omni/entrypoints/openai/tts_adapters/indextts2.py:321 duration_factor=[1.0/speed] with adapter validate on speed in [0.5,2.0] matching INDEXTTS25_MIN/MAX_DURATION_FACTOR; talker _preprocess_prefill:565-569 fail-fast for Omni.generate callers. +- [validated] vllm_omni/model_executor/models/indextts2/indextts2_talker.py:453-465 later decode steps emit only use_gpt_latent; accumulator keeps first-step S_ref/style because missing keys are not replaced (omni_connector_model_runner_mixin.py:920-947). +- [validated] talker2s2mel_full_payload code-only path uses request.output_token_ids when use_gpt_latent is false (stage_input_processors/indextts2.py:237-244) and fail-fast requires meta.use_gpt_latent; IndexTTS-2 talker always emits that flag (indextts2_talker.py:453). +- [validated] gpu_ar_model_runner.py:1723-1747 request-end d2h defer is gated on omni_payload_at_request_end (indextts2_5 only), no prefix cache, and full-payload accumulation; NPU runner left unchanged as the PR states. +- [validated] _patch_empty_hf_config (arg_utils.py:256-265) only fabricates config.json when model_type==indextts2_5, model is a directory, and config.json does not lexist; malformed/local/remote errors still reach the parent loader (tests/engine/test_arg_utils.py). +- [validated] skip_tokenizer_init now supplies _TokenOnlyRenderer (stage_init_utils.py:1140-1146); IndexTTS-2 yaml already had skip_tokenizer_init, so this is shared but matches the 0.27 InputProcessor tokenizer-resolution failure the tests describe. +- [sweep] simplifiability: IndexTTS25Adapter inherits IndexTTS2Adapter and only adds lang/TN/speed; duration-factor constants are now shared (existing thread resolved). end2end _get_stage0_tokenizer_file hasattr/get branches are defensive for dict vs object hf_overrides; not dead. + +`tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:16` [major] — This PR adds IndexTTS 2.5 pipeline/config/codec/recipe unit tests in `test_configuration_and_pipeline.py`, `test_enhanced_codec.py`, and `test_recipe_defaults.py` with no `pytestmark`, while sibling files in the same directory set `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. CUDA and AMD ready/merge lanes run `pytest -sv tests/model_executor -m 'core_model and cpu'`; pyproject.toml documents `core_model` as L1/L2 PR tests and neither `tests/conftest.py` nor pytest `addopts` auto-applies the marker. Those unmarked modules are the only lock for `OMNI_PIPELINES["indextts2_5"] is INDEXTTS25_PIPELINE`, deploy YAML `use_gpt_latent` stage sync, real EnhancedCodec decode plus native `checkpoints/codec.pth` resolution, and `s2mel_cfm_batch_size: 4` (TRITON_ATTN is also asserted in marked `test_s2mel_v2_5.py:1064`). They are collected then deselected while CI stays green. Add the same pytestmark to all three files and run `pytest -sv tests/model_executor/models/indextts2/test_configuration_and_pipeline.py tests/model_executor/models/indextts2/test_enhanced_codec.py tests/model_executor/models/indextts2/test_recipe_defaults.py -m 'core_model and cpu'`. (evidence: tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:16 `def test_indextts25_defaults_are_distinct_from_v2():` — module has no pytestmark. tests/model_executor/models/indextts2/test_enhanced_codec.py:11 `def test_enhanced_codec_decode_restores_hidden_size_and_doubles_time():` — no pytestmark. tests/model_executor/models/indextts2/test_recipe_defaults.py:9 `def test_indextts25_default_recipe_enables_validated_cfm_batching():` — no pytestmark. tests/model_executor/models/indextts2/test_s2mel_v2_5.py:26 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. .buildkite/cuda/test-ready.yml:13 `timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml`. pyproject.toml:247 `core_model: L1&L2 tests (run in each PR)`. pyproject.toml:241-244 `addopts = ["--strict-markers", "--strict-config"]` — no marker auto-apply. tests/conftest.py:20-27 `pytest_plugins = (...)` and no `pytest_collection_modifyitems`. Unique skipped locks: test_configuration_and_pipeline.py:55 `assert OMNI_PIPELINES["indextts2_5"] is INDEXTTS25_PIPELINE`; test_configuration_and_pipeline.py:85-88 `assert [stage["hf_overrides"]["use_gpt_latent"] for stage in config["stages"]] == [use_gpt_latent, use_gpt_latent]`; test_enhanced_codec.py:79-84 `checkpoint = tmp_path / "checkpoints" / "codec.pth"` then `assert preprocess_utils.resolve_model_file(str(tmp_path), "codec.pth") == str(checkpoint)`; test_recipe_defaults.py:14 `assert stage1["hf_overrides"]["s2mel_cfm_batch_size"] == 4`.) + +`vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:237` [major] — `_normalize_with_official_backend` treats Spanish TN as optional: `lang == 'es'` imports `nemo_text_processing` and on `ModuleNotFoundError` `return text` (text_processing_v2_5.py:224-237), while every other non-zh/en/zhen language including `ar` hits the bare `return text` at :238. The advertised `vllm-omni[indextts2]` extra (pyproject.toml:87-91) is only fugashi, unidic-lite, wetext, and unpinned WeTextProcessing — no NeMo — so the documented install never enables Spanish TN, yet default `text_normalization=True` still looks successful. Japanese missing `fugashi`/`unidic_lite` instead raises RuntimeError with that same install command. Add NeMo to the extra (or document a separate install) and fail-fast like Japanese, or spell out that `es`/`ar` are prefix/tokenization only under the extra the recipe actually installs. (evidence: text_processing_v2_5.py:224-238 `if lang == "es":` … `except ModuleNotFoundError as exc:` `if exc.name != "nemo_text_processing":` `raise` `return text` `return text` — Spanish TN silently becomes identity when NeMo is absent; all other non-zh/en/zhen langs including ar skip TN. pyproject.toml:87-91 `indextts2 = [` `"fugashi>=1.2.0",` `"unidic-lite>=1.0.0",` `"wetext>=0.0.9; sys_platform != 'linux'",` `"WeTextProcessing; sys_platform == 'linux'",` `]` — extra has no nemo_text_processing. text_processing_v2_5.py Japanese path `raise RuntimeError(` `"IndexTTS 2.5 Japanese preprocessing requires fugashi and unidic-lite. "` `"Install them with pip install 'vllm-omni[indextts2]'"` `)` — missing ja deps fail-fast with the extra install command.) + +`vllm_omni/model_executor/models/indextts2/preprocess_utils.py:?` [major] — This diff makes local-directory IndexTTS loads fail closed: `load_wav2vec2` raises at preprocess_utils.py:105-106 when `local_dir is None and os.path.isdir(model_path)` and never reaches the `facebook/w2v-bert-2.0` Hub fallback still present at :119-120. The same ungated `isdir` short-circuit was added for RepCodec (:176-177) and CAMPPlus (:206-207). `indextts2_talker.py:138` sets `self.model_path = vllm_config.model_config.model` and calls those loaders (`load_wav2vec2` at :1271, `load_campplus` at :1278), so IndexTTS-2 inherits the contract. A cloned tree or resolved HF snapshot is a directory, so Hub fallback remains reachable only for a non-directory repo id. Gate the fail-fast on the 2.5 native `checkpoints/` bundle (or `model_type==indextts2_5`) and keep the previous sidecar downloads for IndexTTS-2 local snapshots that never vendored w2v-bert. (evidence: preprocess_utils.py:105-106 `if local_dir is None and os.path.isdir(model_path):` / `raise FileNotFoundError(f"IndexTTS Wav2Vec2-BERT assets are missing from local bundle {model_path!r}")` — no model_type or checkpoints/ gate. Dead Hub fallback still follows at preprocess_utils.py:119-120 `with _repo_prefetch_lock("facebook/w2v-bert-2.0"):` / `src_args = dict(pretrained_model_name_or_path="facebook/w2v-bert-2.0")`. Same directory short-circuit at preprocess_utils.py:176-177 `if os.path.isdir(model_path):` / `raise FileNotFoundError(f"IndexTTS RepCodec checkpoint is missing from local bundle {model_path!r}")` before `hf_hub_download("amphion/MaskGCT", ...)`, and preprocess_utils.py:206-207 `if os.path.isdir(model_path):` / `raise FileNotFoundError(f"IndexTTS CAMPPlus checkpoint is missing from local bundle {model_path!r}")` before `hf_hub_download("funasr/campplus", ...)`. Shared IndexTTS-2 consumer: indextts2_talker.py:138 `self.model_path = vllm_config.model_config.model`; indextts2_talker.py:1271 `load_wav2vec2(self.model_path, device)`.) + +`recipes/IndexTeam/IndexTTS-2_5.md:40` [minor] — `recipes/IndexTeam/IndexTTS-2_5.md` Hardware Support is titled `### 1x NVIDIA H20 96GB` (line 40) and `recipes/README.md:53` lists `1x NVIDIA H20 96GB`, but the same recipe Notes (line 290) say the standard recipe was exercised on one NVIDIA H200. Retitle the hardware section and README row to the GPU that was actually run, or split a verified H200 note from an unverified H20 listing so operators do not treat H20 96GB as the measured default. (evidence: recipes/IndexTeam/IndexTTS-2_5.md:40 `### 1x NVIDIA H20 96GB` — recipes/IndexTeam/IndexTTS-2_5.md:290 `- Hardware scope: the standard recipe has been exercised on one NVIDIA H200.` — recipes/README.md:53 `| [`IndexTeam/IndexTTS-2_5.md`](./IndexTeam/IndexTTS-2_5.md) | Online + offline multilingual voice-cloned TTS with native speed and emotion control | 1x NVIDIA H20 96GB |`) + +`.buildkite/amd/test-amd-merge.yml:32` [minor] — Gate report shows `buildkite/vllm-omni-amd-ci` red on this head, and this review cannot tell whether main is also red. Stated overlap with this diff is shared serving/runner/stage-init changes, not the Intel XPU Dockerfile. Is this check red on main too, or is it a PR regression that should block merge? (evidence: AMD CI check buildkite/vllm-omni-amd-ci is red; this review cannot tell whether main is also red.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3642` [minor] — HTTP `OpenAICreateSpeechRequest.validate_streaming_constraints` no longer pydantic-rejects streaming `speed!=1.0`, so the schema change applies to every TTS model. Non-native models still 400 in `serving_speech._validate_speech_streaming_request` (line 3642) unless `adapter.native_speed_control`; only `IndexTTS25Adapter` sets that flag (`IndexTTS2Adapter` inherits base False). WebSocket `StreamingSpeechSessionConfig` still requires `speed=1.0`. Public contract for non-native HTTP streaming is now serving-layer 400 rather than schema rejection. (evidence: protocol/audio.py:311-321 OpenAICreateSpeechRequest.validate_streaming_constraints: `if self.speed is None: self.speed = 1.0` then `return self` — no `speed != 1.0` raise. serving_speech.py:3642-3645 `if request.speed is not None and request.speed != 1.0 and not self._uses_native_speed_control(): return response_format, self.create_error_response(f"{mode_label} is not supported with speed adjustment. ...")`. serving_speech.py:517-519 `_uses_native_speed_control` returns `bool(adapter is not None and adapter.native_speed_control)`. tts_adapters/base.py:119 `native_speed_control: ClassVar[bool] = False`. tts_adapters/indextts2.py:273-276 `class IndexTTS25Adapter(IndexTTS2Adapter): ... native_speed_control: ClassVar[bool] = True` (IndexTTS2Adapter does not override it). protocol/audio.py:558-561 StreamingSpeechSessionConfig: `elif self.speed != 1.0: raise ValueError("WebSocket stream_audio=true currently requires speed=1.0; native speed control is only available through HTTP streaming.")`.) + +`pyproject.toml:91` [minor] — Linux extra `WeTextProcessing` is unpinned (`pyproject.toml:91`) while non-Linux `wetext` is `>=0.0.9`. `_load_wetext_normalizer` on Linux imports `tn.chinese.normalizer.Normalizer` / `tn.english.normalizer.Normalizer` (`text_processing_v2_5.py:104-108`). Pin a lower bound that still exports those symbols; `tests/test_indextts2_optional_dependencies.py` currently asserts an empty specifier. (evidence: pyproject.toml:90 ` "wetext>=0.0.9; sys_platform != 'linux'",` pyproject.toml:91 ` "WeTextProcessing; sys_platform == 'linux'",` — Linux extra has no version pin. text_processing_v2_5.py:104 ` if platform.system() == "Linux":` text_processing_v2_5.py:106 ` from tn.chinese.normalizer import Normalizer` text_processing_v2_5.py:108 ` from tn.english.normalizer import Normalizer` — Linux TN path requires those symbols. tests/test_indextts2_optional_dependencies.py:85 ` ("wetextprocessing", "", 'sys_platform == "linux"'),` — test locks in the empty specifier.) + +`vllm_omni/model_executor/stage_input_processors/indextts2.py:?` [minor] — talker2s2mel_full_payload now raises ValueError when latent-mode `mel_seq`/`latent_seq` lengths differ (`indextts2.py:261`), replacing the previous silent min-length crop. INDEXTTS2_PIPELINE and INDEXTTS25_PIPELINE both wire this helper (`pipeline.py:30` and `:66`). IndexTTS-2 is the affected consumer (`use_gpt_latent` defaults True at `configuration_indextts2.py:147`); IndexTTS-2.5 defaults `use_gpt_latent` False, so the default code-only path never reaches this check. Tests were renamed to expect the raise (`test_talker2s2mel_full_payload_rejects_mel_latent_length_mismatch`). IndexTTS-2 requests that previously synthesized after a silent crop will now fail Stage 1. Document this 2.0 contract change on the IndexTTS-2 notes, or keep crop-and-continue only on the latent path — do not gate the error on the 2.5 code-only path, which has no latent. (evidence: vllm_omni/model_executor/stage_input_processors/indextts2.py:255-264 `if use_gpt_latent:` then `if mel_seq.shape[0] != latent_seq.shape[0]:` `raise ValueError(` `f"IndexTTS mel/latent length mismatch for req={rid}: "` `f"mel={mel_seq.shape[0]} latent={latent_seq.shape[0]}")`. vllm_omni/model_executor/models/indextts2/pipeline.py:30 and :66 `custom_process_next_stage_input_func=f"{_PROC}.talker2s2mel_full_payload"`. vllm_omni/model_executor/models/indextts2/configuration_indextts2.py:147 `"use_gpt_latent": True,` (IndexTTS2Config._DEFAULTS). tests/model_executor/stage_input_processors/test_indextts2.py:251-260 `def test_talker2s2mel_full_payload_rejects_mel_latent_length_mismatch():` `with pytest.raises(ValueError, match="mel/latent length mismatch"):` `talker2s2mel_full_payload(None, payload, SimpleNamespace(request_id="r-crop"))`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py:259` [minor] Production IndexTTS 2.5 tokenization goes through `load_indextts25_encoding` (`tiktoken.Encoding` with a hard 58836 mergeable-rank / 60509 vocab ABI at tokenizer_v2_5.py:259). Repo-wide, that function is only defined and + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.trace.json.gz new file mode 100644 index 000000000..1279a3d2f Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/QUARANTINE_skillleak_r3/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/manifest.json b/eval/dataset/arms/copilot_cb_grok46/manifest.json new file mode 100644 index 000000000..b70019040 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_cb_grok46", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5509.cost.json b/eval/dataset/arms/copilot_cb_grok46/pr5509.cost.json new file mode 100644 index 000000000..6c2d5cbcf --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5509.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1196.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5509/run-20260814-203330-303463", + "recorded_at": "2026-08-14T20:53:25", + "attempts": [ + { + "invocation_id": "inv-bcc55208", + "outcome": "run:run-20260814-203330-303463", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-203330-303463", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-203330-303463", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.539, + "minutes": 19.93, + "ci_minutes": 0, + "input_tokens": 1349103, + "output_tokens": 158815, + "tool_calls": 263, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7524, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.2, + "gate": 1.67, + "review": 1189.66, + "report": 0.06 + }, + "time_to_first_result_s": 1195.57 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 39628 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5509.md b/eval/dataset/arms/copilot_cb_grok46/pr5509.md new file mode 100644 index 000000000..5f88cd53a --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5509.md @@ -0,0 +1,61 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] tests/diffusion/attention/test_trtllm_attn.py:315 Missing TRTLLM-GEN kernel skip: reachable on SM103 for int8 (docs say int8 is SM100-only); skips rather than failing — arm can occur. +- [sweep] trtllm_attn.py:90 HAS_FLASHINFER false → _sage_kernel_available False; SAGE init raises, dense path unchanged. Arm occurs without flashinfer. +- [sweep] trtllm_attn.py:198/296 self.quant.enabled iff dtype_qk is set; both init and forward gated the same way. QuantConfig.enabled ignores dtype_vo-only configs (those serialize from AttnQuantSpec but SAGE stays off). +- [sweep] trtllm_attn.py:199 non-SAGE dtype_qk (bfloat16/float16) raises RuntimeError at impl init; covered by test_quant_rejects_non_sage_dtype. AttentionSpec still accepts those dtypes. +- [sweep] trtllm_attn.py:204 _sage_kernel_available False raises at init (signature probe). test_quant_quantize_requires_flashinfer_routine only covers the later missing-quantize-fn arm, not this one. +- [sweep] trtllm_attn.py:211 missing trtllm_sage_attention_quantize raises; unit test monkeypatches this path. Fail-loud, not silent dense. +- [sweep] data.py:1415/1418 AttnQuantSpec rejects int4 and k_block_size=8; None dtypes allowed. Block sizes always validated even when SAGE is off. +- [sweep] data.py:1447 allows quant on TRTLLM_ATTN and FLASHINFER_ATTN; TORCH_SDPA rejected (test_quant_validation_rejects). FLASHINFER_ATTN has no quant consumer. + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — This diff adds the SAGE section by deleting ## Known Limitations (LTX-2 CUDNN_ATTN + torch.compile crash / #3121 and the FLASHINFER/TORCH_SDPA workaround), the FA4 note, mixed per-role backends, Validation Guidance, and the bench_attention_backends.py --preset hv15 recipe, while leaving line 232 `- **Workaround for known issues**: see Known Limitations above.` There is no remaining ## Known Limitations heading, so that cross-reference is dead, and the LTX-2 FLASHINFER_ATTN example at line 337 no longer states why to use it. Line 188 also points to a FLASHINFER quant section that is not in this file. Restore those sections (the bench script still exists) or rewrite this bullet and keep the LTX-2 compile workaround next to that example; SAGE docs should be additive. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — sole remaining Known Limitations mention; file has no `## Known Limitations` heading and ends at the SageAttention3 example (line 381). Same file:337-339 `DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \` then `--model Lightricks/LTX-2` — no compile-crash rationale. Same file:187-188 ``FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`),` / ``dtype_vo`, and `flashinfer_backend` (see its section).` — no FLASHINFER quant section in this file. benchmarks/diffusion/bench_attention_backends.py:15 `python benchmarks/diffusion/bench_attention_backends.py --preset hv15` — recipe still exists.) + +`vllm_omni/diffusion/data.py:1447` [major] — This PR allowlists `quant` on FLASHINFER_ATTN in `AttentionSpec.__post_init__` (`data.py:1447`) and documents that FLASHINFER already reads `dtype_qk` (`float16`/`bfloat16`), `dtype_vo`, and `flashinfer_backend` and that a wrong-set `dtype_qk` raises (`attention_backends.md:185-188`, including a dangling “see its section”). That contract is unimplemented: `Attention` always forwards `spec.backend_kwargs()` (`layer.py:92-109`), but `FlashInferAttentionImpl.__init__` has no `backend_kwargs` parameter and discards it via unused `**extra_impl_args` (`flashinfer_attn.py:63`), and `forward_cuda` only calls dense `single_prefill_with_kv_cache` with `sm_scale`/`causal`/`custom_mask` (`flashinfer_attn.py:171-178`). Unlike SDPA (`sdpa.py:89-90`), there is no unused-kwargs warning. `AttnQuantSpec` uses one shared dtype set (`data.py:1410`), so `int8` on FLASHINFER is accepted at spec time and `bfloat16` on TRTLLM is only rejected later in `TrtllmAttentionImpl` (`trtllm_attn.py:199-203`). `test_quant_superset_fields_passed_through` only round-trips the dict. Restrict `quant` to TRTLLM_ATTN until FLASHINFER consumes those fields (or implement/reject in FlashInferAttentionImpl and split dtype sets in AttentionSpec), hedge/delete the FLASHINFER docs paragraph, and retarget that test so it does not encode an unimplemented contract. (evidence: vllm_omni/diffusion/data.py:1447-1450 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` / `raise ValueError(` / `f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but "` — FLASHINFER is allowlisted. docs/user_guide/diffusion/attention_backends.md:185-188 `The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the` / `fields that apply and rejects the rest. `TRTLLM_ATTN` (SAGE) reads `dtype_qk` (`int8` / `fp8_e4m3`),` / ``q_block_size`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`),` / ``dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.` — docs claim unimplemented FLASHINFER consumption; grep of this file has no FLASHINFER quant section. vllm_omni/diffusion/attention/backends/flashinfer_attn.py:63 `**extra_impl_args,` — no `backend_kwargs` parameter, unused. flashinfer_attn.py:171-178 `kwargs: dict = {` / `"sm_scale": self.softmax_scale,` / `"causal": self.causal,` / `"return_lse": False,` / `}` / `if custom_mask is not None:` / `kwargs["custom_mask"] = custom_mask if custom_mask.dim() == 2 else custom_mask[b]` / `out = single_prefill_with_kv_cache(query[b], key[b], value[b], **kwargs)` — dense path only. vllm_omni/diffusion/attention/layer.py:92-109 `backend_kwargs = spec.backend_kwargs()` then `self.attn_impl_cls(... backend_kwargs=backend_kwargs)`. vllm_omni/diffusion/attention/backends/sdpa.py:89-90 `if backend_kwargs:` / `logger.warning("SDPAImpl ignoring backend_kwargs: %s", list(backend_kwargs.keys()))` — FLASHINFER has no equivalent. vllm_omni/diffusion/data.py:1410 `_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})` — no per-backend split. vllm_omni/diffusion/attention/backends/trtllm_attn.py:199-203 `if self.quant.dtype_qk not in _QK_QUANT_DTYPES:` / `raise RuntimeError(` / `f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got "` — wrong-set dtype raises only on TRTLLM init. tests/diffusion/attention/test_attention_config.py:65-73 `def test_quant_superset_fields_passed_through(self):` / `AttentionSpec(` / `backend="FLASHINFER_ATTN",` / `quant={"dtype_qk": "bfloat16", "dtype_vo": "fp8_e4m3", "flashinfer_backend": "trtllm-gen"},` / `)` — round-trip only.) + +`tests/diffusion/attention/test_trtllm_attn.py:50` [major] — The four new CPU-safe SAGE tests (`test_quant_config_from_backend_kwargs`, `test_quant_rejects_non_sage_dtype`, `test_quant_quantize_requires_flashinfer_routine`, `test_quant_quantize_calls_routine_and_shapes_sfs`) live in `test_trtllm_attn.py`, module-marked `pytest.mark.cuda` (line 13). Ready/merge Simple · Diffusion Test runs `pytest tests/diffusion -m 'core_model and cpu'` and deselects the file; Diffusion · Other Test runs `tests/diffusion/*.py` (top-level only); `.buildkite` has no `tests/diffusion/attention` job (only `vllm_omni/diffusion/attention/parallel/` as a source_file_dependency). Sibling `test_attention_config.py` / `test_trtllm_calibration.py` are `cpu` and do run in L2 (this PR's AttnQuantSpec tests included) — they do not cover `QuantConfig` / `TrtllmAttentionImpl` SAGE init. Move these four tests to `test_attention_config.py` or mark them `pytest.mark.cpu`; keep `test_sage_quant_matches_sdpa` behind cuda. After the marker change: `pytest -sv tests/diffusion/attention/test_trtllm_attn.py -m 'core_model and cpu' -k quant`. (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` +tests/diffusion/attention/test_trtllm_attn.py:50 `def test_quant_config_from_backend_kwargs():` — followed by test_quant_rejects_non_sage_dtype (60), test_quant_quantize_requires_flashinfer_routine (65), test_quant_quantize_calls_routine_and_shapes_sfs (74); none of these four allocate CUDA. +tests/diffusion/attention/test_attention_config.py:34 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]` +tests/diffusion/attention/test_trtllm_calibration.py:14 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]` +.buildkite/cuda/test-ready.yml:18 `timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu'` +.buildkite/cuda/test-ready.yml:68 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` +.buildkite/cuda/test-merge.yml:18 `timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu'` +.buildkite grep `tests/diffusion/attention`: no job path; only `.buildkite/cuda/test-ready.yml:54` `vllm_omni/diffusion/attention/parallel/` as a source_file_dependency.) + +`tests/diffusion/attention/test_trtllm_attn.py:321` [major] — This diff's only runtime SAGE check is `test_sage_quant_matches_sdpa`, asserting `rel < 0.1` vs SDPA (10× the dense BF16 bound `rel < 0.01` in `test_bf16_dense_matches_sdpa`). Dense already satisfies `< 0.01`, so deleting the `if self.quant.enabled` block in `forward_cuda` (no `trtllm_sage_attention_quantize`, empty `sage_kwargs`) still passes. `@requires_sage` plus `pytest.skip` on `Missing TRTLLM-GEN kernel` means non-Blackwell CI never runs the kernel; `test_quant_quantize_calls_routine_and_shapes_sfs` only asserts a fake `QuantConfig.quantize()` return and never calls `forward_cuda`. No test file contains `sage_attn_sfs` or `trtllm_ragged_attention_deepseek`. Add a CPU test that monkeypatches `trtllm_ragged_attention_deepseek`, drives `_impl(quant={"dtype_qk": "int8"}).forward_cuda`, and asserts `sage_attn_sfs` / `num_elts_per_sage_attn_blk` (and `skip_softmax_threshold_scale_factor` when both are set). On GPU, assert SAGE output differs from dense the way `test_skip_threshold_engages` does (`> 0.02`). Command: `pytest -sv tests/diffusion/attention/test_trtllm_attn.py -k 'quant'` after the CPU test exists. (evidence: tests/diffusion/attention/test_trtllm_attn.py:320 `assert rel < 0.1, f"SAGE({dtype_qk}) rel err {rel:.4f} too high"` vs :295 `assert rel < 0.01, f"BF16 dense rel err {rel:.4f} too high"`; :312-316 `except RuntimeError as e: # pragma: no cover - arch-dependent (int8 is SM100-only)` / `if "Missing TRTLLM-GEN kernel" in str(e):` / `pytest.skip(...)`; :74-84 `test_quant_quantize_calls_routine_and_shapes_sfs` calls only `QuantConfig(dtype_qk="int8").quantize(..., fake_quantize)` and asserts `("qsfs", "ksfs", None, "vsfs")`; :338-345 `test_skip_threshold_engages` asserts `(skipped - dense).abs().mean() / dense.abs().mean() > 0.02`. tests/ has zero hits for `sage_attn_sfs` or `trtllm_ragged_attention_deepseek`. Production wiring is vllm_omni/diffusion/attention/backends/trtllm_attn.py:296-320 `if self.quant.enabled:` / `sage_kwargs["sage_attn_sfs"] = sage_attn_sfs` / `sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes` then `out = trtllm_ragged_attention_deepseek(` ... `**sage_kwargs`.) + +`recipes/Wan-AI/Wan2.2-T2V.md:?` [major] — This diff adds a public `AttentionSpec.quant` block for TRTLLM_ATTN SAGE (`int8`/`fp8_e4m3`) and documents it with a Wan2.2 T2V serve example, but `recipes/Wan-AI/Wan2.2-T2V.md:186` still tells operators ``TRTLLM_ATTN` is BF16-only (no attention-level quantization).` Update or remove that limitation and add the SAGE `quant` example (or explicitly mark T2V unvalidated) so the recipe does not contradict the new API. (evidence: recipes/Wan-AI/Wan2.2-T2V.md:186 `- \`TRTLLM_ATTN\` is BF16-only (no attention-level quantization).` — leftover false limitation. docs/user_guide/diffusion/attention_backends.md:191-193 `vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \\` / `--diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",` / `"quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}'` — PR documents SAGE on T2V. vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None` and data.py:1447 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` — public quant is allowed on TRTLLM_ATTN.) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — This PR adds `AttentionSpec.quant: AttnQuantSpec | None` and `spec_keys = {"backend", "skip_softmax", "quant"}`, plus a SAGE section with JSON/Python examples — but the configuration intro still says `Today the only one is TRTLLM_ATTN's Skip-Softmax` and the only dotted-flag example is `--diffusion-attention-config.default.skip_softmax.target_sparsity`. Update that sentence to mention `quant` (link `#trtllm_attn-sage-quantization`) and add `--diffusion-attention-config.default.quant.dtype_qk` so the public CLI surface matches the typed spec. (evidence: docs/user_guide/diffusion/attention_backends.md:72-77 `A backend that needs configuration exposes it as a typed field on the spec. Today the only one is \`TRTLLM_ATTN\`'s Skip-Softmax` and `--diffusion-attention-config.default.skip_softmax.target_sparsity 0.5`. SAGE section at attention_backends.md:190-193 shows only JSON `--diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN", "quant": {"dtype_qk": "fp8_e4m3", ...}}}'`, not a dotted flag. vllm_omni/diffusion/data.py:1433-1434 `skip_softmax: SkipSoftmaxSpec | None = None` / `quant: AttnQuantSpec | None = None`. data.py:1553 `spec_keys = {"backend", "skip_softmax", "quant"}`.) + +`tests/diffusion/attention/test_trtllm_attn.py:315` [minor] — test_sage_quant_matches_sdpa parametrizes dtype_qk=int8 and turns FlashInfer's 'Missing TRTLLM-GEN kernel' into pytest.skip (test_trtllm_attn.py:314–316). Docs say int8 SAGE kernels are SM100-only (attention_backends.md:210), but AttnQuantSpec and TrtllmAttentionImpl.__init__ still accept int8 with no SM check, so SM103 int8 skips instead of failing as unsupported. Buildkite CUDA jobs are L4/H100 (cc<10), so @requires_sage never enters and neither the skip nor the SM100 int8 success path runs in CI. (evidence: tests/diffusion/attention/test_trtllm_attn.py:305 `@pytest.mark.parametrize("dtype_qk", ["fp8_e4m3", "int8"])` — int8 is not SM100-gated. tests/diffusion/attention/test_trtllm_attn.py:314-316 `except RuntimeError as e: # pragma: no cover - arch-dependent (int8 is SM100-only)` / `if "Missing TRTLLM-GEN kernel" in str(e):` / `pytest.skip(f"no trtllm-gen SAGE kernel for dtype_qk={dtype_qk} on this GPU arch")` — SM103 int8 becomes skip, not fail. docs/user_guide/diffusion/attention_backends.md:208-210 `Kernel availability is arch-dependent: \`fp8_e4m3\` QK has kernels on both **SM100** (B200) and **SM103** (B300); \`int8\` QK kernels are compiled for **SM100 only**.` vllm_omni/diffusion/attention/backends/trtllm_attn.py:198-203 `if self.quant.dtype_qk not in _QK_QUANT_DTYPES:` / `raise RuntimeError(... supports dtype_qk in {sorted(_QK_QUANT_DTYPES)} ...)` — init allows int8 on any arch that has the SAGE symbol. tests/diffusion/attention/test_trtllm_attn.py:19-20 `if torch.cuda.get_device_capability()[0] < 10:` / `return False` — L4/H100 CI never enters @requires_sage.) + +`trtllm_attn.py:90` [minor] — Docs already state SAGE needs FlashInfer ≥ 0.6.16rc1 (`attention_backends.md`), but `requirements/common.txt`, `requirements/cuda.txt`, and `pyproject.toml` still do not pin flashinfer (inherited from vLLM). SAGE is opt-in (`quant.enabled`) and fail-loud: `_sage_kernel_available` returns False when `HAS_FLASHINFER` is False (`trtllm_attn.py:90-91`) or the kernel lacks `sage_attn_sfs`, and `TrtllmAttentionImpl.__init__` then raises (`204-216`). Dense forward omits SAGE kwargs unless quant is enabled (`292-300`), so an older FlashInfer that vLLM ships will not silently break BF16 TRTLLM. Should this repo add a packaging lower bound for the new SAGE APIs, or is inheriting whatever vLLM ships the intended compatibility contract? (evidence: trtllm_attn.py:90-91 `if not HAS_FLASHINFER:` / `return False` — no FlashInfer ⇒ SAGE kernel probe is False. trtllm_attn.py:198-209 `if self.quant.enabled:` … `if not _sage_kernel_available():` `raise RuntimeError(` "TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build does not" — SAGE init is opt-in and fail-loud. trtllm_attn.py:292-300 `# SAGE kwargs are only understood by newer FlashInfer builds; pass them exclusively when` / `sage_kwargs: dict = {}` / `if self.quant.enabled:` … `sage_kwargs["sage_attn_sfs"] = sage_attn_sfs` — dense path does not pass SAGE kwargs. docs/user_guide/diffusion/attention_backends.md:208 `Needs FlashInfer ≥ 0.6.16rc1.` — docs declare the bound; requirements/ and pyproject.toml contain no `flashinfer` pin.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/data.py:1418` [minor] AttnQuantSpec.__post_init__ always rejects q/k_block_size outside {1,4,16} (so 8 and 0 fail even when dtype_qk/dtype_vo are None / SAGE is off). QuantConfig.from_backend_kwargs does not apply that check and uses `int(q.g + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5509.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/pr5509.trace.json.gz new file mode 100644 index 000000000..5b0358205 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5550.cost.json b/eval/dataset/arms/copilot_cb_grok46/pr5550.cost.json new file mode 100644 index 000000000..f109b913f --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5550.cost.json @@ -0,0 +1,110 @@ +{ + "wall_s": 1207.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5550/run-20260814-233134-a14730", + "recorded_at": "2026-08-14T23:51:40", + "attempts": [ + { + "invocation_id": "inv-9f65c85e", + "outcome": "run:run-20260814-233134-a14730", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-203330-5a76e3", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-215450-2bd4a2", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-233134-a14730", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-233134-a14730", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5281, + "minutes": 20.11, + "ci_minutes": 0, + "input_tokens": 1319367, + "output_tokens": 156230, + "tool_calls": 374, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.751, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.94, + "gate": 1.65, + "review": 1200.87, + "report": 0.06 + }, + "time_to_first_result_s": 1206.51 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 109951 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5550.md b/eval/dataset/arms/copilot_cb_grok46/pr5550.md new file mode 100644 index 000000000..e58be2dce --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5550.md @@ -0,0 +1,58 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] checklist-1 correctness: execute_request now validates identity before RPC (multiproc_executor.py:459-460) so envelope mismatches raise instead of becoming worker output; dense-mode metadata rejection still runs only in DiffusionModelRunner._validate_diffusion_kv_metadata after the 4th positional arg is sent, then except Exception at execute_request:574 converts it to DiffusionOutput(error=) with finished=True. +- [sweep] checklist-1 None/empty: paged mode allows metadata=None (W0, no producer); NewRequestData.diffusion_kv_metadata defaults to None; from_state in base_scheduler.py:114 never attaches metadata. +- [sweep] new-branch diffusion_engine.py:199 scheduler_to_close is not None: live because _init_scheduler assigns self.scheduler before initialize(); test_scheduler_initialization_failure_closes_scheduler_and_executor covers initialize() failure. The None arm is only a constructor-failure-before-assignment path. +- [sweep] new-branch multiproc_executor.py:545 metadata is not None: both arms occur; test_dense_request_rpc_keeps_legacy_positional_shape covers 3-tuple, test_request_rpc_sends_attached_metadata_for_only_that_request covers 4-tuple. +- [sweep] new-branch interface.py:184 envelope!=forwarded: reachable and tested by test_request_rpc_rejects_envelope_request_identity_mismatch. +- [sweep] new-branch interface.py:192 metadata.request_id!=forwarded: reachable when envelope matches forwarded but metadata does not; no executor test hits this arm (runner-only test_paged_request_rejects_mismatched_metadata_identity). +- [sweep] new-branch model_runner.py:172 dense+metadata: reachable; tested on runner.execute_model, not on executor.execute_request. +- [sweep] new-branch model_runner.py:175 metadata identity: reachable; tested on runner. + +`vllm_omni/diffusion/executor/multiproc_executor.py:?` [minor] — This diff adds `and not has_diffusion_kv_metadata` to the DLO+AllGather multi-request gate in execute_request so any attached snapshot skips the wave RPC (`args=(reqs_list, od_config, kv_prefetch_job)`, unique_reply_rank=None) and falls through to the per-request loop (`unique_reply_rank=0`, single `req`, optional 4th metadata arg). Those paths are not equivalent: DiffusionWorker.execute_model sets is_batch from isinstance(req, list) and otherwise every rank computes the same request instead of req[dp_rank % len(req)]. OmniDiffusionConfig does not forbid paged_scheduler plus DLO (it only rejects paged_scheduler with need_recv_cache), and this gate is inherited by DLO users such as MiniMax-H3 / examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py via execute_batch. make_executor() hard-codes enable_distributed_layerwise_offload=False and existing DLO tests build NewRequestData with diffusion_kv_metadata=None, so the skip never runs. Fail closed with ValueError for DLO+AllGather multi-req waves that carry metadata until the list RPC can transport per-request snapshots, and add a CPU test with DLO+allgather True, two NewRequestData (metadata on at least one) that this combination does not silently serialize. Guard with: pytest -sv tests/diffusion/diffusion_kv/test_worker_contract.py tests/diffusion/test_multiproc_engine_concurrency.py -m 'core_model and cpu' -k 'dlo or attached_metadata'. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)`; :469-474 `if ( len(new_reqs) > 1 and not has_diffusion_kv_metadata and getattr(self.od_config, "enable_distributed_layerwise_offload", False) and getattr(self.od_config, "dlo_use_allgather", True) )`; :504-510 wave RPC `args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job)` / `unique_reply_rank=None`; :544-551 serial RPC `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None: args += (new_req.diffusion_kv_metadata,)` / `unique_reply_rank=0`; :605-613 DLO DP `return self.execute_request(scheduler_output)`. vllm_omni/diffusion/worker/diffusion_worker.py:444-450 `is_batch = isinstance(req, list)` / `if is_batch:` / `idx = dp_rank % len(req)` / `req = req[idx]`. vllm_omni/diffusion/data.py:941-947 `if self.diffusion_kv_mode is DiffusionKVCacheMode.PAGED_SCHEDULER and self.omni_kv_config.get("need_recv_cache", False): raise ValueError("paged_scheduler Diffusion KV does not support imported AR KV in Phase 1; disable need_recv_cache until connector-aware admission is implemented")`. tests/diffusion/diffusion_kv/test_worker_contract.py:62-64 `enable_distributed_layerwise_offload=False`. tests/diffusion/test_multiproc_engine_concurrency.py:200-207 NewRequestData constructed without diffusion_kv_metadata (interface.py:165 default `None`). examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py:83-84 `enable_distributed_layerwise_offload=True` / `dlo_use_allgather=True`.) + +`tests/diffusion/diffusion_kv/test_metadata.py:?` [minor] — test_diffusion_kv_metadata_uses_native_cache_group_block_ids (and the two CFG context tests) only construct DiffusionKV* dataclasses and assert sequences[0].block_ids / contexts[0].block_ids / context_ids equal the constructor arguments. metadata.py is a plain @dataclass with no __post_init__, so these assertions cannot fail unless field assignment is broken, and they never enter NewRequestData.from_state, validate_new_request_data_identity, or execute_request. Identity validation only compares request_id, so routing through that helper still would not fail on block_ids or context_ids; reuse these richer snapshots (native two-group block_ids, shared vs branch context_ids) on the existing worker-contract RPC path and assert they are the 4th execute_request argument. (evidence: tests/diffusion/diffusion_kv/test_metadata.py:30-38 `metadata = DiffusionKVMetadata(\n request_id="req-0",\n allocation_generation=3,\n sequences=(sequence,),\n contexts=(context,),\n )\n\n assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])\n assert metadata.contexts[0].block_ids == ([7], [11])` — constructor inputs echoed back. vllm_omni/diffusion/diffusion_kv/metadata.py:16-28 `@dataclass\nclass DiffusionKVContextMetadata:` then `block_ids: tuple[list[int], ...]` with no `__post_init__`. vllm_omni/diffusion/sched/interface.py:181-197 `validate_new_request_data_identity` only compares `new_req.request_id` / `metadata.request_id` to `forwarded_request_id`. tests/diffusion/diffusion_kv/test_worker_contract.py:27-40 `make_metadata()` uses `block_ids=([1, 2],)` and no contexts; :238-241 already asserts that simpler snapshot as the 4th RPC arg.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:574` [minor] — Identity mismatches raise at execute_request:459-460 before RPC, but dense-mode metadata is still appended as the 4th positional arg at :545-546 and rejected only later in DiffusionModelRunner._validate_diffusion_kv_metadata:172-173. except Exception at execute_request:574-581 then wraps that worker ValueError into RunnerOutput(finished=True, result=DiffusionOutput(error=str(exc))), so a dense request that attached metadata is reported as a completed request instead of failing closed like the identity helper. Reject dense-mode metadata on the executor before RPC; existing tests only cover runner.execute_model, not executor.execute_request. (evidence: multiproc_executor.py:459-460 `for new_req in new_reqs:\n validate_new_request_data_identity(new_req)` — identity errors raise before RPC. multiproc_executor.py:545-546 `if new_req.diffusion_kv_metadata is not None:\n args += (new_req.diffusion_kv_metadata,)` — 4th positional arg sent with no cache-mode gate. diffusion_model_runner.py:172-173 `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:\n raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")` — dense rejection only after RPC reaches the runner. multiproc_executor.py:574-581 `except Exception as exc:\n runner_outputs.append(\n RunnerOutput(\n request_id=new_req.request_id,\n step_index=None,\n finished=True,\n result=DiffusionOutput(error=str(exc)),\n )\n )` — worker contract ValueError becomes finished output. test_worker_contract.py:103-111 `def test_dense_request_rejects_metadata_before_request_forward()` calls `runner.execute_model(...)`, not `executor.execute_request`.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:1372` [minor] — Live RPC never calls WorkerWrapperBase.execute_model. WorkerProc._execute_rpc (diffusion_worker.py:1036) does wrapper.execute_method(method, *args), and execute_method (1442-1443) always getattr(self.worker, method), so collective_rpc("execute_model", args=(req, od_config, kv_prefetch_job, metadata)) lands on DiffusionWorker.execute_model as four positionals (executor 544-546). test_worker_wrapper_only_extends_paged_request_rpc instead calls wrapper.execute_model(..., diffusion_kv_metadata=metadata); test_dense_worker_call_does_not_extend_model_runner_signature only covers worker.execute_model without metadata. Retarget the positive 4th-argument test at DiffusionWorker.execute_model(req, od_config, kv_prefetch_job, metadata) so signature drift on the real callee cannot pass. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:1036 `result = self.worker.execute_method(method, *args, **kwargs)`. vllm_omni/diffusion/worker/diffusion_worker.py:1442-1443 `func = getattr(self.worker, method)` / `return func(*args, **kwargs)`. vllm_omni/diffusion/executor/multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None:` / `args += (new_req.diffusion_kv_metadata,)`. vllm_omni/diffusion/worker/diffusion_worker.py:421-426 DiffusionWorker.execute_model(req, od_config, kv_prefetch_job=None, diffusion_kv_metadata=None). tests/diffusion/diffusion_kv/test_worker_contract.py:258-259 `wrapper.execute_model(req, od_config)` / `wrapper.execute_model(req, od_config, diffusion_kv_metadata=metadata)`. tests/diffusion/diffusion_kv/test_worker_contract.py:288 `worker.execute_model(req, SimpleNamespace())` with no metadata.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:?` [minor] — DiffusionWorker.execute_model only injects diffusion_kv_metadata into model_runner.execute_model when the snapshot is not None (vllm_omni/diffusion/worker/diffusion_worker.py:462-465). In-tree ARDiffusionModelRunner still declares execute_model(self, req, kv_prefetch_job=None) and calls super().execute_model(req, kv_prefetch_job=kv_prefetch_job) (vllm_omni/experimental/ar_diffusion/runner.py:237-241, :255), while ARDiffusionEngine.default_diffusion_model_runner_cls routes workers to that subclass (vllm_omni/experimental/ar_diffusion/engine.py:13, :40). The first attached envelope therefore TypeErrors on unexpected keyword argument before DiffusionModelRunner._validate_diffusion_kv_metadata. Add the optional parameter on ARDiffusionModelRunner.execute_model and pass it through to super() so the parent mode/identity gate stays the single rejection path. Do not unconditionally pass the kwarg when None — that would break dense runners whose old (req, kv_prefetch_job=None) signature this PR intentionally preserves (test_dense_worker_call_does_not_extend_model_runner_signature). (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `output = self.model_runner.execute_model(req, **kwargs)` — non-None snapshot is forwarded as a keyword. vllm_omni/experimental/ar_diffusion/runner.py:237-241 `def execute_model(` / `self,` / `req: OmniDiffusionRequest,` / `kv_prefetch_job: KVPrefetchJob | None = None,` — no diffusion_kv_metadata parameter. runner.py:243 and :255 `return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` / `output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` — snapshot is not passed through. vllm_omni/experimental/ar_diffusion/engine.py:13 `AR_DIFFUSION_MODEL_RUNNER_CLS = "vllm_omni.experimental.ar_diffusion.runner.ARDiffusionModelRunner"`; engine.py:40 `default_diffusion_model_runner_cls = AR_DIFFUSION_MODEL_RUNNER_CLS` — ARDiffusionEngine workers build that subclass.) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:88` [minor] — This PR adds `_validate_diffusion_kv_metadata` on `execute_model` / `execute_model_batch` / `execute_stepwise`, and paged mode silently accepts a matching snapshot (`diffusion_model_runner.py:172-178`). `test_paged_request_without_metadata_reaches_request_forward` calls `execute_model(req)` with no metadata (`test_worker_contract.py:96-100`); the batch/step tests only cover missing metadata or envelope-id mismatch. A regression that rejected every snapshot, including paged + matching `request_id`, would still pass those tests and the mismatch cases. The executor RPC test that attaches matching metadata never reaches the runner validator. Add `test_paged_request_with_matching_metadata_reaches_request_forward` that passes `make_metadata("req-0")` and asserts `_execute_request_list` is called; add the same matching-metadata accept on `execute_model_batch`; for `execute_stepwise` assert `_supports_step_mode` is reached (that path does not call `_execute_request_list`). Guard with: pytest -sv tests/diffusion/diffusion_kv/test_worker_contract.py -m 'core_model and cpu' -k 'matching_metadata or without_metadata'. (evidence: tests/diffusion/diffusion_kv/test_worker_contract.py:88 `def test_paged_request_without_metadata_reaches_request_forward() -> None:` — paged request-path test exists. +tests/diffusion/diffusion_kv/test_worker_contract.py:96-100 `req = SimpleNamespace(request_id="req-0")\n\n assert runner.execute_model(req) is expected_output\n\n runner._execute_request_list.assert_called_once()` — calls execute_model with no metadata. +tests/diffusion/diffusion_kv/test_worker_contract.py:144-155 `test_batch_path_allows_missing_metadata_before_forward` builds NewRequestData without diffusion_kv_metadata and only asserts `_execute_request_list` was called. +tests/diffusion/diffusion_kv/test_worker_contract.py:173-186 `test_step_path_allows_missing_metadata_before_step_support_check` likewise omits metadata and only proves `_supports_step_mode` ran. +Grep of `diffusion_kv_metadata=make_metadata` in that file hits only reject paths (dense carry, stale id, envelope-id mismatch at lines 110, 123, 135, 164, 195); no matching-metadata accept on the runner. +vllm_omni/diffusion/worker/diffusion_model_runner.py:172-178 `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:\n raise ValueError(...)\n\n if metadata is not None and metadata.request_id != request_id:\n raise ValueError(...)` — paged + matching request_id is a silent accept; only non-paged presence and id mismatch raise. +vllm_omni/diffusion/worker/diffusion_model_runner.py:609-612 `self._validate_diffusion_kv_metadata(\n request_id=req.request_id,\n metadata=diffusion_kv_metadata,\n )` — execute_model validates before `_execute_request_list`. +vllm_omni/diffusion/worker/diffusion_model_runner.py:636-641 and 750-755 — execute_model_batch and execute_stepwise call the same validator; execute_stepwise then hits `_supports_step_mode()`, not `_execute_request_list`.) + +`vllm_omni/diffusion/sched/interface.py:?` [minor] — validate_new_request_data_identity (interface.py:181-197) has two raise arms: envelope request_id vs req.request_id (184), then metadata.request_id vs forwarded_request_id (192). test_request_rpc_rejects_envelope_request_identity_mismatch, test_batch_path_rejects_envelope_request_identity_mismatch, and test_step_path_rejects_envelope_request_identity_mismatch all set request_id="envelope-id" vs req.request_id="actual-request-id", so line 184 always fires and line 192 never runs. test_paged_request_rejects_mismatched_metadata_identity only hits DiffusionModelRunner._validate_diffusion_kv_metadata (match "request mismatch"), not the helper used by execute_request / execute_model_batch / execute_stepwise. Add a case where envelope and req.request_id are both "req-0" but make_metadata("stale") is attached, and assert those three entry points raise "request identity mismatch" with collective_rpc / _execute_request_list not called. (evidence: vllm_omni/diffusion/sched/interface.py:184 `if new_req.request_id != forwarded_request_id:` — first arm raises before metadata is inspected; :192 `if metadata is not None and metadata.request_id != forwarded_request_id:` — second arm never reached by current helper tests. tests/diffusion/diffusion_kv/test_worker_contract.py:132-135 `NewRequestData(request_id="envelope-id", req=req, diffusion_kv_metadata=make_metadata("envelope-id"))` with req.request_id="actual-request-id" (same pattern at :161-164 and :192-195) — first arm always true. :116-124 `runner.execute_model(..., diffusion_kv_metadata=make_metadata("stale"))` with match "request mismatch" — hits runner-local `_validate_diffusion_kv_metadata` (diffusion_model_runner.py:175-177 `f"Diffusion KV metadata request mismatch: expected={request_id!r}, got={metadata.request_id!r}"`), not the helper. Helper callers: multiproc_executor.py:460 `validate_new_request_data_identity(new_req)`; diffusion_model_runner.py:637 and :751 same call before forward/step.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5550.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/pr5550.trace.json.gz new file mode 100644 index 000000000..d3b5aabba Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5610.cost.json b/eval/dataset/arms/copilot_cb_grok46/pr5610.cost.json new file mode 100644 index 000000000..aedf04d4a --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5610.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 907.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5610/run-20260814-203329-249b8a", + "recorded_at": "2026-08-14T20:48:36", + "attempts": [ + { + "invocation_id": "inv-9fb6fc30", + "outcome": "run:run-20260814-203329-249b8a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-203329-249b8a", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-203329-249b8a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2318, + "minutes": 15.12, + "ci_minutes": 0, + "input_tokens": 554131, + "output_tokens": 74754, + "tool_calls": 195, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5267, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.36, + "gate": 2.05, + "review": 901.46, + "report": 0.06 + }, + "time_to_first_result_s": 906.92 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 23350 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5610.md b/eval/dataset/arms/copilot_cb_grok46/pr5610.md new file mode 100644 index 000000000..c9359d29c --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5610.md @@ -0,0 +1,37 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] docs/.nav.yml:121 adds the feature page beside async_chunk.md; the second vae_parallel.md entry at line 122 is pre-existing, not introduced here. +- [sweep] docs/design/index.md:13 adds the same page under Feature Design Documents; async_chunk.md is still absent from that curated index (pre-existing). +- [sweep] docs/source/architecture/qwen3-omni-async-output-step-gap.svg is a self-contained figure; 2.8 ms → 41 µs is internally consistent (~68×). MkDocs-relative src ../../../source/architecture/... resolves from /design/feature// to docs/source/architecture/ on directory URLs. +- [validated] Correctness of described logic: _should_use_async_omni_output() at gpu_ar_model_runner.py:1595-1619 matches the compatibility table (async_scheduling, omni_prefix_cache, speculative_config, async_chunk, enable_return_routed_experts, use_async_omni_output, has_postprocess/eager postprocess). +- [validated] Thinker opt-in: qwen3_omni.py:133-134 sets use_async_omni_output=True with has_postprocess=False; hidden stays in the snapshot because omni_pooler_payload_include_hidden defaults True (gpu_ar_model_runner.py:1592-1631). +- [validated] Talker opt-in: qwen3_omni.py:157-164 and qwen3_tts_talker.py:304,351-353 set use_async_omni_output, eager_omni_postprocess_before_async_output, and omni_pooler_payload_include_hidden=False. +- [validated] Bundled profile enablement: qwen3_omni_moe.yaml:15 async_chunk true, stages 0/1 omit async_scheduling so _build_engine_args + engine_args.get(..., True) at stage_config.py:941 keep AR async scheduling; enable_prefix_caching false; stage 2 async_scheduling false. pipeline.py default_deploy_config_name is qwen3_omni_moe.yaml / qwen3_tts.yaml. +- [validated] NPU remains synchronous: npu_ar_model_runner.py:1273-1323 builds OmniModelRunnerOutput, drains get_omni_connector_output, then wraps AsyncGPUModelRunnerOutput. + +`docs/design/feature/omni_async_output_materialization.md:?` [minor] — Line 58’s unqualified “does not change model computation or generated output” overstates the CUDA/ROCm hash match in PR #4476. `_should_use_async_omni_output()` (gpu_ar_model_runner.py:1595-1619) has no platform predicate, so XPU (`XPUARModelRunner` → `GPUARModelRunner`) and MUSA (`get_omni_ar_worker_cls` → `GPUARWorker`) can take this path when the other guards pass. The intended design is CUDA-only: `_clone_cuda_tensor_payload` appends sources only for `device.type == "cuda"` else `detach().clone()` (73-77); `_copy_tensor_payload_to_cpu` returns non-cuda tensors unchanged (89-90); `_get_or_create_omni_payload_copy_stream` always does `torch.cuda.Stream()` (1721) before snapshot; Event/current_stream are `torch.cuda.*` (135-140). XPU’s `torch_cuda_wrapper` aliases Stream/current_stream/stream only (not Event) and only wraps `super().__init__`, so later sample_tokens snapshot/D2H is not the pinned async path. Scope the invariance claim to the CUDA/ROCm hash-matched configs and state that XPU/MUSA may select the feature without running that D2H design. (evidence: omni_async_output_materialization.md:58 `It does not change model computation or generated output.` md:62-64 `XPU inherits \`GPUARModelRunner\`, and MUSA selects \`GPUARWorker\`, so those backends may enter the asynchronous path when its other guards pass.` gpu_ar_model_runner.py:1595-1619 `_should_use_async_omni_output` checks async_scheduling/prefix cache/speculative/async_chunk/routed experts/model flags only — no platform predicate — then `return True`. gpu_ar_model_runner.py:73-77 `if value.device.type == "cuda":` clone+append sources; else `return value.detach().clone()`. gpu_ar_model_runner.py:89-90 `if value.device.type != "cuda": return value`. gpu_ar_model_runner.py:135-140 `source_stream = torch.cuda.current_stream()` / `ready_event = torch.cuda.Event()` / `with torch.cuda.stream(copy_stream)`. gpu_ar_model_runner.py:1656,1721 snapshot always passes `copy_stream=self._get_or_create_omni_payload_copy_stream()` which does `stream = torch.cuda.Stream()`. xpu_ar_model_runner.py:10-13 `class XPUARModelRunner(GPUARModelRunner)` with `torch_cuda_wrapper()` only around `super().__init__`. xpu/utils.py:10-13 aliases `Stream`/`default_stream`/`current_stream`/`stream` only — not Event. musa/platform.py:26-27 `return "vllm_omni.worker.gpu_ar_worker.GPUARWorker"`. qwen3_omni_moe.yaml:15 `async_chunk: true`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5610.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/pr5610.trace.json.gz new file mode 100644 index 000000000..994fb40fb Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5703.cost.json b/eval/dataset/arms/copilot_cb_grok46/pr5703.cost.json new file mode 100644 index 000000000..88442e5ea --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5703.cost.json @@ -0,0 +1,110 @@ +{ + "wall_s": 1606.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5703/run-20260814-235141-721d69", + "recorded_at": "2026-08-15T00:18:27", + "attempts": [ + { + "invocation_id": "inv-f410c53d", + "outcome": "run:run-20260814-235141-721d69", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-204837-4aa9fd", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-221443-7af57a", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260814-235141-721d69", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-235141-721d69", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7655, + "minutes": 26.76, + "ci_minutes": 0, + "input_tokens": 2021767, + "output_tokens": 199666, + "tool_calls": 326, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9518, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.16, + "gate": 1.79, + "review": 1599.55, + "report": 0.06 + }, + "time_to_first_result_s": 1605.54 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 83597 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5703.md b/eval/dataset/arms/copilot_cb_grok46/pr5703.md new file mode 100644 index 000000000..1e10e586f --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5703.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] pipeline_minimax_h3.py:545-547 constructs MiniMaxH3VideoVAE(..., device=self.device) with self.device = get_local_device() (utils.py:11-16), so encode-time parameter.device follows the omni platform (cuda/musa/npu), not a hardcoded cuda device. +- [validated] DIFF-1b shared sampling generator path is untouched; this is MiniMax-H3-only keyframe seed 42 in vae.py, consumed only by pipeline_minimax_h3 encode helpers. +- [validated] MiniMaxH3VideoVAE is the only get_device_module caller; blast radius is MiniMax H3 video-VAE encode, not a global platform default. +- [validated] pytestmark on test_minimax_h3_contract.py is core_model, cpu, diffusion, so CUDA/AMD cpu diffusion lanes execute MiniMaxH3VideoVAE.__init__ (get_device_module) via test_video_vae_keeps_reference_fp32_weights. +- [validated] Gate report: buildkite/vllm-omni-amd-ci is red; AMD ready/merge yml runs pytest tests/diffusion -m 'core_model and cpu', which includes this file. +- [validated] CUDA still takes the accelerator-RNG path because device.type==cuda satisfies != cpu; FL2VA/Ref2VA CUDA callers are not left on the old skip path. +- [validated] MiniMaxH3AudioVAE._AudioVAEDeterminismContext remains CUDA-backend specific; out of this PR's video-VAE statement of work. +- [validated] Serving knowledge route was a title/body misroute from the recipe HTTP mention; no serving protocol/default change. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This diff replaces the CUDA-only gate `parameter.device.type == "cuda"` with `!= "cpu"` in both `encode_image()` and `encode_video()`, then seeds via `self.device_module` captured as `torch.get_device_module()` with no device in `__init__`. That is not the CUDA/MUSA allowlist the PR states: every non-CPU type, including Ascend `npu` (`NPUOmniPlatform.get_torch_device` → `torch.device("npu")`; MiniMax-H3-NPU.md runs CPU-only PyTorch + `torch_npu`), now enters `torch.random.fork_rng(devices=...)` and `device_module.manual_seed`, whereas the old `== "cuda"` predicate left those encodes on the CPU generator. `pipeline_minimax_h3.py` still calls `video_vae.encode_image` (rank 0) and `encode_video` (Ref2VA / distributed VPP). `fork_rng(devices=devices)` also omits `device_type`, so save/restore stays on the CUDA default while seeding uses the cached module; `--enable-cpu-offload` can move this module after init. Gate with `parameter.device.type in {"cuda", "musa"}`, pass `device_type=parameter.device.type` into `fork_rng`, and call `torch.get_device_module(parameter.device)` at encode time. (evidence: vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` (same predicate vae.py:214). vae.py:120 `self.device_module = torch.get_device_module()`. vae.py:172 `with torch.random.fork_rng(devices=devices):` then vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`. platforms/npu/platform.py:161-163 `def get_torch_device(...)` / `return torch.device("npu")`. pipeline_minimax_h3.py:963 `rows = self.video_vae.encode_image(image) if rank == 0 else None`. pipeline_minimax_h3.py:1066 `self.video_vae.encode_video(load_video_frames(item["prepared_path"]))`. recipes/MiniMaxAI/MiniMax-H3-NPU.md:18-19 `Runs on a CPU-only (aarch64) PyTorch build with `torch_npu`; no CUDA runtime is present.`) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — The only serve block is introduced as the validated TP4 Ref2VA profile (`The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs`) but then `export MODEL="${MODEL_ROOT}/FL2VA"` (line 76). Two paragraphs above, the same file says to set MODEL to Ref2VA for Ref2VA; MiniMax-H3.md switches with `export MODEL="${MODEL_ROOT}/Ref2VA"`; and `_resolve_task` rejects a task the loaded partition does not list. Operators reproducing the claimed TP4 Ref2VA HTTP 200 (638.6s) will load the T2VA/FL2VA partition. Change this export to Ref2VA, or split FL2VA vs Ref2VA command blocks the way MiniMax-H3.md does. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-73 `The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs`; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76 `export MODEL="${MODEL_ROOT}/FL2VA"`; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:69-70 `Set MODEL to FL2VA for T2VA and FL2VA, or to Ref2VA for Ref2VA.`; recipes/MiniMaxAI/MiniMax-H3.md:168 `export MODEL="${MODEL_ROOT}/Ref2VA"`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:595-597 `if task not in self.supported_tasks: raise OmniClientError(f"checkpoint partition {self.partition!r} supports {sorted(self.supported_tasks)}, got task={task!r}")`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:?` [major] — This diff adds accelerator RNG seeding on the non-CPU branch of MiniMaxH3VideoVAE.encode_image/encode_video (vae.py:170-176 and 214-220) but ships no test files. test_video_vae_encode_uses_configured_parallel_tiling builds via object.__new__ (test_minimax_h3_contract.py:344) with a CPU Parameter (line 335), so devices is [] and device_module is unset and never used; there are no encode_image tests under tests/diffusion/models/minimax_h3/, test_minimax_h3_vae.py is absent, and greps find no torchaudio/soundfile/ffmpeg/TorchCodec coverage there despite MiniMax-H3-MUSA.md claiming that soundfile/TorchCodec fallback was validated. Add a core_model and cpu contract test that stubs parameter.device.type for cuda/musa/npu/cpu, asserts non-cpu devices (code is `!= "cpu"`) are passed to fork_rng with matching device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED), and asserts CPU plus device RNG state restore on exit; drop the recipe audio-loader claims or land those tests. Guard with: pytest -q -c /dev/null tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py -k 'video_vae'. (evidence: vae.py:120 `self.device_module = torch.get_device_module()`. vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`. vae.py:219-220 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`. test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))`. test_minimax_h3_contract.py:344 `video_vae = object.__new__(MiniMaxH3VideoVAE)`. test_minimax_h3_contract.py:353 `rows, shape = video_vae.encode_video("frames")`. tests/diffusion/models/minimax_h3 contains test_minimax_h3_contract.py, test_minimax_h3_e2e.py, test_minimax_h3_packing.py, test_minimax_h3_parallel.py, test_minimax_h3_quantization.py, test_minimax_h3_quantization_quality.py — no test_minimax_h3_vae.py; no encode_image, fork_rng, device_module, or MINIMAX_H3_KEYFRAME_ENCODE_SEED matches under that tree.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — MiniMaxH3VideoVAE.__init__ now calls no-arg torch.get_device_module() (vae.py:120). test_video_vae_keeps_reference_fp32_weights constructs MiniMaxH3VideoVAE(..., device=cpu) after mocking loaders (test_minimax_h3_contract.py:319-322) under pytestmark core_model+cpu+diffusion (line 12), so AMD .buildkite/amd/test-amd-ready.yml:19 `pytest ... tests/diffusion ... -m 'core_model and cpu'` executes this on that lane. Pass the constructor `device` into get_device_module(device) instead of the process default; do not treat a red AMD check as caused by this until the job log shows it. (evidence: vae.py:120 `self.device_module = torch.get_device_module()` — runs at the end of MiniMaxH3VideoVAE.__init__. test_minimax_h3_contract.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`. test_minimax_h3_contract.py:319-322 `video_vae = vae_module.MiniMaxH3VideoVAE(\n "unused",\n device=torch.device("cpu"),\n )` — constructs the class so __init__ (and line 120) runs. .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — MiniMaxH3VideoVAE.__init__ now always does `self.device_module = torch.get_device_module()` (vae.py:120), with no CPU/accelerator gate. `test_video_vae_keeps_reference_fp32_weights` constructs `MiniMaxH3VideoVAE("unused", device=torch.device("cpu"))`, so CPU contract tests execute this new init-time call. Please confirm `buildkite/vllm-omni-amd-ci` is red for a pre-existing reason, not this call or the rewritten non-CPU RNG path (`self.device_module.device` / `manual_seed`). (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — unconditional in MiniMaxH3VideoVAE.__init__. tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:319-322 `video_vae = vae_module.MiniMaxH3VideoVAE(\n "unused",\n device=torch.device("cpu"),\n )` — CPU contract test runs that __init__.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — `MiniMaxH3VideoVAE.__init__` now always calls `torch.get_device_module()` (`vae.py:120`). `test_video_vae_keeps_reference_fp32_weights` is the L2 `core_model`+`cpu` path that actually constructs the class (`MiniMaxH3VideoVAE("unused", device=torch.device("cpu"))`), so that CPU suite now executes the new init-time API. A red `buildkite/vllm-omni-amd-ci` check is not proof this hunk is at fault; confirm the failure is pre-existing on main before attributing it here. (evidence: vae.py:120 `self.device_module = torch.get_device_module()` — last line of MiniMaxH3VideoVAE.__init__, no device-type guard. test_minimax_h3_contract.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`. test_minimax_h3_contract.py:319-321 `video_vae = vae_module.MiniMaxH3VideoVAE(` / `"unused",` / `device=torch.device("cpu"),` — real constructor, not object.__new__.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — vae.py:120 calls no-arg `torch.get_device_module()` inside `MiniMaxH3VideoVAE.__init__`. This repo does not pin `torch>=` (`pyproject.toml` uses dynamic deps via requirements/; `requirements/cuda.txt` has no torch package). A torch that lacks that API (added in 2.5) would fail at init, including CPU `test_video_vae_keeps_reference_fp32_weights`, which constructs `MiniMaxH3VideoVAE("unused", device=torch.device("cpu"))`. The NPU recipe documents 2.10.0+cpu, which is new enough. Please pin torch>=2.5 on supported install paths or guard this call so `__init__` cannot AttributeError. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — unguarded no-arg call in MiniMaxH3VideoVAE.__init__. requirements/cuda.txt (full file) has no torch package: `-r common.txt` / `onnxruntime>=1.23.2` / `fa3-fwd==0.0.3` plus a cublas comment. pyproject.toml:14 `requires-python = ">=3.10,<3.14"` and `dynamic = ["version", "dependencies"]` with no torch>= pin. tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:296 `def test_video_vae_keeps_reference_fp32_weights(monkeypatch):` constructs `video_vae = vae_module.MiniMaxH3VideoVAE("unused", device=torch.device("cpu"))`. recipes/MiniMaxAI/MiniMax-H3-NPU.md:42 `- PyTorch: 2.10.0+cpu`. Repo-wide grep: `get_device_module` appears only at vae.py:120.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5703.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/pr5703.trace.json.gz new file mode 100644 index 000000000..1eb413118 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5715.cost.json b/eval/dataset/arms/copilot_cb_grok46/pr5715.cost.json new file mode 100644 index 000000000..64a76af26 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5715.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 445.9, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5715/run-20260814-205226-7c1ff1", + "recorded_at": "2026-08-14T20:59:51", + "attempts": [ + { + "invocation_id": "inv-86178251", + "outcome": "run:run-20260814-205226-7c1ff1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-205226-7c1ff1", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-205226-7c1ff1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0584, + "minutes": 7.42, + "ci_minutes": 0, + "input_tokens": 102822, + "output_tokens": 27823, + "tool_calls": 62, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2717, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.35, + "gate": 1.81, + "review": 440.06, + "report": 0.06 + }, + "time_to_first_result_s": 445.26 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 12850 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5715.md b/eval/dataset/arms/copilot_cb_grok46/pr5715.md new file mode 100644 index 000000000..abdeca5cd --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5715.md @@ -0,0 +1,39 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] README.md:19 0.26.0 news: MiniMax H3 recipe exists and is a joint video/audio DiT; layerwise offload exists in-tree; MiniCPM full-duplex exists under vllm_omni/experimental/fullduplex and examples/online_serving/minicpmo, but the news hyperlink target recipes/OpenBMB/MiniCPM-o-4_5.md has zero duplex/realtime coverage. +- [sweep] README.md:54–60 feature/model lists: full-duplex marked experimental and listed last as requested; MiniCPM-o 4.5 added under omni; MiniMax H3 added under diffusion — matches recipes and docs/models/supported_models.md (MiniMaxH3Pipeline / MiniCPMO45OmniForConditionalGeneration). +- [sweep] docs/README.md:55–60: feature and model lists stay in sync with README.md (no Latest News section on the docs landing page). +- [sweep] docs/configuration/README.md:5: vLLM engine link moved from docs.vllm.ai/en/v0.16.0 to /en/v0.26.0/configuration/index.html; no other docs.vllm.ai/en/v0.16.0 consumers remain (live HTTP of the 0.26 URL not checked — no network permission). +- [sweep] docs/getting_started/installation/README.md:3–4: 0.26.x↔0.26.x compatibility note; vllm is not a pip pin in requirements/common.txt (peer install via uv pip install vllm==0.26.0 in cuda.inc.md), so the note is a documented contract not an installer constraint. +- [sweep] docs/getting_started/installation/gpu/cuda.inc.md:7 and :127: matching-major/minor wording plus BASE_IMAGE=vllm/vllm-openai:v0.26.0; docker/Dockerfile.cuda:1 already defaults to the same image — docs example now matches the Dockerfile. +- [sweep] docs/getting_started/installation/npu/npu.inc.md:56–69: from-main snippet now cds into clone dirs (fixes the old cd /vllm-workspace/vllm-omni after a relative clone) but clones vllm-ascend from releases/v0.26.0rc while lines 12–13 advertise quay.io/atlas-ci/vllm-ascend:v0.26.0. +- [validated] README.md:19 MiniMax H3 claim: recipes/MiniMaxAI/MiniMax-H3.md:10–14 and :27 call it a CFG-distilled joint video/audio diffusion transformer (T2VA/FL2VA/Ref2VA) — diffusion classification is correct. + +`README.md:?` [minor] — This diff's 0.26.0 news bullet advertises an experimental full-duplex realtime runtime for MiniCPM-o 4.5 and points that claim at recipes/OpenBMB/MiniCPM-o-4_5.md. That recipe only documents OpenAI /v1/chat/completions omni chat and never mentions duplex, realtime, or minicpmo_4_5_duplex.yaml, so the headline feature lands on the wrong user path. Point the MiniCPM-o 4.5 news link at examples/online_serving/minicpmo/README.md (duplex overlay + Realtime demo) or vllm_omni/experimental/fullduplex/README.md, or add a duplex section to the recipe and keep this link. (evidence: README.md:19: 'an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md)'. recipes/OpenBMB/MiniCPM-o-4_5.md:12-13: 'Mode: Online serving via the OpenAI-compatible `/v1/chat/completions` API (plus Gradio demo), and offline inference via `Omni.generate`'. examples/online_serving/minicpmo/README.md:40: '| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |'. vllm_omni/experimental/fullduplex/README.md:6-7: 'the MiniCPM-o 4.5 native audio path used by `/v1/duplex` and `/v1/realtime?duplex=1`'.) + +`docs/getting_started/installation/npu/npu.inc.md:62` [minor] — This diff retargets the NPU from-main source install to `git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git` while the same file's recommended release path already advertises stable `quay.io/atlas-ci/vllm-ascend:v0.26.0` images and clones vLLM/omni on `v0.26.0`. The comment on the previous line calls this 'the v0.26 release line', so readers will treat the RC branch as the 0.26 stable pin. Use the same git ref as the Docker tags (`v0.26.0`) if that tag exists; if from-main must stay on `releases/v0.26.0rc`, say so next to the clone command (and why it differs from the v0.26.0 images above). (evidence: npu.inc.md:56-57: '# Pin vLLM and vLLM-Ascend to the v0.26 release line' / 'git clone -b v0.26.0 https://github.com/vllm-project/vllm.git'. npu.inc.md:62: 'git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git'. npu.inc.md:12-13: '# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0' / '# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3'.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5715.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/pr5715.trace.json.gz new file mode 100644 index 000000000..4c0c752b0 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5840.cost.json b/eval/dataset/arms/copilot_cb_grok46/pr5840.cost.json new file mode 100644 index 000000000..08acd09d4 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5840.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1409.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5840/run-20260814-205326-b4c289", + "recorded_at": "2026-08-14T21:16:55", + "attempts": [ + { + "invocation_id": "inv-1e9b7371", + "outcome": "run:run-20260814-205326-b4c289", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-205326-b4c289", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-205326-b4c289", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6845, + "minutes": 23.49, + "ci_minutes": 0, + "input_tokens": 1699276, + "output_tokens": 205203, + "tool_calls": 407, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8702, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.33, + "gate": 1.78, + "review": 1404.02, + "report": 0.06 + }, + "time_to_first_result_s": 1409.18 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 41105 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5840.md b/eval/dataset/arms/copilot_cb_grok46/pr5840.md new file mode 100644 index 000000000..a4249cb54 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5840.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep][validated] extract_minimax_h3_context (extractors.py:1252-1395) copies MiniMaxH3DiTModel.forward (minimax_h3_transformer.py:1077-1196): same kwargs contract, _embed, blocks loop with packed_total/video_layout, sp_prepare/sp_gather, final_layer, infer_out_pos/audio_pos select, update_mask. test_extractor_matches_native_forward covers the uncached path. +- [sweep][validated] enable_minimax_h3_teacache (backend.py:76-101) hooks only pipeline.transformer; combined Ref2VA uses _transformer_for_task -> transformers_ref (pipeline_minimax_h3.py:736-739, 1498). ref2va raises; fl2va/combined tested in test_cache_backends.py. +- [sweep][validated] DiffusionCacheConfig.rel_l1_thresh None (data.py:422) is resolved in TeaCacheConfig.__post_init__ (config.py:128-133). Qwen empty-config path still gets 0.2 (test_enable_uses_generic_default_threshold). MiniMax empty backend config gets 0.17. +- [sweep] Breaking consumers of the old 0.2 default: async_omni_engine.py:853-856 still injects rel_l1_thresh=0.2 when cache_config is omitted; CLI help and teacache.md still say 0.2; examples pass 0.2 explicitly (unchanged, still valid for non-H3). Omni() goes through AsyncOmniEngine (omni_base.py:189) so the inject hits offline too. +- [sweep] quality=high still returns CacheDiTRequestSpec (quality_policy.py:149-151) with no tea_cache check; prepare() will CacheDiTBackend.enable on the same pipeline (cachedit/runtime.py:71-74). test_model_policy_owns_request_cache_target matrix (test_minimax_h3_quality_policy.py:163-170) has cache_dit/none only, not tea_cache. Recipe 'mutually exclusive' is only a server-backend note. +- [sweep] Polynomial intercept 0.217 > default 0.17: np.poly1d(coeffs)(0)=2.173e-01; hook accumulates abs(rescaled) then compares < thresh (hook.py:230-236). test_teacache_reuses_h3_block_residual uses 0.3 so a zero-distance second forward can skip; 0.17 would recompute. +- [sweep] Simplifiability: extractor duplication of forward is the TeaCache pattern; not dead. partition==combined warning can occur. partition None would raise; pipeline always sets partition in __init__. +- [sweep] Rebase/merge: no dropped MiniMax TeaCache symbols; EXTRACTOR_REGISTRY and CUSTOM_TEACACHE_ENABLERS both gained MiniMaxH3*. Known-limitations Ref2VA bullets look like merge docs, not TeaCache logic. + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:?` [major] — This PR registers MiniMaxH3Pipeline in CUSTOM_TEACACHE_ENABLERS (backend.py:127) and apply_teacache_hook(pipeline.transformer) in enable_minimax_h3_teacache, and the recipe says TeaCache and Cache-DiT cannot share a server. MiniMaxH3QualityPolicy.resolve still always installs Cache-DiT when quality=="high" (quality_policy.py:149-151) and only consults cache_backend for omitted quality (line 153). pipeline_minimax_h3.py:1766 then calls _cache_dit_runtime.prepare(quality_plan.cache_dit); runtime.py:71-74 constructs CacheDiTBackend and enable()s the same pipeline/transformer TeaCache already hooked. quality=lossless only returns cache_dit=None, which disables Cache-DiT and does not remove the TeaCache hook. Reject quality=high / force-refresh when cache_backend=="tea_cache", document that TeaCache is process-lifetime, stop the request-scoped quality recipe from telling tea_cache users to send quality=high, and extend test_model_policy_owns_request_cache_target with cache_backend="tea_cache". (evidence: quality_policy.py:149-153 `if quality == "high":` / `base_key = MINIMAX_H3_HIGH_CACHE_KEY` / `base_config = _high_quality_cache_config()` / `elif self._configured_backend == "cache_dit" and quality is None:` — high ignores tea_cache. quality_policy.py:127-131 docstring: `In either case, ``lossless`` selects no cache and ``high`` selects H3's high-quality profile.` / `the startup backend only controls the omitted-quality default.` teacache/backend.py:91 `apply_teacache_hook(pipeline.transformer, teacache_config)` and backend.py:127 `"MiniMaxH3Pipeline": enable_minimax_h3_teacache,`. pipeline_minimax_h3.py:1766 `self._cache_dit_runtime.prepare(quality_plan.cache_dit)`. cachedit/runtime.py:71-74 `if self._backend is None:` / `backend = CacheDiTBackend(spec.cache_config)` / `backend.enable(self._pipeline)`. recipes/MiniMaxAI/MiniMax-H3.md: `No Cache-DiT startup option is required; H3 installs its conservative profile when a \`high\` request arrives` vs `TeaCache and Cache-DiT cannot be enabled on the same server.` test_minimax_h3_quality_policy.py matrix includes `("none", "high", "minimax_h3.high")` and omits tea_cache.) + +`vllm_omni/engine/async_omni_engine.py:?` [major] — This PR changes DiffusionCacheConfig.rel_l1_thresh to None and TeaCacheConfig.__post_init__ resolves None to MiniMaxH3DiTModel 0.17 (else 0.2). AsyncOmniEngine._get_default_cache_config still returns {"rel_l1_thresh": 0.2} when cache_config is omitted (async_omni_engine.py:853-856, 881-882), and OmniBase constructs this engine (omni_base.py:189) via _create_default_diffusion_stage_cfg (async_omni_engine.py:954-958), so Omni()/vllm serve never reach 0.17. Empty cache_config={} skips the injector and would get 0.17; tests use DiffusionCacheConfig() and do not cover _normalize_cache_config. Return {} from the tea_cache default dict and add a test that _normalize_cache_config("tea_cache", None) leaves rel_l1_thresh unset. (evidence: async_omni_engine.py:853-856 `if cache_backend == "tea_cache": return { "rel_l1_thresh": 0.2, }`. async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"): cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`. async_omni_engine.py:954-958 `cache_backend = normalized_kwargs.get("cache_backend", "none"); cache_config = AsyncOmniEngine._normalize_cache_config(cache_backend, normalized_kwargs.get("cache_config", None))`. omni_base.py:189 `self.engine = AsyncOmniEngine(`. data.py:421-422 `# None defers to the model-specific TeaCache default (0.2 fallback). / rel_l1_thresh: float | None = None`. teacache/config.py:96-98 `_DEFAULT_REL_L1_THRESH = 0.2` / `_MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, }`. teacache/config.py:128-133 `if threshold is None: threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)` then `self.rel_l1_thresh = threshold`. tests/diffusion/cache/test_cache_backends.py:504-508 `config = ( DiffusionCacheConfig() if configured_threshold is None else DiffusionCacheConfig(rel_l1_thresh=configured_threshold) )` — no _normalize_cache_config coverage.) + +`vllm_omni/diffusion/cache/teacache/config.py:98` [major] — This PR registers MiniMaxH3DiTModel coefficients ending in 2.173782527946167e-01 (config.py:92) and _MODEL_DEFAULT_REL_L1_THRESH["MiniMaxH3DiTModel"]=0.17 (config.py:98). TeaCacheHook sets rescale_func=np.poly1d(coefficients) (hook.py:63), then accumulated += abs(rescale_func(rel_distance)) and skips only when accumulated < rel_l1_thresh (hook.py:229-236). poly(0) is already ~0.217 > 0.17, so identical consecutive steps cannot skip at the shipped default. The only automated skip test (test_teacache_reuses_h3_block_residual) sets rel_l1_thresh=0.3 (test_teacache_extractors.py:688) so that intercept can hit, and it asserts block_calls/shapes rather than assert_close of cache-hit vs compute outputs. Re-fit a mapping whose zero-distance value is below the shipped threshold, or raise the default above the intercept, and assert a skip at that shipped value (keep the 0.3 identical-input case). (evidence: config.py:92 `2.173782527946167e-01,` — MiniMaxH3DiTModel poly1d intercept; config.py:98 `"MiniMaxH3DiTModel": 0.17,` — shipped default below that intercept; hook.py:63 `self.rescale_func = np.poly1d(config.coefficients)`; hook.py:229-236 `rescaled_distance = float(self.rescale_func(rel_distance))` / `state.accumulated_rel_l1_distance += abs(rescaled_distance)` / `if state.accumulated_rel_l1_distance < rel_l1_thresh: return False`; test_teacache_extractors.py:688 `rel_l1_thresh=0.3,` — skip test does not use 0.17; test_teacache_extractors.py:693-695 `assert block_calls == 1` plus shape checks only (assert_close is in test_extractor_matches_native_forward at 642-643, the uncached path).) + +`vllm_omni/entrypoints/cli/serve.py:586` [minor] — This PR changes DiffusionCacheConfig.rel_l1_thresh to float|None=None; TeaCacheConfig then resolves MiniMaxH3DiTModel to 0.17 and every other transformer to 0.2. serve.py --cache-config help still shows TeaCache: '{"rel_l1_thresh": 0.2}', and teacache.md still documents type float / default 0.2. Copying that CLI example pins 0.2 and overrides the H3 0.17 default — update the help/docs to omit-key/None plus the model-specific fallback. (evidence: serve.py:586 `"TeaCache: '{\"rel_l1_thresh\": 0.2}'. "` — --cache-config help still illustrates a single 0.2. config.py:122 `rel_l1_thresh: float | None = None`; config.py:129-130 `if threshold is None: threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)` with `_DEFAULT_REL_L1_THRESH = 0.2` and `_MODEL_DEFAULT_REL_L1_THRESH = {"MiniMaxH3DiTModel": 0.17}`. data.py:422 `rel_l1_thresh: float | None = None`. teacache.md:122 `| \`rel_l1_thresh\` | float | \`0.2\` | Similarity threshold for cache reuse. ... |`) + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:152` [minor] — PR title is '[MiniMax-H3] TeaCache support and Cache-DiT validation', but this diff only lands FL2VA TeaCache (extractor/backend/tests/docs) plus the shared rel_l1_thresh default. quality_policy.py still resolves only Cache-DiT (`quality=="high"` / `cache_backend=="cache_dit"`) and is not in the diff; existing Cache-DiT tests in test_minimax_h3_quality_policy.py are untouched. Is Cache-DiT validation intentionally out of scope here (e.g. #5997) despite the title? (evidence: quality_policy.py:149 `if quality == "high":`; quality_policy.py:152 `elif self._configured_backend == "cache_dit" and quality is None:`; quality_policy.py:158 `return MiniMaxH3QualityPlan(cache_dit=None)` — resolve() only builds CacheDiTRequestSpec, no TeaCache. docs/user_guide/diffusion_features.md:162 `| **MiniMax-H3** | ✅ (FL2VA) | ✅ |` — this PR flips TeaCache to ✅ (FL2VA); Cache-DiT was already ✅. test_minimax_h3_e2e.py:155 `cache_backend="tea_cache",` — added smoke is TeaCache, not Cache-DiT. Issue #5700 snapshot: `Referenced by PR #5840: [MiniMax-H3] TeaCache support and Cache-DiT validation`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:170` [minor] — test_minimax_h3_t2va_teacache_single_gpu_smoke sets cache_backend="tea_cache" and cache_config={"rel_l1_thresh": 0.17} (test_minimax_h3_e2e.py:155-156) but uses num_inference_steps=2 (line 170) and only _assert_joint_output (line 186), which checks frames.shape==(107, 256, 448, 3) (line 22). TeaCacheHook._should_compute_full_transformer always computes when state.cnt==0 (hook.py:211-213), so two denoise steps allow at most one cache decision. A silent always-compute TeaCache still passes that shape contract. Assert an actual skip (block call count or hook compute vs cnt) on this path at the shipped 0.17 operating point; this 2-step smoke cannot fail if the named TeaCache skip path is a no-op. Docs also mark TeaCache as not for runs < 20 steps (teacache.md). (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:155-156 `cache_backend="tea_cache",` `cache_config={"rel_l1_thresh": 0.17},`; :170 `num_inference_steps=2,`; :186 `_assert_joint_output(outputs)`; :22 `assert frames.shape == (107, 256, 448, 3)`. vllm_omni/diffusion/cache/teacache/hook.py:211-213 `if state.cnt == 0:` `state.accumulated_rel_l1_distance = 0.0` `return True`. docs/user_guide/diffusion/cache_acceleration/teacache.md:158 `- Very short inference runs (< 20 steps) where caching overhead may outweigh benefits`) + +`tests/diffusion/cache/test_cache_backends.py:490` [nit] — The new MiniMax TeaCache cases in tests/diffusion/cache/test_cache_backends.py inherit the file-level `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (line 32). AMD ready.yml runs `pytest tests/diffusion tests/model_executor -m 'core_model and cpu'` and only `--ignore`s test_teacache_extractors.py (line 19), so these tests are collected in that job while extractor tests are not. Attribute a red buildkite/vllm-omni-amd-ci check from the job log (these cases vs pre-existing) before treating it as a product regression. (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — file-level marks on the new MiniMax tests including line 490 `def test_minimax_h3_only_enables_fl2va_teacache(`. .buildkite/amd/test-amd-ready.yml:19 `timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml` — collects that file; only extractors are ignored.) + +`docs/user_guide/diffusion/cache_acceleration/teacache.md:?` [nit] — This PR makes omitted `rel_l1_thresh` `None` on `DiffusionCacheConfig` (`data.py:421-422`; empty-dict example prints `None` at `data.py:416-417`). `TeaCacheConfig` then resolves `None` to `MiniMaxH3DiTModel` `0.17` or the `0.2` fallback (`config.py:96-98`). `docs/user_guide/diffusion/cache_acceleration/teacache.md:122` still documents type `float` and a single default of `0.2`. Update that table to `float | None`, default None/model-specific (`0.2` fallback; MiniMax-H3 FL2VA `0.17`). (evidence: docs/user_guide/diffusion/cache_acceleration/teacache.md:122 `| `rel_l1_thresh` | float | `0.2` | Similarity threshold for cache reuse. Lower values prioritize quality (less caching), higher values prioritize speed (more caching). Suggested range: 0.1-0.8 |` — vllm_omni/diffusion/data.py:421-422 `# None defers to the model-specific TeaCache default (0.2 fallback).` / `rel_l1_thresh: float | None = None` — data.py:416-417 `>>> default_config = DiffusionCacheConfig.from_dict({})` / `>>> print(default_config.rel_l1_thresh) # None` — vllm_omni/diffusion/cache/teacache/config.py:96-98 `_DEFAULT_REL_L1_THRESH = 0.2` / `_MODEL_DEFAULT_REL_L1_THRESH = {` / ` "MiniMaxH3DiTModel": 0.17,`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5840.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/pr5840.trace.json.gz new file mode 100644 index 000000000..bf4d09d20 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5863.cost.json b/eval/dataset/arms/copilot_cb_grok46/pr5863.cost.json new file mode 100644 index 000000000..049f6175e --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5863.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 520.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5863/run-20260814-205952-8b419f", + "recorded_at": "2026-08-14T21:08:32", + "attempts": [ + { + "invocation_id": "inv-7eb0fbb2", + "outcome": "run:run-20260814-205952-8b419f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-205952-8b419f", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-205952-8b419f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0714, + "minutes": 8.66, + "ci_minutes": 0, + "input_tokens": 149494, + "output_tokens": 28175, + "tool_calls": 68, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3089, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.61, + "gate": 2.61, + "review": 513.21, + "report": 0.06 + }, + "time_to_first_result_s": 519.48 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 14524 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5863.md b/eval/dataset/arms/copilot_cb_grok46/pr5863.md new file mode 100644 index 000000000..14f4754e2 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5863.md @@ -0,0 +1,37 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] Text-encoder TP 2/4/8 divides Qwen3-VL 64 heads (MiniMax-H3.md valid set 1,2,4,8). VAE patch size equals DiT world size, matching H3 'size 1 or full group' rule. +- [validated] Online FP8 note matches MiniMax-H3.md known limitation (layerwise offload). This recipe does not enable DLO on the 6000 path. + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340` [major] — This diff adds a copy-paste Ref2VA curl that will not work for readers of the recipe. `/v1/videos/sync` parses `audio_reference` as JSON and then requires `UrlAudioReference.audio_url` to be an HTTP(S) or data URL (`vllm_omni/entrypoints/openai/protocol/videos.py` AudioReference = UrlAudioReference with only `audio_url`; `api_server.py` `_parse_form_json` then `decode_audio_url`). The in-tree contract is the MiniMax-H3.md example `audio_reference={\"audio_url\":\"${AUDIO_URL}\"}` plus a local http.server, not a host-specific `/root/audio_ref.json` whose schema is never shown. The same block also uses `/root/hand.jpg`, `--max-time 1200` (T2VA uses 1800; this recipe says Ref2VA is unmeasured, and MiniMax-H3.md already reports 784 s for two-video Ref2VA on 4×B300), and 60 steps vs the measured 50-step T2VA table. Replace this example with placeholders and the `audio_url` JSON from MiniMax-H3.md, or drop it and link to MiniMax-H3.md#http-api-examples the way MiniMax-H3-NPU.md:140 already does. (evidence: Diff hunk: `-F "audio_reference="`, so the 27th unique value is `` / `` (`chr(97+26)` is `'{'`). Callers are `_normalize_zh_or_en` protecting `_PINYIN_TONE_PATTERN` and `_NAME_PATTERN` matches before WeText (`:194-206`); restore is `str.replace` with no missing-marker check (`:166-168,207-212`). Switch both helpers to a numeric marker such as `<{prefix}_{index}>`. (evidence: vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:154 `marker = f"<{prefix}_{chr(ord('a') + index)}>"` — index 26 is chr(123)='{', so the 27th unique value is `` / ``. Same scheme at :167 in `_restore_values`: `marker = f"<{prefix}_{chr(ord('a') + index)}>"` then `text = text.replace(marker, transform(value) if transform else value)` with no missing-marker check. Callers at :194-206: `pinyin_values = [match.group(0) for match in _PINYIN_TONE_PATTERN.finditer(text)]` then `_protect_values(..., prefix="pinyin")` and `_protect_values(_NAME_PATTERN.findall(text), prefix="n")`, then `result = _load_wetext_normalizer(normalizer_lang).normalize(text)`.) + +`vllm_omni/model_executor/stage_input_processors/indextts2.py:223` [minor] — Shared `talker2s2mel_full_payload` is now fail-fast: missing `meta.use_gpt_latent` raises at indextts2.py:223, and a mel/latent length skew raises instead of cropping (`test_talker2s2mel_full_payload_rejects_mel_latent_length_mismatch`). IndexTTS-2 still wires this processor (pipeline.py:30) and IndexTTS-2.5 aliases the same `IndexTTS2S2MelDecoder` (registry.py:378-381), which also owns the rewritten CFM `solve_euler` per-row `prompt_mask`/`masked_fill_` and `_run_cfm_groups`. IndexTTS-2 talker does emit `use_gpt_latent` on the happy path, so line 223 is the incomplete-payload path; the latent-mode crop removal and shared CFM rewrite still apply to IndexTTS-2. Run IndexTTS-2 offline `end2end.py` at this head, or state explicitly that IndexTTS-2 payload/CFM behavior is intentionally tightened. (evidence: vllm_omni/model_executor/stage_input_processors/indextts2.py:222-223 `if use_gpt_latent_value is None:` / `raise ValueError(f"IndexTTS payload is missing meta.use_gpt_latent for req={rid}")`; indextts2.py:261-264 `if mel_seq.shape[0] != latent_seq.shape[0]: raise ValueError(f"IndexTTS mel/latent length mismatch for req={rid}: ...")`; vllm_omni/model_executor/models/indextts2/pipeline.py:30 `custom_process_next_stage_input_func=f"{_PROC}.talker2s2mel_full_payload"`; vllm_omni/model_executor/models/registry.py:378-381 `"IndexTTS25S2MelDecoder": ("indextts2", "indextts2_s2mel_decoder", "IndexTTS2S2MelDecoder")`; tests/model_executor/stage_input_processors/test_indextts2.py:251 `def test_talker2s2mel_full_payload_rejects_mel_latent_length_mismatch():`; flow_matching.py:184 `prompt_mask = prompt_positions.unsqueeze(0) < prompt_lens.unsqueeze(1)` and :189 `prompt_x[..., : prompt.size(-1)] = prompt * prompt_mask.unsqueeze(1).to(prompt.dtype)`; indextts2_s2mel_decoder.py:587 `mel = self._run_cfm_groups(` (shared decoder, not v2.5-gated); indextts2_talker.py:453 `req_meta: dict[str, Any] = {"use_gpt_latent": self.use_gpt_latent}`) + +**Additional observations (beyond the comment budget):** +- `pyproject.toml:91` [minor] Linux extra `indextts2` leaves `WeTextProcessing` unpinned (`pyproject.toml:91`) while non-Linux `wetext` is `>=0.0.9`. Linux `_load_wetext_normalizer` constructs `tn.chinese.normalizer.Normalizer(remove_interjections=Fa + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5957.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/pr5957.trace.json.gz new file mode 100644 index 000000000..7309fba1b Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5976.cost.json b/eval/dataset/arms/copilot_cb_grok46/pr5976.cost.json new file mode 100644 index 000000000..b3f50a537 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5976.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 2003.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46/runs/pr5976/run-20260814-211656-a7e015", + "recorded_at": "2026-08-14T21:50:19", + "attempts": [ + { + "invocation_id": "inv-5132c066", + "outcome": "run:run-20260814-211656-a7e015", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-211656-a7e015", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-211656-a7e015", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6808, + "minutes": 33.38, + "ci_minutes": 0, + "input_tokens": 1721020, + "output_tokens": 196440, + "tool_calls": 466, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0065, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 5.3, + "gate": 1.68, + "review": 1995.53, + "report": 0.06 + }, + "time_to_first_result_s": 2002.56 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 67504 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5976.md b/eval/dataset/arms/copilot_cb_grok46/pr5976.md new file mode 100644 index 000000000..54f9d1fa6 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46/pr5976.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] 1 correctness: AR _update_request_as_session:627-629 seeds stale then :669-671 calls mixin which seeds again at omni_scheduler_mixin.py:105; num_in_flight_tokens is not cleared, so replace_streaming_prompt double-counts. Generation scheduler drains (:385-388) but has no segment-stop seed (only mixin via _update_request_as_session). Chunk adapter does not seed stale. +- [sweep] 2 simplifiability: async_tokens_to_discard is write-only after this diff (ar_scheduler.py:479 and :634); grep of vllm_omni found no readers — DELETE. hasattr fallbacks in test_omni_ar_scheduler_stale_drain.py:59-64 are 0.26 shims on a 0.27-targeted rebase. +- [sweep] 3 breaking: register_omni_models_to_vllm now unconditionally overwrites upstream archs (arg_utils.py) — intentional, paired with qwen3/qwen2.5 model_stage default. FusedMoE in-repo imports updated to FusedMoEFactory; diffusion consumers still go through vllm_omni.diffusion.layers.fused_moe.FusedMoE adapter. CUDA_FLASH_ATTENTION_BACKEND_ATTEMPTS extraction does not change order. LingBot extra_args size is a restore, covered by new tests. +- [sweep] 4 rebase/merge: MistralToolCall import move, is_interleaved rehome, _copy_missing_attrs rehome, mamba_utils import path, XPU default native->vllm_c look like mechanical 0.27 drift not duplicated logic. +- [sweep] 5 tests: stale-drain tests call _update_request_as_session without replace_streaming_prompt so they cannot catch double-seed; test_explicit_streaming_payload_replaces_placeholder_prompt (streaming.py:252) exercises replace but does not assert num_stale_output_tokens. Qwen3 forward-contract tests bind real forward/make_omni_output. Replica-cache tests cover scoped_spawn_device_env. SSIM 0.97->0.96 only on matched SDPA fallback. Bagel goldens re-probed with full mismatch table. Quant quality both arms enforce_eager=True. WER still open per PR body (nightly 2957 0.3916). +- [sweep] 6 docs: scheduler comments claim seed==drain exactly; false on the replace path. Qwen3MoeLLMModel.forward annotation still says Tensor|IntermediateTensors but returns a 2-tuple (:579-582). fused_moe.py still says factory function while importing FusedMoEFactory. +- [sweep] 7 invariants: drain treats any num_stale_output_tokens>0 as dropping the whole frame and asserts leftover>=0 (ar_scheduler.py:336-339); leftover 1 plus a new prefill of 5 tokens kills the engine-core. Replica 0 keeps shared VLLM_CACHE_ROOT (`if not replica_id or replica_id <= 0`). StageEngineCoreProcManager starts processes inside scoped_spawn_device_env so children inherit the isolated env. +- [sweep] 8 scope: purpose claims every 2952-2957 regression fixed; seed-TTS WER remains open in the PR body. MiniCPM-o h100_2, shm-size 8gb, inductor CantSplit, Qwen3 bare-tensor, use_all2all, replica cache, extra_args size are in the diff. + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [major] — OmniARScheduler._update_request_as_session seeds num_stale_output_tokens from num_in_flight_tokens (omni_ar_scheduler.py:627-629) before the replace_streaming_prompt branch calls mixin _replace_streaming_session, which adds the same uncleared num_in_flight_tokens again (omni_scheduler_mixin.py:105) and returns. One in-flight frame becomes stale=2; update_from_output (336-353) drops every frame while stale>0, so the new segment's first valid frame is swallowed, or a later prefill underflows assert num_stale_output_tokens >= 0. MiniCPM-o 4.5 sets that flag on assistant-turn handoff (minicpmo_4_5_omni.py:953-954). OmniGenerationScheduler seeds only via the mixin (omni_generation_scheduler.py:604). Keep the mixin seed; move the AR seed to the non-replace/super() branch only. test_omni_ar_scheduler_stale_drain.py's helper uses a bare StreamingUpdate and never enters the mixin. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0:` / `session.num_stale_output_tokens += in_flight_tokens` — vllm_omni/core/sched/omni_ar_scheduler.py:669-671 `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)` / `return` — vllm_omni/core/sched/omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` (mixin never zeros num_in_flight_tokens) — vllm_omni/core/sched/omni_ar_scheduler.py:336-339 `if request is not None and request.num_stale_output_tokens > 0:` / `output_is_stale = True` / `request.num_stale_output_tokens -= num_tokens_scheduled` / `assert request.num_stale_output_tokens >= 0` then 348-353 `if output_is_stale: continue` — vllm_omni/core/sched/omni_generation_scheduler.py:604 `self._replace_streaming_session(session, update)` — vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:953-954 `if not is_native_duplex_handoff or native_turn_start:` / `handoff_meta["replace_streaming_prompt"] = True`) + +`vllm_omni/engine/arg_utils.py:?` [major] — Dropping `if arch not in supported_archs` makes every `_OMNI_MODELS` entry overwrite upstream, including `Qwen2_5OmniForConditionalGeneration` (registry.py:9-12). Qwen3's combined thinker now returns `self.thinker(...)` — a bare tensor when capture is off (qwen3_omni.py:417-424) — so stock `GPUModelRunner` can index `hidden_states[logit_indices]`. Qwen2.5's combined thinker still `return OmniOutput(...)` (qwen2_5_omni.py:326-329); `OmniOutput` is a NamedTuple (output_templates.py:9), so that index is `TypeError: only integer tensors of a single element can be converted to an index` — the failure this PR documents for Qwen3. `getattr(model_stage) or "thinker"` (qwen2_5_omni.py:86) only survives engine-core init. Mirror the Qwen3 contract or stop implying non-staged `vllm serve` works for Qwen2.5; add a CPU forward-shape test like `test_qwen3_omni_forward_contract.py`. (evidence: vllm_omni/engine/arg_utils.py:116-117 `for arch, (mod_folder, mod_relname, cls_name) in _OMNI_MODELS.items():` / `ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")` — no `supported_archs` guard. vllm_omni/model_executor/models/registry.py:9-12 `"Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni", "qwen2_5_omni", "Qwen2_5OmniForConditionalGeneration")`. vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py:79-86 `# this class is now also constructed for non-staged runs (its arch overrides upstream's in the registry)` / `self.model_stage = getattr(vllm_config.model_config, "model_stage", None) or "thinker"`. vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py:326-329 `return OmniOutput(\n text_hidden_states=(text_hidden_states.reshape(-1, text_hidden_states.shape[-1])),\n multimodal_outputs=None,\n )`. vllm_omni/model_executor/models/output_templates.py:9 `class OmniOutput(NamedTuple):`. vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py:417-424 `return self.thinker(` — comment: `Returns (text_hidden_states, captured_layer_dict) when capturing, a bare tensor otherwise.`) + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:?` [minor] — GPU `_dummy_run` now takes `randomize_inputs: bool = False` (AR forwards it at gpu_model_runner.py:1134; generation accepts but does not forward at gpu_generation_model_runner.py:847). Both NPU overrides still end at `profile_cpp` — npu_generation_model_runner.py:656 and npu_model_runner.py:127 — and `vllm_omni/platforms/npu` has zero `randomize_inputs` hits. A keyword call from vLLM 0.27 kernel_warmup will TypeError the same way CUDA did in build 2952. Accept the kwarg on both NPU signatures; on generation, do not forward it. (evidence: gpu_generation_model_runner.py:580 `randomize_inputs: bool = False,` — GPU generation accepts the 0.27 kwarg; gpu_generation_model_runner.py:847 `self.maybe_randomize_inputs(input_ids, inputs_embeds),` does not forward it. gpu_model_runner.py:881 `randomize_inputs: bool = False,`; gpu_model_runner.py:1134 `self.maybe_randomize_inputs(input_ids, inputs_embeds, randomize_inputs=randomize_inputs),` AR forwards it. npu_generation_model_runner.py:641 `def _dummy_run(` last params :655 `profile_seq_lens: int | None = None,` :656 `profile_cpp: bool = False,` — no randomize_inputs. npu_model_runner.py:112 `def _dummy_run(` ends `profile_seq_lens: int | None = None,` / `profile_cpp: bool = False,` — same omission. Grep of vllm_omni/platforms/npu: zero `randomize_inputs` hits.) + +`tests/core/sched/test_omni_ar_scheduler_stale_drain.py:?` [minor] — This diff's new `test_omni_ar_scheduler_stale_drain.py` helper is named `_replace_streaming_session` but only seeds through `OmniARScheduler._update_request_as_session` (`tests/core/sched/test_omni_ar_scheduler_stale_drain.py:84`) with `stage_id=0`, `chunk_transfer_adapter=None`, and a `StreamingUpdate` that never sets `replace_streaming_prompt`, so `omni_ar_scheduler.py:669-671` never calls mixin `_replace_streaming_session`. Drain then uses unbound `OmniARScheduler.update_from_output` on a MagicMock (`:132`), which cannot reach the stopped-but-not-finished segment-stop seed at `omni_ar_scheduler.py:490-491`. Generation always seeds via mixin (`omni_generation_scheduler.py:604` → `omni_scheduler_mixin.py:105`). Add cases for (1) `OmniGenerationScheduler._update_request_as_session` / mixin replace and (2) a finished-but-not-finished segment-stop that hits `:490`. Run `pytest tests/core/sched/test_omni_ar_scheduler_stale_drain.py -m 'core_model and cpu'`. (evidence: tests/core/sched/test_omni_ar_scheduler_stale_drain.py:71 `sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=0))`; :74 `sched.chunk_transfer_adapter = None`; :84 `sched._update_request_as_session(session, update)` — seed site is AR _update_request_as_session, not mixin. :132 `OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output)` — drain on a MagicMock, no segment-stop path. vllm_omni/core/sched/omni_ar_scheduler.py:669-672 `if replace_streaming_prompt: self._replace_streaming_session(session, update); return` then `super()._update_request_as_session`. omni_ar_scheduler.py:490-491 `if request.num_in_flight_tokens > 0: request.num_stale_output_tokens += request.num_in_flight_tokens`. omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`. omni_generation_scheduler.py:604 `self._replace_streaming_session(session, update)` — generation always uses mixin replace. Grep of tests for num_stale_output_tokens hits only this new file plus logprobs/chunk-transfer dummy fields.) + +`vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py:?` [minor] — This diff changes `Qwen2_5OmniVideoProcessorItems.get_processor_data` from iterating `self.data` to `self.get()` so `MediaWithBytes` unwraps before the HF processor (comment at 299-309: skipping unwrap became a 400 IndexError on every video request). `tests/` has no `Qwen2_5OmniVideoProcessorItems` / `get_processor_data` / `MediaWithBytes` reference; `test_per_video_audio_mask.py` only stubs `get_mm_hashes`. Add a CPU test under `tests/model_executor/models/qwen2_5_omni/` that puts a wrapper (`.data` is not a `(frames, metadata)` tuple; `.get()` returns the tuple) into the items object and asserts `get_processor_data()["videos"]` is non-empty. Command: `pytest tests/model_executor/models/qwen2_5_omni/ -m 'core_model and cpu'`. (evidence: qwen2_5_omni_thinker.py:312 `items = [self.get(index) for index in range(self.get_count())]` then :313 `videos = [frames for frames, _ in items]`. Comment :302-306 `Iterating self.data skips that unwrap` … `IndexError: list index out of range`. Grep of tests/ for `Qwen2_5OmniVideoProcessorItems`, `get_processor_data`, and `MediaWithBytes` returned no matches. test_per_video_audio_mask.py:271 `inputs.get_mm_hashes = MagicMock(return_value={"video": ["v0"], "audio": ["a0"]})` — does not call `get_processor_data`.) + +`ming_flash_omni.py:98` [nit] — After this diff's unconditional `_OMNI_MODELS` overwrite, Qwen2.5/Qwen3 guard `model_stage` with `getattr(...) or "thinker"`, but `ming_flash_omni.py:98`, `voxtral_tts.py:112`, and `step_audio2.py:57` still read `model_config.model_stage` bare (also minicpmo, mammoth, cosyvoice3, omnivoice, covo). If those archs are in upstream vLLM 0.27, plain `vllm serve` will AttributeError. Confirm they are not in the 0.27 registry, or give them the same getattr default. (evidence: Qwen2.5/Qwen3 gained getattr(model_stage) or thinker, but ming_flash_omni.py:98, voxtral_tts.py:112, step_audio2.py:57 still read model_config.model_stage bare — only a bug if those archs are also in upstream 0.27 (unproven; not filed). Other _OMNI_MODELS still read model_stage unguarded (voxtral, minicpmo, step_audio2, ming, mammoth, cosyvoice3, omnivoice, covo) — unproven those archs are in vLLM 0.27, not posted.) + +`vllm_omni/core/sched/omni_ar_scheduler.py:479` [nit] — Dead writes: `request.async_tokens_to_discard = outstanding_async_tokens` at AR segment-stop (`omni_ar_scheduler.py:479`) and `session.async_tokens_to_discard = 1` (`:634`) after this PR's comments say a0c092ee72 removed that `Request` field in favor of `num_stale_output_tokens`. The drain (`:336-353`) only subtracts `num_stale_output_tokens`; a `vllm_omni` grep has no production readers. Delete both writes (they do not participate in the new drain) and drop the matching asserts in `tests/core/sched/test_omni_ar_scheduler_streaming.py` (:124, :221, :243). (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:336-338 `if request is not None and request.num_stale_output_tokens > 0:` / `output_is_stale = True` / `request.num_stale_output_tokens -= num_tokens_scheduled` — drain never reads `async_tokens_to_discard`. vllm_omni/core/sched/omni_ar_scheduler.py:479 `request.async_tokens_to_discard = outstanding_async_tokens`. vllm_omni/core/sched/omni_ar_scheduler.py:634 `session.async_tokens_to_discard = 1`. Package grep of `async_tokens_to_discard` under vllm_omni: only those two assignments plus comments at omni_ar_scheduler.py:328,475; omni_scheduler_mixin.py:100; omni_generation_scheduler.py:377.) + +`vllm_omni/worker/gpu_generation_model_runner.py:?` [nit] — Generation `_dummy_run` takes `randomize_inputs: bool = False` for vLLM 0.27 API compat but calls `maybe_randomize_inputs(input_ids, inputs_embeds)` without the flag (thinker runner forwards `randomize_inputs=randomize_inputs`). That omission is the code2wav fix (vocab-uniform ids as RVQ codes). `tests/` has zero `randomize_inputs` hits, so copying the thinker call here would re-break warmup. Pin that `_dummy_run(..., randomize_inputs=True)` does not pass True into `maybe_randomize_inputs`; extend `tests/worker/test_capture_replay_contract.py`. (evidence: vllm_omni/worker/gpu_generation_model_runner.py:580 `randomize_inputs: bool = False,` — kwarg accepted. gpu_generation_model_runner.py:840-847 comment `Never randomize generation-stage dummy ids` then `self.maybe_randomize_inputs(input_ids, inputs_embeds),` — no randomize_inputs kwarg. Contrast vllm_omni/worker/gpu_model_runner.py:1134 `self.maybe_randomize_inputs(input_ids, inputs_embeds, randomize_inputs=randomize_inputs),`. Grep of tests/ for `randomize_inputs` returned no matches.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46/pr5976.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46/pr5976.trace.json.gz new file mode 100644 index 000000000..d82ca9be6 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/manifest.json b/eval/dataset/arms/copilot_cb_grok46_r2/manifest.json new file mode 100644 index 000000000..1a1b8bba9 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_cb_grok46_r2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.cost.json b/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.cost.json new file mode 100644 index 000000000..f5735f37d --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1090.9, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r2/runs/pr5509/run-20260814-203313-e73ae8", + "recorded_at": "2026-08-14T20:51:23", + "attempts": [ + { + "invocation_id": "inv-fd3d4760", + "outcome": "run:run-20260814-203313-e73ae8", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-203313-e73ae8", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-203313-e73ae8", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4455, + "minutes": 18.17, + "ci_minutes": 0, + "input_tokens": 1073458, + "output_tokens": 141502, + "tool_calls": 251, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6817, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.87, + "gate": 1.69, + "review": 1084.25, + "report": 0.06 + }, + "time_to_first_result_s": 1089.85 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 39517 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.md b/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.md new file mode 100644 index 000000000..55e6d7e5c --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] tests/diffusion/attention/test_trtllm_attn.py:315 Missing TRTLLM-GEN kernel skip: reachable on SM103 for int8 (docs: int8 kernels SM100-only); skip is arch-gated not a swallowed assert; GPU test still requires finite output and rel<0.1 when the kernel exists. +- [sweep] trtllm_attn.py:90 _sage_kernel_available HAS_FLASHINFER False → False; SAGE init then raises at :204 rather than silently disabling. Dense path still ImportErrors in forward_cuda if FlashInfer is missing. Both arms occur. +- [sweep] trtllm_attn.py:198/296 quant.enabled (dtype_qk is not None): disabled keeps sage_kwargs empty so older FlashInfer dense calls stay compatible (commit 4a6ab1a0); enabled quantizes then passes sage_attn_sfs + num_elts_per_sage_attn_blk. Both arms occur. +- [sweep] trtllm_attn.py:199 dtype_qk not in {int8, fp8_e4m3}: raises RuntimeError; tested by test_quant_rejects_non_sage_dtype (bfloat16). Arm occurs; fail-loud. +- [sweep] trtllm_attn.py:204 _sage_kernel_available False: raises (no sage_attn_sfs in signature). Arm occurs on old FlashInfer. Fail-loud, not a silent dense fallback. +- [sweep] trtllm_attn.py:211 _sage_quantize_fn is None: raises; test_quant_quantize_requires_flashinfer_routine monkeypatches this arm. Fail-loud. +- [sweep] data.py:1415 invalid dtype_qk/dtype_vo: rejects int4 (test_quant_validation_rejects). Union set {float16,bfloat16,int8,fp8_e4m3} is shared, so TRTLLM+bfloat16 is legal at spec and only fails later at impl. +- [sweep] data.py:1418 block sizes not in {1,4,16}: rejects k_block_size=8 (tested). Always applied even when SAGE is off. + +`vllm_omni/diffusion/data.py:?` [major] — This diff allowlists `quant` on `FLASHINFER_ATTN` (`data.py:1447`) and documents a shared `AttnQuantSpec` that `FLASHINFER_ATTN` "reads" (`dtype_qk` float16/bfloat16, `dtype_vo`, `flashinfer_backend`, "see its section") and that "each backend rejects the rest" / "a dtype_qk from the wrong set raises". `AttnQuantSpec._VALID_DTYPES` is the union `{float16,bfloat16,int8,fp8_e4m3}` (`data.py:1410`), so FLASHINFER+int8 and TRTLLM+bfloat16 pass spec construction. `QuantConfig.from_backend_kwargs` (`trtllm_attn.py:121-127`) only reads `dtype_qk`/`q_block_size`/`k_block_size` and ignores leftover FLASHINFER keys. `FlashInferAttentionImpl.__init__` (`flashinfer_attn.py:55-66`) never binds `backend_kwargs` — `layer.py:109` passes them and they land in `**extra_impl_args` and are dropped with no warning (unlike `SDPAImpl` at `sdpa.py:89-90`). `test_quant_superset_fields_passed_through` only round-trips the dict. Restrict `quant` to `TRTLLM_ATTN` until FLASHINFER consumes it, drop `dtype_vo`/`flashinfer_backend` from the public spec/docs (there is no "its section"), and reject leftover keys in `QuantConfig` rather than claiming backends reject the rest. (evidence: vllm_omni/diffusion/data.py:1447 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` — allowlists FLASHINFER. data.py:1410 `_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})` — union, no per-backend split. vllm_omni/diffusion/attention/layer.py:109 `backend_kwargs=backend_kwargs,` passed into impl. vllm_omni/diffusion/attention/backends/flashinfer_attn.py:55-66 `def __init__(self, num_heads, head_size, softmax_scale, causal=False, num_kv_heads=None, prefix="", **extra_impl_args) -> None: self.causal = causal; self.softmax_scale = softmax_scale` — no `backend_kwargs`/`quant` bind. vllm_omni/diffusion/attention/backends/trtllm_attn.py:121-127 `q = (backend_kwargs or {}).get("quant") or {}; return cls(dtype_qk=q.get("dtype_qk"), q_block_size=int(q.get("q_block_size", 1) or 1), k_block_size=int(q.get("k_block_size", 16) or 16))` — leftover keys ignored. docs/user_guide/diffusion/attention_backends.md:185-188 `The \`quant\` block is a shared \`AttnQuantSpec\` consumed by more than one backend; each reads the fields that apply and rejects the rest. ... \`FLASHINFER_ATTN\` reads \`dtype_qk\` (\`float16\` / \`bfloat16\`), \`dtype_vo\`, and \`flashinfer_backend\` (see its section).` — no FLASHINFER quant section exists in that file. tests/diffusion/attention/test_attention_config.py:65-73 `test_quant_superset_fields_passed_through` only asserts dict round-trip. vllm_omni/diffusion/attention/backends/sdpa.py:89-90 `if backend_kwargs: logger.warning("SDPAImpl ignoring backend_kwargs: %s", list(backend_kwargs.keys()))` — contrast warning FLASHINFER lacks.) + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — This diff inserts `## TRTLLM_ATTN SAGE Quantization` into `docs/user_guide/diffusion/attention_backends.md` and deletes Known Limitations (LTX-2 `CUDNN_ATTN`+torch.compile crash / FLASHINFER|TORCH_SDPA workaround, FA4 note), mixed-backend and FLASH_ATTN examples, Validation Guidance, and the `bench_attention_backends.py` repro. Choosing a Backend Manually still says "see Known Limitations above" (line 232) with no such heading, and Configuration still claims the only typed spec field is Skip-Softmax (lines 72–73) even though SAGE is enabled via a typed `quant`/`AttnQuantSpec` block. Restore those sections (or retarget the bullet) and update the Configuration sentence to include `quant`; SAGE is the case that most needs the deleted "not numerically interchangeable with TORCH_SDPA" guidance. (evidence: docs/user_guide/diffusion/attention_backends.md:72-73 `A backend that needs configuration exposes it as a typed field on the spec. Today the only one is \`TRTLLM_ATTN\`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):` — docs/user_guide/diffusion/attention_backends.md:170 `## TRTLLM_ATTN SAGE Quantization` — docs/user_guide/diffusion/attention_backends.md:174 `Enable it through the typed \`quant\` block on the attention spec:` — docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — file has no `## Known Limitations` / Validation Guidance / mixed-backends / bench repro and ends at docs/user_guide/diffusion/attention_backends.md:381 ` --output outputs/hv15_sage3.mp4` — vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None` — benchmarks/diffusion/bench_attention_backends.py:15 `python benchmarks/diffusion/bench_attention_backends.py --preset hv15`) + +`vllm_omni/diffusion/attention/layer.py:?` [major] — SAGE is applied only in TrtllmAttentionImpl.forward_cuda (`if self.quant.enabled: ... self.quant.quantize(...)` then `**sage_kwargs`). Ring SP never reaches that impl: `_forward_impl` calls `_run_ring_attention` when `use_ring`, which dispatches FA/SDPA ring kernels. `layer.py` already raises NotImplementedError if skip-softmax is configured on that path (so skip cannot be silently dropped); QuantConfig has no matching check, so `quant` + `ring_degree > 1` runs unquantized ring attention while the user believes SAGE is on. Raise the same NotImplementedError when `getattr(self.attention, "quant", None)` is enabled, and extend `test_ring_sp_with_skip_softmax_rejected`. (evidence: layer.py:281-284 `if self.use_ring and strategy is not self._no_parallel_strategy: out = self._run_ring_attention(...)` else `out = self._run_local_attention(...)` (local is `self.attention.forward`). layer.py:320-326 `skip = getattr(self.attention, "skip", None)` / `if skip is not None and getattr(skip, "configured", False): raise NotImplementedError("Skip-Softmax (TRTLLM_ATTN) is not supported with ring sequence parallelism: the ring path bypasses the backend, so the skip config would be silently ignored.")` — no `quant`/`enabled` check. trtllm_attn.py:296-299 `if self.quant.enabled: q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)`. ring.py:130-133 / 157 `if backend_pref in {"sdpa", "torch", "torch_sdpa"}: return ring_pytorch_attn_func(...)` else `return ring_flash_attn_func(...)`. tests/diffusion/attention/test_trtllm_attn.py:175-182 `test_ring_sp_with_skip_softmax_rejected` uses `_impl(target_sparsity=0.5)` only.) + +`tests/diffusion/attention/test_trtllm_attn.py:315` [minor] — `test_sage_quant_matches_sdpa` catches `Missing TRTLLM-GEN kernel` and `pytest.skip`s for both `fp8_e4m3` and `int8` with no arch gate. Docs say int8 kernels are SM100-only (SM103 expected miss), so an SM100 int8 kernel miss is a silent pass. `@requires_sage` already skips capability major<10 (L4/Ada sm_89), so CI cannot fail this GPU path. Numeric bound is also `rel < 0.1` vs BF16 `rel < 0.01`. Skip int8 only off SM100; fail on SM100 if the kernel is missing. (evidence: tests/diffusion/attention/test_trtllm_attn.py:315-316 `if "Missing TRTLLM-GEN kernel" in str(e):` / `pytest.skip(f"no trtllm-gen SAGE kernel for dtype_qk={dtype_qk} on this GPU arch")` — skip is arch-unconditional. tests/diffusion/attention/test_trtllm_attn.py:305 `@pytest.mark.parametrize("dtype_qk", ["fp8_e4m3", "int8"])`. docs/user_guide/diffusion/attention_backends.md:209-210 ``fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103** (B300); `int8` QK kernels are compiled for **SM100 only**.`. tests/diffusion/attention/test_trtllm_attn.py:19-20 `if torch.cuda.get_device_capability()[0] < 10: return False` — L4/Ada never reaches the GPU SAGE body. tests/diffusion/attention/test_trtllm_attn.py:295 `assert rel < 0.01` vs :321 `assert rel < 0.1`.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:204` [minor] — Docs pin FlashInfer ≥0.6.16rc1 for SAGE (`attention_backends.md`), but `pyproject.toml` and `requirements/` have no flashinfer pin (it arrives via vLLM). Dense `forward_cuda` starts `sage_kwargs` empty and only fills it when `self.quant.enabled` (`trtllm_attn.py:295-300`); `__init__` raises at 204-216 if SAGE is requested without `sage_attn_sfs` / `trtllm_sage_attention_quantize`. Missing lower bound is not a dense-path break — either declare ≥0.6.16rc1 as an install bound, or treat this runtime raise as the compatibility contract and say so in the SAGE docs. (evidence: docs/user_guide/diffusion/attention_backends.md:208 `**Requirements.** Needs FlashInfer ≥ 0.6.16rc1.` — docs-only pin. trtllm_attn.py:204-209 `if not _sage_kernel_available(): raise RuntimeError("TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build does not expose the trtllm-gen sage_attn_sfs kernel path. ...")` and :211-216 `if self._sage_quantize_fn is None: raise RuntimeError("... lacks trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). ...")` — SAGE-on-old-FlashInfer is a hard raise. trtllm_attn.py:292-300 `# SAGE kwargs are only understood by newer FlashInfer builds; pass them exclusively when SAGE quant is active ... so the dense path stays compatible with older builds that lack these parameters.` / `sage_kwargs: dict = {}` / `if self.quant.enabled:` — dense path does not pass SAGE kwargs. pyproject.toml and requirements/*.txt contain no `flashinfer` string (grep, zero matches).) + +`tests/diffusion/attention/test_attention_config.py:?` [minor] — `test_quant_and_skip_softmax_coexist` only asserts `backend_kwargs` contains both `target_sparsity` and `quant.dtype_qk`. `forward_cuda` still always passes `skip_softmax_threshold_scale_factor=_skip_factor` and, when `quant.enabled`, `**sage_kwargs` (`sage_attn_sfs` / `num_elts_per_sage_attn_blk`) into `trtllm_ragged_attention_deepseek`, but no test calls that path with both set: `test_sage_quant_matches_sdpa` uses only `quant={dtype_qk,q_block_size,k_block_size}`, and skip CUDA tests use only `skip_softmax_threshold`. Add a composition test (monkeypatch the kernel, or a CUDA numerical that skips on Missing TRTLLM-GEN kernel) that the kernel receives both SAGE kwargs and a non-None skip factor. (evidence: tests/diffusion/attention/test_attention_config.py:75-79 `def test_quant_and_skip_softmax_coexist(self):` / `bk = AttentionSpec(backend="TRTLLM_ATTN", quant={"dtype_qk": "int8"}, skip_softmax={"target_sparsity": 0.5}).backend_kwargs()` / `assert bk["target_sparsity"] == 0.5 and bk["quant"]["dtype_qk"] == "int8"` — dict-only, no kernel. vllm_omni/diffusion/attention/backends/trtllm_attn.py:290 `_skip_factor = self._resolve_skip_factor(kv_len)` ; 296-299 `if self.quant.enabled:` then `sage_kwargs["sage_attn_sfs"] = sage_attn_sfs` / `sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes` ; 319-320 `skip_softmax_threshold_scale_factor=_skip_factor,` / `**sage_kwargs,`. tests/diffusion/attention/test_trtllm_attn.py:311 `impl = _impl(quant={"dtype_qk": dtype_qk, "q_block_size": 1, "k_block_size": 16})` — SAGE CUDA test has no skip_softmax keys; no other test monkeypatches `trtllm_ragged_attention_deepseek` with both.) + +`recipes/Wan-AI/Wan2.2-T2V.md:?` [minor] — This PR adds typed SAGE quantization for TRTLLM_ATTN (quant.dtype_qk int8/fp8_e4m3) and documents it on Wan-AI/Wan2.2-T2V-A14B-Diffusers, but recipes/Wan-AI/Wan2.2-T2V.md:186 still says "TRTLLM_ATTN is BF16-only (no attention-level quantization)". That bullet is now false. Replace it with the SAGE contract or a pointer to docs/user_guide/diffusion/attention_backends.md. (evidence: recipes/Wan-AI/Wan2.2-T2V.md:186 `- `TRTLLM_ATTN` is BF16-only (no attention-level quantization).` — stale under the same Wan2.2-T2V-A14B TRTLLM_ATTN section. vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None`. vllm_omni/diffusion/data.py:1447 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):`. vllm_omni/diffusion/attention/backends/trtllm_attn.py:108-110 `_QK_QUANT_DTYPES = { "int8": torch.int8, "fp8_e4m3": torch.float8_e4m3fn, }`. docs/user_guide/diffusion/attention_backends.md:172-173 `TRTLLM_ATTN can run **SAGE** (SageAttention) — the trtllm-gen FMHA kernel with per-block quantized Q/K and per-channel fp8 V`. docs/user_guide/diffusion/attention_backends.md:191-193 `vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers` with `"quant": {"dtype_qk": "fp8_e4m3", ...}`.) + +`tests/diffusion/attention/test_trtllm_attn.py:74` [minor] — SAGE init/packing tests added to `tests/diffusion/attention/test_trtllm_attn.py` never run in CI: the file is module-marked `cuda` (L2 `pytest tests/diffusion -m 'core_model and cpu'` deselects it) and Diffusion Other Test uses `pytest tests/diffusion/*.py` (does not recurse into `attention/`; `.buildkite` has no `tests/diffusion/attention` path). `test_quant_quantize_calls_routine_and_shapes_sfs` only asserts a fake `quantize_fn` packing and never calls `forward_cuda`, so `sage_attn_sfs` / `num_elts_per_sage_attn_blk` at trtllm_attn.py:296-320 are untested. Move CPU-safe SAGE tests into cpu-marked `test_attention_config.py` (already collected), monkeypatch `trtllm_ragged_attention_deepseek` to assert those kwargs, and keep `test_sage_quant_matches_sdpa` as SM100-only (`_has_trtllm_attn` requires capability major>=10; L4 CI is `l4_1`). (evidence: test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]`. .buildkite/cuda/test-ready.yml:18 `pytest -sv tests/diffusion -m 'core_model and cpu'`. .buildkite/cuda/test-ready.yml:68 `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` with :69 `mirror_hardwares: l4_1`. tests/diffusion/*.py is only top-level files (e.g. test_attention_compile.py), not tests/diffusion/attention/. test_trtllm_attn.py:79-85 `return "qq", "kq", "vq", "qsfs", "ksfs", "vsfs"` then `QuantConfig(dtype_qk="int8").quantize(..., fake_quantize)` asserting `("qq", "kq", "vq", ("qsfs", "ksfs", None, "vsfs"), (1, 16, 0, 1))` with no forward_cuda. trtllm_attn.py:296-320 `if self.quant.enabled: ... sage_kwargs["sage_attn_sfs"] = sage_attn_sfs; sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes` then `trtllm_ragged_attention_deepseek(..., **sage_kwargs)`. test_attention_config.py:34 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]`. test_sage_attn3.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]`. test_trtllm_attn.py:19 `if torch.cuda.get_device_capability()[0] < 10: return False`; :298-300 `requires_sage = pytest.mark.skipif(not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None), ...)`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.trace.json.gz new file mode 100644 index 000000000..cf8fb4771 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r2/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.cost.json b/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.cost.json new file mode 100644 index 000000000..13c777d9d --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1179.3, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r2/runs/pr5550/run-20260814-203314-9d6041", + "recorded_at": "2026-08-14T20:52:52", + "attempts": [ + { + "invocation_id": "inv-f9d2579e", + "outcome": "run:run-20260814-203314-9d6041", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-203314-9d6041", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-203314-9d6041", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4914, + "minutes": 19.63, + "ci_minutes": 0, + "input_tokens": 1144941, + "output_tokens": 165658, + "tool_calls": 373, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7272, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.5, + "gate": 1.72, + "review": 1172.36, + "report": 0.09 + }, + "time_to_first_result_s": 1177.63 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 36296 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.md b/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.md new file mode 100644 index 000000000..ae6b3f277 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] tests/diffusion/diffusion_kv/test_metadata.py:37-38 sequences[0]/contexts[0] — constructor round-trip on a 1-element tuple the test itself built; no producer/consumer; safe only as a shape smoke. +- [sweep] test_metadata.py:68,109 sequences[0].context_ids — same fixture assert; documents CFG shared vs branch-specific context_ids but does not enter a CFG path. +- [sweep] test_worker_contract.py:221 calls[0][1][0].prepared_layout — execute_request ran one new_req so index 0 is the dense RPC; prepared_layout identity is the object the test attached. +- [sweep] diffusion_engine.py:199 scheduler_to_close is not None — reachable if StepScheduler()/RequestScheduler() raises before assignment; _init_scheduler assigns self.scheduler before initialize(), so initialize() failure always closes; arm is defensive not dead. +- [sweep] multiproc_executor.py:545 metadata is not None — live; test_request_rpc_sends_attached_metadata_for_only_that_request builds a 4-tuple only for that request. +- [sweep] interface.py:184 envelope vs forwarded id — live; test_request_rpc_rejects_envelope_request_identity_mismatch; raises before collective_rpc so execute_request cannot wrap it as DiffusionOutput(error=...). +- [sweep] interface.py:192 metadata.request_id vs forwarded id — live; runner tests cover stale metadata.request_id. +- [sweep] diffusion_model_runner.py:172 dense rejects metadata — live; test_dense_request_rejects_metadata_before_request_forward. + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — This diff adds `and not has_diffusion_kv_metadata` to the DLO+AllGather multi-request predicate in MultiprocDiffusionExecutor.execute_request (vllm_omni/diffusion/executor/multiproc_executor.py:469-473). Any attached snapshot skips the list RPC (`args=(reqs_list, od_config, kv_prefetch_job)` at 506-509, no metadata slot) and falls through to per-request 4-tuples (544-546). That gate is untested: `make_executor()` hard-codes `enable_distributed_layerwise_offload=False` (tests/diffusion/diffusion_kv/test_worker_contract.py:62-64), and DLO concurrency tests build NewRequestData without metadata so the field stays at its None default (vllm_omni/diffusion/sched/interface.py:165; tests/diffusion/test_multiproc_engine_concurrency.py:200-207). Deleting the new conjunct would not fail current tests, and later attached snapshots would either serialize the DLO wave or, if the guard is removed, drop allocations on the list RPC. Add a CPU test with DLO+AllGather, `len(new_reqs)>1`, and attached metadata that asserts `collective_rpc` is two per-request 4-tuples rather than one list RPC. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)`; :469-473 `if (\n len(new_reqs) > 1\n and not has_diffusion_kv_metadata\n and getattr(self.od_config, "enable_distributed_layerwise_offload", False)\n and getattr(self.od_config, "dlo_use_allgather", True)\n ):`; :506-509 list RPC `args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job)` with no metadata; :544-546 `if new_req.diffusion_kv_metadata is not None:\n args += (new_req.diffusion_kv_metadata,)`. tests/diffusion/diffusion_kv/test_worker_contract.py:62-64 `executor.od_config = SimpleNamespace(\n enable_distributed_layerwise_offload=False,\n dlo_use_allgather=True,\n )`. vllm_omni/diffusion/sched/interface.py:165 `diffusion_kv_metadata: DiffusionKVMetadata | None = None`. tests/diffusion/test_multiproc_engine_concurrency.py:200-207 `_make_sched_output` constructs `NewRequestData(request_id=rid, req=...)` with no metadata kwarg.) + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [minor] — DiffusionWorker.execute_model builds kwargs and adds `diffusion_kv_metadata` only when it is not None, then calls `self.model_runner.execute_model(req, **kwargs)` (vllm_omni/diffusion/worker/diffusion_worker.py:462-465). ARDiffusionModelRunner.execute_model (vllm_omni/experimental/ar_diffusion/runner.py:237-241) is the only in-repo DiffusionModelRunner override and still accepts only `(req, kv_prefetch_job=None)`, so a request-mode RPC that carries metadata TypeErrors on unexpected keyword before `super().execute_model`. Accept `diffusion_kv_metadata=None` (or `**kwargs`) and forward it to `super()`. DreamZero/AR-diffusion request mode selects this runner via ARDiffusionEngine.default_diffusion_model_runner_cls. Batch/step overrides only raise and are unaffected. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `output = self.model_runner.execute_model(req, **kwargs)`. vllm_omni/experimental/ar_diffusion/runner.py:237-241 `def execute_model(` / `self,` / `req: OmniDiffusionRequest,` / `kv_prefetch_job: KVPrefetchJob | None = None,` / `) -> DiffusionOutput:` — no `diffusion_kv_metadata` and no `**kwargs`. Only DiffusionModelRunner subclass in-repo; ARDiffusionEngine.default_diffusion_model_runner_cls points here.) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:?` [minor] — Live request RPC never calls WorkerWrapperBase.execute_model. WorkerProc._execute_rpc does `self.worker.execute_method(method, *args)` (diffusion_worker.py:1036), and execute_method always does `func = getattr(self.worker, method); return func(*args)` (1442-1443), so the executor 4-tuple `(req, od_config, kv_prefetch_job, metadata)` (multiproc_executor.py:544-546) binds onto DiffusionWorker.execute_model. `test_worker_wrapper_only_extends_paged_request_rpc` instead calls `wrapper.execute_model(..., diffusion_kv_metadata=metadata)` as a keyword (test_worker_contract.py:258-259), and `test_dense_worker_call_does_not_extend_model_runner_signature` only drives the no-metadata 2-arg arm (line 288). A DiffusionWorker that dropped the 4th positional would not fail these tests. Drive `wrapper.execute_method("execute_model", req, od_config, None, metadata)` (or DiffusionWorker.execute_model with that 4-tuple) and assert the runner kwargs include `diffusion_kv_metadata`. (evidence: tests/diffusion/diffusion_kv/test_worker_contract.py:258-259 `wrapper.execute_model(req, od_config)` / `wrapper.execute_model(req, od_config, diffusion_kv_metadata=metadata)` — keyword call on the wrapper method, not execute_method. vllm_omni/diffusion/worker/diffusion_worker.py:1036 `result = self.worker.execute_method(method, *args, **kwargs)` — live RPC entry. Same file:1442-1443 `func = getattr(self.worker, method)` / `return func(*args, **kwargs)` — lookup is on the inner worker, skipping WorkerWrapperBase.execute_model. Same file:855-866 `_create_worker` returns WorkerWrapperBase wrapping DiffusionWorker. vllm_omni/diffusion/executor/multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None:` / `args += (new_req.diffusion_kv_metadata,)` — live RPC sends a positional 4-tuple. tests/diffusion/diffusion_kv/test_worker_contract.py:288 `worker.execute_model(req, SimpleNamespace())` — dense path only exercises the no-metadata arm.) + +`tests/diffusion/diffusion_kv/test_metadata.py:37` [minor] — test_metadata.py:37-38 only round-trips constructor-built `sequences[0].block_ids`/`contexts[0].block_ids`; the CFG share/branch tests at 68-69 and 109-110 likewise assert `context_ids` on objects this file constructed. `context_ids` is unused outside the dataclass and this file. `test_worker_contract.make_metadata` (27-40) already sends metadata through `NewRequestData`/`execute_request`, but with empty `context_ids` and no `contexts`, so a path that dropped or reshaped CFG share/branch contexts would not fail CI. Keep this file as W0 constructor shape-smoke, or add one RPC test that attaches a shared or branch-specific `contexts` payload. (evidence: tests/diffusion/diffusion_kv/test_metadata.py:33-38 `sequences=(sequence,),` / `contexts=(context,),` then `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])` / `assert metadata.contexts[0].block_ids == ([7], [11])` — 1-element constructor round-trip, no producer/consumer. tests/diffusion/diffusion_kv/test_metadata.py:67-69 `assert metadata.contexts == (shared_context,)` / `assert metadata.sequences[0].context_ids == (shared_context.context_id,)` / `assert metadata.sequences[1].context_ids == (shared_context.context_id,)` — CFG share never leaves the constructor. tests/diffusion/diffusion_kv/test_worker_contract.py:27-40 `sequences=(DiffusionKVSequenceMetadata(..., block_ids=([1, 2],),),),` with no `contexts=`/`context_ids=` (defaults to `()`). tests/diffusion/diffusion_kv/test_worker_contract.py:231-239 `NewRequestData(..., diffusion_kv_metadata=metadata_0)` then `assert [call[1] for call in calls] == [(req_0, executor.od_config, None, metadata_0), ...]` — RPC forwards that sequences-only object.) + +`vllm_omni/diffusion/sched/base_scheduler.py:114` [minor] — Live producer still does `scheduled_new_reqs.append(NewRequestData.from_state(state))` with no metadata (W0-intentional: runner says allocator/installer are unwired). `from_state` defaults `diffusion_kv_metadata=None`, so production RPCs stay the historical 3-tuple and `and not has_diffusion_kv_metadata` never skips DLO until W1 attaches snapshots. Confirm allocator/geometry attach is linked-issue remainder, not an accidental omission from this wire. (evidence: base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — only live producer, no metadata kwarg. interface.py:165/172 `diffusion_kv_metadata: DiffusionKVMetadata | None = None` (field and from_state default). multiproc_executor.py:544-546 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` then `if new_req.diffusion_kv_metadata is not None: args += (new_req.diffusion_kv_metadata,)` — live None keeps the 3-tuple. multiproc_executor.py:462/471 `has_diffusion_kv_metadata = any(...)` and `and not has_diffusion_kv_metadata` — skip is live-unreachable while producer never attaches. diffusion_model_runner.py:170-171 `# W0 does not wire the Scheduler allocator or Worker installer yet, so validate an allocation snapshot when present without requiring one.`) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:88` [minor] — Paged mode documents accepting an allocation snapshot when present without requiring one (`_validate_diffusion_kv_metadata` at vllm_omni/diffusion/worker/diffusion_model_runner.py:170-172; `execute_model` at 609-612). `test_paged_request_without_metadata_reaches_request_forward` only calls `execute_model(req)` with no metadata (tests/diffusion/diffusion_kv/test_worker_contract.py:98). Sibling tests cover dense+metadata rejection (108-111) and request_id mismatch (121-123). No test supplies matching metadata to `DiffusionModelRunner.execute_model` and asserts `_execute_request_list` still runs. Add `runner.execute_model(req, diffusion_kv_metadata=make_metadata())` under `PAGED_SCHEDULER` so present metadata cannot start being required or dropped without a failing test. (evidence: tests/diffusion/diffusion_kv/test_worker_contract.py:98 `assert runner.execute_model(req) is expected_output` — paged happy path omits metadata. tests/diffusion/diffusion_kv/test_worker_contract.py:108-111 `runner.execute_model(SimpleNamespace(request_id="req-0"), diffusion_kv_metadata=make_metadata())` is DENSE_LEGACY reject only. tests/diffusion/diffusion_kv/test_worker_contract.py:121-123 `diffusion_kv_metadata=make_metadata("stale")` is mismatch reject only. vllm_omni/diffusion/worker/diffusion_model_runner.py:171-172 `# validate an allocation snapshot when present without requiring one.` vllm_omni/diffusion/worker/diffusion_model_runner.py:609-612 `self._validate_diffusion_kv_metadata(request_id=req.request_id, metadata=diffusion_kv_metadata)` then `_execute_request_list`.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:460` [minor] — execute_request validates every envelope with validate_new_request_data_identity before selecting a dispatch path (multiproc_executor.py:454-460) so identity errors stay outside the RPC error wrapper. execute_batch still forwards the whole scheduler_output on the fused path (len(scheduled_new_reqs)>1, not DLO DP AllGather) via collective_rpc("execute_model_batch", args=(scheduler_output, self.od_config)) at :615-619, and execute_step always does collective_rpc("execute_stepwise", args=(scheduler_output,)) at :651-655. Identity is only checked after RPC in DiffusionModelRunner.execute_model_batch (:636-641) and execute_stepwise (:750-755). Call validate_new_request_data_identity on scheduled_new_reqs in execute_batch and execute_step before collective_rpc. (evidence: multiproc_executor.py:454-460 `# Validate every envelope before selecting a dispatch path. In particular, keep identity errors outside the RPC error wrapper so an invalid request cannot be forwarded or converted into a worker output.` then `for new_req in new_reqs: validate_new_request_data_identity(new_req)`. multiproc_executor.py:600-601 `if len(scheduler_output.scheduled_new_reqs) <= 1: return self.execute_request(scheduler_output)` — only the serial wave reuses that pre-RPC check. multiproc_executor.py:615-619 `result = self.collective_rpc("execute_model_batch", args=(scheduler_output, self.od_config), unique_reply_rank=0, exec_all_ranks=True)` — no validate_new_request_data_identity before RPC. multiproc_executor.py:651-655 `result = self.collective_rpc("execute_stepwise", args=(scheduler_output,), unique_reply_rank=0, exec_all_ranks=True)` — no pre-RPC identity check. diffusion_model_runner.py:636-641 `for new_req in scheduler_output.scheduled_new_reqs: validate_new_request_data_identity(new_req)` inside execute_model_batch (worker, after RPC). diffusion_model_runner.py:750-755 same loop inside execute_stepwise (worker, after RPC).) + +`vllm_omni/diffusion/sched/interface.py:?` [minor] — validate_new_request_data_identity’s second branch (interface.py:192) rejects metadata.request_id != forwarded req.request_id, and execute_request runs it before the per-request try/except that wraps failures as DiffusionOutput(error=...) (multiproc_executor.py:459-460 vs 574-581). test_request_rpc_rejects_envelope_request_identity_mismatch only mismatches envelope vs req (envelope-id vs actual-request-id), so the first branch raises first; test_paged_request_rejects_mismatched_metadata_identity hits the runner check and never asserts executor calls == []. Add an executor test with matching NewRequestData.request_id and req.request_id but a different metadata.request_id, expecting ValueError and no collective_rpc. (evidence: vllm_omni/diffusion/sched/interface.py:184 `if new_req.request_id != forwarded_request_id:` — first branch. vllm_omni/diffusion/sched/interface.py:192 `if metadata is not None and metadata.request_id != forwarded_request_id:` — second branch, reached only if envelope IDs already match. vllm_omni/diffusion/executor/multiproc_executor.py:459-460 `for new_req in new_reqs:` / `validate_new_request_data_identity(new_req)` — before RPC. vllm_omni/diffusion/executor/multiproc_executor.py:574-580 `except Exception as exc:` then `result=DiffusionOutput(error=str(exc)),` — RPC failures become worker output. tests/diffusion/diffusion_kv/test_worker_contract.py:131-135 `req = SimpleNamespace(request_id="actual-request-id")` then `NewRequestData(request_id="envelope-id", req=req, diffusion_kv_metadata=make_metadata("envelope-id"))` — first branch only. tests/diffusion/diffusion_kv/test_worker_contract.py:121-124 `runner.execute_model(SimpleNamespace(request_id="req-0"), diffusion_kv_metadata=make_metadata("stale"))` — runner path, not executor; only executor `assert calls == []` is line 141 of the envelope-mismatch test.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.trace.json.gz new file mode 100644 index 000000000..096b81175 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r2/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.cost.json b/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.cost.json new file mode 100644 index 000000000..a35d84bfa --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1280.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r2/runs/pr5610/run-20260814-203313-b8a133", + "recorded_at": "2026-08-14T20:54:33", + "attempts": [ + { + "invocation_id": "inv-1221e96f", + "outcome": "run:run-20260814-203313-b8a133", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-203313-b8a133", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-203313-b8a133", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3355, + "minutes": 21.33, + "ci_minutes": 0, + "input_tokens": 837268, + "output_tokens": 99478, + "tool_calls": 264, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.684, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.62, + "gate": 2.22, + "review": 1274.14, + "report": 0.06 + }, + "time_to_first_result_s": 1280.02 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 21012 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.md b/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.md new file mode 100644 index 000000000..5bd565585 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] 1. Correctness: `_should_use_async_omni_output()` at gpu_ar_model_runner.py:1595-1619 matches the compatibility table (async scheduling, prefix cache, spec decode, async_chunk, routed experts, opt-in, eager postprocess). +- [validated] 2. Simplifiability: no new runtime branches; docs-only. +- [validated] 3. Breaking behavior: no public CLI/YAML flag added; use_async_omni_output remains a model contract (omni_async_output_materialization.md:362-365). +- [validated] 4. Rebase/merge: nav.yml inserts the page beside async_chunk.md:120-121; design/index.md:13 adds the index link; no dropped related-file paths. +- [validated] 5. Tests: documented tests exist in tests/worker/test_gpu_ar_model_runner.py (guard, snapshot, connector-after-build, background reraise); docs-only profile skipped pytest. mkdocs --strict is blocked by the pre-existing qwen-asr/transformers pin noted in the PR body; CI gates are clean. +- [sweep] 6. Docs staleness: new page links to async_chunk.md:15-18, but async_chunk.md Related Files (line 262) has no reverse pointer. +- [validated] 7. Invariants: prefix-cache incompatibility (gpu_ar_model_runner.py:1598-1599), connector drain after payload build (gpu_ar_model_runner.py:1899-1900), and unreachable full-payload branch under async_chunk (omni_connector_model_runner_mixin.py:60-61) are documented and match code. +- [validated] 8. Scope: changed files are the feature page, nav, index, and SVG; stated purpose is delivered. Prior threads on platform scope, live connector state, image path, SVG accumulation, and prefix cache rechecked at 50126edf and are fixed. + +`docs/design/feature/omni_async_output_materialization.md:110` [minor] — The c=64 table (+22% throughput, -4% TTFP, -25% RTF from 9.3→11.3 req/s, 655→631 ms, 0.63→0.47) is arithmetically consistent, but the page cites only the Qwen3-Omni blog for methodology and results; these figures do not appear in any in-repo result JSON. If the docs bar requires a reproducible in-repo artifact, cite it here; otherwise the blog citation is the sole evidence. (evidence: docs/design/feature/omni_async_output_materialization.md:106 `results at concurrency 64:` +docs/design/feature/omni_async_output_materialization.md:110 `| CUDA Graph + async chunk | 9.3 req/s | 655 ms | 0.63 |` +docs/design/feature/omni_async_output_materialization.md:111 `| + async output materialization | 11.3 req/s | 631 ms | 0.47 |` +docs/design/feature/omni_async_output_materialization.md:112 `| Change | **+22%** | **-4%** | **-25%** |` +docs/design/feature/omni_async_output_materialization.md:119 `[Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization)` +docs/design/feature/omni_async_output_materialization.md:120 `for the complete benchmark methodology and results.` — 11.3/9.3-1=+21.5%≈+22%; (631-655)/655=-3.7%≈-4%; (0.47-0.63)/0.63=-25.4%≈-25%. Repo-wide grep of `9.3 req/s`, `11.3 req/s`, and `655 ms` hits only this file; the only `*result*.json` files are `tests/dfx/perf/scripts/result_omni_template.json` and `diffusion_result_template.json`.) + +`docs/design/feature/async_chunk.md:262` [minor] — The new page links to async_chunk.md (Overview line 16 and Related Files line 382), but async_chunk.md Related Files (line 262) has no reverse pointer. Add a backlink to omni_async_output_materialization.md so the related-files lists stay bidirectional. (evidence: omni_async_output_materialization.md:16 `It complements [Async Chunk](async_chunk.md): async chunk pipelines partial`; omni_async_output_materialization.md:382 `- [Async Chunk](async_chunk.md): Inter-stage chunking and scheduling design.`; async_chunk.md:262 `## Related Files` through async_chunk.md:275 `- \`vllm_omni/config/model.py\`: Model config with async_chunk field` — the list is only Python modules and never mentions omni_async_output_materialization.md.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.trace.json.gz new file mode 100644 index 000000000..29bddf199 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r2/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.cost.json b/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.cost.json new file mode 100644 index 000000000..90c2e4a64 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1081.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r2/runs/pr5703/run-20260814-205124-1d05ff", + "recorded_at": "2026-08-14T21:09:25", + "attempts": [ + { + "invocation_id": "inv-61dba213", + "outcome": "run:run-20260814-205124-1d05ff", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-205124-1d05ff", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-205124-1d05ff", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5719, + "minutes": 18.02, + "ci_minutes": 0, + "input_tokens": 1461262, + "output_tokens": 161248, + "tool_calls": 346, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7317, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.67, + "gate": 2.16, + "review": 1075.03, + "report": 0.06 + }, + "time_to_first_result_s": 1080.91 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 27609 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.md b/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.md new file mode 100644 index 000000000..6df5ff8e9 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] pipeline_minimax_h3.py:958-963 `_encode_visual_condition` and :1040-1066 `_encode_video_conditions` call the changed methods on rank 0 (or all VPP ranks when distributed); no extra load_to_device around VAE encode, so `parameter.device` at encode time is whatever residency CPU-offload left. +- [validated] recipes/MiniMaxAI/MiniMax-H3-NPU.md:117 shows the correct follow-up `export MODEL="${MODEL_ROOT}/Ref2VA"`; CUDA MiniMax-H3.md:168 does the same. The new MUSA recipe omits that follow-up. +- [validated] vae.py:172-176 still seeds CPU `default_generator` inside `fork_rng`, then device_module seed; that matches the old CUDA structure and is what MUSA validation claims restores CPU+device state. +- [upstream-verify] consumers named in changed_symbol_consumers for encode_image/encode_video outside minimax_h3 are different classes; MiniMax callers are pipeline_minimax_h3.py:963 and :1066 plus the contract test fake. +- [sweep] vae.py:120 stores torch.get_device_module() with no device arg; only production call site in the repo. +- [sweep] vae.py:171 and vae.py:215 replace parameter.device.type == "cuda" with != "cpu" in encode_image/encode_video; both still wrap torch.random.fork_rng + device_module.manual_seed. +- [sweep] recipes/MiniMaxAI/MiniMax-H3-MUSA.md:1-147 new MUSA recipe; serve block at 72-93. +- [sweep] recipes/README.md:48-49 adds MiniMax-H3-MUSA.md row and rewords the CUDA MiniMax-H3.md summary; table link resolves. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This diff replaces the CUDA-only gate with `parameter.device.type != "cpu"` in both `encode_image` (vae.py:170) and `encode_video` (vae.py:214). That is broader than CUDA/MUSA: Ascend uses `torch.device("npu")` (platforms/npu/platform.py:163), `MiniMaxH3VideoVAE` is constructed with `device=self.device` (pipeline_minimax_h3.py:547), and rank-0 FL2VA/Ref2VA still call `encode_image`/`encode_video` (pipeline_minimax_h3.py:963, :1066) on the published NPU recipe. Previously NPU used `fork_rng(devices=[])` and only seeded `default_generator`; now it enters `fork_rng(devices=[npu:...])` plus `self.device_module.manual_seed`. Restrict the gate to `device.type in {"cuda", "musa"}` (and pass `device_type` into `fork_rng` if MUSA needs it) so other accelerators keep the old CPU-seed path until hardware-validated. (evidence: vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` — same at vae.py:214. vae.py:120 `self.device_module = torch.get_device_module()`. vae.py:172-176 `with torch.random.fork_rng(devices=devices):` then `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`. platforms/npu/platform.py:163 `return torch.device("npu")`. pipeline_minimax_h3.py:547 `device=self.device` (MiniMaxH3VideoVAE). pipeline_minimax_h3.py:963 `rows = self.video_vae.encode_image(image) if rank == 0 else None`. pipeline_minimax_h3.py:1066 `self.video_vae.encode_video(load_video_frames(item["prepared_path"]))`. recipes/MiniMaxAI/MiniMax-H3-NPU.md is present as a published NPU serving path.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — The only serve command is introduced as the validated Ref2VA TP4 profile (`MiniMax-H3-MUSA.md:72-73`) but then `export MODEL="${MODEL_ROOT}/FL2VA"` (`:76`). The same page already says one server loads one partition and Ref2VA must set `MODEL` to `Ref2VA` (`:69-70`); FL2VA cannot serve that profile. CUDA `MiniMax-H3.md:168` and NPU `MiniMax-H3-NPU.md:117` keep FL2VA for the first serve command and restart with `export MODEL="${MODEL_ROOT}/Ref2VA"`. Point this command at Ref2VA, or relabel the block as FL2VA and add that partition switch so the documented TP4 command matches the claimed validation. (evidence: MiniMax-H3-MUSA.md:72-73 `The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs and offloads inactive model components to CPU:`; MiniMax-H3-MUSA.md:76 `export MODEL="${MODEL_ROOT}/FL2VA"`; MiniMax-H3-MUSA.md:69-70 `One server loads one checkpoint partition. Set \`MODEL\` to \`FL2VA\` for T2VA and FL2VA, or to \`Ref2VA\` for Ref2VA.`; MiniMax-H3-MUSA.md:11 `- Hardware: 4x Moore Threads MTT S5000 for the validated Ref2VA profile`; MiniMax-H3.md:168 `export MODEL="${MODEL_ROOT}/Ref2VA"` after `To serve Ref2VA, stop the FL2VA server and restart the same one- or four-GPU command`; MiniMax-H3-NPU.md:117 `export MODEL="${MODEL_ROOT}/Ref2VA"` after the FL2VA serve block. MUSA recipe has no Ref2VA MODEL export.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — This diff adds `self.device_module = torch.get_device_module()` with no device argument in `MiniMaxH3VideoVAE.__init__` (vae.py:120), then uses that captured module for `device_module.device(device)` / `manual_seed` on whatever `parameter.device` encode picks (vae.py:175-176 and encode_video at 219-220). `__init__` already takes `device: torch.device` and `MiniMaxH3Pipeline` passes `device=self.device` (pipeline_minimax_h3.py:545-548). Bind with `torch.get_device_module(device)` (or `get_device_module(parameter.device)` at encode time) so a MUSA VAE cannot seed through `torch.cuda` if the process default accelerator differs. (evidence: vae.py:120 `self.device_module = torch.get_device_module()`. vae.py:104 `device: torch.device`. pipeline_minimax_h3.py:545-548 `self.video_vae = MiniMaxH3VideoVAE(\n os.path.join(model_path, "video_vae"),\n device=self.device,\n )`. vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`. vae.py:219-220 same pair in encode_video.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — This diff widens conditioned VAE RNG selection from `parameter.device.type == "cuda"` to `!= "cpu"` and seeds via `self.device_module.manual_seed` (vae.py:170-176 and :214-220) but adds no tests of that branch. The only in-tree `encode_video` unit test (`test_video_vae_encode_uses_configured_parallel_tiling`) uses `object.__new__(MiniMaxH3VideoVAE)` so `__init__` never assigns `device_module`, plus a CPU `Parameter` (test_minimax_h3_contract.py:335, :344-353), so `devices=[]` and the new loop never runs. `encode_image` has no tests under tests/diffusion/models/minimax_h3/; `test_minimax_h3_vae.py` does not exist; tests/ has no `device_module` or `fork_rng` hits. Extend that test (keep the CPU/core_model case) with stubs for `fork_rng`/`device_module`, parametrize `parameter.device.type` across cpu/cuda/musa/npu so a too-wide gate fails, and assert non-cpu devices enter `fork_rng` with that device, `manual_seed(42)`, and CPU RNG restore. Guard: `pytest -q -c /dev/null tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py -k 'video_vae'`. (evidence: vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` (same gate at vae.py:214). vae.py:176 `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` (same seed at vae.py:220). test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))`. test_minimax_h3_contract.py:344 `video_vae = object.__new__(MiniMaxH3VideoVAE)`. test_minimax_h3_contract.py:353 `rows, shape = video_vae.encode_video("frames")` — CPU Parameter => devices=[], new seed loop never runs. tests/diffusion/models/minimax_h3 has no test_minimax_h3_vae.py; grep encode_image there and grep device_module/fork_rng under tests/ returned no matches.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — `torch.get_device_module()` at vae.py:120 is the first in-repo use. `requirements/cuda.txt`, `requirements/musa.txt`, and `requirements/common.txt` do not pin a torch lower bound (setup.py loads platform extras dynamically). The NPU recipe documents PyTorch 2.10.0+cpu; this checkout does not state which minimum torch (including ROCm) exposes `get_device_module`. Which minimum torch is this call safe on? (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — sole in-repo use. requirements/cuda.txt is only `-r common.txt`, `onnxruntime>=1.23.2`, `fa3-fwd==0.0.3` (no torch pin). requirements/musa.txt has `torchada>=0.1.52` and no torch pin. recipes/MiniMaxAI/MiniMax-H3-NPU.md:42 `- PyTorch: 2.10.0+cpu`. vae.py:172 `with torch.random.fork_rng(devices=devices):` — no `device_type=`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [nit] — Head CI lists `buildkite/vllm-omni-amd-ci` red. This diff adds `torch.get_device_module()` in `MiniMaxH3VideoVAE.__init__` (vae.py:120), and `.buildkite/amd/test-amd-ready.yml:19` runs `tests/diffusion -m core_model and cpu`, which includes that constructor. Reviewers did not attribute the red check to this PR (ROCm still uses device.type `cuda`) and did not have logs. Can you confirm the red AMD job is unrelated to this constructor call? (evidence: gate: buildkite/vllm-omni-amd-ci is red; .buildkite/amd/test-amd-ready.yml:19 runs tests/diffusion -m core_model and cpu, which includes MiniMaxH3VideoVAE().__init__ now calling get_device_module().) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.trace.json.gz new file mode 100644 index 000000000..463cdcf5b Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r2/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.cost.json b/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.cost.json new file mode 100644 index 000000000..fa4499478 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 394.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r2/runs/pr5715/run-20260814-205252-f3e45a", + "recorded_at": "2026-08-14T20:59:26", + "attempts": [ + { + "invocation_id": "inv-292860ab", + "outcome": "run:run-20260814-205252-f3e45a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-205252-f3e45a", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-205252-f3e45a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0518, + "minutes": 6.56, + "ci_minutes": 0, + "input_tokens": 94452, + "output_tokens": 23944, + "tool_calls": 38, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2457, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.69, + "gate": 1.83, + "review": 387.64, + "report": 0.07 + }, + "time_to_first_result_s": 393.22 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 11939 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.md b/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.md new file mode 100644 index 000000000..0714eb20e --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.md @@ -0,0 +1,39 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Enumerated paths: README 0.26 news/cadence/skills removal; feature list full-duplex; model examples MiniCPM-o 4.5 + MiniMax H3; docs/README mirror; config URL 0.16→0.26; install 0.26.x↔0.26.x note; cuda.inc matching-minor prose + docker BASE_IMAGE 0.22.1→0.26.0; npu.inc vLLM/Ascend pins + cd path. +- [validated] README.md:19 0.26.0 release sentence + MiniMax H3 recipe header (joint video/audio diffusion). +- [validated] README.md feature list last item experimental full-duplex; DESIGN.md:365 `The full-duplex implementation remains experimental:`; examples/online_serving/minicpmo/README.md:40 duplex overlay. +- [validated] docs/README.md feature/model lists match README.md. +- [validated] docs/configuration/README.md:5 is the only `docs.vllm.ai/en/v0.*` configuration link. +- [validated] docs/getting_started/installation/README.md:4 0.26.x↔0.26.x note. +- [validated] cuda.inc.md:127 BASE_IMAGE matches docker/Dockerfile.cuda:1. +- [validated] npu.inc.md `cd vllm-omni` after clone is correct; vLLM clone is `v0.26.0`. + +`README.md:?` [minor] — This diff’s 0.26.0 news bullet advertises “an experimental full-duplex realtime runtime for MiniCPM-o 4.5” and points that claim at recipes/OpenBMB/MiniCPM-o-4_5.md. That recipe never mentions duplex or realtime serving, so the announcement link does not land on the feature it names. Point this news link at examples/online_serving/minicpmo/README.md (duplex overlay + realtime demo) or add a duplex pointer in the recipe. (evidence: README.md:19: “an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md)”. grep of recipes/OpenBMB/MiniCPM-o-4_5.md for duplex|realtime|full-duplex: no matches. examples/online_serving/minicpmo/README.md:40: “| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |”. vllm_omni/experimental/fullduplex/DESIGN.md:365: “The full-duplex implementation remains experimental:”.) + +`docs/getting_started/installation/npu/npu.inc.md:62` [minor] — This diff retargets the NPU from-source Ascend pin from v0.18.0rc1 to `releases/v0.26.0rc` under a new comment that this is the v0.26 release line. The same page’s recommended images and docker/Dockerfile.npu already use the stable `v0.26.0` tag, so from-source users are told to mix RC git with stable images. Clone a matching non-RC ref (`v0.26.0` or `releases/v0.26.0`) or change that comment to state that source installs still need the RC branch while images are on v0.26.0. (evidence: npu.inc.md:56–62: “# Pin vLLM and vLLM-Ascend to the v0.26 release line” then “git clone -b v0.26.0 https://github.com/vllm-project/vllm.git” and “git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git”. npu.inc.md:12–13: “# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0” / “# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3”. docker/Dockerfile.npu:2: “ARG VLLM_ASCEND_TAG=v0.26.0”.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.trace.json.gz new file mode 100644 index 000000000..4d3f01df6 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r2/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.cost.json b/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.cost.json new file mode 100644 index 000000000..e56794039 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1340.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r2/runs/pr5840/run-20260814-205433-4bf5ab", + "recorded_at": "2026-08-14T21:16:54", + "attempts": [ + { + "invocation_id": "inv-deb2d2c2", + "outcome": "run:run-20260814-205433-4bf5ab", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-205433-4bf5ab", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-205433-4bf5ab", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7034, + "minutes": 22.33, + "ci_minutes": 0, + "input_tokens": 1731121, + "output_tokens": 214515, + "tool_calls": 547, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8589, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 5.15, + "gate": 1.69, + "review": 1333.08, + "report": 0.06 + }, + "time_to_first_result_s": 1339.97 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 43305 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.md b/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.md new file mode 100644 index 000000000..aba7f4d31 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] correctness: extract_minimax_h3_context mirrors MiniMaxH3DiTModel.forward (kwargs contract, _embed, sp_prepare/blocks/sp_gather, final_layer + masks); uncached test_extractor_matches_native_forward asserts close. +- [validated] vllm_omni/diffusion/cache/teacache/extractors.py:1341-1348 modulated_input uses blocks[0].adaln_proj+norm1+_modulate_scale_shift, matching MiniMaxH3DiTBlock.forward:700-711 on the full pre-SP decoder_input (standard TeaCache proxy; USP=1 is the calibrated path). +- [validated] extractors.py:1270 empty blocks raises before blocks[0]; x.shape[0]!=1 / token_tags / inverse_indices checks copy native forward:1124-1130. +- [validated] backend.py:79-93 ref2va raises, unknown partition raises, combined hooks only pipeline.transformer and warns; pipeline._transformer_for_task:737-739 routes ref2va to transformers_ref so combined Ref2VA stays uncached as documented. +- [validated] TeaCacheBackend.enable sets enabled=True only after the custom enabler returns; test_minimax_h3_rejects_ref2va_teacache asserts enabled is False and apply_teacache_hook is not called. +- [sweep] simplifiability: partition==ref2va vs not in {fl2va,combined} vs combined can all occur via _minimax_h3_partition_for_task (fl2va/combined/ref2va); keep the three arms. skip_mask_out_condition and update_audio_mask match native. TeaCacheConfig None-then-<=0 is required for the new default. +- [sweep] breaking defaults: DiffusionCacheConfig.rel_l1_thresh 0.2->None and TeaCacheConfig model default MiniMaxH3DiTModel=0.17; Qwen/other types still resolve to 0.2 in TeaCacheConfig.__post_init__. Stale injector: AsyncOmniEngine._get_default_cache_config still returns {rel_l1_thresh:0.2} and Omni/vllm serve both construct AsyncOmniEngine. +- [validated] tests/diffusion/cache/test_cache_backends.py:481-482 generic DiffusionCacheConfig() still yields 0.2 for QwenImage; :516 H3 None->0.17 and explicit 0.2 preserved. + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:?` [major] — This PR registers MiniMaxH3Pipeline → enable_minimax_h3_teacache, which apply_teacache_hook(pipeline.transformer) (backend.py:91), and the recipe now says TeaCache and Cache-DiT are mutually exclusive. MiniMaxH3QualityPolicy.resolve() still maps quality=="high" to an unconditional Cache-DiT spec (quality_policy.py:149-151) with no tea_cache check; pipeline.forward always then calls _cache_dit_runtime.prepare(quality_plan.cache_dit) (pipeline_minimax_h3.py:1766). On a --cache-backend tea_cache server the request-scoped runtime has not adopted a backend, so prepare constructs CacheDiTBackend and enable()s the same FL2VA transformer (runtime.py:71-73). quality=lossless only returns cache_dit=None and cannot remove the process-scoped TeaCache hook. In resolve(), if _configured_backend=="tea_cache", raise OmniClientError on quality="high" and Cache-DiT force-refresh args; extend test_model_policy_owns_request_cache_target with tea_cache rows; update the recipe quality section to say TeaCache is server-scoped and quality=high/lossless do not switch it off. (evidence: quality_policy.py:149-151 `if quality == "high":` / `base_key = MINIMAX_H3_HIGH_CACHE_KEY` / `base_config = _high_quality_cache_config()` — no `_configured_backend` / tea_cache gate. quality_policy.py:140 `self._configured_backend = str(getattr(od_config, "cache_backend", "none") or "none").lower()` is only used for omitted-quality cache_dit (line 152), so tea_cache+high still emits a CacheDiTRequestSpec. pipeline_minimax_h3.py:1766 `self._cache_dit_runtime.prepare(quality_plan.cache_dit)` is unconditional after resolve. cachedit/runtime.py:71-73 `if self._backend is None:` / `backend = CacheDiTBackend(spec.cache_config)` / `backend.enable(self._pipeline)` — tea_cache startup never adopt()s, so this path runs. teacache/backend.py:91 `apply_teacache_hook(pipeline.transformer, teacache_config)` and backend.py:127 `"MiniMaxH3Pipeline": enable_minimax_h3_teacache`. recipes/MiniMaxAI/MiniMax-H3.md: `TeaCache and Cache-DiT are mutually exclusive; pick one cache backend per server.` vs still documenting `-F 'quality=high' # H3's conservative Cache-DiT profile.` tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py parametrize for test_model_policy_owns_request_cache_target includes `("none", "high", "minimax_h3.high")` but no tea_cache rows.) + +`vllm_omni/diffusion/cache/teacache/config.py:98` [major] — This diff changes DiffusionCacheConfig.rel_l1_thresh default from 0.2 to None and registers _MODEL_DEFAULT_REL_L1_THRESH MiniMaxH3DiTModel=0.17 in TeaCacheConfig.__post_init__ (config.py:96-133); the recipe advertises 0.17 as the MiniMax-H3 default. That value never reaches Omni()/vllm serve when cache_config is omitted: both construct AsyncOmniEngine, whose _create_default_diffusion_stage_cfg calls _normalize_cache_config, which still injects {"rel_l1_thresh": 0.2} from _get_default_cache_config("tea_cache") (async_omni_engine.py:853-856, 881-882, 954-957) before DiffusionCacheConfig is built. enable_minimax_h3_teacache then passes that 0.2 into TeaCacheConfig (backend.py:86-88), so H3 never sees None. Omitting --cache-config therefore serves uncalibrated 0.2 (more aggressive reuse than 0.17), while cache_config={} skips the inject and would get 0.17 — omitted vs {} now diverge. Tests that assert 0.17 construct DiffusionCacheConfig() directly (test_cache_backends.py:504-508) and never hit this injector; the H3 smoke/recipe examples pass 0.17 explicitly. Omit rel_l1_thresh from the tea_cache engine default (return {}), add a _create_default_diffusion_stage_cfg({"cache_backend": "tea_cache"}) assertion, and align teacache.md / --cache-config help which still publish 0.2 as the sole default; keep explicit user cache_config as an override. (evidence: vllm_omni/diffusion/cache/teacache/config.py:96-98 `_DEFAULT_REL_L1_THRESH = 0.2` / `_MODEL_DEFAULT_REL_L1_THRESH = {` / `"MiniMaxH3DiTModel": 0.17,`. config.py:128-130 `threshold = self.rel_l1_thresh` / `if threshold is None:` / `threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)`. vllm_omni/engine/async_omni_engine.py:853-856 `if cache_backend == "tea_cache":` `return {` `"rel_l1_thresh": 0.2,`. async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"):` `cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`. async_omni_engine.py:954-957 `cache_backend = normalized_kwargs.get("cache_backend", "none")` / `cache_config = AsyncOmniEngine._normalize_cache_config(` / `cache_backend,` / `normalized_kwargs.get("cache_config", None),`. vllm_omni/entrypoints/omni_base.py:189 `self.engine = AsyncOmniEngine(`. vllm_omni/diffusion/cache/teacache/backend.py:86-88 `teacache_config = TeaCacheConfig(` / `transformer_type="MiniMaxH3DiTModel",` / `rel_l1_thresh=config.rel_l1_thresh,`. tests/diffusion/cache/test_cache_backends.py:504-508 `config = (` / `DiffusionCacheConfig()` / `if configured_threshold is None`. docs/user_guide/diffusion/cache_acceleration/teacache.md:122 `| \`rel_l1_thresh\` | float | \`0.2\` | Similarity threshold for cache reuse.`. vllm_omni/entrypoints/cli/serve.py:586 `"TeaCache: '{\"rel_l1_thresh\": 0.2}'. "`) + +`tests/diffusion/cache/test_teacache_extractors.py:?` [minor] — test_teacache_reuses_h3_block_residual calls TeaCacheHook.new_forward twice with identical sample_inputs and rel_l1_thresh=0.3 (test_teacache_extractors.py:688,693-694), so the second call is a guaranteed hit, but it only asserts block_calls==1 and logit dim0 (696-698). The hook applies cache as ctx.hidden_states + previous_residual then ctx.postprocess(output) (hook.py:147,189); H3 postprocess index_selects video/audio rows from that packed hidden (extractors.py:1377), so a wrong residual still keeps dim0. Capture the first-pass video/audio tensors and assert_close them to the second-pass logits. The uncached extractor path already does this at 642-643. This is the CPU test CUDA L1/L2 selects (pytestmark core_model at 41, class @pytest.mark.cpu at 569; .buildkite/cuda/test-ready.yml:18). (evidence: tests/diffusion/cache/test_teacache_extractors.py:693 ` hook.new_forward(minimax_h3_module, **sample_inputs)` — first-pass logits discarded. tests/diffusion/cache/test_teacache_extractors.py:694 ` video_logits, audio_logits = hook.new_forward(minimax_h3_module, **sample_inputs)`. tests/diffusion/cache/test_teacache_extractors.py:696 ` assert block_calls == 1`. tests/diffusion/cache/test_teacache_extractors.py:697 ` assert video_logits.shape[0] == sample_inputs["img_pos_info"]["position_ids"].shape[0]`. tests/diffusion/cache/test_teacache_extractors.py:698 ` assert audio_logits.shape[0] == sample_inputs["audio_pos_info"]["position_ids"].shape[0]`. tests/diffusion/cache/test_teacache_extractors.py:642 ` torch.testing.assert_close(actual_video, expected_video)` and :643 ` torch.testing.assert_close(actual_audio, expected_audio)` only on the uncached extractor path. vllm_omni/diffusion/cache/teacache/hook.py:147 ` ctx.hidden_states = ctx.hidden_states + state.previous_residual` then :189 ` return ctx.postprocess(output)`. vllm_omni/diffusion/cache/teacache/extractors.py:1377 ` video_logits = video_logits.index_select(0, infer_out_pos.to(device))` — dim0 comes from row ids, not residual correctness. tests/diffusion/cache/test_teacache_extractors.py:41 `pytestmark = [pytest.mark.core_model]`; class TestMiniMaxH3Extractor at :570 is under `@pytest.mark.cpu`. .buildkite/cuda/test-ready.yml:18 `timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu'`.) + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149` [minor] — The title includes Cache-DiT validation, but this PR's 10-file diff only adds TeaCache (extractor/backend/tests), rel_l1_thresh default plumbing, and recipe/feature-matrix text. Cache-DiT is unchanged: MiniMaxH3QualityPolicy.resolve() still owns request-scoped Cache-DiT, MiniMaxH3DiTModel already registers CacheDiTAdapterConfig, and test_lifecycle_applies_to_every_declared_dit already covers H3. The body already says no Cache-DiT implementation change was required and cites #5700—keep Cache-DiT as description-only validation of the existing adapter, not as in-diff merge work. (evidence: quality_policy.py:149 `if quality == "high":` — first branch of pre-existing MiniMaxH3QualityPolicy.resolve() (file not in this PR's 10-file diff). quality_policy.py:152 `elif self._configured_backend == "cache_dit" and quality is None:` — omitted quality still installs Cache-DiT from the server backend. minimax_h3_transformer.py:810 `_cache_dit_adapter_config = CacheDiTAdapterConfig(` — pre-existing Pattern_3 adapter. test_cache_backends.py:96 `def test_lifecycle_applies_to_every_declared_dit(self, mock_cache_dit, mock_block_adapter):` and test_cache_backends.py:99 `pipeline.__class__.__name__ = "MiniMaxH3Pipeline"` — pre-existing Cache-DiT lifecycle test, not added here. New tests in this diff are TeaCache-only (`test_minimax_h3_only_enables_fl2va_teacache`).) + +`vllm_omni/diffusion/cache/teacache/extractors.py:1342` [minor] — extract_minimax_h3_context decides TeaCache reuse from blocks[0] AdaLN on the full unsharded decoder_input (extractors.py:1341-1349) before any SP. The only sp_prepare/sp_gather calls are inside run_transformer_blocks (extractors.py:1351-1368); those modules are identity in Python but are the USP split/gather boundaries via MiniMaxH3DiTModel._sp_plan. TeaCacheHook cache hits add previous_residual and skip that callable (hook.py:143-150), so the USP plan never runs on a hit. Native MiniMaxH3DiTModel.forward is always sp_prepare then blocks, with no pre-SP modulated_input (minimax_h3_transformer.py:1157-1173). The new TeaCache e2e smoke pins ulysses_degree=1 (test_minimax_h3_e2e.py:157), while diffusion_features.md marks MiniMax-H3 TeaCache ✅ (FL2VA) next to SP ✅ (line 162) and the global TeaCache×Ulysses-SP cell stays ✅ (line 188). Document H3 TeaCache as ulysses_degree=1 only, or add a same-seed USP A/B before inheriting that combination. (evidence: extractors.py:1341-1342 `shift_msa, scale_msa, *_ = module.blocks[0].adaln_proj(t_emb)` / `modulated_hidden = module.blocks[0].norm1(decoder_input)` — cache decision tensor is AdaLN on unsharded decoder_input. extractors.py:1351-1353 / 1368 `def run_transformer_blocks()` / `hidden, block_rope, block_combined = module.sp_prepare(decoder_input, ...)` / `hidden = module.sp_gather(hidden)` — USP boundaries run only on the compute path. hook.py:143-147 `if not should_compute and state.previous_residual is not None:` / `ctx.hidden_states = ctx.hidden_states + state.previous_residual` — cache hit skips run_transformer_blocks. minimax_h3_transformer.py:790-806 MiniMaxH3SPPrepare/SPGather `return hidden_states, ...` / `return hidden_states` (identity Python) while :832-838 `_sp_plan = { "sp_prepare": { 0: SequenceParallelInput(split_dim=0, ...)` is the USP split. minimax_h3_transformer.py:1157-1173 native `hidden, block_rope, block_combined = self.sp_prepare(...)` then `for block in self.blocks` then `hidden = self.sp_gather(hidden)` — no pre-SP modulated_input. test_minimax_h3_e2e.py:157 `parallel_config=DiffusionParallelConfig(ulysses_degree=1),` in `test_minimax_h3_t2va_teacache_single_gpu_smoke`. diffusion_features.md:162 MiniMax-H3 `| ✅ (FL2VA) | ... | ✅ |` (TeaCache and SP). diffusion_features.md:188 `| **🔀Ulysses-SP** | ✅ |` (TeaCache × Ulysses-SP).) + +`vllm_omni/diffusion/cache/teacache/config.py:?` [minor] — config.py:86-98 registers MiniMaxH3DiTModel coefficients [2.283704065852778e03, -7.775977277886368e02, 9.408414741359490e01, -4.232669906169421e00, 2.173782527946167e-01] and rel_l1_thresh 0.17. test_minimax_h3_only_enables_fl2va_teacache already receives the post_init TeaCacheConfig but only asserts teacache_config.rel_l1_thresh (test_cache_backends.py:516); no test pins coefficients (no teacache_config.coefficients assert, and the five-float list is absent from tests/). test_enable_with_coefficients uses QwenImagePipeline with dummy [1.0, 0.5, 0.2, 0.1, 0.05]. Extend the H3 backend test to assert teacache_config.coefficients equals that list when DiffusionCacheConfig() is used — a polynomial typo would still yield thresh 0.17 and a green CPU run while every cache decision uses the wrong rescale. (evidence: vllm_omni/diffusion/cache/teacache/config.py:87-98 `"MiniMaxH3DiTModel": [\n 2.283704065852778e03,\n -7.775977277886368e02,\n 9.408414741359490e01,\n -4.232669906169421e00,\n 2.173782527946167e-01,\n ],\n}\n\n_DEFAULT_REL_L1_THRESH = 0.2\n_MODEL_DEFAULT_REL_L1_THRESH = {\n "MiniMaxH3DiTModel": 0.17,` — calibrated polynomial and 0.17 thresh registered. tests/diffusion/cache/test_cache_backends.py:516 `assert teacache_config.rel_l1_thresh == expected_threshold` — H3 enable test pins thresh only, not coefficients. tests/diffusion/cache/test_cache_backends.py:546 `config = DiffusionCacheConfig(rel_l1_thresh=0.3, coefficients=[1.0, 0.5, 0.2, 0.1, 0.05])` — coefficient override test is QwenImagePipeline with a dummy list. Grep of tests/ found no `2.283704065852778e03` and no `teacache_config.coefficients`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:?` [minor] — test_minimax_h3_t2va_teacache_single_gpu_smoke enables TeaCache (`cache_backend="tea_cache"`, `rel_l1_thresh=0.17`) but uses `num_inference_steps=2` and only `_assert_joint_output` shape checks (test_minimax_h3_e2e.py:154-187). TeaCacheHook always computes on the first timestep (`state.cnt == 0` and `previous_modulated_input is None` both return True; hook.py:207-216), so two steps yield at most one skip decision and the test still passes if the hook never skips. TeaCacheState tracks `cnt` but has no hit/skip counter, so either assert a skip after generate (once counted) or use the 50-step 107-frame recipe. This test is also never selected by CI: pytestmark is local_model/cuda/diffusion/parallel (not core_model), skipif unless VLLM_TEST_MINIMAX_H3_FL2VA_MODEL is set, .buildkite has no local_model jobs and never names this e2e module; test-nightly.yml:433/471/510 run test_minimax_h3_t2va_similarity.py, test_minimax_h3_vllm_omni.json, and test_minimax_h3_quantization_quality.py instead. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:155-156 `cache_backend="tea_cache",` / `cache_config={"rel_l1_thresh": 0.17},`; :170 `num_inference_steps=2,`; :187 `_assert_joint_output(outputs)` which only checks `frames.shape == (107, 256, 448, 3)` (:22). vllm_omni/diffusion/cache/teacache/hook.py:207-216 `# First timestep: always compute` / `if state.cnt == 0:` / `return True` then `if state.previous_modulated_input is None:` / `return True`. tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:9-14 `pytestmark = [pytest.mark.local_model, pytest.mark.cuda, pytest.mark.diffusion, pytest.mark.parallel]`; :139-142 skipif unless `VLLM_TEST_MINIMAX_H3_FL2VA_MODEL`. .buildkite has no `test_minimax_h3_e2e` and no `local_model`; test-nightly.yml:433/471/510 run `test_minimax_h3_t2va_similarity.py`, `test_minimax_h3_vllm_omni.json`, `test_minimax_h3_quantization_quality.py`.) + +`tests/diffusion/cache/test_cache_backends.py:?` [nit] — New MiniMax-H3 TeaCacheBackend tests in tests/diffusion/cache/test_cache_backends.py inherit module pytestmark `[pytest.mark.core_model, pytest.mark.cpu]` at line 32. AMD Simple Test (.buildkite/amd/test-amd-ready.yml) runs `pytest tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py`, so test_enable_uses_generic_default_threshold, test_minimax_h3_only_enables_fl2va_teacache, and test_minimax_h3_rejects_ref2va_teacache are selected. Attribute any AMD Simple Test failure from that job log rather than guessing whether these new cases caused it. (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — module marker applies to the new tests. tests/diffusion/cache/test_cache_backends.py:490 `def test_minimax_h3_only_enables_fl2va_teacache(` — one of the new cases. .buildkite/amd/test-amd-ready.yml Simple Test command: `timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py` — ignores extractors only, so test_cache_backends.py is collected.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.trace.json.gz new file mode 100644 index 000000000..75720e63c Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r2/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5863.cost.json b/eval/dataset/arms/copilot_cb_grok46_r2/pr5863.cost.json new file mode 100644 index 000000000..06a4fa418 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5863.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 597.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r2/runs/pr5863/run-20260814-205926-055c81", + "recorded_at": "2026-08-14T21:09:24", + "attempts": [ + { + "invocation_id": "inv-d5b28133", + "outcome": "run:run-20260814-205926-055c81", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-205926-055c81", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-205926-055c81", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1252, + "minutes": 9.95, + "ci_minutes": 0, + "input_tokens": 338523, + "output_tokens": 30761, + "tool_calls": 99, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3666, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.92, + "gate": 1.6, + "review": 591.54, + "report": 0.07 + }, + "time_to_first_result_s": 597.11 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 14787 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5863.md b/eval/dataset/arms/copilot_cb_grok46_r2/pr5863.md new file mode 100644 index 000000000..27eb33b2d --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r2/pr5863.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:136-160 Eight-GPU TP2× USP4 and vllm_omni/platforms/cuda/platform.py:184-226 TRTLLM_ATTN defaults only when compute_capability.major==10; SM120 (RTX PRO 6000) defaults to CUDNN_ATTN, so the explicit --diffusion-attention-backend CUDNN_ATTN pin is correct and TRTLLM_ATTN would raise. +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:81-101 two-GPU command vs MiniMax-H3.md:97-127 same flag set minus DLO/eager, plus CUDNN_ATTN. +- [validated] vllm_omni/diffusion/worker/diffusion_model_runner.py:351-355 peak_memory_mb is CUDA max_memory_reserved, not nvidia-smi. + +`recipes/README.md:?` [minor] — This diff adds recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md as a standalone hardware recipe but does not add it to the Available Recipes table or point readers at the family API page. Insert a row after the MUSA entry for 2/4/8× RTX PRO 6000, and add a See also on the new page to MiniMax-H3.md (input matrix, FL2VA, limits) the way MiniMax-H3-MUSA.md already does. (evidence: recipes/README.md:48-49 currently lists only MiniMax-H3.md and MiniMax-H3-MUSA.md. recipes/MiniMaxAI/MiniMax-H3-MUSA.md:143 already has `- [MiniMax-H3.md](MiniMax-H3.md) — full model and API guide`. Grep of recipes/MiniMaxAI found no link to MiniMax-H3-RTX-PRO-6000.md from any sibling.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [minor] — This diff documents the unmeasured two-server layout with `--cpunodebind` / `--membind` written beside `--port`, which are numactl switches, not vllm serve flags. Rewrite as `numactl --cpunodebind=0 --membind=0 -- vllm serve ... --port 8091` (and the node-1 counterpart); leaving them unqualified will make a copy-paste `vllm serve ... --cpunodebind=0` fail as an unrecognized argument. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201-203: `Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and `--port 8091` for the FL2VA server.` Grep of vllm_omni/entrypoints for `cpunodebind` returned no matches; serve.py registers `--usp`/`--ulysses-degree` and `--port`, not NUMA bind flags.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:328` [minor] — This diff tells readers to reproduce peak HBM by reading `X-Peak-Memory-MB`, but the table above is defined as 1 Hz `nvidia-smi --query-gpu=memory.used` across every device. Split the two metrics: keep nvidia-smi for the GiB columns; if you mention the header, label it as per-rank CUDA `max_memory_reserved` in MB, which will not match 77.49/66.44/61.07 GiB. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:254-255: `Peak memory is the maximum of `nvidia-smi --query-gpu=memory.used` sampled at 1 Hz across every device`. vllm_omni/diffusion/worker/diffusion_model_runner.py:351-355: `Uses max_memory_reserved (CUDA memory pool high-water mark)` / `peak_reserved_bytes = current_omni_platform.max_memory_reserved()`. vllm_omni/entrypoints/openai/api_server.py:3511: `"X-Peak-Memory-MB": f"{peak_memory_mb:.3f}"`.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:339` [minor] — This diff adds a Ref2VA curl that hard-codes `/root/hand.jpg` and `audio_reference=0 / missing ids for scoped_spawn_device_env. tests/diffusion/test_inductor_divisibility_patch.py drives _provably_factorable_multiple and install/idempotency. tests/diffusion/distributed/test_expert_parallel_layout.py asserts EP use_all2all and version-guards the kwarg. +- [validated] qwen3_omni.py:417-424 returns thinker output directly (bare tensor vs capture tuple); make_omni_output:513-518 accepts both. qwen3_omni.py:145 is_staged_run gates capture. gpu_generation_model_runner.py:847 accepts randomize_inputs but does not forward it into maybe_randomize_inputs. +- [validated] init_vllm_model_parallel_group EP path passes use_all2all=True at parallel_state.py:964-969. stage_engine_startup.py:1046-1065 forwards stage_id/replica_id into scoped_spawn_device_env. + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [major] — This diff seeds stale twice on the AR replace_streaming_prompt arm. OmniARScheduler._update_request_as_session does `session.num_stale_output_tokens += in_flight_tokens` (omni_ar_scheduler.py:627-629) and does not zero `num_in_flight_tokens`, then `if replace_streaming_prompt: self._replace_streaming_session(...)` (669-670). Mixin `_replace_streaming_session` does a second `+= int(num_in_flight_tokens)` (omni_scheduler_mixin.py:105), so in_flight=1 becomes stale=2. The new drain then swallows the first valid new-segment frame or underflows `assert num_stale_output_tokens >= 0` — the leftover-counter hole this PR claims to close. MiniCPM-o 4.5 sets `handoff_meta["replace_streaming_prompt"] = True` when `not is_native_duplex_handoff or native_turn_start` (minicpmo_4_5_omni.py:953-954). Generation is safe (it only calls the mixin). Seed on only one of the two AR sites (assign rather than `+=`, then zero `num_in_flight_tokens`), and extend `tests/core/sched/test_omni_ar_scheduler_stale_drain.py` to drive `_update_request_as_session` with `replace_streaming_prompt=True` and `num_in_flight_tokens>0`, asserting stale == in_flight not 2*in_flight. (evidence: omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0:` / `session.num_stale_output_tokens += in_flight_tokens` — seeds stale and does not zero `num_in_flight_tokens`. omni_ar_scheduler.py:669-670 `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)` — same method then calls the mixin. omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` — second += from the still-nonzero counter. Drain omni_ar_scheduler.py:336-350 `if request is not None and request.num_stale_output_tokens > 0:` then `request.num_stale_output_tokens -= num_tokens_scheduled` / `assert request.num_stale_output_tokens >= 0` then `if output_is_stale: continue`. omni_generation_scheduler.py:604 `self._replace_streaming_session(session, update)` — generation seeds only via mixin. minicpmo_4_5_omni.py:953-954 `if not is_native_duplex_handoff or native_turn_start:` / `handoff_meta["replace_streaming_prompt"] = True`. test_omni_ar_scheduler_stale_drain.py:77-84 StreamingUpdate has no replace_streaming_prompt meta and calls `_update_request_as_session` only.) + +`vllm_omni/core/sched/omni_scheduler_mixin.py:?` [major] — This PR seeds `num_stale_output_tokens` from `num_in_flight_tokens` in mixin `_replace_streaming_session` (omni_scheduler_mixin.py:105-107) and in `_update_request_as_session` (omni_ar_scheduler.py:627-629), but MiniCPM-o Talker never uses those as the prompt-replace site. Stage 1 is LLM_AR with `async_chunk: true` (pipeline.py:55-58, minicpmo_4_5.yaml:3) and LLM_AR defaults `async_scheduling=True` (omni_config.py:1407-1408). Thinker sets `handoff_meta["replace_streaming_prompt"] = True` (minicpmo_4_5_omni.py:953-954). Downstream async-chunk `_update_request_as_session` returns before `_replace_streaming_session` and defers prompt replace to the connector (omni_ar_scheduler.py:639-658). The live replace is `chunk_transfer_adapter.py:237-242` → `construct_next_stage_streaming_input_prompt`, which on that flag clears tokens and sets `num_computed_tokens = 0` with no stale seed (adapter.py:235-247). `update_from_output` skips append only when `num_stale_output_tokens > 0` (omni_ar_scheduler.py:336-353), so late in-flight talker frames are appended onto the new turn. Seed stale from `num_in_flight_tokens` (and zero placeholders/spec ids) on that replace_prompt arm, and pin it with a chunk-adapter test that a post-replace leftover in-flight frame is not delivered. (evidence: vllm_omni/core/sched/omni_scheduler_mixin.py:105-107 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `session.num_output_placeholders = 0` / `session.spec_token_ids = []`. vllm_omni/core/sched/omni_ar_scheduler.py:639-646 `if self.chunk_transfer_adapter and self.chunk_transfer_adapter.receives_chunks:` … `if stage_id != 0:` … `# Do not replace prompt/additional_information here; the next upstream chunk will populate them in chunk transfer adapter.` then `return` at 658. vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py:237-242 `if self.model_mode == "ar":` / `replace_prompt = meta.get("replace_streaming_prompt") is True` / `if getattr(request, "resumable", False) and (chunk_id > 0 or replace_prompt):` / `construct_next_stage_streaming_input_prompt(payload_data, request)`. vllm_omni/distributed/omni_connectors/adapter.py:235-247 `if replace_prompt and isinstance(next_stage_prompt_len, int) and next_stage_prompt_len > 0:` … `request._output_token_ids.clear()` … `request.num_computed_tokens = 0` … `return` (no `num_stale_output_tokens` write). vllm_omni/core/sched/omni_ar_scheduler.py:336-353 `if request is not None and request.num_stale_output_tokens > 0:` … `if output_is_stale:` … `continue`. vllm_omni/config/omni_config.py:1407-1408 `if "async_scheduling" not in kwargs and execution_type == StageExecutionType.LLM_AR:` / `kwargs["async_scheduling"] = True`. vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py:55-58 stage 1 `execution_type=StageExecutionType.LLM_AR`. vllm_omni/deploy/minicpmo_4_5.yaml:3 `async_chunk: true`. vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:953-954 `handoff_meta["replace_streaming_prompt"] = True`.) + +`tests/core/sched/test_omni_ar_scheduler_stale_drain.py:?` [major] — This file claims to pin the seed-TTS stale-drain / talker-codec contract, but `_replace_streaming_session` only drives OmniARScheduler._update_request_as_session with `stage_id=0` and `StreamingUpdate(mm_features=None)` (no `replace_streaming_prompt` meta) at tests/core/sched/test_omni_ar_scheduler_stale_drain.py:68-84. AR calls mixin `_replace_streaming_session` only when `replace_streaming_prompt` is true (omni_ar_scheduler.py:669-670); these tests therefore hit AR's own seed (omni_ar_scheduler.py:627-629) and OmniARScheduler.update_from_output, not the talker path. Downstream replacement is OmniGenerationScheduler._update_request_as_session -> mixin seed (omni_generation_scheduler.py:604, omni_scheduler_mixin.py:105), and generation `update_from_output` is the drain that would swallow new-segment frames. `test_omni_generation_scheduler_update_session.py` has no `num_stale_output_tokens` coverage. Add one case that calls mixin `_replace_streaming_session` then `OmniGenerationScheduler.update_from_output` before citing these tests as WER evidence. (evidence: tests/core/sched/test_omni_ar_scheduler_stale_drain.py:71 `sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=0))` +tests/core/sched/test_omni_ar_scheduler_stale_drain.py:77-84 `update = StreamingUpdate(mm_features=None, prompt_token_ids=[10, 20], ...)` then `sched._update_request_as_session(session, update)` +tests/core/sched/test_omni_ar_scheduler_stale_drain.py:132 `OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output)` +vllm_omni/core/sched/omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0:` / `session.num_stale_output_tokens += in_flight_tokens` +vllm_omni/core/sched/omni_ar_scheduler.py:669-670 `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)` +vllm_omni/core/sched/omni_generation_scheduler.py:604 `self._replace_streaming_session(session, update)` +vllm_omni/core/sched/omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` +tests/core/sched/test_omni_generation_scheduler_update_session.py: grep `num_stale` → no matches; update_from_output only at line 160 with no stale-drain assertions) + +`tests/e2e/accuracy/test_qwen_image.py:38` [minor] — SSIM_THRESHOLD_SDPA_FALLBACK = 0.96 is applied only in test_qwen_image_matches_diffusers (test_qwen_image.py:38, 262). test_qwen_image_2512_matches_diffusers_pixelwise uses the same FA3/SDPA matching helpers (_omni_server_env, _set_reference_attention_backend) but still grades with assert_images_pixel_close (mean<=3e-2, p99<=4e-1) and no SDPA-specific relaxation (test_qwen_image.py:186, 223, 282-288). Build 2954's 0.9687 SSIM is the 512 Qwen-Image SSIM arm, not 2512, so it does not transfer as a 2512 fail; it also does not make 0.96 FA3-parity evidence. This PR already forces both LPIPS arms to enforce_eager=True; these accuracy tests do not. Either run both Qwen-Image accuracy arms eager (keep 0.97) or measure 2512 under SDPA and set an explicit pixel fallback. Nightly: pytest -sv tests/e2e/accuracy/test_qwen_image.py -m 'full_model and diffusion'. (evidence: tests/e2e/accuracy/test_qwen_image.py:38 `SSIM_THRESHOLD_SDPA_FALLBACK = 0.96` — tests/e2e/accuracy/test_qwen_image.py:262 `ssim_threshold=SSIM_THRESHOLD if _flash_attn3_hub_available() else SSIM_THRESHOLD_SDPA_FALLBACK` (only SSIM-arm consumer; grep hits 32/38/262 only) — tests/e2e/accuracy/test_qwen_image.py:186 `with OmniServer(model, server_args, use_omni=True, env_dict=_omni_server_env()) as omni_server:` — tests/e2e/accuracy/test_qwen_image.py:223 `_set_reference_attention_backend(pipe)` — tests/e2e/accuracy/test_qwen_image.py:282-288 `assert_images_pixel_close(` / `mean_threshold=MEAN_ABS_DIFF_THRESHOLD_2512,` / `p99_threshold=P99_ABS_DIFF_THRESHOLD_2512,` (no FA3/SDPA branch) — tests/diffusion/quantization/test_quantization_quality.py:525 `bl_kwargs: dict = {"model": config.baseline_ref(), "enforce_eager": True}`) + +`vllm_omni/core/sched/omni_generation_scheduler.py:385` [minor] — OmniGenerationScheduler drains leftover stale tokens at omni_generation_scheduler.py:385-401 but never seeds them on connector segment-stop: the `if not finished` block at 467-470 only discards from segment_finished_requests. The only generation seed is mixin `_replace_streaming_session` (omni_scheduler_mixin.py:105), reached from `_update_request_as_session`; `_handle_stopped_request` (56-67) says connector chunks do not use StreamingUpdate. OmniARScheduler seeds the matching stop site at omni_ar_scheduler.py:490-491. Is seeding that connector generation path in scope for this leftover-counter fix, or explicitly out of scope? (evidence: omni_generation_scheduler.py:385-388 `if request is not None and request.num_stale_output_tokens > 0:\n output_is_stale = True\n request.num_stale_output_tokens -= num_tokens_scheduled\n assert request.num_stale_output_tokens >= 0` — drain exists. omni_generation_scheduler.py:62-67 `# Downstream async-chunk stages receive the next segment from the\n # connector, not from an API StreamingUpdate.` then `return False`. omni_generation_scheduler.py:467-470 `if not finished:\n # for streaming input request only\n if self.chunk_transfer_adapter:\n self.chunk_transfer_adapter.segment_finished_requests.discard(req_id)` — no num_stale_output_tokens seed. omni_generation_scheduler.py:604 `self._replace_streaming_session(session, update)` — only StreamingUpdate seed path. omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`. Contrast omni_ar_scheduler.py:490-491 `if request.num_in_flight_tokens > 0:\n request.num_stale_output_tokens += request.num_in_flight_tokens`.) + +`async_omni_engine.py:568` [minor] — async_omni_engine.py:542 imports private `vllm.config.multimodal._get_mm_hasher_algorithm` and passes it as the first argument to `MultiModalHasher.hash_kwargs` at :568 with no older-vLLM fallback. `use_all2all` is inspect-guarded at parallel_state.py:537 because "Older vLLM has neither the kwarg nor the requirement", and tests still shim `num_stale_output_tokens` for 0.26 local. Dockerfile.ci:21 pins `VLLM_BASE_TAG=v0.27.0`. Pick one floor: either 0.27 (drop the inspect-guard / 0.26 shims) or give `hash_kwargs` the same compatibility guard. (evidence: async_omni_engine.py:542 `from vllm.config.multimodal import _get_mm_hasher_algorithm` — unconditional private import. async_omni_engine.py:568-571 `base_uuid = MultiModalHasher.hash_kwargs(\n _get_mm_hasher_algorithm(),\n model_id=model_id,\n **{modality: item},\n )` — first arg is that helper, no fallback. parallel_state.py:537 `if use_all2all and "use_all2all" in inspect.signature(vllm_parallel_state.init_model_parallel_group).parameters:` — older-vLLM inspect guard. docker/Dockerfile.ci:21 `ARG VLLM_BASE_TAG=v0.27.0` — CI pin.) + +`vllm_omni/benchmarks/patch/patch.py:?` [minor] — probe_task = asyncio.create_task(probe_loop()) at patch.py:1714, but probe_stop.set(); await probe_task runs only after await asyncio.gather(*tasks) (1779-1783). If gather raises, probe_loop keeps issuing requests — it checks probe_stop only between request_func and sleep (1707-1709). Wrap gather in try/finally that sets probe_stop and awaits/cancels probe_task. (evidence: vllm_omni/benchmarks/patch/patch.py:1711-1714 `probe_task: asyncio.Task | None = None` / `if probe_request_rate > 0:` / `print(f"Probe request rate: {probe_request_rate} req/s")` / `probe_task = asyncio.create_task(probe_loop())`. vllm_omni/benchmarks/patch/patch.py:1779-1783 `outputs: list[MixRequestFuncOutput] = await asyncio.gather(*tasks)` / `if probe_task is not None:` / `probe_stop.set()` / `await probe_task` — no try/finally around gather. vllm_omni/benchmarks/patch/patch.py:1707-1709 `while not probe_stop.is_set():` / `probe_outputs.append(await request_func(request_func_input=probe_input, session=session))` / `await asyncio.sleep(interval)` — stop is checked only between probe requests.) + +`vllm_omni/worker/gpu_generation_model_runner.py:?` [minor] — Both Omni `_dummy_run` overrides accept `randomize_inputs: bool = False` (`gpu_generation_model_runner.py:580`, `gpu_model_runner.py:881`) so v0.27 FlashInfer autotune does not TypeError. The thinker runner forwards it (`gpu_model_runner.py:1134`); the generation runner calls `maybe_randomize_inputs(input_ids, inputs_embeds)` with no flag (`gpu_generation_model_runner.py:847`) so vocab-uniform ids never hit code2wav RVQ. `tests/worker/test_gpu_generation_model_runner.py` only has `test_sample_tokens_*` and `test_execute_model_skips_non_positive_scheduled_span`; `test_omni_gpu_model_runner.py` has no `_dummy_run` coverage; `tests/` has zero `randomize_inputs` hits. Add CPU tests that `inspect.signature` of both `_dummy_run` includes `randomize_inputs`, and that generation `_dummy_run(randomize_inputs=True)` does not pass `randomize_inputs=True` into `maybe_randomize_inputs`. Run: pytest -sv tests/worker/test_gpu_generation_model_runner.py tests/worker/test_omni_gpu_model_runner.py -m 'core_model and cpu'. (evidence: gpu_generation_model_runner.py:580 ` randomize_inputs: bool = False,` — generation `_dummy_run` accepts the kwarg. gpu_generation_model_runner.py:847 ` self.maybe_randomize_inputs(input_ids, inputs_embeds),` — generation does not forward `randomize_inputs`. gpu_model_runner.py:1134 ` self.maybe_randomize_inputs(input_ids, inputs_embeds, randomize_inputs=randomize_inputs),` — thinker forwards it. tests/worker/test_gpu_generation_model_runner.py:53/63/73/83/121 are only `test_sample_tokens_tensor_output`, `test_sample_tokens_list_output`, `test_sample_tokens_list_allows_none_output`, `test_sample_tokens_dict_output`, `test_execute_model_skips_non_positive_scheduled_span`. Grep `tests/` for `randomize_inputs`: no matches.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py:312` [minor] `Qwen2_5OmniVideoProcessorItems.get_processor_data` now iterates `self.get()` (qwen2_5_omni_thinker.py:312-313) so `VideoProcessorItems._unwrap` strips `MediaWithBytes` before the HF processor; walking `self.data` skippe +- `tests/worker/test_worker_base.py:38` [minor] The fake now yields `total_consumed=torch_peak + non_torch` (tests/worker/test_worker_base.py:38) and `OmniGPUWorkerBase.determine_available_memory` assigns `self.total_consumed = profile_result.total_consumed` (vllm_omn + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r2/pr5976.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r2/pr5976.trace.json.gz new file mode 100644 index 000000000..a2cd9dce6 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r2/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.cost.json new file mode 100644 index 000000000..56058433a --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1140.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5703/run-20260814-230742-e9c430", + "recorded_at": "2026-08-14T23:26:42", + "attempts": [ + { + "invocation_id": "inv-d456669e", + "outcome": "run:run-20260814-230742-e9c430", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-230742-e9c430", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-230742-e9c430", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6366, + "minutes": 19.0, + "ci_minutes": 0, + "input_tokens": 1741292, + "output_tokens": 151310, + "tool_calls": 290, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7753, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.69, + "gate": 1.63, + "review": 1134.64, + "report": 0.06 + }, + "time_to_first_result_s": 1140.0 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 27551 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.md b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.md new file mode 100644 index 000000000..aa209ae65 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] CUDA encode path: old type=="cuda" and new type!="cpu" both include cuda devices; get_device_module() on CUDA is torch.cuda so device()/manual_seed match the removed torch.cuda calls (vae.py:170-176). +- [validated] pipeline_minimax_h3.py:963 and :1066 still call encode_image/encode_video with unchanged signatures; rank-0 encode + broadcast contract untouched. +- [validated] test_video_vae_keeps_reference_fp32_weights constructs via __init__ on CPU, so it now executes torch.get_device_module() (vae.py:120) but never the device RNG loop. +- [validated] Existing fork_rng call sites (minicpmo batched_token2wav.py:81, ming byte5_encoder.py:110) pass CUDA devices only; this PR is the first !=cpu expansion. +- [upstream-verify] PR validation lists test_minimax_h3_vae.py; that file is absent from tests/diffusion/models/minimax_h3/. +- [validated] CUDA MiniMax H3 still takes the device-RNG branch because `"cuda" != "cpu"`; ROCm typically reports `device.type == "cuda"` so it already did before this PR. +- [validated] Existing CPU encode_video contract test still reaches encode_video (test_minimax_h3_contract.py:353) because FakeModel weights are CPU so devices=[]. +- [validated] pipeline_minimax_h3.py:545-547 is the only MiniMaxH3VideoVAE() construction; T2VA does not call these encode paths. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This diff replaces the CUDA-only gate with `parameter.device.type != "cpu"` in `encode_image`/`encode_video` (`vae.py:170` and `:214`) and seeds via init-time `self.device_module = torch.get_device_module()` (`vae.py:120`) plus `torch.random.fork_rng(devices=devices)` with no `device_type` (`vae.py:172`, `:216`). That is not a CUDA+MUSA allowlist. `MiniMaxH3Pipeline` constructs `MiniMaxH3VideoVAE(..., device=self.device)` (`pipeline_minimax_h3.py:545-547`) and Ref2VA already calls `encode_image` (`:963`) and `encode_video` (`:1066`), so the shipped Ascend path (`MiniMax-H3-NPU.md:18-19`: CPU-only aarch64 PyTorch + `torch_npu`, no CUDA runtime) now hands `npu` devices to CUDA-default `fork_rng`. Keep CPU as the no-device-RNG path, allow only `{"cuda", "musa"}`, resolve the module at encode time with `torch.get_device_module(parameter.device)`, and pass `device_type=parameter.device.type` into `fork_rng`. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()`; vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:172 `with torch.random.fork_rng(devices=devices):`; vae.py:174-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` (mirrored at vae.py:214-220). pipeline_minimax_h3.py:545 `self.video_vae = MiniMaxH3VideoVAE(` and :547 `device=self.device,`; :963 `rows = self.video_vae.encode_image(image) if rank == 0 else None`; :1066 `self.video_vae.encode_video(load_video_frames(item["prepared_path"]))`. recipes/MiniMaxAI/MiniMax-H3-NPU.md:18-19 `- Runs on a CPU-only (aarch64) PyTorch build with `torch_npu`; no CUDA runtime is present.`) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — The only copy-paste serve block is labeled as the validated Ref2VA TP4 profile but sets MODEL to FL2VA (recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76). This file already states the partitions are independently served (lines 16-20) and later reports Ref2VA TP4 measurements, but unlike MiniMax-H3.md / MiniMax-H3-NPU.md it never shows `export MODEL="${MODEL_ROOT}/Ref2VA"`. Set the validated block to Ref2VA and add that explicit partition restart so copy-paste cannot load the wrong checkpoint. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:18-20 `- \`FL2VA\` serves text-to-video+audio (\`t2va\`) and first-frame-to-video+audio (\`fl2va\`).` / `- \`Ref2VA\` serves image/audio or video-reference generation (\`ref2va\`).` recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76 `The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs and offloads inactive model components to CPU:` then `export MODEL="${MODEL_ROOT}/FL2VA"`. Grep of this file for Ref2VA hits only prose (lines 9, 11, 19, 40, 70, 72, 114, 124), never an export. recipes/MiniMaxAI/MiniMax-H3.md:168 `export MODEL="${MODEL_ROOT}/Ref2VA"` after `To serve Ref2VA, stop the FL2VA server and restart...`; recipes/MiniMaxAI/MiniMax-H3-NPU.md:117 same restart.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — vae.py:170 and :214 now gate `fork_rng` with `parameter.device.type != "cpu"` and seed via `self.device_module.manual_seed` (vae.py:175-176 and :219-220), but no test exercises that branch. `test_video_vae_encode_uses_configured_parallel_tiling` builds `MiniMaxH3VideoVAE` with `object.__new__` (test_minimax_h3_contract.py:344) so `device_module` is never set, and FakeModel's CPU `Parameter` makes `devices` always `[]`, skipping the new loop. There is no `test_minimax_h3_vae.py`; tests/ never mention `device_module`, `fork_rng`, or `MINIMAX_H3_KEYFRAME_ENCODE_SEED`. Extend that `core_model and cpu` contract test to stub `parameter.device.type` for `cuda`/`musa`/`npu`/`cpu`, assert only cuda/musa enter `fork_rng` + `manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`, and run `pytest -q -c /dev/null tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py -m 'core_model and cpu'`. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` (same pair at vae.py:214 and :219-220). tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`; :344 `video_vae = object.__new__(MiniMaxH3VideoVAE)` then Module.__init__ only, so `device_module` unset; FakeModel `self.weight = torch.nn.Parameter(torch.ones(1))` is CPU so `devices` is `[]`; :353 `rows, shape = video_vae.encode_video("frames")` never enters the non-CPU seed loop. tests/diffusion/models/minimax_h3 has no test_minimax_h3_vae.py; tests/ grep for `device_module`, `fork_rng`, and `MINIMAX_H3_KEYFRAME_ENCODE_SEED` is empty.) + +`reference_video.py:477` [minor] — This PR does not add test files, and tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py is absent (directory has only contract/e2e/packing/parallel/quantization tests). MiniMax tests grep empty for load_audio_file/soundfile/torchaudio/ffmpeg. load_audio_file (vllm_omni/diffusion/models/minimax_h3/reference_video.py:477) is pre-existing torchaudio→soundfile→ffmpeg fallback and is not in this diff, which only adds device-aware VAE RNG plus the MUSA recipe. Is that audio-loading remainder deferred, or should those tests land before merge? (evidence: vllm_omni/diffusion/models/minimax_h3/reference_video.py:458 `len(timestamps),` — not load_audio_file; that function is at :477 `def load_audio_file(path: str) -> tuple[torch.Tensor, int]:` then :486-493 `import torchaudio` / `return torchaudio.load(path)` / `return _soundfile_to_waveform(path)` with ffmpeg demux after. tests/diffusion/models/minimax_h3/ listing: test_minimax_h3_contract.py, test_minimax_h3_e2e.py, test_minimax_h3_packing.py, test_minimax_h3_parallel.py, test_minimax_h3_quantization.py, test_minimax_h3_quantization_quality.py — no test_minimax_h3_vae.py. Grep of that directory for load_audio_file|soundfile|torchaudio|ffmpeg: no matches.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — This diff adds unguarded `torch.get_device_module()` at `MiniMaxH3VideoVAE.__init__` (`vae.py:120`) with no hasattr/version/try guard. `requirements/common.txt`, `requirements/`, and `pyproject.toml` have no `torch` pin (`torch>=` matches nowhere; common.txt lists `torchsde` only; dependencies are dynamic via setup.py/platform extras). Oldest in-repo torch bound is undeclared. What is the oldest supported torch for this call, and does it need a pin or guard? (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — assigned in MiniMaxH3VideoVAE.__init__ with no hasattr/try/version guard; consumed unguarded at encode_image/encode_video (`with self.device_module.device(device):` / `self.device_module.manual_seed(...)`). requirements/common.txt:21 `torchsde>=0.2.6` — torch itself is not listed among common deps. pyproject.toml:11 `dynamic = ["version", "dependencies"]` and pyproject.toml:32 `# Dependencies are now managed dynamically via setup.py based on detected hardware platform.` Repo-wide grep for `torch>=` returned no matches.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [nit] — This diff adds unguarded `self.device_module = torch.get_device_module()` in `MiniMaxH3VideoVAE.__init__` (`vae.py:120`), which CPU test `test_video_vae_keeps_reference_fp32_weights` now executes. Gate report lists `buildkite/vllm-omni-amd-ci` failing, and `.buildkite/amd/test-amd-ready.yml` runs `pytest tests/diffusion -m 'core_model and cpu'`, which selects this MiniMax contract file. Is that red check already on main, or does this new init-time API fail in that lane? (evidence: New line vllm_omni/diffusion/models/minimax_h3/vae.py:120: `self.device_module = torch.get_device_module()`. CPU constructor coverage at tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:319-322: `video_vae = vae_module.MiniMaxH3VideoVAE("unused", device=torch.device("cpu"))`. AMD selection at .buildkite/amd/test-amd-ready.yml:19: `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`. Gate report: FAILING CHECKS (1): ['buildkite/vllm-omni-amd-ci'].) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.trace.json.gz new file mode 100644 index 000000000..4658de8af Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5703/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.cost.json new file mode 100644 index 000000000..02a0cefbb --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1189.3, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5884/run-20260814-232416-3a394c", + "recorded_at": "2026-08-14T23:44:05", + "attempts": [ + { + "invocation_id": "inv-54a67076", + "outcome": "run:run-20260814-232416-3a394c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-232416-3a394c", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-232416-3a394c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.61, + "minutes": 19.81, + "ci_minutes": 0, + "input_tokens": 1556891, + "output_tokens": 172392, + "tool_calls": 336, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7793, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.68, + "gate": 1.74, + "review": 1183.17, + "report": 0.06 + }, + "time_to_first_result_s": 1188.64 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5884.trace.json.gz", + "trace_bytes": 24247 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.md b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.md new file mode 100644 index 000000000..5eea77859 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5884, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5dc24e7f6a0c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] backend.py:45 `if not isinstance(name, str)`: reachable only if a pipeline puts a non-str in `_dit_modules`; production ClassVar values are list[str]. Skip is defensive, not dead, and matches the old filter. +- [sweep] backend.py:51 `if module is not None`: distinct from AttributeError; a declared path that exists and is None (optional second DiT) is dropped. Same None-skip as ModuleDiscovery._collect_modules. +- [sweep] backend.py:61 `if not BlockAdapter.is_cached(transformer)`: not redundant with cache_dit.summary — summary still proceeds for parallelized/quantized uncached modules (summary.py:82-89). Needed so Cosmos3 UND (`transformer.language_model`) is not summarized when only GEN `transformer` is cached via the custom enabler. +- [validated] Correctness: `_dit_module_names` + `_make_pipeline_transformer_getter` + `cache_summary` now use attrgetter; Bagel `_dit_modules=['language_model.model']` (pipeline_bagel.py:159) and `self.transformer = self.language_model.model` (pipeline_bagel.py:254) resolve to Qwen2MoTModel which owns `_cache_dit_adapter_config`/`BagelCachedAdapter` (bagel_transformer.py:890-895). Missing nested paths still AttributeError-continue then enable() raises if empty (backend.py:252-253). +- [validated] Simplifiability: nested resolution duplicates ModuleDiscovery._collect_modules (module_collector.py:108-112) rather than importing the offloader helper; appropriate layering. No dead branch among the three new arms. +- [validated] Breaking/consumers: `cache_summary` signature unchanged; callers remain diffusion_model_runner.py:532 and tests. `_dit_module_names` is private. In-repo nested production `_dit_modules`: Bagel, Cosmos3 (custom enabler for enable; summary uses discovery), SoulX. +- [validated] Rebase/merge: single-commit 5dc24e7f; no duplicated helpers or dangling getattr in the touched backend functions. `_default_get_pipeline_transformer` still getattr('transformer') for custom enablers (Cosmos3/Krea2) — intentional. +- [validated] Tests: new tests drive real `cache_summary`/`CacheDiTBackend.enable` with SimpleNamespace dotted paths (not pre-transformed names). `test_summary_skips_uncached_nested_dit` uses real BlockAdapter.is_cached (getattr `_is_cached`, block_adapters.py:601-619) and fails if the skip is removed while both modules resolve. File keeps `pytestmark = [core_model, cpu]` (test_cache_backends.py:32). Nested enable test proves refresh_context gets the nested module (test_cache_backends.py:156-160). PR body claims Cache-DiT unit tests 30 passed, Bagel E2E 1 passed; this agent did not rerun them. + +`vllm_omni/diffusion/cache/cachedit/backend.py:48` [minor] — Replacing getattr with operator.attrgetter in `_dit_module_names` (backend.py:48) and `_make_pipeline_transformer_getter` (backend.py:81) makes dotted `_dit_modules` entries resolve on the generic CacheDiTBackend.enable loop instead of being dropped. FlowMatchingAudioPipeline declares `_dit_modules = ["cfm_decoder.model.diff_estimator"]` (pipeline_soulx_singer_base.py:63); DiffLlama already has `_cache_dit_adapter_config` with Pattern_3/`layers` (llama.py:115-119). SoulX class names are not in CUSTOM_DIT_ENABLERS (Wan22*/Cosmos3*/Krea2Pipeline only) or `_NO_CACHE_ACCELERATION` (NextStep11Pipeline, AudioXPipeline only), so enable will now build a BlockAdapter and call enable_cache instead of raising "has no declared DiT modules". The new unit test only covers the two-segment `language_model.model` fixture. Extend `test_enable_and_refresh_nested_declared_dit` (or add a sibling) with a three-segment SimpleNamespace matching `cfm_decoder.model.diff_estimator`, and record this newly unblocked SoulX enable path in the PR body. (evidence: backend.py:48 `module = attrgetter(name)(pipeline)` — dotted `_dit_modules` now resolve instead of getattr dropping them. backend.py:81 `return cast(nn.Module, attrgetter(name)(pipeline))`. backend.py:239-253 `for name in _dit_module_names(pipeline):` then `_maybe_build_block_adapter` / `enable_cache_for_dit`; empty list raises `Pipeline {pipeline_name} has no declared DiT modules for Cache-DiT`. pipeline_soulx_singer_base.py:63 `_dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]`. llama.py:115-119 `_cache_dit_adapter_config = CacheDiTAdapterConfig(block_forward_patterns={"layers": ForwardPattern.Pattern_3,},)`. model_specific.py:846-858 CUSTOM_DIT_ENABLERS.update keys are only Wan22*/Cosmos3*/Krea2Pipeline. registry.py:332-336 `_NO_CACHE_ACCELERATION = {"NextStep11Pipeline", "AudioXPipeline"}`. test_cache_backends.py:145-147 `pipeline = SimpleNamespace(_dit_modules=["language_model.model"], language_model=SimpleNamespace(model=transformer),)` — two-segment only.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:48` [minor] — This diff switches `_dit_module_names` from getattr to attrgetter, so dotted `_dit_modules` now resolve on the generic `CacheDiTBackend.enable` loop, which still wraps every resolved name (backend.py:239). Cosmos3 declares both `transformer.language_model` and `transformer`; only `Cosmos3VFMTransformer` has `_cache_dit_adapter_config`. Today `CUSTOM_DIT_ENABLERS['Cosmos3OmniDiffusersPipeline']` bypasses that loop via `enable_cache_for_cosmos3`, which wraps only the default `transformer`. A subclass or missed registration would now attempt Cache-DiT on the nested UND first — the case getattr previously hid. Filter nested-inside-another-listed-DiT before enable (reuse `outermost_dits`), or add a unit test that generic enable on a Cosmos3-shaped list does not call `enable_cache` on the nested UND module. (evidence: backend.py:48 ` module = attrgetter(name)(pipeline)` — dotted names now resolve. backend.py:233 ` custom_enabler = CUSTOM_DIT_ENABLERS.get(pipeline_name)` then else backend.py:239 ` for name in _dit_module_names(pipeline):` still enables every resolved name. pipeline_cosmos3.py:816 ` _dit_modules: ClassVar[list[str]] = ["transformer.language_model", "transformer"]`. transformer_cosmos3.py:878 `class Cosmos3LanguageModel(nn.Module):` has no `_cache_dit_adapter_config`; transformer_cosmos3.py:971 `class Cosmos3VFMTransformer(nn.Module):` / 992 ` _cache_dit_adapter_config = CacheDiTAdapterConfig(` is the only Cosmos3 adapter config. model_specific.py:781 `def enable_cache_for_cosmos3` uses `_maybe_build_block_adapter(pipeline)` (default `transformer` getter) then a single `enable_cache_for_dit`; model_specific.py:853 ` "Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,`. module_collector.py:26 ` def outermost_dits(self)` documents Cosmos3 listing both `transformer` and nested `transformer.language_model`. tests/diffusion/cache/test_cache_backends.py:129 `_dit_modules=["transformer.language_model", "transformer"]` is summary-only; :146 `_dit_modules=["language_model.model"]` is nested Bagel enable, not generic enable on the mixed list.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:48` [minor] — Cache-DiT `_dit_module_names` now resolves dotted `_dit_modules` via `attrgetter` (backend.py:48), but compile/SP/LoRA still `getattr`/`hasattr` the raw names and skip nested DiTs: `_compile_transformer` at diffusion_model_runner.py:153 (`getattr(..., None)` then return), SP at registry.py:451 (`getattr(model, attr)`), LoRA at lora/manager.py:388 (`hasattr` continue). Offload already uses `attrgetter`. The documented contract (interface.py) says all component names support dotted paths. This remainder is pre-existing and outside issue-5879 / Cache-DiT enable-refresh-summary; please confirm it is intentionally unfixed here or file follow-up. (evidence: backend.py:48 `module = attrgetter(name)(pipeline)` — Cache-DiT now resolves dotted `_dit_modules`. diffusion_model_runner.py:153 `model = getattr(self.pipeline, attr_name, None)` — compile still cannot resolve dotted names. registry.py:451 `transformer = getattr(model, attr)` — SP still getattr the raw name. lora/manager.py:386-388 `if not hasattr(self.pipeline, component_name): continue` / `component = getattr(self.pipeline, component_name)` — LoRA skips nested DiTs. interface.py:97-98 `All attribute names support dotted paths for nested submodules (e.g. "pipe.transformer").`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:61` [minor] — The new `BlockAdapter.is_cached` skip in shared `cache_summary` (backend.py:61) is inherited by every `--enable-cache-dit-summary` run. The only production caller is `diffusion_model_runner.py:532`, gated by `enable_cache_dit_summary` (default False in OrchestratorArgs). Uncached declared DiTs are omitted for every pipeline, not only Cosmos3's mixed `_dit_modules` list. Confirm that protocol change is intended for all summary users of this helper. (evidence: backend.py:56-63 `def cache_summary(...):` / `for transformer in transformers:` / `if not BlockAdapter.is_cached(transformer):` / `continue` / `cache_dit.summary(transformer, details=details)` — skip is inside the shared helper with no pipeline-type or Cosmos3 gate. diffusion_model_runner.py:526-532 `if (self.cache_backend is not None and self.cache_backend.is_enabled() and od_config.cache_backend == "cache_dit" and od_config.enable_cache_dit_summary): cache_summary(self.pipeline, details=True)` — sole production caller, gated only by enable_cache_dit_summary. arg_utils.py:489 `enable_cache_dit_summary: bool = False` — CLI/orchestrator default off.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:61` [nit] — cache_summary now skips transformers that fail BlockAdapter.is_cached (backend.py:61-63), but its docstring still says it logs statistics for every transformer on the pipeline (backend.py:57). Update the docstring to say every transformer with an active Cache-DiT context. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:57 `"""Log Cache-DiT statistics for every transformer on the pipeline."""` — vllm_omni/diffusion/cache/cachedit/backend.py:61-63 `if not BlockAdapter.is_cached(transformer):` / `continue` / `cache_dit.summary(transformer, details=details)` — uncached declared DiTs are omitted while the docstring still claims every transformer.) + +`tests/diffusion/cache/test_cache_backends.py:121` [nit] — This diff adds `test_summary_skips_uncached_nested_dit` and `test_enable_and_refresh_nested_declared_dit` on a file marked `pytest.mark.core_model` and `pytest.mark.cpu`. Gate report lists `buildkite/vllm-omni-amd-ci` failing, and that lane runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` (`.buildkite/amd/test-amd-ready.yml`). Is that red already present on main, or did these new cases / the discovery change start it? (evidence: test_cache_backends.py:32: `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. New tests at lines 121 and 139. `.buildkite/amd/test-amd-ready.yml:19`: `timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py`. CUDA L2 equivalent is `.buildkite/cuda/test-ready.yml:18`. Gate report: FAILING CHECKS (1): ['buildkite/vllm-omni-amd-ci'].) + +`vllm_omni/diffusion/cache/cachedit/backend.py:61` [nit] — The new skip calls `BlockAdapter.is_cached(transformer)`, while production `BagelCachedAdapter` sets `_is_cached` on `block_adapter.pipe.__class__` (`model_specific.py:645`). cache-dit is pinned at `==1.3.0`, but this review could not read that pin's `is_cached` / `BlockAdapter.pipe` default. If `pipe` is the pipeline rather than `language_model.model`, would Bagel `--enable-cache-dit-summary` skip the only DiT? The new skip test sets instance `_is_cached` on a Mock, which may not match that class-level production flag. (evidence: unverified: BagelCachedAdapter sets `_is_cached` on block_adapter.pipe.__class__ (model_specific.py:645) while cache_summary checks is_cached(transformer); if pipe is the pipeline not language_model.model, Bagel --enable-cache-dit-summary would skip the only DiT. Not posted: could not read cache-dit 1.3.0 BlockAdapter.pipe default.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.trace.json.gz new file mode 100644 index 000000000..718be3d5d Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5884/pr5884.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.cost.json new file mode 100644 index 000000000..4dcac985c --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 2589.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5957/run-20260814-232643-77d6cd", + "recorded_at": "2026-08-15T00:09:51", + "attempts": [ + { + "invocation_id": "inv-fb2241a7", + "outcome": "run:run-20260814-232643-77d6cd", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-232643-77d6cd", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-232643-77d6cd", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2415, + "minutes": 43.14, + "ci_minutes": 0, + "input_tokens": 3462605, + "output_tokens": 278754, + "tool_calls": 601, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3303, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.48, + "gate": 2.3, + "review": 2581.66, + "report": 0.06 + }, + "time_to_first_result_s": 2588.48 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5957.trace.json.gz", + "trace_bytes": 127184 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.md b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.md new file mode 100644 index 000000000..5256fc33d --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.md @@ -0,0 +1,58 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5957, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 19a6e513f28e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] 1 correctness: serving native-speed skips CreateAudio resample (serving_speech.py:521-524) and streaming encodes speed=1.0 (serving_speech.py:2273); Stage 1 applies duration_factor=1/speed. Non-native stream+speed still 400 at serving_speech.py:3642. +- [validated] request-end payload: omni_payload_at_request_end only for model_type==indextts2_5 (indextts2_talker.py:171); later decode meta is {use_gpt_latent} only (test_talker_v2_5.py) and accumulator REPLACE keeps first-step meta.S_ref/duration_factor because those keys are absent on later steps (omni_connector_model_runner_mixin.py:920-947). +- [validated] talker2s2mel_full_payload code-only uses output_token_ids and fail-fast missing use_gpt_latent/S_ref (stage_input_processors/indextts2.py:211-277); IndexTTS-2 tests updated to emit use_gpt_latent. +- [sweep] 2 simplifiability: end2end _get_stage0_tokenizer_file hasattr/get chain is example-only; not worth a comment. duration_factor validation on shared talker defaults to 1.0 for IndexTTS-2 so it is not dead, just 2.5-branded. +- [sweep] 3 breaking: OpenAICreateSpeechRequest no longer 422s stream+speed!=1.0 (protocol/audio.py:317-318); serving rejects non-native (400). tests/entrypoints/openai_api/test_serving_speech.py loosened to 400|422. WebSocket still requires speed=1.0 (audio.py:556-558). build_bench_args extra `--` strip covered by test_bench_tts_cli.py. +- [sweep] 4 rebase: 0.26/0.27 shims (MistralToolCall, renderer.warmup getattr, num_stale_output_tokens, TokenOnlyRenderer, XPU eager Qwen2.5) look intentional, not dropped hunks. +- [sweep] 5 tests: L2 CPU tests bind real entry points (adapter.validate/build, talker2s2mel, duration bounds, skip_tokenizer renderer). E2E in PR body is Whisper transcription + speed 1.0 vs 2.0 duration, not official beam-search parity (docs disclose num_beams=1). +- [validated] skip_tokenizer_init TokenOnlyRenderer is shared by indextts2, indextts2_5, indextts2_low_latency, personaplex; all already skip tokenizer and submit prompt_token_ids. + +`vllm_omni/entrypoints/openai/serving_speech.py:3642` [major] — Moving the streaming `speed != 1.0` rejection out of OpenAICreateSpeechRequest.validate_streaming_constraints into OmniOpenAIServingSpeech._validate_speech_streaming_request left the diffusion path uncovered. create_speech returns `_create_diffusion_speech` at serving_speech.py:3676-3677 when `_diffusion_mode` is true (set by for_diffusion at :412) before the validator is invoked at :3700/:3721. Diffusion TTS (OmniVoice) therefore accepts stream=true/stream_format with speed!=1.0, ignores streaming, and still applies generic CreateAudio stretch via `_audio_encode_speed` at :3600. Call `_validate_speech_streaming_request` (or reject streaming+speed when `_uses_native_speed_control()` is false) before that early return, and add a `_diffusion_mode=True` streaming+speed test. (evidence: vllm_omni/entrypoints/openai/protocol/audio.py:319-321 `if self.speed is None: self.speed = 1.0` then `return self` with no `speed != 1.0` raise. vllm_omni/entrypoints/openai/serving_speech.py:412 `instance._diffusion_mode = True`. serving_speech.py:3642 `if request.speed is not None and request.speed != 1.0 and not self._uses_native_speed_control():`. serving_speech.py:3676-3677 `if self._diffusion_mode: return await self._create_diffusion_speech(request)` before serving_speech.py:3700/3721 `response_format, error = self._validate_speech_streaming_request(`. serving_speech.py:3600 in `_create_diffusion_speech` `speed=self._audio_encode_speed(request)` with no `request.is_streaming()` check.) + +`tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:16` [major] — New IndexTTS-2.5 modules test_configuration_and_pipeline.py, test_enhanced_codec.py, and test_recipe_defaults.py have no pytestmark, unlike siblings such as test_s2mel_v2_5.py:26. L2 `.buildkite/cuda/test-ready.yml:13` runs `pytest -sv tests/model_executor -m 'core_model and cpu'`; tests/model_executor/conftest.py is a comment stub and tests/ has no pytest_collection_modifyitems, so these modules are silently deselected. That drops CI coverage of IndexTTS25Config defaults (vocab 60509, enhanced codec, use_gpt_latent=false), OMNI_PIPELINES['indextts2_5'], EnhancedCodec decode/layout plus native checkpoints/ resolution, and the only assert that s2mel_cfm_batch_size == 4 (test_recipe_defaults.py:14). Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` to all three files. (evidence: test_configuration_and_pipeline.py:16 `def test_indextts25_defaults_are_distinct_from_v2():` — first test; file has no pytestmark. test_enhanced_codec.py:11 `def test_enhanced_codec_decode_restores_hidden_size_and_doubles_time():` — no pytestmark. test_recipe_defaults.py:14 `assert stage1["hf_overrides"]["s2mel_cfm_batch_size"] == 4` — only batch-size==4 assert; no pytestmark. test_s2mel_v2_5.py:26 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. .buildkite/cuda/test-ready.yml:13 `timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml`. tests/model_executor/conftest.py: `"""Reserved for model_executor-only hooks.` (comment stub, no marker hook). No `pytest_collection_modifyitems` under tests/.) + +`vllm_omni/model_executor/stage_input_processors/indextts2.py:?` [major] — IndexTTS-2 is an untested rider of this shared-path tightening. INDEXTTS2_PIPELINE still binds `talker2s2mel_full_payload` (`pipeline.py:30`) and `IndexTTS25S2MelDecoder` is a registry alias of `IndexTTS2S2MelDecoder`. The processor now fail-fasts on missing `meta.use_gpt_latent` (`indextts2.py:222-223`) and mel/latent mismatch (`indextts2.py:262-263`), and forwards `sampling_params.seed` into Stage 1 (`indextts2.py:285-287`). Shared `compute_fbank` now raises `RuntimeError("IndexTTS requires torchaudio Kaldi fbank support")` instead of returning zeros (`preprocess_utils.py:281-282`). `recipes/IndexTeam/IndexTTS-2.md` still describes the old 2-only serving contract and does not mention CFM seed or fail-fast handoff. Run a same-seed IndexTTS-2 online/offline smoke on this head, or isolate the tightening so IndexTTS-2 is not an untested rider. (evidence: vllm_omni/model_executor/models/indextts2/pipeline.py:30 `custom_process_next_stage_input_func=f"{_PROC}.talker2s2mel_full_payload"` — INDEXTTS2_PIPELINE still binds the shared processor. vllm_omni/model_executor/models/registry.py:378-382 `"IndexTTS25S2MelDecoder": ("indextts2", "indextts2_s2mel_decoder", "IndexTTS2S2MelDecoder")` — 2.5 decoder is the shared class. vllm_omni/model_executor/stage_input_processors/indextts2.py:222-223 `if use_gpt_latent_value is None: raise ValueError(f"IndexTTS payload is missing meta.use_gpt_latent for req={rid}")`. same file:262-263 `if mel_seq.shape[0] != latent_seq.shape[0]: raise ValueError(f"IndexTTS mel/latent length mismatch for req={rid}:`. same file:285-287 `seed = _request_seed(request); if seed is not None: additional_information["seed"] = seed`. vllm_omni/model_executor/models/indextts2/preprocess_utils.py:281-282 `except (ImportError, RuntimeError, OSError) as exc: raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc`. recipes/IndexTeam/IndexTTS-2.md title `# IndexTTS-2 for voice-cloned TTS on 1x GPU` with no CFM seed or fail-fast handoff text.) + +`recipes/IndexTeam/IndexTTS-2_5.md:40` [minor] — The new recipe heading `### 1x NVIDIA H20 96GB` and `recipes/README.md` advertise H20 96GB, but Notes say the standard recipe was exercised on one NVIDIA H200. Change the heading and README table to H200, or add a separate H20 run before advertising that profile. (evidence: recipes/IndexTeam/IndexTTS-2_5.md:40 `### 1x NVIDIA H20 96GB`. recipes/IndexTeam/IndexTTS-2_5.md:290 `- Hardware scope: the standard recipe has been exercised on one NVIDIA H200.`. recipes/README.md:53 `| [`IndexTeam/IndexTTS-2_5.md`](./IndexTeam/IndexTTS-2_5.md) | Online + offline multilingual voice-cloned TTS with native speed and emotion control | 1x NVIDIA H20 96GB |`) + +`vllm_omni/entrypoints/openai/api_server.py:1001` [minor] — The renderer.warmup getattr skip at api_server.py:1001–1003 sits in shared omni_init_app_state next to OmniOpenAIServingChat construction gated only on "generate" in supported_tasks — not IndexTTS. Generation output is unchanged. On 0.26 accelerator images whose OnlineRenderer lacks warmup, init no longer fails and the first chat request pays the template-processing cost warmup would have prepaid. Confirm this shared 0.26-compat skip belongs on the IndexTTS-2.5 vehicle, or gate/split it to the v0.27 accelerator path. (evidence: api_server.py:993 `state.openai_serving_chat = OmniOpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None` — chat serving is task-generic, no IndexTTS gate. api_server.py:999-1003 `# Upstream f5ffc59b6a moved warmup onto OnlineRenderer. Accelerator\n # images can temporarily lag that renderer API, where warmup is optional.\n renderer_warmup = getattr(state.online_renderer, "warmup", None)\n if renderer_warmup is not None:\n renderer_warmup()` — shared getattr skip; missing warmup is a no-op, so first-request chat-template cost is not prepaid.) + +`vllm_omni/entrypoints/openai/protocol/audio.py:558` [minor] — StreamingSpeechSessionConfig still hard-rejects speed!=1.0 for every model, including IndexTTS 2.5 (`audio.py:558-561`), while HTTP `OpenAICreateSpeechRequest` no longer does (`audio.py:319-321`) and `IndexTTS25Adapter.native_speed_control` is True (`tts_adapters/indextts2.py:276`). The new test `test_streaming_speech_session_config_scopes_native_speed_to_http` and the IndexTTS recipe/client (`/v1/audio/speech` only) match that HTTP-only split. If WebSocket native speed is out of scope this release, the current gate is consistent; otherwise the session validator needs an adapter-aware exception like HTTP serving. (evidence: audio.py:558-561 `elif self.speed != 1.0: raise ValueError("WebSocket stream_audio=true currently requires speed=1.0; native speed control is only available through HTTP streaming.")` — WebSocket rejects non-1.0 speed with no adapter/native_speed_control check. audio.py:319-321 HTTP streaming only defaults missing speed: `if self.speed is None: self.speed = 1.0` / `return self` (no speed!=1.0 raise). tts_adapters/indextts2.py:276 `native_speed_control: ClassVar[bool] = True`. test_serving_speech.py:2707-2716 `test_streaming_speech_session_config_scopes_native_speed_to_http` expects ValidationError matching `native speed control is only available through HTTP streaming` for `StreamingSpeechSessionConfig(stream_audio=True, response_format="pcm", speed=1.5)`. speech_client.py:138 `url = f"{args.api_base}/v1/audio/speech"`; recipes/IndexTeam/IndexTTS-2_5.md:9 `Mode: Online serving with the OpenAI-compatible `/v1/audio/speech` API`.) + +`vllm_omni/deploy/indextts2_5.yaml:20` [minor] — vllm_omni/deploy/indextts2_5.yaml Stage 0 sets skip_tokenizer_init: true with no tokenizer pin (line 21); INDEXTTS25_PIPELINE extras are the same. IndexTTS-2 pins tokenizer: gpt2 in both indextts2.yaml:17 and pipeline.py:29. OmniEngineArgs.create_model_config still auto-detects the first immediate subdirectory with tokenizer_config.json whenever self.tokenizer is unset (arg_utils.py:327-333) and never consults skip_tokenizer_init. HF-converted bundles place qwen0.6bemo4-merge at MODEL root (configuration_indextts2.py:142), so Stage 0 can bind that QwenEmotion tokenizer path before vLLM config is built. Skip auto-detect when skip_tokenizer_init is true, or pin a dummy tokenizer like v2. (evidence: indextts2_5.yaml:21 `skip_tokenizer_init: true` with no tokenizer key on Stage 0. indextts2.yaml:16-17 `skip_tokenizer_init: true` / `tokenizer: gpt2`. pipeline.py:29 v2 extras `{"skip_tokenizer_init": True, "tokenizer": "gpt2"}` vs pipeline.py:65 v2.5 extras `{"skip_tokenizer_init": True}`. arg_utils.py:327-333 `if not self.tokenizer and self.model:` then first child dir with tokenizer_config.json does `self.tokenizer = candidate` with no skip_tokenizer_init check. configuration_indextts2.py:142 `"qwen_emo_path": "qwen0.6bemo4-merge/"`.) + +`vllm_omni/entrypoints/openai/protocol/audio.py:?` [minor] — OpenAICreateSpeechRequest.validate_streaming_constraints (audio.py:319-321) no longer raises on stream=true with speed≠1.0, so pydantic now accepts that payload for every TTS model. Non-native adapters still fail later in OmniOpenAIServingSpeech._validate_speech_streaming_request (serving_speech.py:3642-3647) via create_error_response (`{mode} is not supported with speed adjustment`), which HTTP tests treat as 400 rather than the previous pydantic 422 `Speed adjustment is not supported when streaming`. Document this shared /v1/audio/speech status and message change for Qwen3/Fish/GLM/IndexTTS-2/Higgs clients that matched the old ValidationError; only IndexTTS25Adapter sets native_speed_control=True. (evidence: vllm_omni/entrypoints/openai/protocol/audio.py:319-321 `if self.speed is None:` / `self.speed = 1.0` / `return self` — no `speed != 1.0` raise. vllm_omni/entrypoints/openai/serving_speech.py:3642-3647 `if request.speed is not None and request.speed != 1.0 and not self._uses_native_speed_control():` / `return response_format, self.create_error_response(` / `f"{mode_label} is not supported with speed adjustment. "` / `"Use a non-streaming request or remove the speed parameter."`. vllm_omni/entrypoints/openai/tts_adapters/base.py:119 `native_speed_control: ClassVar[bool] = False`. vllm_omni/entrypoints/openai/tts_adapters/indextts2.py:276 `native_speed_control: ClassVar[bool] = True`. tests/entrypoints/openai_api/test_serving_speech.py:2428 `assert response.status_code in (400, 422)`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/common/qwen3_code_predictor.py:577` [minor] This IndexTTS 2.5 PR also disables torch.compile on XPU in shared `qwen3_code_predictor._setup_compile` (`if current_omni_platform.is_xpu() or not current_omni_platform.supports_torch_inductor():` then `self._compiled_mo +- `vllm_omni/engine/stage_init_utils.py:1141` [minor] Shared `build_stage0_input_processor` now does `InputProcessor(vllm_config=..., renderer=_build_token_only_renderer(...))` whenever `skip_tokenizer_init` is true (`stage_init_utils.py:1140-1143`), and `_build_token_only_ +- `tests/e2e/offline_inference/test_indextts2_expansion.py:24` [minor] IndexTTS 2.5's default Stage 0→1 path is code-only (`vllm_omni/deploy/indextts2_5.yaml` `use_gpt_latent: false`; `talker2s2mel_full_payload` reads `request.output_token_ids` when latent is off). The only Omni GPU job, `t + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.trace.json.gz new file mode 100644 index 000000000..a6210af5e Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/QUARANTINE_skillleak_pr5957/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/manifest.json b/eval/dataset/arms/copilot_cb_grok46_r3/manifest.json new file mode 100644 index 000000000..dc2ae5910 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_cb_grok46_r3", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.cost.json new file mode 100644 index 000000000..9ee32674b --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.cost.json @@ -0,0 +1,102 @@ +{ + "wall_s": 1327.4, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5509/run-20260815-001259-e8d9df", + "recorded_at": "2026-08-15T00:35:06", + "attempts": [ + { + "invocation_id": "inv-80df6462", + "outcome": "run:run-20260815-001259-e8d9df", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-224628-d8debb", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260815-001259-e8d9df", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-001259-e8d9df", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6158, + "minutes": 22.11, + "ci_minutes": 0, + "input_tokens": 1611377, + "output_tokens": 164312, + "tool_calls": 301, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8207, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.74, + "gate": 1.72, + "review": 1321.23, + "report": 0.06 + }, + "time_to_first_result_s": 1326.73 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 72161 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.md b/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.md new file mode 100644 index 000000000..22513d2f1 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.md @@ -0,0 +1,56 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] test_trtllm_attn.py:315 Missing TRTLLM-GEN kernel skip: reachable on SM103 for int8; converts a kernel-miss into skip rather than fail; GPU test is not in CI anyway. +- [sweep] trtllm_attn.py:90 HAS_FLASHINFER false: _sage_kernel_available returns False; quant.enabled then raises at init (204) rather than silently degrading. +- [sweep] trtllm_attn.py:198 quant.enabled: dtype_qk is not None; dense path leaves sage_kwargs empty so older FlashInfer still works. +- [sweep] trtllm_attn.py:199 dtype_qk not in _QK_QUANT_DTYPES: raises RuntimeError; covered by test_quant_rejects_non_sage_dtype (bfloat16). +- [sweep] trtllm_attn.py:204 _sage_kernel_available false: raises; inspect.signature miss or no FlashInfer. +- [sweep] trtllm_attn.py:211 _sage_quantize_fn is None: raises; covered by monkeypatch test. +- [sweep] trtllm_attn.py:296 forward SAGE: quantize then sage_attn_sfs/num_elts_per_sage_attn_blk; packing is (q_sfs,k_sfs,None,v_sfs) and (q_block,k_block,0,1). +- [sweep] data.py:1415 dtype not in union: ValueError; union is float16/bfloat16/int8/fp8_e4m3 — not per-backend. + +`tests/diffusion/attention/test_trtllm_attn.py:50` [major] — The four CPU-pure QuantConfig tests (`test_quant_config_from_backend_kwargs`, `test_quant_rejects_non_sage_dtype`, `test_quant_quantize_requires_flashinfer_routine`, `test_quant_quantize_calls_routine_and_shapes_sfs`) live in `tests/diffusion/attention/test_trtllm_attn.py`, which is file-marked `pytest.mark.cuda` (line 13). L2 Ready/Merge runs `pytest -sv tests/diffusion -m 'core_model and cpu'` (`.buildkite/cuda/test-ready.yml:18`, `test-merge.yml:18`) and therefore deselects that file. CUDA Other runs `pytest -sv tests/diffusion/*.py` (`test-ready.yml:68`), which does not recurse into `attention/`. Grep of `.buildkite` has no `tests/diffusion/attention` collector. Move the four cases into `test_attention_config.py` (already `core_model`+`cpu` at line 34) so merge CI runs the init/error/packing paths. Keep `test_sage_quant_matches_sdpa` (`@requires_sage`; `_has_trtllm_attn` returns False when `torch.cuda.get_device_capability()[0] < 10`) on an SM100 job — an L4 `l4_1` runner would skip it. (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` ; tests/diffusion/attention/test_trtllm_attn.py:50 `def test_quant_config_from_backend_kwargs():` ; tests/diffusion/attention/test_trtllm_attn.py:19 `if torch.cuda.get_device_capability()[0] < 10:` ; tests/diffusion/attention/test_attention_config.py:34 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]` ; .buildkite/cuda/test-ready.yml:18 `timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu'` ; .buildkite/cuda/test-ready.yml:68 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` ; grep of `.buildkite` for `tests/diffusion/attention` returns no matches.) + +`vllm_omni/diffusion/data.py:?` [major] — This diff allowlists FLASHINFER_ATTN in AttentionSpec.__post_init__ (data.py:1447) and documents that FLASHINFER_ATTN reads dtype_qk (float16/bfloat16), dtype_vo, and flashinfer_backend, with a wrong-set dtype_qk raising. FlashInferAttentionImpl.__init__ (flashinfer_attn.py:55-64) only takes **extra_impl_args and never reads backend_kwargs['quant']; forward_cuda always calls dense single_prefill_with_kv_cache. Attention() still passes backend_kwargs=spec.backend_kwargs() (layer.py:109). test_quant_superset_fields_passed_through only round-trips the dict. Restrict the allowlist and AttnQuantSpec FLASHINFER fields to TRTLLM_ATTN until flashinfer_attn.py consumes quant, or reject unused quant at impl init the way TRTLLM rejects non-SAGE dtypes — otherwise a user who follows the new docs gets silent dense attention. QuantConfig.enabled (trtllm_attn.py:130-131) is `dtype_qk is not None`, so a dtype_vo-only block is also silent dense on TRTLLM. (evidence: data.py:1447-1450 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"): raise ValueError(...)`. docs/user_guide/diffusion/attention_backends.md:185-188 `The quant block is a shared AttnQuantSpec consumed by more than one backend; each reads the fields that apply and rejects the rest. ... FLASHINFER_ATTN reads dtype_qk (float16 / bfloat16), dtype_vo, and flashinfer_backend (see its section). A dtype_qk from the wrong set raises.` flashinfer_attn.py:55-66 `__init__(..., **extra_impl_args)` stores only `self.causal`/`self.softmax_scale` — no `backend_kwargs`/`quant` use. flashinfer_attn.py:178 `out = single_prefill_with_kv_cache(query[b], key[b], value[b], **kwargs)` dense path with no quant kwargs. layer.py:93 `backend_kwargs = spec.backend_kwargs()`; layer.py:109 `backend_kwargs=backend_kwargs`. tests/diffusion/attention/test_attention_config.py:65-73 only asserts FLASHINFER-style fields round-trip on AttentionSpec. trtllm_attn.py:130-131 `def enabled(self) -> bool: return self.dtype_qk is not None`.) + +`docs/user_guide/diffusion/attention_backends.md:223` [major] — This diff inserts `## TRTLLM_ATTN SAGE Quantization` then deletes everything after the BF16 table: `## Known Limitations` (LTX-2.0 `CUDNN_ATTN` + torch.compile crash / #3121 and the `FLASHINFER_ATTN`/`TORCH_SDPA` workaround), `### FA4 not yet integrated`, mixed-backend and FA comparison examples, `## Validation Guidance`, and `## Reproducing the Benchmark Table`. `attention_backends.md:232` still says 'see Known Limitations above', but that heading is gone and grep finds no relocated LTX-2 CUDNN limitation. `benchmarks/diffusion/bench_attention_backends.py` still exists (`--preset hv15`). Restore those sections intact (or move them); do not ship SAGE docs by dropping unrelated operator guidance. (evidence: docs/user_guide/diffusion/attention_backends.md:222 `Pattern: mask-heavy DiTs (HV-1.5, Qwen-Image) favor `CUDNN_ATTN`; lighter-mask DiTs and TP-saturated configs (Wan 2.2, FLUX.2 TP=2) tie within noise.` then :226 `## Choosing a Backend Manually` with no `## Known Limitations` (or FA4 / Validation Guidance / Reproducing the Benchmark Table) in this file. :232 `- **Workaround for known issues**: see Known Limitations above.` — dangling pointer. benchmarks/diffusion/bench_attention_backends.py:15 `python benchmarks/diffusion/bench_attention_backends.py --preset hv15` still exists.) + +`vllm_omni/diffusion/data.py:1479` [minor] — AttentionSpec.backend_kwargs() serializes `quant.dtype_vo` and `quant.flashinfer_backend` (data.py:1479-1482) into the dict backends consume, but nothing in this tree reads them: TRTLLM QuantConfig.from_backend_kwargs only takes dtype_qk/q_block_size/k_block_size (trtllm_attn.py:123-126), and FlashInferAttentionImpl.__init__ swallows **extra_impl_args unused (flashinfer_attn.py:63-66). Repo-wide, those keys appear only in this serializer, a round-trip test, and docs that claim FLASHINFER_ATTN consumes them with no matching impl/section. Drop the unused remainder from this SAGE PR, or keep it only as an explicit #5344 deferral. (evidence: vllm_omni/diffusion/data.py:1479-1482 `if q.dtype_vo is not None:` / `quant_kw["dtype_vo"] = q.dtype_vo` / `if q.flashinfer_backend is not None:` / `quant_kw["flashinfer_backend"] = q.flashinfer_backend` — serialized into backend_kwargs. vllm_omni/diffusion/attention/backends/trtllm_attn.py:123-126 `return cls(` / `dtype_qk=q.get("dtype_qk"),` / `q_block_size=int(q.get("q_block_size", 1) or 1),` / `k_block_size=int(q.get("k_block_size", 16) or 16),` — SAGE QuantConfig never reads dtype_vo or flashinfer_backend. vllm_omni/diffusion/attention/backends/flashinfer_attn.py:63-66 `**extra_impl_args,` / `) -> None:` / `self.causal = causal` / `self.softmax_scale = softmax_scale` — FLASHINFER_ATTN never unpacks quant/dtype_vo/flashinfer_backend.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:143` [minor] — `QuantConfig.quantize` hardcodes the FlashInfer SAGE kwargs as `sage_attn_sfs=(q_sfs, k_sfs, None, v_sfs)` and `num_elts_per_sage_attn_blk=(q_block, k_block, 0, 1)` with no comment on why the third (O) slots are `None`/`0`. User docs cover V fp8-per-channel and K-smoothing as non-knobs, not this 4-tuple ABI. `test_quant_quantize_calls_routine_and_shapes_sfs` only checks that packing via a fake that returns strings (`"qq"` passthrough is tautological) and cannot catch a real `trtllm_sage_attention_quantize` scale-tensor layout mismatch. Please add a one-line QKVO comment at the packing site. (evidence: trtllm_attn.py:143 `sage_attn_sfs = (q_sfs, k_sfs, None, v_sfs)` — third slot hardcoded None, no comment. trtllm_attn.py:144 `num_elts_per_sage_attn_blk = (self.q_block_size, self.k_block_size, 0, 1)` — third slot hardcoded 0. attention_backends.md:182-183 `V is always quantized per-channel to fp8_e4m3 and K-smoothing is applied inside the routine, so neither is a user knob.` — documents knobs, not the 4-tuple ABI. test_trtllm_attn.py:81 `return "qq", "kq", "vq", "qsfs", "ksfs", "vsfs"` then :84 `assert (q_q, k_q, v_q, sfs, blk) == ("qq", "kq", "vq", ("qsfs", "ksfs", None, "vsfs"), (1, 16, 0, 1))` — packing asserted only through fake string returns.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:130` [minor] — TRTLLM `QuantConfig.enabled` is only `dtype_qk is not None` (`trtllm_attn.py:130-131`), while shared `AttnQuantSpec.enabled` is `dtype_qk or dtype_vo` (`data.py:1424-1425`) and `backend_kwargs()` still serializes a dtype_vo-only mapping (`data.py:1472-1480`). That config therefore skips SAGE init/forward (`trtllm_attn.py:198`, `296`) and runs dense with empty `sage_kwargs`. Unsupported `dtype_qk` already fail-louds (`trtllm_attn.py:199-203`); dtype_vo-only should reject as an unsupported TRTLLM field the same way, not stay silently dense. (evidence: trtllm_attn.py:130-131 `def enabled(self) -> bool:\n return self.dtype_qk is not None` — SAGE on/off is dtype_qk-only, dtype_vo is never read. data.py:1424-1425 `def enabled(self) -> bool:\n return self.dtype_qk is not None or self.dtype_vo is not None` — shared spec treats dtype_vo-only as enabled. data.py:1472-1480 `if self.quant is not None and self.quant.enabled:` then `quant_kw["dtype_qk"] = q.dtype_qk` and `if q.dtype_vo is not None: quant_kw["dtype_vo"] = q.dtype_vo` — dtype_vo-only still reaches the backend. trtllm_attn.py:198 `if self.quant.enabled:` — dtype_qk is None so SAGE init/API checks are skipped. trtllm_attn.py:295-296 `sage_kwargs: dict = {}\n if self.quant.enabled:` — forward stays dense with empty sage kwargs.) + +`docs/user_guide/diffusion/attention_backends.md:170` [minor] — This PR adds typed `quant: AttnQuantSpec | None` on `AttentionSpec` (`vllm_omni/diffusion/data.py:1434`) and documents it under `## TRTLLM_ATTN SAGE Quantization`, but Configuration still says the only typed spec field is Skip-Softmax and only shows a `skip_softmax` dotted-flag example (`docs/user_guide/diffusion/attention_backends.md:72-78`). Update that sentence and add a `quant` dotted-flag example so operators see `quant` as a first-class spec key. (evidence: docs/user_guide/diffusion/attention_backends.md:72-73 `A backend that needs configuration exposes it as a typed field on the spec. Today the only / one is \`TRTLLM_ATTN\`'s Skip-Softmax`; :76-77 `--diffusion-attention-config.default.backend TRTLLM_ATTN \\` / `--diffusion-attention-config.default.skip_softmax.target_sparsity 0.5`; :170 `## TRTLLM_ATTN SAGE Quantization`; :174 `Enable it through the typed \`quant\` block on the attention spec:`; vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None`.) + +`tests/diffusion/attention/test_trtllm_attn.py:?` [minor] — test_sage_quant_matches_sdpa (test_trtllm_attn.py:321) asserts only `rel < 0.1` vs SDPA — 10× the dense BF16 bound of `rel < 0.01` in test_bf16_dense_matches_sdpa:295 on the same B,S,H,D. A SAGE no-op that still ran dense TRTLLM would pass. The test also `pytest.skip`s on `Missing TRTLLM-GEN kernel` (314–317). Contrast test_skip_threshold_engages:345, which asserts skipped-vs-dense divergence `> 0.02`. On SM100 (int8 kernels are SM100-only per attention_backends.md:210), also compare SAGE vs `_impl().forward_cuda` and assert they differ; keep a tighter vs-SDPA bound if int8 stays near dense. Config-only test_quant_and_skip_softmax_coexist (test_attention_config.py:75–79) only serializes `backend_kwargs()` and never hits the kernel. (evidence: test_trtllm_attn.py:295 `assert rel < 0.01, f"BF16 dense rel err {rel:.4f} too high"` (same B,S,H,D=2,256,8,128 as the SAGE test). test_trtllm_attn.py:314-321 `except RuntimeError as e:` / `if "Missing TRTLLM-GEN kernel" in str(e):` / `pytest.skip(...)` / `assert rel < 0.1, f"SAGE({dtype_qk}) rel err {rel:.4f} too high"`. test_trtllm_attn.py:345 `assert (skipped - dense).abs().mean() / dense.abs().mean() > 0.02`. test_attention_config.py:75-79 `def test_quant_and_skip_softmax_coexist(self):` / `bk = AttentionSpec(...).backend_kwargs()` / `assert bk["target_sparsity"] == 0.5 and bk["quant"]["dtype_qk"] == "int8"` — no `forward_cuda`. test_trtllm_attn.py:19 `if torch.cuda.get_device_capability()[0] < 10:`. attention_backends.md:210 ``int8` QK kernels are compiled for **SM100 only`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/attention/backends/trtllm_attn.py:214` [minor] TRTLLM SAGE imports `flashinfer.trtllm_sage_attention_quantize` and, when `quant` is set, passes `sage_attn_sfs` / `num_elts_per_sage_attn_blk` into `trtllm_ragged_attention_deepseek`. The claimed floor is only a runtime + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.trace.json.gz new file mode 100644 index 000000000..66bfe11b7 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.cost.json new file mode 100644 index 000000000..a10ec5d2d --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1274.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5550/run-20260814-224628-2f4f5b", + "recorded_at": "2026-08-14T23:07:41", + "attempts": [ + { + "invocation_id": "inv-f3c6a0a7", + "outcome": "run:run-20260814-224628-2f4f5b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-224628-2f4f5b", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-224628-2f4f5b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6149, + "minutes": 21.22, + "ci_minutes": 0, + "input_tokens": 1549695, + "output_tokens": 178603, + "tool_calls": 359, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8054, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.84, + "gate": 1.68, + "review": 1267.43, + "report": 0.06 + }, + "time_to_first_result_s": 1273.0 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 40955 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.md b/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.md new file mode 100644 index 000000000..9b4b5a8e3 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] correctness: execute_request validates identity before RPC and outside the DiffusionOutput(error=) wrapper; runner rejects dense+metadata and request_id mismatch; paged allows missing metadata as documented W0; DLO list RPC omitted when any metadata present so snapshots cannot be dropped or applied to the wrong DP-selected request. +- [sweep] simplifiability: WorkerWrapperBase.execute_model kwargs-if-present duplicates DiffusionWorker.execute_model; WorkerProc RPC uses execute_method -> getattr(self.worker, method) so the wrapper method is not the production execute_model boundary (comment). +- [sweep] breaking: NewRequestData.diffusion_kv_metadata defaults None; in-repo NewRequestData(...) callers use keywords and remain valid; DiffusionWorker.execute_model adds optional 4th arg; ARDiffusionModelRunner.execute_model does not accept the new kwarg (comment). +- [sweep] rebase/merge: no dropped/duplicated producer; from_state still omits metadata and schedule() still calls NewRequestData.from_state(state) only — matches stated W0 (no allocator yet). +- [sweep] tests: new cpu/core_model/diffusion pytestmark files exist; request-RPC 3-tuple vs 4-tuple and identity mismatches are covered; DLO+metadata arm is not; metadata CFG tests are constructor round-trips (comment). +- [sweep] docs: metadata.py and NewRequestData docstrings match W0 trusted-DTO scope; WorkerWrapperBase.execute_model docstring omits the new parameter; execute_method docstring still claims wrapper methods are preferred but code always dispatches to the inner worker. +- [sweep] invariants: block_ids is tuple[list[int], ...] (native cache-group order, unvalidated in W0); context_ids are unchecked references; first-element test indexes are locally constructed singles, not runtime contracts. +- [sweep] scope: engine init cleanup is extra vs RFC W0 title but tested and resource-symmetric; described W1 allocator/installer/attention consumption is explicitly out of scope, not dropped from a claimed deliverable. + +`vllm_omni/diffusion/executor/multiproc_executor.py:?` [minor] — This diff adds `and not has_diffusion_kv_metadata` to the DLO+AllGather multi-request predicate in `execute_request` (multiproc_executor.py:469-473), so any wave with even one `NewRequestData.diffusion_kv_metadata` skips the list RPC `args=(reqs_list, od_config, kv_prefetch_job)` (no per-request snapshot slot, `unique_reply_rank=None`) and falls through to per-request 4-tuples with `unique_reply_rank=0`. That fallback replaces rank-picks-`req[rank % N]` (diffusion_worker.py:444-450) with all-ranks-same-request execution. `make_executor` hardcodes `enable_distributed_layerwise_offload=False`, and `_make_sched_output` (test_multiproc_engine_concurrency.py:200-207) builds `NewRequestData(request_id=rid, req=...)` with metadata None, so existing DLO tests never make the new conjunct the deciding skip. Add a CPU test with `enable_distributed_layerwise_offload=True`, `dlo_use_allgather=True`, two envelopes (including the mixed case where `any(...)` disables the whole wave), and assert `collective_rpc` is two per-request 4-tuples rather than one list-args call; log or fail-closed if paged metadata disables DLO DP concurrency. (evidence: multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)` — wave-wide any(); :469-473 `if ( len(new_reqs) > 1 and not has_diffusion_kv_metadata and getattr(self.od_config, "enable_distributed_layerwise_offload", False) and getattr(self.od_config, "dlo_use_allgather", True) ):` — metadata present skips DLO DP wave; :506-510 `results = self.collective_rpc( "execute_model", timeout=_DLO_DP_WAVE_TIMEOUT_S, args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job), unique_reply_rank=None,` — list RPC has no per-request metadata slot; :544-551 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / `if new_req.diffusion_kv_metadata is not None: args += (new_req.diffusion_kv_metadata,)` / `unique_reply_rank=0` — fallback is serial 4-tuples. diffusion_worker.py:444-450 `is_batch = isinstance(req, list)` / `idx = dp_rank % len(req)` / `req = req[idx]` — rank-pick only on list args. tests/diffusion/diffusion_kv/test_worker_contract.py:63-64 `enable_distributed_layerwise_offload=False` / `dlo_use_allgather=True`. tests/diffusion/test_multiproc_engine_concurrency.py:200-207 `NewRequestData(request_id=rid, req=OmniDiffusionRequest(...))` — metadata omitted (None); DLO tests at :366-368/:393 use that helper. vllm_omni/diffusion/data.py:699 `enable_distributed_layerwise_offload: bool = False`; :705 `dlo_use_allgather: bool = True`; :941-947 rejects only `PAGED_SCHEDULER` + `need_recv_cache`, not DLO.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:454` [minor] — Pre-RPC `validate_new_request_data_identity` exists only in `execute_request` (multiproc_executor.py:454-460). `_init_execute_fn` binds the engine to `execute_batch`/`execute_step`, not `execute_request`. `execute_batch` validates only on the `execute_request` fallback (`len(scheduled_new_reqs) <= 1` or DLO DP); the N>1 fused path does `collective_rpc("execute_model_batch", args=(scheduler_output, self.od_config))` with no pre-pass. `execute_step` RPCs `execute_stepwise` unvalidated. Worker `execute_model_batch` runs `lora_manager.set_active_adapter` before runner validation; identity errors then hit the async RPC wrapper (`AsyncDiffusionOutput(..., error=str(e))` when `rpc_id` is set). New tests call `runner.execute_model_batch`/`execute_stepwise` directly, never `executor.execute_batch`/`execute_step`. Run the same validate loop in `execute_batch` and `execute_step` before `collective_rpc`. (evidence: multiproc_executor.py:459-460 `for new_req in new_reqs:\n validate_new_request_data_identity(new_req)` — only inside execute_request. multiproc_executor.py:600-601 `if len(scheduler_output.scheduled_new_reqs) <= 1:\n return self.execute_request(scheduler_output)` — N>1 fused path does not take this. multiproc_executor.py:615-620 `result = self.collective_rpc(\n "execute_model_batch",\n args=(scheduler_output, self.od_config),\n unique_reply_rank=0,\n exec_all_ranks=True,\n )` with no identity pre-pass. multiproc_executor.py:651-655 `result = self.collective_rpc(\n "execute_stepwise",\n args=(scheduler_output,),\n unique_reply_rank=0,\n exec_all_ranks=True,\n )` with no identity check. diffusion_engine.py:289-292 `if self.execution_mode == DiffusionExecutionMode.STEP_BATCH:\n self.execute_fn = self.executor.execute_step\n else:\n self.execute_fn = self.executor.execute_batch`. diffusion_worker.py:484-495 `self.lora_manager.set_active_adapter(...)` then `output = self.model_runner.execute_model_batch(scheduler_output, od_config)`. diffusion_worker.py:1119-1127 `AsyncDiffusionOutput(kind=AsyncOutputKind.RPC_RESULT, rpc_id=rpc_id, error=str(e))`.) + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [minor] — This diff makes DiffusionWorker.execute_model build kwargs = {"kv_prefetch_job": kv_prefetch_job} and, when the snapshot is not None, set kwargs["diffusion_kv_metadata"] before self.model_runner.execute_model(req, **kwargs) (diffusion_worker.py:462-465). ARDiffusionModelRunner.execute_model still accepts only (req, kv_prefetch_job=None) and calls super().execute_model(req, kv_prefetch_job=kv_prefetch_job) on both paths (experimental/ar_diffusion/runner.py:237-243 and :255); it is the only in-repo DiffusionModelRunner subclass and is selected via ARDiffusionEngine.default_diffusion_model_runner_cls (engine.py:40). test_dense_worker_call_does_not_extend_model_runner_signature only calls worker.execute_model without metadata (test_worker_contract.py:288), so the first paged AR-Diffusion request that attaches a snapshot TypeErrors on the unexpected keyword before DiffusionModelRunner._validate_diffusion_kv_metadata runs. Add the optional argument on ARDiffusionModelRunner.execute_model and forward it to super(). (evidence: diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / `output = self.model_runner.execute_model(req, **kwargs)`. experimental/ar_diffusion/runner.py:237-241 `def execute_model(self, req: OmniDiffusionRequest, kv_prefetch_job: KVPrefetchJob | None = None) -> DiffusionOutput:` with no diffusion_kv_metadata. runner.py:243 `return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` and runner.py:255 `output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)`. Repo grep `class \w+\(.*DiffusionModelRunner` matches only ARDiffusionModelRunner. engine.py:40 `default_diffusion_model_runner_cls = AR_DIFFUSION_MODEL_RUNNER_CLS`. test_worker_contract.py:288 `worker.execute_model(req, SimpleNamespace())` — metadata=None only.) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:172` [minor] — `_validate_diffusion_kv_metadata` (diffusion_model_runner.py:172) rejects attached metadata whenever `diffusion_kv_mode is not PAGED_SCHEDULER`. Default `OmniDiffusionConfig.diffusion_kv_mode` is `DENSE_LEGACY` (data.py:666), and the platform default runner is this shared `DiffusionModelRunner` (platforms/interface.py:180), so every current diffusion pipeline inherits the optional `execute_model` kwarg and the dense-mode reject. Paged requests still work without snapshots. RPC wiring is mp-only (`ray`/`external_launcher` still NotImplemented in abstract.py:37-44). Worker-contract tests construct `DiffusionModelRunner` via `object.__new__` only. Confirm whether W0 intends this global dense reject, or whether validation should be gated to `paged_scheduler`. (evidence: diffusion_model_runner.py:169-173 `cache_mode = getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)` / `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:` / `raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")`. data.py:666 `diffusion_kv_mode: DiffusionKVCacheMode = DiffusionKVCacheMode.DENSE_LEGACY`. platforms/interface.py:180 `return "vllm_omni.diffusion.worker.diffusion_model_runner.DiffusionModelRunner"`. abstract.py:37-44 `elif distributed_executor_backend == "ray": raise NotImplementedError("ray backend is not yet supported.")` / `elif distributed_executor_backend == "external_launcher": raise NotImplementedError("external_launcher backend is not yet supported.")`. test_worker_contract.py:43-45 `def make_runner(...): runner = object.__new__(DiffusionModelRunner)`.) + +`vllm_omni/diffusion/sched/base_scheduler.py:114` [minor] — W0 still does not produce an allocation: `base_scheduler.py:114` calls `NewRequestData.from_state(state)` with no metadata (defaults to None). The runner validates then drops the snapshot — `execute_model`/`execute_model_batch` never pass it into `_execute_request_list`, and `execute_stepwise` never reads it in `_update_states`. `block_ids` and `context_ids` are unvalidated DTO fields. In-code W0 note says the allocator/installer are not wired yet. Confirm that remainder is the tracked W1 follow-up, not an in-diff omission. (evidence: base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — no metadata kwarg. interface.py:172 `diffusion_kv_metadata: DiffusionKVMetadata | None = None` — from_state defaults to None. diffusion_model_runner.py:170-171 `# W0 does not wire the Scheduler allocator or Worker installer yet, so` / `# validate an allocation snapshot when present without requiring one.` diffusion_model_runner.py:609-618 `_validate_diffusion_kv_metadata(...)` then `_execute_request_list([req], ...)` with no metadata argument. diffusion_model_runner.py:642 `reqs = [nr.req for nr in scheduler_output.scheduled_new_reqs]` then `_execute_request_list(reqs, ...)` — metadata stripped. diffusion_model_runner.py:768 `states, new_request_ids = self._update_states(scheduler_output)` after validation; `_update_states` at 668-678 copies `request_id`/`req` fields only, never `diffusion_kv_metadata`. metadata.py:6-8 `Cache geometry and block-table validation stay with the native cache builders and Worker installation path.` metadata.py:42-43 `block_ids: tuple[list[int], ...]` / `context_ids: tuple[str, ...] = ()` — no `__post_init__` or reference check.) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:?` [minor] — test_worker_wrapper_only_extends_paged_request_rpc calls WorkerWrapperBase.execute_model with a keyword, but production RPC never uses that method: WorkerProc does `self.worker.execute_method(method, *args)` (diffusion_worker.py:1036) and WorkerWrapperBase.execute_method does `getattr(self.worker, method)(*args, **kwargs)` (1442-1443), binding DiffusionWorker.execute_model to the executor's positional 4-tuple from multiproc_executor.py:544-546. The dense test only omits metadata, so DiffusionWorker.execute_model's `if diffusion_kv_metadata is not None` branch (463-465) is untested. Drive `wrapper.execute_method("execute_model", req, od_config, None, metadata)` (or DiffusionWorker.execute_model with that 4-tuple), assert the model_runner received diffusion_kv_metadata, and keep the dense no-kwarg case. (evidence: tests/diffusion/diffusion_kv/test_worker_contract.py:258-259 `wrapper.execute_model(req, od_config)` / `wrapper.execute_model(req, od_config, diffusion_kv_metadata=metadata)` — keyword call on the wrapper, not execute_method. vllm_omni/diffusion/executor/multiproc_executor.py:544-549 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` then `if new_req.diffusion_kv_metadata is not None: args += (new_req.diffusion_kv_metadata,)` then `collective_rpc("execute_model", args=args, ...)`. vllm_omni/diffusion/worker/diffusion_worker.py:1036 `result = self.worker.execute_method(method, *args, **kwargs)`. vllm_omni/diffusion/worker/diffusion_worker.py:1442-1443 `func = getattr(self.worker, method)` / `return func(*args, **kwargs)` — looks up execute_model on the inner worker, skipping WorkerWrapperBase.execute_model. vllm_omni/diffusion/worker/diffusion_worker.py:462-465 `kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / `if diffusion_kv_metadata is not None:` / `kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata`. tests/diffusion/diffusion_kv/test_worker_contract.py:288-290 `worker.execute_model(req, SimpleNamespace())` / `assert calls == [(req, None)]` — metadata omitted, present-branch untested.) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:88` [minor] — Paged W0 validates metadata when present but does not require or install it (`diffusion_model_runner.py:163-178`; `execute_model` at 609-620 then calls `_execute_request_list` without metadata). `test_paged_request_without_metadata_reaches_request_forward` (L88), `test_batch_path_allows_missing_metadata_before_forward` (L144), and `test_step_path_allows_missing_metadata_before_step_support_check` (L173) cover the omitted-metadata path; `test_paged_request_rejects_mismatched_metadata_identity` (L116) covers runner mismatch. No test calls `DiffusionModelRunner.execute_model` or `execute_model_batch` with matching `DiffusionKVMetadata` and asserts `_execute_request_list`; no `execute_stepwise` matching-metadata case asserts validation passed (`_supports_step_mode` reached). Separately, `validate_new_request_data_identity`'s second arm (`interface.py:191-196`, `metadata.request_id != forwarded_request_id`) is untested on `execute_request`: `test_request_rpc_rejects_envelope_request_identity_mismatch` (L129) uses envelope-id vs actual-request-id, so it raises on the first check. Add (1) paged matching-metadata success on request/batch/step, and (2) `execute_request` with envelope_id == req.request_id and `metadata.request_id="stale"` expecting "request identity mismatch" and `calls == []`. Command: `pytest -sv tests/diffusion/diffusion_kv/test_worker_contract.py -m 'core_model and cpu'`. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:170-178 `# W0 does not wire the Scheduler allocator or Worker installer yet, so / validate an allocation snapshot when present without requiring one.` then only `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None: raise` and `if metadata is not None and metadata.request_id != request_id: raise` — no require-metadata path. vllm_omni/diffusion/worker/diffusion_model_runner.py:609-619 `self._validate_diffusion_kv_metadata(request_id=req.request_id, metadata=diffusion_kv_metadata)` then `runner_output = self._execute_request_list([req], od_config=self.od_config, allow_single_output=True, require_request_batch_support=False, kv_prefetch_job=kv_prefetch_job, record_name="pipeline_forward")` — metadata is not forwarded. tests/diffusion/diffusion_kv/test_worker_contract.py:88-100 `def test_paged_request_without_metadata_reaches_request_forward()` then `assert runner.execute_model(req) is expected_output` with `req = SimpleNamespace(request_id="req-0")` and no metadata kwarg. tests/diffusion/diffusion_kv/test_worker_contract.py:116-126 `test_paged_request_rejects_mismatched_metadata_identity` calls `execute_model(..., diffusion_kv_metadata=make_metadata("stale"))` and `assert_not_called`. tests/diffusion/diffusion_kv/test_worker_contract.py:144-155 and 173-185 cover batch/step missing-metadata only. vllm_omni/diffusion/sched/interface.py:184-196 `if new_req.request_id != forwarded_request_id: raise ValueError("Diffusion request identity mismatch: ...")` then `if metadata is not None and metadata.request_id != forwarded_request_id: raise ValueError(...)`. tests/diffusion/diffusion_kv/test_worker_contract.py:132-135 `NewRequestData(request_id="envelope-id", req=req, diffusion_kv_metadata=make_metadata("envelope-id"))` with `req.request_id="actual-request-id"` — first-arm raise. Grep of tests for `diffusion_kv_metadata`: matching success only on executor RPC (`test_request_rpc_sends_attached_metadata_for_only_that_request` L224) and wrapper (L244), never on DiffusionModelRunner.execute_model/batch/stepwise.) + +`tests/diffusion/diffusion_kv/test_metadata.py:?` [nit] — This diff adds `tests/diffusion/diffusion_kv/test_metadata.py` that constructs `DiffusionKVContextMetadata` / `DiffusionKVSequenceMetadata` / `DiffusionKVMetadata` and asserts the same field values just passed in (e.g. line 37 `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])`; lines 41–69 shared `context_ids`; lines 72–110 branch-specific `context_ids`). `metadata.py` is dataclasses with no `__post_init__`, and W0 still has no producer: `base_scheduler.py:114` is `NewRequestData.from_state(state)` (metadata defaults to None). `test_worker_contract.make_metadata` is a single sequence with no `contexts`/`context_ids`, and `from_state` / `execute_request` only assert that blob’s identity, so CFG sharing cannot be caught if dropped at the envelope this PR actually added. Keep the DTO examples, but drive one shared-context and one branch-specific envelope through `NewRequestData.from_state` and `validate_new_request_data_identity` (or the request RPC). Command: `pytest -sv tests/diffusion/diffusion_kv/test_metadata.py tests/diffusion/diffusion_kv/test_worker_contract.py -m 'core_model and cpu'`. (evidence: tests/diffusion/diffusion_kv/test_metadata.py:37 `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])` — tautological constructor round-trip. tests/diffusion/diffusion_kv/test_metadata.py:67-69 `assert metadata.contexts == (shared_context,)` / `assert metadata.sequences[0].context_ids == (shared_context.context_id,)` / `assert metadata.sequences[1].context_ids == (shared_context.context_id,)` — same in-memory CFG share, never an envelope. vllm_omni/diffusion/diffusion_kv/metadata.py:16-57 `@dataclass` classes with fields only; no `__post_init__`. vllm_omni/diffusion/sched/base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — no `diffusion_kv_metadata`. vllm_omni/diffusion/sched/interface.py:165,172 `diffusion_kv_metadata: DiffusionKVMetadata | None = None` default. tests/diffusion/diffusion_kv/test_worker_contract.py:27-40 `make_metadata` builds one sequence `block_ids=([1, 2],)` and no `contexts`. tests/diffusion/diffusion_kv/test_worker_contract.py:82-85 `NewRequestData.from_state(state, diffusion_kv_metadata=metadata)` then `assert new_req.diffusion_kv_metadata is metadata` — identity only. tests/diffusion/diffusion_kv/test_worker_contract.py:238-241 RPC asserts `(req_0, ..., metadata_0)` using that same no-context blob.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.trace.json.gz new file mode 100644 index 000000000..b1669d335 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.cost.json new file mode 100644 index 000000000..812c2fff2 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1627.3, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5610/run-20260814-224627-5c825c", + "recorded_at": "2026-08-14T23:13:34", + "attempts": [ + { + "invocation_id": "inv-cf079648", + "outcome": "run:run-20260814-224627-5c825c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-224627-5c825c", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-224627-5c825c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5861, + "minutes": 27.11, + "ci_minutes": 0, + "input_tokens": 1587135, + "output_tokens": 143238, + "tool_calls": 382, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8839, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.45, + "gate": 2.49, + "review": 1620.69, + "report": 0.06 + }, + "time_to_first_result_s": 1626.68 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 24724 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.md b/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.md new file mode 100644 index 000000000..4bf0deef4 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.md @@ -0,0 +1,41 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] 1. Correctness of changed logic: no Python/runtime logic in the diff; activation, snapshot, connector-drain, and NPU claims checked against gpu_ar_model_runner.py / npu_ar_model_runner.py. +- [validated] docs/design/feature/omni_async_output_materialization.md:336-350 compatibility table matches GPUARModelRunner._should_use_async_omni_output at vllm_omni/worker/gpu_ar_model_runner.py:1595-1619 (async_scheduling, omni_prefix_cache, speculative_config, async_chunk, enable_return_routed_experts, use_async_omni_output, has_postprocess+eager flag). +- [validated] Qwen3-Omni YAML excerpt (async_chunk, enable_prefix_caching false, Code2Wav async_scheduling false, AR async_scheduling default) matches vllm_omni/deploy/qwen3_omni_moe.yaml:15-60 and StageDeployConfig async_scheduling default True. +- [validated] Qwen3-Omni Thinker/Talker opt-in at qwen3_omni.py:133-164; Talker postprocess writes hidden_states[-1,:] as hidden_states.last (qwen3_omni.py:683-687); omni_pooler_payload_include_hidden=False on Talker. +- [validated] Qwen3-TTS Talker opt-in at qwen3_tts_talker.py:351-353; default deploy qwen3_tts.yaml:14,43-53 (async_chunk true, Talker async_scheduling true, prefix cache false); Code2Wav is StageExecutionType.LLM_GENERATION in pipeline.py:36-39. +- [validated] NPU path: npu_ar_model_runner.py:1273-1328 builds OmniModelRunnerOutput, drains get_omni_connector_output, then wraps AsyncGPUModelRunnerOutput. +- [validated] Accumulation unreachable on the feature path: should_accumulate_full_payload_output returns False when async_chunk is True (omni_connector_model_runner_mixin.py:60-61). +- [validated] Connector drain after streaming payload: _build_omni_model_runner_output_from_snapshot calls get_omni_connector_output at gpu_ar_model_runner.py:1899-1900; execute_model keeps recv/flush synchronous at 1040-1048. + +`docs/design/feature/omni_async_output_materialization.md:?` [minor] — The platform note says XPU inherits GPUARModelRunner so it may enter the asynchronous path when other guards pass, but Architecture's overlapping D2H is CUDA-only. `_should_use_async_omni_output()` has no device/platform check, so XPU can still construct OmniAsyncGPUModelRunnerOutput. Snapshot helpers only treat `device.type == "cuda"` as copy sources (`_clone_cuda_tensor_payload`) and `_snapshot_tensor_payload_to_cpu_async` returns a no-stream snapshot when `cuda_sources` is empty, so XPU skips the overlapping memcpy. Qualify the note: inheritance selects the async object/builder, not CUDA async memcpy. (evidence: docs/design/feature/omni_async_output_materialization.md:62-64 `This is a validation scope, not a runtime platform guard: XPU inherits GPUARModelRunner, and MUSA selects GPUARWorker, so those backends may enter the asynchronous path when its other guards pass.` vllm_omni/platforms/xpu/worker/xpu_ar_model_runner.py:10 `class XPUARModelRunner(GPUARModelRunner):` vllm_omni/worker/gpu_ar_model_runner.py:1595-1619 `_should_use_async_omni_output` has no CUDA/`device.type` guard (returns True from scheduling/async_chunk/opt-in checks only). gpu_ar_model_runner.py:73-77 `if value.device.type == "cuda": cloned = value.detach().clone(); sources.append(cloned); return cloned` else `return value.detach().clone()` — non-CUDA tensors are not copy sources. gpu_ar_model_runner.py:89-90 `if value.device.type != "cuda": return value`. gpu_ar_model_runner.py:132-133 `if not cuda_sources: return _AsyncCPUPayloadSnapshot(cloned, None, cuda_sources)`. gpu_ar_model_runner.py:2126 `async_output_cls = OmniAsyncGPUModelRunnerOutput if use_async_omni_output else AsyncGPUModelRunnerOutput`.) + +`docs/design/feature/omni_async_output_materialization.md:89` [nit] — L89 says the feature is enabled for both Qwen3-Omni AR stages as if that is always true. `_should_use_async_omni_output()` still returns False without async_scheduling, a null prefix cache, and async_chunk (gpu_ar_model_runner.py:1596-1607). Captioner/thinker-only is a single LLM_AR stage (pipeline.py:77-95,103), and merge_pipeline_deploy forces async_chunk=False when len(pipeline.stages) <= 1 (stage_config.py:902-904). qwen3_omni_moe_thinking.yaml sets enable_prefix_caching: true and async_scheduling: false. Scope L89 to the bundled full Instruct profile already described in Enabling (qwen3_omni_moe.yaml / Example 1). (evidence: docs/design/feature/omni_async_output_materialization.md:89 `For Qwen3-Omni, the feature is enabled for both AR stages:` vs Enabling L277 `With this profile, async output materialization activates automatically for:`. gpu_ar_model_runner.py:1596-1599 `if not self.use_async_scheduling: return False` / `if self.omni_prefix_cache is not None: return False`; 1606-1607 `if not bool(getattr(model_config, "async_chunk", False)): return False`. stage_config.py:902-904 `# async_chunk is irrelevant for single-stage pipelines, so we always disable it` / `if len(pipeline.stages) <= 1:` / `deploy.async_chunk = False`. pipeline.py:77-86 `QWEN3_OMNI_THINKER_ONLY_PIPELINE` / `stages=(` / `StagePipelineConfig(` / `stage_id=0,` / `model_stage="thinker",` / `execution_type=StageExecutionType.LLM_AR,`; pipeline.py:103 `e.g., Qwen3-Omni-30B-A3B-Captioner, are thinker only.`. qwen3_omni_moe_thinking.yaml:5 `enable_prefix_caching: true`; yaml:13 `async_scheduling: false`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.trace.json.gz new file mode 100644 index 000000000..8ee5e497a Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.cost.json new file mode 100644 index 000000000..7a5c21845 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.cost.json @@ -0,0 +1,102 @@ +{ + "wall_s": 1314.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5703/run-20260815-001259-b346ef", + "recorded_at": "2026-08-15T00:34:53", + "attempts": [ + { + "invocation_id": "inv-18ff413d", + "outcome": "run:run-20260815-001259-b346ef", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-230742-e9c430", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260815-001259-b346ef", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-001259-b346ef", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6362, + "minutes": 21.88, + "ci_minutes": 0, + "input_tokens": 1615914, + "output_tokens": 181711, + "tool_calls": 317, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8251, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.56, + "gate": 1.71, + "review": 1307.67, + "report": 0.07 + }, + "time_to_first_result_s": 1312.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 54183 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.md b/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.md new file mode 100644 index 000000000..7cb84885d --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] 1. Correctness: encode_image/encode_video now take any non-CPU device into torch.random.fork_rng(devices=...) with no device_type= and seed via a module captured at init; NPU device type is npu, so the documented MiniMax H3 NPU path newly enters this branch. +- [sweep] 2. Simplifiability: the same RNG block is duplicated at encode_image and encode_video; self.device_module is redundant if get_device_module(parameter.device) is resolved at encode time. Ask is to DELETE the init cache and the != cpu expansion, not document them. +- [sweep] 3. Breaking behavior: in-repo NPU consumer is recipes/MiniMaxAI/MiniMax-H3-NPU.md plus pipeline MiniMaxH3VideoVAE(..., device=self.device) where self.device comes from get_local_device() -> torch.device("npu", rank). Diff does not update NPU recipe or gate npu out. DreamZero/Wan/Lance encode_image/encode_video hits are different classes; only pipeline_minimax_h3.py:963 and :1066 call this VAE. +- [sweep] 4. Rebase/merge damage: squashed net diff has no test files; PR body still claims test_minimax_h3_vae.py and torchaudio/soundfile/ffmpeg coverage that are absent at this head. +- [sweep] 5. Tests: TEST FILES TOUCHED=NONE. test_video_vae_encode_uses_configured_parallel_tiling uses object.__new__ and a CPU Parameter, so devices=[] and self.device_module is never read. test_video_vae_keeps_reference_fp32_weights does call __init__ (now get_device_module) on CPU only. +- [sweep] 6. Docs: MiniMax-H3-MUSA.md validated-Ref2VA command exports MODEL_ROOT/FL2VA; CUDA recipe MiniMax-H3.md:168 correctly uses Ref2VA. recipes/README.md index row is consistent. vae.py has no RNG docstring to stale. +- [sweep] 7. Undocumented invariants: != cpu stands in for an unpublished allowlist; get_device_module() with no args assumes init-time current accelerator equals encode-time parameter.device. +- [sweep] 8. Scope: stated purpose is conditioned VAE RNG + MUSA recipe. Diff matches that file set, but the body also claims audio-loader tests that this diff does not deliver. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — vae.py:170 and :214 replace the CUDA-only RNG gate with `parameter.device.type != "cpu"`, so every non-CPU device is passed to `torch.random.fork_rng(devices=devices)` with no `device_type=` and then `self.device_module.manual_seed`. MiniMax H3 constructs this VAE as `MiniMaxH3VideoVAE(..., device=self.device)` after `self.device = get_local_device()` (pipeline_minimax_h3.py:481,545-547); `NPUOmniPlatform.get_torch_device` returns `torch.device("npu", local_rank)` (platforms/npu/platform.py:163-164). On Ascend, `"npu" != "cuda"` previously skipped this path; `"npu" != "cpu"` now enters it for live `encode_image`/`encode_video` callers. Gate on `parameter.device.type in ("cuda", "musa")` in both methods, pass `device_type=parameter.device.type` into `fork_rng`, and do not infer NPU/XPU/MPS RNG restore from the MUSA smoke. (evidence: vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` then vae.py:172 `with torch.random.fork_rng(devices=devices):` and vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(...)` (same gate at vae.py:214-220). pipeline_minimax_h3.py:481 `self.device = get_local_device()`; :545-547 `self.video_vae = MiniMaxH3VideoVAE(..., device=self.device)`; :963 `self.video_vae.encode_image(image)`; :1066 `self.video_vae.encode_video(...)`. vllm_omni/platforms/npu/platform.py:163-164 `return torch.device("npu")` / `return torch.device("npu", local_rank)`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [major] — This diff stores `self.device_module = torch.get_device_module()` in `MiniMaxH3VideoVAE.__init__` with no device argument, then encode_image/encode_video do `with self.device_module.device(device): self.device_module.manual_seed(...)` over `parameter.device`. `__init__` already receives `device: torch.device` (pipeline passes `device=self.device`) and encode already has `parameter = next(self.parameters())`; no-arg `get_device_module()` is the process default accelerator, not that constructor device or `parameter.device`. Delete the instance attribute and call `torch.get_device_module(parameter.device)` at encode time after the non-CPU gate so CPU construction (`test_video_vae_keeps_reference_fp32_weights` uses `device=torch.device("cpu")`), later device moves, and MUSA/CUDA encode cannot bind the wrong module. (evidence: vae.py:100-120 `def __init__(self, component_path: str, *, device: torch.device) -> None:` then `self.remote.eval().to(device=device, dtype=torch.float32)` then `self.device_module = torch.get_device_module()`. vae.py:166-176 `parameter = next(self.parameters())` / `devices = [parameter.device] if parameter.device.type != "cpu" else []` / `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`. vae.py:210-220 repeats the same `parameter.device` vs `self.device_module` seed loop in `encode_video`. pipeline_minimax_h3.py:545-548 `self.video_vae = MiniMaxH3VideoVAE(..., device=self.device)`. test_minimax_h3_contract.py:319-322 `MiniMaxH3VideoVAE("unused", device=torch.device("cpu"))`.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — This new recipe introduces the only serve snippet as the validated Ref2VA TP4 configuration (`The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs`) but then `export MODEL="${MODEL_ROOT}/FL2VA"`. The same page defines FL2VA as the t2va/fl2va partition and Ref2VA as image/audio/video-reference; the CUDA recipe switches with `export MODEL="${MODEL_ROOT}/Ref2VA"` after stopping FL2VA. Change this block's MODEL export to Ref2VA (keep a distinct FL2VA command only where the text is T2VA/FL2VA) so copy-paste does not load the wrong checkpoint for the 638.6s TP4 profile this page claims to validate. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76 `The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs and offloads inactive model components to CPU:` immediately followed by `export MODEL="${MODEL_ROOT}/FL2VA"`. Same file:17-19 ``FL2VA` serves text-to-video+audio (`t2va`) and first-frame-to-video+audio (`fl2va`).` / ``Ref2VA` serves image/audio or video-reference generation (`ref2va`).`. recipes/MiniMaxAI/MiniMax-H3.md:164-168 `To serve Ref2VA, stop the FL2VA server and restart the same one- or four-GPU command with:` then `export MODEL="${MODEL_ROOT}/Ref2VA"`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This diff changes the seeded encode path in encode_image()/encode_video() (vae.py:170 and :214) but adds no tests. The only in-repo encode_video exercise (test_video_vae_encode_uses_configured_parallel_tiling at test_minimax_h3_contract.py:328) uses object.__new__(MiniMaxH3VideoVAE) plus a CPU Parameter (:336, :344), so devices=[] and device_module is never read on the encode path — CPU CI cannot catch a wrong != "cpu" allowlist, a missing fork_rng(device_type=...), or device_module.device/manual_seed failing. encode_image (called from pipeline_minimax_h3.py:963 on rank 0) has zero tests; tests/diffusion/models/minimax_h3/ has no test_minimax_h3_vae.py. Extend that contract test (keep core_model+cpu+diffusion) with a fake Parameter on cuda/musa vs cpu/npu that asserts which types enter fork_rng/manual_seed and that CPU still skips. Guard with pytest -q -c /dev/null tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py -k video_vae. (evidence: vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []` (same at vae.py:170 in encode_image). vae.py:120 `self.device_module = torch.get_device_module()`; vae.py:218-220 `with self.device_module.device(device):` / `self.device_module.manual_seed(...)`. test_minimax_h3_contract.py:336 `self.weight = torch.nn.Parameter(torch.ones(1))`; :344 `video_vae = object.__new__(MiniMaxH3VideoVAE)`; :353 `rows, shape = video_vae.encode_video("frames")`. pipeline_minimax_h3.py:963 `rows = self.video_vae.encode_image(image) if rank == 0 else None`. tests/diffusion/models/minimax_h3/ listing has no test_minimax_h3_vae.py.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — MiniMaxH3VideoVAE.__init__ newly calls torch.get_device_module() with no device argument (vae.py:120; only in-repo use). The CPU constructor test test_video_vae_keeps_reference_fp32_weights now depends on that API. pyproject.toml marks dependencies dynamic via setup.py, and requirements/cuda.txt / requirements/npu.txt do not pin torch>=, so the oldest allowed torch is not checkable in-tree. The NPU recipe documents PyTorch 2.10.0+cpu, which is not a lower bound for CUDA/MUSA. What minimum torch is guaranteed for every flavor that constructs MiniMaxH3VideoVAE? (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` — only in-repo call, inside MiniMaxH3VideoVAE.__init__. tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` and :319-322 `video_vae = vae_module.MiniMaxH3VideoVAE(\n "unused",\n device=torch.device("cpu"),\n )` — CPU __init__ path. pyproject.toml:10 `dynamic = ["version", "dependencies"]`. requirements/cuda.txt:1-5 `-r common.txt` / `onnxruntime>=1.23.2` / `fa3-fwd==0.0.3` / cublas comment only — no torch pin. recipes/MiniMaxAI/MiniMax-H3-NPU.md:42 `- PyTorch: 2.10.0+cpu`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:12` [nit] — gate_report shows buildkite/vllm-omni-amd-ci red on this head. AMD L2 runs `pytest tests/diffusion -m 'core_model and cpu'`, which selects this module (`pytestmark` core_model+cpu+diffusion) including `MiniMaxH3VideoVAE()` construction that now calls `torch.get_device_module()` in `__init__`. Attribution of that red job to this diff is unproven — can you confirm whether the failure is explained by that new CPU constructor call, or is independent of this hunk? (evidence: gate_report: buildkite/vllm-omni-amd-ci is red; MiniMax H3 contract tests are pytest.mark.cpu and now call get_device_module() in MiniMaxH3VideoVAE.__init__. / AMD CI failure attribution to this diff is unproven; only overlap is the new get_device_module() call on CPU MiniMax H3 tests. / .buildkite/amd/test-amd-ready.yml runs pytest tests/diffusion -m 'core_model and cpu', which selects test_minimax_h3_contract.py including MiniMaxH3VideoVAE().__init__ (get_device_module).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.trace.json.gz new file mode 100644 index 000000000..8dac73633 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.cost.json new file mode 100644 index 000000000..e585ea3ea --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 424.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5715/run-20260814-230838-c043eb", + "recorded_at": "2026-08-14T23:15:41", + "attempts": [ + { + "invocation_id": "inv-e3693594", + "outcome": "run:run-20260814-230838-c043eb", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-230838-c043eb", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-230838-c043eb", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0856, + "minutes": 7.06, + "ci_minutes": 0, + "input_tokens": 221896, + "output_tokens": 23339, + "tool_calls": 68, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2759, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.11, + "gate": 1.73, + "review": 417.67, + "report": 0.06 + }, + "time_to_first_result_s": 423.56 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 12556 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.md b/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.md new file mode 100644 index 000000000..22bd2eb76 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.md @@ -0,0 +1,50 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] README.md:19 MiniMax H3 joint video/audio claim matches recipes/MiniMaxAI/MiniMax-H3.md:1-8 (CFG-distilled joint video/audio DiT; T2VA/FL2VA/Ref2VA). +- [validated] recipes/OpenBMB/MiniCPM-o-4_5.md and recipes/cosmos3/Cosmos3-Nano.md exist for the new/retained recipe links. +- [validated] docker/Dockerfile.cuda:1 ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0 matches cuda.inc.md:127 --build-arg BASE_IMAGE=vllm/vllm-openai:v0.26.0. +- [validated] cuda.inc.md:23/45 uv pip install vllm==0.26.0 and installation/README.md:4 0.26.x requires vLLM 0.26.x are consistent with each other. +- [validated] NPU installation-main now has cd vllm / cd vllm-ascend / cd vllm-omni around the three editable installs (npu.inc.md:57-66), fixing the old pip -e . in the clone parent. +- [validated] docs/configuration/README.md:5 is the only docs.vllm.ai/en/v0.* config link; no leftover v0.16.0 config URL. +- [validated] Full-duplex marked experimental in README.md:54 and docs/README.md:55; DESIGN.md:365 states the implementation remains experimental. +- [upstream-verify] vLLM tag v0.26.0 and vllm-ascend branch releases/v0.26.0rc were not fetched (no gh/web in this step); author Test Result claims they exist. + +`README.md:?` [minor] — This diff's 0.26.0 news bullet advertises an experimental full-duplex realtime runtime and hyperlinks recipes/OpenBMB/MiniCPM-o-4_5.md. That recipe documents ordinary /v1/chat/completions omni chat+TTS only and has no duplex, realtime, or /v1/realtime coverage, so readers following the new announcement never reach the duplex overlay or WebSocket demo. Point the MiniCPM-o 4.5 full-duplex mention at examples/online_serving/minicpmo/README.md (minicpmo_4_5_duplex.yaml and the Realtime demo) or add a duplex pointer on the recipe page. (evidence: README.md:19: featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md) +recipes/OpenBMB/MiniCPM-o-4_5.md grep duplex|realtime|/v1/realtime → no matches +examples/online_serving/minicpmo/README.md:40: | `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |) + +`README.md:?` [nit] — This diff adds full-duplex as a top-level 'flexible and easy to use' bullet beside the OpenAI-compatible API server, qualified only as experimental. Ordinary --omni deployments do not load that runtime; the 0.26 news bullet already scopes it to MiniCPM-o 4.5. Name MiniCPM-o 4.5 in this bullet (and the copy at docs/README.md:55) so the landing page does not read as a general serving mode. (evidence: README.md:54: - Full-duplex realtime serving with streaming audio input and output (experimental) +docs/README.md:55: - Full-duplex realtime serving with streaming audio input and output (experimental) +vllm_omni/experimental/fullduplex/DESIGN.md:365: The full-duplex implementation remains experimental: +vllm_omni/experimental/fullduplex/DESIGN.md:512: constructing the experimental duplex handler. Ordinary deployments do not load) + +`docs/getting_started/installation/gpu/cuda.inc.md:8` [nit] — This diff replaces the unversioned 'depends vLLM' sentence with an explicit 0.26.x↔0.26.x pin, and configuration/README.md was moved from docs.vllm.ai/en/v0.16.0 to /en/v0.26.0. The next paragraph in this same CUDA page still sends build-from-source users to docs.vllm.ai/en/stable, which will drift off the 0.26 line. Point that link at /en/v0.26.0/... to match the new pin. (evidence: docs/getting_started/installation/gpu/cuda.inc.md:8: vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.26.x release line uses vLLM 0.26.x. +docs/getting_started/installation/gpu/cuda.inc.md:15: See [build-from-source-vllm](https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#build-wheel-from-source) +docs/configuration/README.md:5: [vLLM 0.26 configuration guide](https://docs.vllm.ai/en/v0.26.0/configuration/index.html)) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.trace.json.gz new file mode 100644 index 000000000..5902567a5 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.cost.json new file mode 100644 index 000000000..b9a290192 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1729.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5840/run-20260814-231335-b1e957", + "recorded_at": "2026-08-14T23:42:24", + "attempts": [ + { + "invocation_id": "inv-1041e9a2", + "outcome": "run:run-20260814-231335-b1e957", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-231335-b1e957", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-231335-b1e957", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7502, + "minutes": 28.82, + "ci_minutes": 0, + "input_tokens": 1946542, + "output_tokens": 204216, + "tool_calls": 479, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9754, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.45, + "gate": 1.8, + "review": 1723.9, + "report": 0.06 + }, + "time_to_first_result_s": 1729.2 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 39907 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.md b/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.md new file mode 100644 index 000000000..f67eafed5 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] 1 correctness: extract_minimax_h3_context mirrors MiniMaxH3DiTModel.forward (kwargs contract, _embed, sp_prepare/blocks/sp_gather, final_layer, update masks); uncached test_extractor_matches_native_forward uses assert_close. +- [sweep] 1 correctness: enable_minimax_h3_teacache rejects partition=ref2va and unknown partitions; combined hooks only pipeline.transformer; Ref2VA uses transformers_ref via _transformer_for_task. +- [sweep] 1 correctness: TeaCacheConfig.__post_init__ maps None→0.17 for MiniMaxH3DiTModel else 0.2; hook.py assert rel_l1_thresh is not None is after post_init. +- [sweep] 2 simplifiability: extractor duplicates native forward; required by TeaCache CacheContext split, not dead. combined warning+FL2VA-only hook are distinct arms. update_audio_mask hoist is equivalent. +- [sweep] 3 breaking: DiffusionCacheConfig.rel_l1_thresh default 0.2→None. In-repo readers of the raw field besides TeaCacheConfig are backend.py (passes through), hook.py (post_init), hunyuan tea_cache_config (TeaCacheConfig). Stale consumer: AsyncOmniEngine._get_default_cache_config still injects rel_l1_thresh 0.2. +- [validated] vllm_omni/engine/arg_utils.py:489 cache_config: str | None = None; CLI --cache-config default=None; Omni→AsyncOmniEngine._create_default_diffusion_stage_cfg→_normalize_cache_config(None)→{"rel_l1_thresh": 0.2} at async_omni_engine.py:853-856,881-882. +- [validated] tests/diffusion/cache/test_cache_backends.py:481-482 Qwen DiffusionCacheConfig() still yields TeaCacheConfig 0.2; MiniMax None→0.17 only via TeaCacheBackend, not Omni. +- [sweep] 4 rebase: no dropped extractor registry entry; CUSTOM_TEACACHE_ENABLERS includes MiniMaxH3Pipeline; EXTRACTOR_REGISTRY includes MiniMaxH3DiTModel. + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:?` [major] — quality=="high" always emits a CacheDiTRequestSpec (quality_policy.py:149-151) and only the omitted-quality default checks cache_backend=="cache_dit". After this PR TeaCache is enabled on MiniMaxH3Pipeline.transformer at startup, but the runner adopts request-scoped Cache-DiT only when cache_backend=="cache_dit" (diffusion_model_runner.py:316-317), leaving _cache_dit_runtime._backend None. pipeline_minimax_h3.py:1766 still calls prepare(quality_plan.cache_dit), and RequestScopedCacheDiTRuntime.prepare constructs CacheDiTBackend(spec.cache_config).enable(self._pipeline) when _backend is None (runtime.py:71-73), so a tea_cache server serving quality=high installs Cache-DiT on the same FL2VA transformer. test_model_policy_owns_request_cache_target includes ("none","high") but never ("tea_cache","high"). Reject or no-op Cache-DiT when the configured backend is tea_cache, and add that pair to the matrix. (evidence: quality_policy.py:149 ` if quality == "high":` — no cache_backend gate; 152 ` elif self._configured_backend == "cache_dit" and quality is None:` is the only backend check, so high always falls through to CacheDiTRequestSpec. diffusion_model_runner.py:316 ` self.cache_backend.enable(self.pipeline)` then 317 ` if str(self.od_config.cache_backend).lower() == "cache_dit" and adopt_request_scoped_cache_dit(` — tea_cache enables but is not adopted. pipeline_minimax_h3.py:1766 ` self._cache_dit_runtime.prepare(quality_plan.cache_dit)` with no tea_cache guard (no `tea_cache` under models/minimax_h3). runtime.py:71-73 ` if self._backend is None:` / ` backend = CacheDiTBackend(spec.cache_config)` / ` backend.enable(self._pipeline)`. teacache/backend.py:127 ` "MiniMaxH3Pipeline": enable_minimax_h3_teacache,`. test_minimax_h3_quality_policy.py:164 ` ("none", "high", "minimax_h3.high"),` — matrix has none/high, not tea_cache/high.) + +`vllm_omni/diffusion/data.py:422` [major] — This diff changes `DiffusionCacheConfig.rel_l1_thresh` from `0.2` to `None` and registers `_MODEL_DEFAULT_REL_L1_THRESH["MiniMaxH3DiTModel"]=0.17` so omitted thresh is supposed to resolve in `TeaCacheConfig.__post_init__`. `AsyncOmniEngine._get_default_cache_config` still returns `{"rel_l1_thresh": 0.2}` when `cache_config is None` (`async_omni_engine.py:853-882`), so Omni() and `vllm serve --cache-backend tea_cache` never pass None through; `cache_config={}` skips the injector and would get 0.17. `test_minimax_h3_only_enables_fl2va_teacache` constructs `TeaCacheBackend(DiffusionCacheConfig())` and never calls `_normalize_cache_config`; the new e2e smoke and recipe examples pass `{"rel_l1_thresh": 0.17}` explicitly; the PR online log shows `rel_l1_thresh=0.2`. Omit `rel_l1_thresh` from the tea_cache default (return `{}`) so the model map applies, add a test that `_normalize_cache_config("tea_cache", None)` plus MiniMaxH3 enable yields 0.17, and then update `teacache.md` / `--cache-config` help which still document an unconditional 0.2. (evidence: vllm_omni/diffusion/data.py:422: rel_l1_thresh: float | None = None +vllm_omni/diffusion/cache/teacache/config.py:97-98: _MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, } +vllm_omni/engine/async_omni_engine.py:853-856: if cache_backend == "tea_cache": return { "rel_l1_thresh": 0.2, } +vllm_omni/engine/async_omni_engine.py:881-882: if cache_config is None and cache_backend not in (None, "", "none"): cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend) +tests/diffusion/cache/test_cache_backends.py:505-516: DiffusionCacheConfig() ... assert teacache_config.rel_l1_thresh == expected_threshold # None -> 0.17 +tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:156: cache_config={"rel_l1_thresh": 0.17}) + +`tests/diffusion/cache/test_teacache_extractors.py:696` [minor] — test_teacache_reuses_h3_block_residual (tests/diffusion/cache/test_teacache_extractors.py:696) calls TeaCacheHook.new_forward twice on identical kwargs with rel_l1_thresh=0.3, discards the first logits, and only asserts `block_calls == 1` plus output shapes. The skip path (hook.py:143-147) does `ctx.hidden_states = ctx.hidden_states + state.previous_residual` then postprocess; skip is decided before that add, so a broken residual still skips the block and keeps those shapes. Capture the first forward's video/audio logits and `torch.testing.assert_close` them to the second (keep `block_calls==1`). MiniMaxH3DiTModel coefficients end at 2.173782527946167e-01 (config.py:87-93), so poly(0)≈0.217; hook.py accumulates abs(rescaled) then compares to thresh, so this identical-input fixture cannot cache-hit at the shipped default 0.17 (config.py:98). Either add a consecutive-timestep case that hits at 0.17, or state in the test why 0.3 is required. (evidence: tests/diffusion/cache/test_teacache_extractors.py:696 `assert block_calls == 1` — cache-hit test checks skip count only. Same test immediately follows with shape-only checks (`assert video_logits.shape[0] == sample_inputs["img_pos_info"]["position_ids"].shape[0]` and the audio-row analogue); first `hook.new_forward(...)` return is discarded, second unpacks logits. tests/diffusion/cache/test_teacache_extractors.py:642-643 `torch.testing.assert_close(actual_video, expected_video)` / `torch.testing.assert_close(actual_audio, expected_audio)` — numeric check exists only on the uncached extractor path. vllm_omni/diffusion/cache/teacache/hook.py:143 `if not should_compute and state.previous_residual is not None:`. vllm_omni/diffusion/cache/teacache/hook.py:147 `ctx.hidden_states = ctx.hidden_states + state.previous_residual` then later `return ctx.postprocess(output)`. Same hook: `rescaled_distance = float(self.rescale_func(rel_distance))` then `state.accumulated_rel_l1_distance += abs(rescaled_distance)` then `if state.accumulated_rel_l1_distance < rel_l1_thresh`. vllm_omni/diffusion/cache/teacache/config.py:87-93 `"MiniMaxH3DiTModel": [..., 2.173782527946167e-01]` — np.poly1d intercept poly(0)≈0.217. config.py:98 `"MiniMaxH3DiTModel": 0.17` — shipped default; 0.217 < 0.17 is false so identical inputs cannot hit at 0.17. Test constructs `TeaCacheConfig(transformer_type="MiniMaxH3DiTModel", rel_l1_thresh=0.3)`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:170` [minor] — `test_minimax_h3_t2va_teacache_single_gpu_smoke` (line 170) sets `num_inference_steps=2` and only `_assert_joint_output` (107-frame stereo). `TeaCacheHook._should_compute_full_transformer` always returns True when `state.cnt==0` (hook.py:211-213), so two denoise steps allow at most one cache decision; if that second step still computes, TeaCache is a no-op and this `local_model+cuda` smoke still passes. After generate, assert TeaCacheHook skipped at least one DiT block (CPU `test_teacache_reuses_h3_block_residual` already covers identical-kwargs skip), or keep this as plumbing-only and have the recipe 50-step TeaCache example record skip counts. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:9-14 `pytestmark = [pytest.mark.local_model, pytest.mark.cuda, pytest.mark.diffusion, pytest.mark.parallel]` — local_model+cuda. tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:21 `assert frames.shape == (107, 256, 448, 3)` — `_assert_joint_output` is shape-only. tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:170 `num_inference_steps=2,` — two denoise steps. tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:186 `_assert_joint_output(outputs)` — no skip/hit check after generate. vllm_omni/diffusion/cache/teacache/hook.py:211-213 `if state.cnt == 0:` / `state.accumulated_rel_l1_distance = 0.0` / `return True` — first timestep always fully computes.) + +`tests/diffusion/cache/test_cache_backends.py:490` [nit] — AMD L2 Simple Test (.buildkite/amd/test-amd-ready.yml:19) runs `pytest tests/diffusion tests/model_executor -m 'core_model and cpu'` and `--ignore`s only `tests/diffusion/cache/test_teacache_extractors.py`. `test_cache_backends.py` is module-marked `core_model`+`cpu` (line 32), so the new `test_enable_uses_generic_default_threshold` and `test_minimax_h3_only_enables_fl2va_teacache` (line 490) are collected. If `buildkite/vllm-omni-amd-ci` is red, inspect that job log for those names rather than attributing the failure to the ignored extractor file. (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — module marker matches AMD L2 `-m 'core_model and cpu'`. tests/diffusion/cache/test_cache_backends.py:490 ` def test_minimax_h3_only_enables_fl2va_teacache(` — new MiniMax-H3 case in this collected file, not the ignored extractor file. .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — only extractors ignored.) + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149` [nit] — This PR's title includes Cache-DiT validation, but the diff adds no Cache-DiT tests or code (Cache-DiT already exists in quality_policy.py; purpose/body is TeaCache-only). Is Cache-DiT validation deferred from this change, and should the title/scope be narrowed to MiniMax-H3 TeaCache/FL2VA? (evidence: [sweep] 8 scope: title includes Cache-DiT validation; diff has no Cache-DiT test/code; Cache-DiT already exists in quality_policy.py. Purpose/body is TeaCache-only.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.trace.json.gz new file mode 100644 index 000000000..44e94dbb9 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5863.cost.json b/eval/dataset/arms/copilot_cb_grok46_r3/pr5863.cost.json new file mode 100644 index 000000000..859005a21 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5863.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 514.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_cb_grok46_r3/runs/pr5863/run-20260814-231542-d6b034", + "recorded_at": "2026-08-14T23:24:16", + "attempts": [ + { + "invocation_id": "inv-276e0238", + "outcome": "run:run-20260814-231542-d6b034", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-231542-d6b034", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260814-231542-d6b034", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1091, + "minutes": 8.56, + "ci_minutes": 0, + "input_tokens": 286677, + "output_tokens": 28816, + "tool_calls": 56, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3256, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.65, + "gate": 1.63, + "review": 508.11, + "report": 0.06 + }, + "time_to_first_result_s": 513.44 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 15053 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5863.md b/eval/dataset/arms/copilot_cb_grok46_r3/pr5863.md new file mode 100644 index 000000000..847c403d3 --- /dev/null +++ b/eval/dataset/arms/copilot_cb_grok46_r3/pr5863.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:100,130,159 explicit CUDNN_ATTN; vllm_omni/platforms/cuda/platform.py TRTLLM default requires major==10, else Blackwell cuDNN>=9.5 → CUDNN_ATTN. +- [validated] pipeline_minimax_h3.py:132,150 partition path FL2VA/Ref2VA with model_index.json auto-detects; serving commands omitting --task-type are valid. +- [validated] videos.py aspect_ratio: T2VA requires a named ratio; Ref2VA defaults to 16:9 — matches the new T2VA/Ref2VA notes. +- [validated] api_server.py:3510-3511 X-Stage-Durations and X-Peak-Memory-MB exist. +- [validated] MiniMax-H3.md known limitation: VAE patch parallel size must be 1 or full DiT group; all three configs set vae-patch-parallel-size == num-gpus; text-encoder-tp 2/4/8 divide Qwen3-VL 64/8 heads. + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:1` [minor] — This diff adds a standalone RTX PRO 6000 recipe but never catalogues it or points 96 GiB workstation users at it from the family guide. Add a `recipes/README.md` row (alongside the existing MiniMax-H3.md / MiniMax-H3-MUSA.md entries), link this file from `MiniMax-H3.md` next to the four-GPU B300 command (that profile is USP4 without TP and will not fit 96 GiB), and add the same Additional-resources footer MUSA already has so readers can reach the HTTP API matrix. (evidence: recipes/README.md:48-49: `| [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |` and `| [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |`. MiniMax-H3.md four-GPU command uses `--num-gpus 4 --usp 4` with no `--tensor-parallel-size`. MiniMax-H3-MUSA.md Additional resources: `- [MiniMax-H3.md](MiniMax-H3.md) — full model and API guide`.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:4` [minor] — This opening sentence compares the new 96 GiB recipe to "the RTX PRO 5000 profile", but a repo-wide search for `RTX PRO 5000` returns only this line. Rephrase against a real in-repo profile (the 32 GiB RTX 5090/DLO recipe, or "cards below 96 GiB") or drop the named-profile comparison. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:4 is the only `RTX PRO 5000` hit. Closest sibling is recipes/MiniMaxAI/MiniMax-H3-5090.md:1 `# MiniMax-H3 on RTX 5090` with `GPU HBM | 32 GiB`.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [minor] — This two-server layout pastes `--cpunodebind=0 --membind=0` next to `--port 8091` as if they were `vllm serve` flags. `vllm_omni` has no `cpunodebind`; those options belong to `numactl`. Write `numactl --cpunodebind=0 --membind=0 vllm serve ...` or drop the flags so a copy-paste does not die on an unknown argument. (evidence: Grep of `vllm_omni` for `cpunodebind` returned no matches. The same sentence also uses `--port 8091`, which is a real serve flag (`vllm_omni/entrypoints/cli/serve.py`).) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340` [minor] — This Ref2VA example posts `audio_reference` from a host-local file via curl ` FusedMoEFactory, MistralToolCall import move, mamba_utils import, is_interleaved rehome, copy_missing_attrs rehome, xpu_kernels -> vllm_c, MultiModalHasher.hash_kwargs extra algorithm arg, OnlineRenderer.warmup — in-repo greps show tests/callers updated; no leftover xpu_kernels. async_tokens_to_discard is still assigned (omni_ar_scheduler.py:479,634) for omni-only tests. +- [sweep] 5 tests: new tests/core/sched/test_omni_ar_scheduler_stale_drain.py drives OmniARScheduler._update_request_as_session without replace_streaming_prompt/additional_information.meta, so mixin is not entered and double-seed cannot fail the test. tests/core/sched/test_omni_ar_scheduler_streaming.py:263,300 do take the mixin path but never set num_in_flight_tokens or assert num_stale_output_tokens. Generation segment-stop seed is untested. Qwen3 forward-contract tests bind to real forward/make_omni_output. Cache-env tests cover replica 0 vs >0. Inductor patch tests bind to _provably_factorable_multiple and SizeVarAllocator. +- [sweep] 6 docs: drain comments claim both segment-stop and session-replacement seed the counter; that is true for AR, false for generation. Mixin comment is accurate for generation-only replace. Qwen3 comments match the new is_staged_run/bare-tensor contract. +- [sweep] 7 invariants: drain treats a whole frame as stale iff counter>0, then subtracts the full num_tokens_scheduled and asserts >=0 — leftover 1 plus a 5-token new-segment prefill kills the engine-core. Seed is in scheduled-token units; placeholder vs scheduled asymmetry is the bug they intended to fix, but double-seed reintroduces leftover. Test compile.py:107 indexes blocks[0] only after constructing a one-block model. qwen2_5_omni_thinker.py:308 videos[0] is in a comment, not new indexing. + +`vllm_omni/core/sched/omni_ar_scheduler.py:629` [major] — OmniARScheduler._update_request_as_session always seeds `session.num_stale_output_tokens += in_flight_tokens` (omni_ar_scheduler.py:627-629) and then, when `replace_streaming_prompt` is True, calls OmniSchedulerMixin._replace_streaming_session which adds the same still-nonzero `num_in_flight_tokens` again (omni_scheduler_mixin.py:105) with no intervening zero. MiniCPM-o 4.5 sets that meta (minicpmo_4_5_omni.py:954); the AR method then returns at 669-671, so the drain counter is 2× in-flight. Leftover stale count swallows the new segment's first frames or underflows `assert num_stale_output_tokens >= 0` — the hole this PR claims to close. `test_omni_ar_scheduler_stale_drain.py` builds a StreamingUpdate with no such meta so it never takes this branch; `test_omni_ar_scheduler_streaming.py` drives `replace_streaming_prompt=True` but does not set or assert in-flight/stale counts. Seed in exactly one site (skip the AR pre-seed on the mixin path, or skip the mixin seed when the AR override already did), and add a CPU case with `replace_streaming_prompt=True` that asserts `num_stale_output_tokens == in_flight` after replacement and that the first new-segment frame is delivered. (evidence: omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0:` / `session.num_stale_output_tokens += in_flight_tokens` — AR always seeds before the replace check; omni_ar_scheduler.py:669-671 `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)` / `return` — no zero of num_in_flight_tokens between the two sites; omni_scheduler_mixin.py:105-107 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `session.num_output_placeholders = 0` / `session.spec_token_ids = []` — mixin adds again and still does not clear num_in_flight_tokens; minicpmo_4_5_omni.py:954 `handoff_meta["replace_streaming_prompt"] = True`; test_omni_ar_scheduler_stale_drain.py:77-84 StreamingUpdate has prompt_token_ids/max_tokens/arrival_time/sampling_params only, no replace_streaming_prompt meta; test_omni_ar_scheduler_streaming.py:254-264 drives `receives_chunks=False` plus `"meta": {"replace_streaming_prompt": True}` without asserting num_stale_output_tokens.) + +`vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py:?` [major] — This PR defaults a missing model_stage to "thinker" in Qwen2_5OmniForConditionalGeneration.__init__ (qwen2_5_omni.py:86) because arg_utils.py:116-117 now unconditionally overwrites every omni arch, including Qwen2_5OmniForConditionalGeneration (plugin: pyproject.toml:149 vllm_omni_register_models). Unlike the Qwen3 fix in this same PR (qwen3_omni.py:148 is_staged_run; :414-421 `return self.thinker(...)`; thinker returns a bare tensor when capture is off), the Qwen2.5 thinker-stage forward still always `return OmniOutput(` at qwen2_5_omni.py:326-329 with no is_staged_run gate and no make_omni_output. Stock GPUModelRunner indexes the forward output as a tensor, so plain `vllm serve Qwen/Qwen2.5-Omni-*` after omni registration hits OmniOutput. Mirror the Qwen3 contract and add a CPU test like tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py. (evidence: qwen2_5_omni.py:86 `self.model_stage = getattr(vllm_config.model_config, "model_stage", None) or "thinker"` — no is_staged_run. qwen2_5_omni.py:312-318 `thinker_output = self.thinker(...)` then :326-329 `return OmniOutput(\n text_hidden_states=(text_hidden_states.reshape(-1, text_hidden_states.shape[-1])),\n multimodal_outputs=None,\n )` with no non-staged/tensor gate. arg_utils.py:108-117 comment `Unconditionally (re)register every omni arch` and `ModelRegistry.register_model(arch, ...)` with the old `if arch not in supported_archs` removed. registry.py:9-12 `_OMNI_MODELS` includes `Qwen2_5OmniForConditionalGeneration`. pyproject.toml:149 `vllm_omni_register_models = "vllm_omni.engine.arg_utils:register_omni_models_to_vllm"`. Contrast qwen3_omni.py:148 `self.is_staged_run = getattr(vllm_config.model_config, "model_stage", None) is not None` and :414-421 `return self.thinker(`.) + +`vllm_omni/platforms/xpu/platform.py:?` [minor] — This PR changes XPUOmniPlatform.get_default_ir_op_priority from ["xpu_kernels", "native"] to ["vllm_c", "native"] for every non-inductor XPU model (vllm_omni/platforms/xpu/platform.py:139). diffusion_worker.py:184 applies current_omni_platform.get_default_ir_op_priority, so that name is the diffusion default on XPU. CUDA already uses ["vllm_c", "native"] (cuda/platform.py:315) and pins it in tests/platforms/test_cuda_ir_op_priority.py; repo grep has no remaining xpu_kernels and tests/ has no XPUOmniPlatform coverage. Add the XPU counterpart (or scope the rename) so a wrong 0.27 kernel name cannot silently become the platform default. (evidence: vllm_omni/platforms/xpu/platform.py:139 `default = ["native"] if using_inductor else ["vllm_c", "native"]` — non-inductor XPU default is now vllm_c (diff replaced xpu_kernels). vllm_omni/diffusion/worker/diffusion_worker.py:184 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`. vllm_omni/platforms/cuda/platform.py:315 `default = ["vllm_c", "native"]`. tests/platforms/test_cuda_ir_op_priority.py exists; grep XPUOmniPlatform under tests/: no matches; grep xpu_kernels: no matches.) + +`vllm_omni/core/sched/omni_generation_scheduler.py:470` [minor] — OmniGenerationScheduler.update_from_output copies the stale drain (385-396) but this file never does `num_stale_output_tokens +=`. The live async_chunk segment-continue path (467-470) only `segment_finished_requests.discard`s after `_handle_stopped_request` returns False for resumable chunk receivers (55-67). AR’s matching `if not finished` block seeds from `num_in_flight_tokens` (omni_ar_scheduler.py:490-491). Generation’s only seed is the mixin streaming-replace path (omni_scheduler_mixin.py:105). Segment-stop seed on generation is untested (tests cover OmniARScheduler only). (evidence: omni_generation_scheduler.py:385-388 `if request is not None and request.num_stale_output_tokens > 0:\n output_is_stale = True\n request.num_stale_output_tokens -= num_tokens_scheduled\n assert request.num_stale_output_tokens >= 0` — drain only. omni_generation_scheduler.py:55-67 `_handle_stopped_request` returns False for resumable chunk receivers. omni_generation_scheduler.py:465-470 `finished = self._handle_stopped_request(request)\n is_segment_finished = not finished\n if not finished:\n # for streaming input request only\n if self.chunk_transfer_adapter:\n self.chunk_transfer_adapter.segment_finished_requests.discard(req_id)` — no seed. omni_ar_scheduler.py:490-491 `if request.num_in_flight_tokens > 0:\n request.num_stale_output_tokens += request.num_in_flight_tokens` — AR matching site does seed. omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` — only other +=, on streaming replace.) + +`vllm_omni/engine/stage_runtime.py:?` [minor] — scoped_spawn_device_env now takes stage_id/replica_id and _scoped_replica_compile_cache_env no-ops when `not replica_id or replica_id <= 0` (stage_engine_startup.py:896-901, :936). StageRuntime._scoped_spawn_device_env still calls `scoped_spawn_device_env(physical_devices, self._spawn_device_lock)` with neither id (stage_runtime.py:310-313), and _stage_device_scope(stage_id, runtime_cfg) does not forward stage_id (stage_runtime.py:327-330). That wrapper is the only scoped_spawn_device_env on the diffusion path: _initialize_local_diffusion_replica uses _stage_device_scope while plan.replica_id is available (stage_runtime.py:648, :660), and launch_diffusion_stage_replica never calls scoped_spawn_device_env (unlike launch_stage_replica at stage_engine_startup.py:1147). Forward both ids so replica_id>0 diffusion spawns cannot silently skip the TritonBundler cache isolation this PR added. (evidence: vllm_omni/engine/stage_engine_startup.py:936 `if not replica_id or replica_id <= 0:` then yield/return — isolation skipped when replica_id is None. vllm_omni/engine/stage_runtime.py:310-313 `with scoped_spawn_device_env(\n physical_devices,\n self._spawn_device_lock,\n ):` — neither stage_id nor replica_id passed. vllm_omni/engine/stage_runtime.py:327-330 `_stage_device_scope(self, stage_id, runtime_cfg)` then `with self._scoped_spawn_device_env(physical_devices):` — stage_id not forwarded. vllm_omni/engine/stage_runtime.py:648 `with self._stage_device_scope(plan.metadata.stage_id, plan.metadata.runtime_cfg):` wrapping launch_diffusion_stage_replica(..., replica_id=plan.replica_id) at :660. Grep of scoped_spawn_device_env in vllm_omni/engine: diffusion launcher has no call; only launch_stage_replica at stage_engine_startup.py:1147 forwards `stage_id=stage_id, replica_id=replica_id`.) + +`tests/core/sched/test_omni_ar_scheduler_stale_drain.py:84` [minor] — test_omni_ar_scheduler_stale_drain.py's helper is named _replace_streaming_session but calls OmniARScheduler._update_request_as_session with stage_id=0 and a StreamingUpdate that has no replace_streaming_prompt meta (test:71,84), then drains via OmniARScheduler.update_from_output (test:132). That AR path seeds stale tokens then super()-extends the prompt (omni_ar_scheduler.py:627-629,668-672) and never enters mixin._replace_streaming_session. Production talker replacement is OmniGenerationScheduler._update_request_as_session → mixin._replace_streaming_session (omni_generation_scheduler.py:604; mixin.py:105). TestReplaceSessionWithStreamingUpdate already hits that seed site without asserting num_stale_output_tokens or the frame-by-frame drain (test_omni_generation_scheduler_update_session.py:198). Drive the drain cases through OmniGenerationScheduler.update_from_output after mixin replacement so the talker/codec path is the one under test. Run: pytest -s -v tests/core/sched/test_omni_ar_scheduler_stale_drain.py tests/core/sched/test_omni_generation_scheduler_update_session.py -m 'core_model and cpu'. (evidence: tests/core/sched/test_omni_ar_scheduler_stale_drain.py:71 `sched.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_id=0))` — :84 `sched._update_request_as_session(session, update)` — :132 `OmniARScheduler.update_from_output(sched, scheduler_output, model_runner_output)`. vllm_omni/core/sched/omni_ar_scheduler.py:668-672 `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)` / `return` / `super()._update_request_as_session(session, update)` — test StreamingUpdate has no replace_streaming_prompt meta, so mixin is skipped. vllm_omni/core/sched/omni_generation_scheduler.py:604 `self._replace_streaming_session(session, update)` (unconditional). vllm_omni/core/sched/omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`. tests/core/sched/test_omni_generation_scheduler_update_session.py:198 `sched._update_request_as_session(session, update)` asserts prompt reset only, not num_stale_output_tokens.) + +`vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py:312` [minor] — Qwen2_5OmniVideoProcessorItems.get_processor_data now does `items = [self.get(index) for index in range(self.get_count())]` (thinker.py:312) so MediaWithBytes unwraps before HF make_batched_videos. tests/ has zero MediaWithBytes, get_processor_data, or VideoProcessorItems hits; test_per_video_audio_mask.py:271 only mocks get_mm_hashes and never constructs wrapped video items. Add a CPU unit test that stores MediaWithBytes-wrapped (frames, metadata) tuples and asserts get_processor_data()['videos'] is non-empty. Run: pytest -s -v tests/model_executor/models/qwen2_5_omni/ -m 'core_model and cpu'. (evidence: qwen2_5_omni_thinker.py:312 `items = [self.get(index) for index in range(self.get_count())]` then :313 `videos = [frames for frames, _ in items]`. tests/ grep MediaWithBytes, get_processor_data, and VideoProcessorItems: no matches. test_per_video_audio_mask.py:271 `inputs.get_mm_hashes = MagicMock(return_value={"video": ["v0"], "audio": ["a0"]})` — hashes mocked, no wrapped video items.) + +`arg_utils.py:116` [nit] — register_omni_models_to_vllm now unconditionally overwrites every _OMNI_MODELS arch (arg_utils.py:115-116); old code skipped archs vLLM 0.27 already owned. Other .model_stage direct reads (voxtral_tts:112, minicpmo_4_5_omni:94, ming_flash_omni:98, step_audio2, omnivoice, mimo_audio, ming_tts, covo_audio, cosyvoice3, mammoth_moda2) crash only if those archs are in vLLM 0.27's registry — overlap is unproven. Which of these archs does 0.27 already register, and do those paths need the same missing-model_stage default or stock-runner contract this PR applied to Qwen3/Qwen2.5? (evidence: [sweep] 9 blast radius] Other .model_stage direct reads (voxtral_tts:112, minicpmo_4_5_omni:94, ming_flash_omni:98, step_audio2, omnivoice, mimo_audio, ming_tts, covo_audio, cosyvoice3, mammoth_moda2) crash only if those archs are in vLLM 0.27's registry — unproven overlap, not posted.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_cb_grok46_r3/pr5976.trace.json.gz b/eval/dataset/arms/copilot_cb_grok46_r3/pr5976.trace.json.gz new file mode 100644 index 000000000..839544529 Binary files /dev/null and b/eval/dataset/arms/copilot_cb_grok46_r3/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.cost.json b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.cost.json new file mode 100644 index 000000000..05b51bf0c --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1672.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5509/run-20260814-214244-28a698", + "recorded_at": "2026-08-14T22:10:33", + "attempts": [ + { + "invocation_id": "inv-0193b9bf", + "outcome": "run:run-20260814-214244-28a698", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-214244-28a698", + "llm_calls": 95, + "input_tokens": 602008, + "output_tokens": 192003, + "usd": 0.471226, + "source": "spans" + } + ], + "attempt_usd_total": 0.471226, + "llm_calls": 95, + "input_tokens": 602008, + "output_tokens": 192003, + "usd": 0.471226, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-214244-28a698", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4712, + "minutes": 27.82, + "ci_minutes": 0, + "input_tokens": 602008, + "output_tokens": 192003, + "tool_calls": 285, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8423, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 95, + "by_role": { + "moa_member": { + "usd": 0.048934, + "input_tokens": 60247, + "output_tokens": 5822, + "calls": 11 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.365849, + "input_tokens": 500244, + "output_tokens": 145116, + "calls": 80 + }, + "reducer": { + "usd": 0.056443, + "input_tokens": 41517, + "output_tokens": 41065, + "calls": 3 + } + }, + "cache_read_tokens": 3200448, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.36, + "gate": 1.59, + "review": 1664.36, + "report": 0.06 + }, + "time_to_first_result_s": 1669.35 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 42317 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.md b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.md new file mode 100644 index 000000000..8b8aaad76 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.md @@ -0,0 +1,62 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] trtllm_attn.py:194-216 — SAGE init fail-fast checks dtype_qk set, `sage_attn_sfs` kernel param, and `trtllm_sage_attention_quantize` before forward +- [validated] trtllm_attn.py:295-320 — sage kwargs passed only when `self.quant.enabled`; dense path omits them for older FlashInfer +- [sweep] trtllm_attn.py:90 — `if not HAS_FLASHINFER` in `_sage_kernel_available` returns False; covered by existing import guard +- [sweep] trtllm_attn.py:198-211 — quant init branches all reachable; invalid dtype / missing kernel / missing quantize fn tested +- [sweep] trtllm_attn.py:296 — per-forward quantize intentional; `_sage_quantize_fn` pinned at init per compile comment +- [validated] data.py:1472-1483 — `backend_kwargs()` always emits default q/k block sizes when quant enabled +- [sweep] data.py:1415-1420 — block size 0 rejected at spec; `QuantConfig.from_backend_kwargs` `or 1` only matters for direct kwargs bypass +- [sweep] data.py:1447 — quant allowed on FLASHINFER_ATTN but impl does not consume it (see review comment) + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — docs/user_guide/diffusion/attention_backends.md:232 still says 'see Known Limitations above', but this diff deletes the '## Known Limitations' section (LTX-2 CUDNN_ATTN/torch.compile crash workaround #3121, FA4 note), Validation Guidance, mixed-backend examples, and the bench-repro block. The LTX-2 FLASHINFER_ATTN example at line 338 is now left without the crash rationale. Restore those sections or retarget this pointer and the example. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.`; docs/user_guide/diffusion/attention_backends.md:338 ` --model Lightricks/LTX-2 \`. A grep for `Known Limitations` in the file returns only line 232 (the stale bullet), confirming the `## Known Limitations` heading and its LTX-2 workaround no longer exist.) + +`vllm_omni/diffusion/data.py:1425` [major] — This diff adds AttnQuantSpec.enabled as dtype_qk or dtype_vo, while TRTLLM QuantConfig.enabled is only dtype_qk and from_backend_kwargs ignores dtype_vo/flashinfer_backend. AttentionSpec(backend="TRTLLM_ATTN", quant={"dtype_vo": "fp8_e4m3"}) therefore parses as enabled, serializes a quant block, and then TrtllmAttentionImpl runs dense BF16 with no error — contradicting the new docs line that each backend “reads the fields that apply and rejects the rest.” Reject FLASHINFER-only keys on TRTLLM_ATTN at AttentionSpec (and unknown keys in QuantConfig) rather than dropping them. (evidence: vllm_omni/diffusion/data.py:1424-1425: `def enabled(self) -> bool: return self.dtype_qk is not None or self.dtype_vo is not None`. vllm_omni/diffusion/attention/backends/trtllm_attn.py:130-131: `def enabled(self) -> bool: return self.dtype_qk is not None`. trtllm_attn.py:121-127 QuantConfig.from_backend_kwargs reads only dtype_qk/q_block_size/k_block_size. Docs added at attention_backends.md:185-188: `each reads the fields that apply and rejects the rest.` This is the TRTLLM-side hole; FLASHINFER_ATTN allowlist silent-ignore remains on discussion_r3681666323 and is not re-asked here.) + +`vllm_omni/diffusion/data.py:?` [major] — Confirmed: AttentionSpec.__post_init__ (vllm_omni/diffusion/data.py:1447) now accepts `quant` for FLASHINFER_ATTN and backend_kwargs() serializes it, but FlashInferAttentionImpl.__init__ (vllm_omni/diffusion/attention/backends/flashinfer_attn.py:63) only binds **extra_impl_args without storing any quant fields, and forward_cuda (lines 171-178) passes only sm_scale/causal/return_lse/custom_mask to single_prefill_with_kv_cache. So `backend="FLASHINFER_ATTN", quant={...}` silently runs dense. Either consume FLASHINFER_ATTN quant in this PR, or reject `quant` for FLASHINFER_ATTN and drop the docs claim and the round-trip-only test (tests/diffusion/attention/test_attention_config.py:65-73) so the user-facing option cannot silently no-op. (evidence: vllm_omni/diffusion/data.py:1447 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):`; vllm_omni/diffusion/attention/backends/flashinfer_attn.py:63 `**extra_impl_args,`; vllm_omni/diffusion/attention/backends/flashinfer_attn.py:65-66 `self.causal = causal` / `self.softmax_scale = softmax_scale` (no quant stored); vllm_omni/diffusion/attention/backends/flashinfer_attn.py:171-178 `kwargs: dict = {"sm_scale": self.softmax_scale, "causal": self.causal, "return_lse": False}` ... `out = single_prefill_with_kv_cache(query[b], key[b], value[b], **kwargs)` (no quant kwargs).) + +`vllm_omni/diffusion/attention/layer.py:?` [major] — This diff adds `quant`/`AttnQuantSpec` and applies SAGE quantization only in `TrtllmAttentionImpl.forward_cuda` (vllm_omni/diffusion/attention/backends/trtllm_attn.py:296-298), but `Attention._run_ring_attention` (vllm_omni/diffusion/attention/layer.py:319-330) only guards `skip_softmax` and then delegates to `RingParallelAttention.run_attention`, which uses ring_flash/ring_sdpa kernels and never reaches the SAGE path. A `TRTLLM_ATTN + quant` config under `ring_degree > 1` is therefore silently ignored (dense ring attention runs with no error/warning). Add the same guard for `getattr(self.attention, "quant", None)` when `quant.enabled` is True — e.g. `raise NotImplementedError("TRTLLM_ATTN quant (SAGE) is not supported with ring sequence parallelism: the ring path bypasses the backend, so the quant config would be silently ignored. Use Ulysses SP instead, or remove the quant config.")` — or make SAGE work through the ring path. (evidence: ['vllm_omni/diffusion/attention/layer.py:320 `skip = getattr(self.attention, "skip", None)`', 'vllm_omni/diffusion/attention/layer.py:323-324 `"Skip-Softmax (TRTLLM_ATTN) is not supported with ring sequence parallelism: " / "the ring path bypasses the backend, so the skip config would be silently ignored. "`', 'vllm_omni/diffusion/attention/layer.py:328-330 `if self.ring_runner is not None: return self.ring_runner.run_attention(query, key, value, attn_metadata, softmax_scale=self.softmax_scale, causal=self.causal)` — no quant check before delegating.', "vllm_omni/diffusion/attention/backends/trtllm_attn.py:296-297 `if self.quant.enabled: q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)` — SAGE runs only in the backend's own forward path.", 'vllm_omni/diffusion/attention/parallel/ring.py:157 `return ring_flash_attn_func(` — ring path uses ring flash kernels, bypassing the backend impl.']) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — Confirmed: docs/user_guide/diffusion/attention_backends.md:72-73 still says 'Today the only one is TRTLLM_ATTN's Skip-Softmax' although this PR adds `quant`/`AttnQuantSpec` and a SAGE quantization section in the same file; docs/design/module/dit_module.md:543 repeats 'Today the only such field is skip_softmax'; and recipes/Wan-AI/Wan2.2-T2V.md:186 says 'TRTLLM_ATTN is BF16-only (no attention-level quantization)'. Update all three to mention `quant`. (evidence: docs/user_guide/diffusion/attention_backends.md:72 `A backend that needs configuration exposes it as a typed field on the spec. Today the only` +docs/user_guide/diffusion/attention_backends.md:73 `one is `TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):` +docs/design/module/dit_module.md:543 `A backend that needs configuration exposes it as a typed field on `AttentionSpec`; `AttentionSpec.backend_kwargs()` serializes it into the `backend_kwargs` the impl consumes. Today the only such field is `skip_softmax` (TRTLLM_ATTN), so e.g. `target_sparsity` can be set without changing model code.` +recipes/Wan-AI/Wan2.2-T2V.md:186 ` - `TRTLLM_ATTN` is BF16-only (no attention-level quantization).`) + +`tests/diffusion/attention/test_trtllm_attn.py:306` [minor] — PR body's 'Composition with Skip-Softmax' table claims INT8 SAGE + skip-softmax cross-validation on Wan2.2-I2V, but the PR adds no GPU test exercising both paths on one impl: test_sage_quant_matches_sdpa (tests/diffusion/attention/test_trtllm_attn.py:306) sets only `quant` (line 311), and test_quant_and_skip_softmax_coexist (tests/diffusion/attention/test_attention_config.py:75) asserts only serialized kwargs. Add a `@requires_sage` GPU test that constructs `_impl(quant={...}, skip_softmax_threshold=...)`, calls forward_cuda, and asserts finite output with reasonable SDPA-relative error, guarding the combined kwargs path in forward_cuda. (evidence: tests/diffusion/attention/test_trtllm_attn.py:311 `impl = _impl(quant={"dtype_qk": dtype_qk, "q_block_size": 1, "k_block_size": 16})` — SAGE-only, no skip_softmax; tests/diffusion/attention/test_attention_config.py:79 `assert bk["target_sparsity"] == 0.5 and bk["quant"]["dtype_qk"] == "int8"` — coexistence asserted only on serialized kwargs; vllm_omni/diffusion/attention/backends/trtllm_attn.py:319 `skip_softmax_threshold_scale_factor=_skip_factor,` and :320 `**sage_kwargs,` — both passed together in forward_cuda.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:?` [minor] — In `vllm_omni/diffusion/attention/backends/trtllm_attn.py`, the init gate at line 204 calls `_sage_kernel_available()`, which at line 93 only checks `"sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters`. That is arch/dtype-blind, so on SM103 an `int8` SAGE config passes the line-199 dtype check and the signature check, then only fails on the first forward (`Missing TRTLLM-GEN kernel`), after the model is loaded. This contradicts the forward-path comment claiming SAGE availability is 'checked at init'. Make the init gate arch/dtype aware (e.g. reject int8 on non-SM100 device capability) so the config fails at config time. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:93 `return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters` — signature check only, no arch/dtype awareness; vllm_omni/diffusion/attention/backends/trtllm_attn.py:199 `if self.quant.dtype_qk not in _QK_QUANT_DTYPES:` — dtype check accepts int8; vllm_omni/diffusion/attention/backends/trtllm_attn.py:204 `if not _sage_kernel_available():` — init gate relies on signature check; tests/diffusion/attention/test_trtllm_attn.py:315 `if "Missing TRTLLM-GEN kernel" in str(e):` — test explicitly tolerates the late first-forward failure instead of config-time rejection.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:?` [minor] — Confirmed: `TrtllmAttentionImpl.__init__` raises RuntimeError at trtllm_attn.py:204-208 when `_sage_kernel_available()` is False and quant is enabled, and the only quant-branch test (tests/diffusion/attention/test_trtllm_attn.py:65-71) monkeypatches `_sage_kernel_available` to True. Add a CPU test with `monkeypatch.setattr(mod, '_sage_kernel_available', lambda: False)` and `quant={'dtype_qk': 'int8'}` asserting RuntimeError match `'sage_attn_sfs kernel path'`. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:204-208 `if not _sage_kernel_available():` / `raise RuntimeError(` / `"TRTLLM_ATTN quant (SAGE) was requested but this FlashInfer build does not "` / `"expose the trtllm-gen sage_attn_sfs kernel path. Install a FlashInfer build "` / `"that provides it, or remove the quant config."`; tests/diffusion/attention/test_trtllm_attn.py:68 `monkeypatch.setattr(mod, "_sage_kernel_available", lambda: True)` — grep shows no test-side occurrence forcing `_sage_kernel_available` to False.) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/attention/test_trtllm_attn.py:298` [minor] test_sage_quant_matches_sdpa (test_trtllm_attn.py:305-321) is the only real-kernel SAGE fidelity check (rel<0.1 vs SDPA), and both requires_trtllm_attn (via _has_trtllm_attn's capability major>=10 gate at lines 16-17) an +- `tests/diffusion/attention/test_trtllm_attn.py:315` [minor] tests/diffusion/attention/test_trtllm_attn.py:315 — the skip is over-broad: it matches any RuntimeError containing 'Missing TRTLLM-GEN kernel', so a real missing-kernel regression would be silently skipped. Gate the skip +- `requirements/common.txt:1` [minor] Docs require FlashInfer >=0.6.16rc1 for SAGE (attention_backends.md:210), but requirements/common.txt and cuda.txt declare no flashinfer lower bound (grep under requirements/ returns no matches; it arrives via vLLM). The +- `vllm_omni/diffusion/data.py:1410` [minor] `AttnQuantSpec` accepts `float16`/`bfloat16` for any backend with `quant`, but TRTLLM SAGE only supports `int8`/`fp8_e4m3` and fails later with `RuntimeError` in `TrtllmAttentionImpl.__init__`. Add backend-aware validati + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.trace.json.gz new file mode 100644 index 000000000..a99e5a5da Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.cost.json b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.cost.json new file mode 100644 index 000000000..db8f00557 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.cost.json @@ -0,0 +1,127 @@ +{ + "wall_s": 1593.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5509/run-20260814-230058-e85d63", + "recorded_at": "2026-08-14T23:27:30", + "attempts": [ + { + "invocation_id": "inv-ef000c46", + "outcome": "run:run-20260814-230058-e85d63", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-214244-28a698", + "llm_calls": 95, + "input_tokens": 602008, + "output_tokens": 192003, + "usd": 0.471226, + "source": "spans" + }, + { + "run_dir": "run-20260814-230058-e85d63", + "llm_calls": 109, + "input_tokens": 674483, + "output_tokens": 177518, + "usd": 0.522512, + "source": "spans" + } + ], + "attempt_usd_total": 0.993738, + "llm_calls": 109, + "input_tokens": 674483, + "output_tokens": 177518, + "usd": 0.522512, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-230058-e85d63", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5225, + "minutes": 26.52, + "ci_minutes": 0, + "input_tokens": 674483, + "output_tokens": 177518, + "tool_calls": 326, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8478, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 109, + "by_role": { + "moa_member": { + "usd": 0.055193, + "input_tokens": 73339, + "output_tokens": 5380, + "calls": 13 + }, + "agent": { + "usd": 0.436345, + "input_tokens": 577744, + "output_tokens": 149755, + "calls": 93 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "reducer": { + "usd": 0.030974, + "input_tokens": 23400, + "output_tokens": 22383, + "calls": 2 + } + }, + "cache_read_tokens": 4912000, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.67, + "gate": 1.92, + "review": 1585.45, + "report": 0.08 + }, + "time_to_first_result_s": 1591.08 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 83063 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.md b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.md new file mode 100644 index 000000000..fae6548e5 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [validated] trtllm_attn.py:90 — `_sage_kernel_available()` returns False when `HAS_FLASHINFER` is False; quant init raises before forward. +- [validated] trtllm_attn.py:198-216 — quant init rejects unsupported `dtype_qk`, missing `sage_attn_sfs` kernel param, and missing `trtllm_sage_attention_quantize`; quantize fn cached on `self._sage_quantize_fn`. +- [validated] trtllm_attn.py:296-320 — `sage_kwargs` only populated when `self.quant.enabled`; dense/skip-only forwards omit SAGE kwargs for older FlashInfer builds. +- [validated] data.py:1413-1421 — `AttnQuantSpec.__post_init__` rejects invalid dtypes and block sizes {1,4,16}. +- [validated] data.py:1447-1451 — `quant` allowed on TRTLLM_ATTN and FLASHINFER_ATTN at spec level. +- [validated] data.py:1472-1483 — `backend_kwargs()` serializes quant defaults and FLASHINFER-specific fields when `quant.enabled`. +- [validated] layer.py:92-109 — resolved `spec.backend_kwargs()` passed to every backend impl including FlashInfer. +- [sweep] flashinfer_attn.py:54-64 — `FlashInferAttentionImpl.__init__` has no `backend_kwargs` param; `backend_kwargs=` lands in `**extra_impl_args` and is silently discarded (unlike SDPA/Sage which warn). + +`docs/user_guide/diffusion/attention_backends.md:187` [major] — This PR lets AttentionSpec accept quant for FLASHINFER_ATTN (data.py:1447-1450) and documents that FLASHINFER_ATTN reads dtype_qk/dtype_vo/flashinfer_backend, but FlashInferAttentionImpl.__init__ (flashinfer_attn.py:54-66) discards **extra_impl_args and forward_cuda hard-codes sm_scale/causal/return_lse/custom_mask; layer.py:109 passes backend_kwargs into that discarded slot. A user's FLASHINFER_ATTN quant block therefore parses and serializes successfully, then silently runs dense attention. Either consume/reject quant in flashinfer_attn.py, or constrain AttentionSpec validation and the doc to TRTLLM_ATTN until the FLASHINFER path lands. (evidence: ['vllm_omni/diffusion/data.py:1447-1449 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"): raise ValueError(f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but "`', 'vllm_omni/diffusion/attention/layer.py:109 `backend_kwargs=backend_kwargs,`', 'vllm_omni/diffusion/attention/backends/flashinfer_attn.py:54-66 `class FlashInferAttentionImpl(AttentionImpl): def __init__(self, num_heads: int, head_size: int, softmax_scale: float, causal: bool = False, num_kv_heads: int | None = None, prefix: str = "", **extra_impl_args) -> None: self.causal = causal; self.softmax_scale = softmax_scale`']) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — docs/user_guide/diffusion/attention_backends.md:232 still says '- **Workaround for known issues**: see Known Limitations above.', but this PR deletes the only '## Known Limitations' section (including the LTX-2.0 CUDNN_ATTN/torch.compile workaround for #3121, FA4 note, Validation Guidance, and Reproducing the Benchmark Table). Restore the removed sections, or if the cleanup is intentional, retarget/remove the dangling bullet and keep the LTX-2.0 workaround somewhere. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — grep for `## Known Limitations` in the same file returns no matches, and the PR diff deletes the only such heading plus the LTX-2.0/cuDNN workaround, Validation Guidance, Reproducing the Benchmark Table, and the Mixed/Compare subsections.) + +`vllm_omni/diffusion/data.py:?` [minor] — This diff adds `quant` to `spec_keys` in `_flatten_per_role_entry` (vllm_omni/diffusion/data.py:1553), but no test covers a nested per_role entry with a `quant` block. Add a case mirroring `test_constructor_flattens_nested_per_role_tree` (tests/diffusion/attention/test_attention_config.py:113) using `per_role: { self: { backend: TRTLLM_ATTN, quant: {...} } }` so nested/CLI configs round-trip and resolve. (evidence: vllm_omni/diffusion/data.py:1553 `spec_keys = {"backend", "skip_softmax", "quant"}` — `quant` is in the spec-key set used by `_flatten_per_role_entry`. tests/diffusion/attention/test_attention_config.py:113 `def test_constructor_flattens_nested_per_role_tree(self):` followed by `per_role={` / `"ltx2": {` / `"audio_self": {"backend": "FLASH_ATTN"},` / `"audio_to_video": {"backend": "SAGE_ATTN"},` — nested per-role coverage exercises only `backend`, with no `quant` block.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:93` [minor] — `_sage_kernel_available` (trtllm_attn.py:93) checks only `sage_attn_sfs` in `trtllm_ragged_attention_deepseek`'s signature, but `forward_cuda` passes `num_elts_per_sage_attn_blk` too (trtllm_attn.py:299) whenever SAGE quant is enabled; a build exposing one parameter without the other would pass `__init__` and fail at forward. Please confirm whether these two parameters always ship together in FlashInfer, and if not, require both in the availability check. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:93 `return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:299 `sage_kwargs["num_elts_per_sage_attn_blk"] = sage_block_sizes`) + +`pyproject.toml:32` [minor] — No flashinfer minimum is declared in pyproject.toml/setup.py/requirements (dependencies are dynamic via setup.py), and the new SAGE quant path relies on runtime feature detection (_sage_kernel_available/_sage_quantize_fn) with a clear RuntimeError when missing. Minor: consider an explicit flashinfer>=0.6.16rc1 floor for SAGE support instead of runtime detection alone. (evidence: ['pyproject.toml:11 `dynamic = ["version", "dependencies"]`', 'pyproject.toml:32 `# Dependencies are now managed dynamically via setup.py based on detected hardware platform.`', 'setup.py and requirements/*: grep `flashinfer` -> no matches (no declared version floor in any dependency manifest)', 'vllm_omni/diffusion/attention/backends/trtllm_attn.py:214 `"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "`']) + +`tests/diffusion/attention/test_attention_config.py:65` [minor] — `test_quant_superset_fields_passed_through` (tests/diffusion/attention/test_attention_config.py:65-73) only asserts the serialized `backend_kwargs()['quant']` keys. The FLASHINFER_ATTN backend (vllm_omni/diffusion/attention/backends/flashinfer_attn.py) has no `quant`/`backend_kwargs` consumption path, so the spec allowlist admits a FLASHINFER_ATTN quant config that is silently dropped at runtime and this test does not catch it. Consider either adding runtime consumption/rejection in `FlashInferAttentionImpl` plus a test, or documenting that FLASHINFER_ATTN quant is currently serialization-only. (evidence: tests/diffusion/attention/test_attention_config.py:70-73 `).backend_kwargs()["quant"]` / `assert bk["dtype_qk"] == "bfloat16"` / `assert bk["dtype_vo"] == "fp8_e4m3"` / `assert bk["flashinfer_backend"] == "trtllm-gen"` — the test only checks serialization output, with no FLASHINFER_ATTN runtime consumption or rejection assertion.) + +`vllm_omni/diffusion/data.py:1424` [minor] — `AttnQuantSpec.enabled` (data.py:1424-1425) is true for a dtype_vo-only quant block, so `AttentionSpec(backend="TRTLLM_ATTN", quant={"dtype_vo": "fp8_e4m3"})` passes spec validation and serializes `quant`, but `QuantConfig.enabled` (trtllm_attn.py:130-131) only keys on `dtype_qk`; TRTLLM_ATTN therefore silently runs dense. Add per-backend validation in `AttentionSpec.__post_init__` (TRTLLM_ATTN requires `dtype_qk` in {"int8", "fp8_e4m3"}; FLASHINFER_ATTN requires its own dtype set) or tighten `AttnQuantSpec.enabled`/serialization so a dtype_vo-only quant block cannot reach TRTLLM_ATTN. (evidence: vllm_omni/diffusion/data.py:1424-1425 `return self.dtype_qk is not None or self.dtype_vo is not None`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:131 `return self.dtype_qk is not None`) + +`docs/user_guide/diffusion/attention_backends.md:208` [minor] — Docs present SAGE as a general TRTLLM_ATTN feature with a Wan2.2-T2V fp8 example and SM103 availability, but the PR evidence validates only Wan2.2-A14B I2V on B200/GB200. State the validation scope (e.g. 'validated on Wan2.2-A14B I2V; other models at your own risk') or add T2V and SM103 fp8 smoke evidence. (evidence: docs/user_guide/diffusion/attention_backends.md:208 `**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel`; docs/user_guide/diffusion/attention_backends.md:209 `availability is arch-dependent: `fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103**`; docs/user_guide/diffusion/attention_backends.md:191 `vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:296 ` if self.quant.enabled:`; PR body: `Wan2.2-A14B, 1280×720 / 81 frames / 50 steps` and `## I2V — official vLLM-Omni examples (B200, image-anchored)`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.trace.json.gz new file mode 100644 index 000000000..e516c042a Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/QUARANTINE_skillleak_pr5509_regen1/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/manifest.json b/eval/dataset/arms/copilot_moa_cgm/manifest.json new file mode 100644 index 000000000..9439d3f60 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_moa_cgm", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "always", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5509.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5509.cost.json new file mode 100644 index 000000000..e5e7b8259 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5509.cost.json @@ -0,0 +1,129 @@ +{ + "wall_s": 1351.3, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5509/run-20260814-233054-44b004", + "recorded_at": "2026-08-14T23:53:23", + "attempts": [ + { + "invocation_id": "inv-fb67d0a8", + "outcome": "run:run-20260814-233054-44b004", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-214244-28a698", + "llm_calls": 95, + "input_tokens": 602008, + "output_tokens": 192003, + "usd": 0.471226, + "source": "spans" + }, + { + "run_dir": "run-20260814-230058-e85d63", + "llm_calls": 109, + "input_tokens": 674483, + "output_tokens": 177518, + "usd": 0.522512, + "source": "spans" + }, + { + "run_dir": "run-20260814-233054-44b004", + "llm_calls": 62, + "input_tokens": 502728, + "output_tokens": 123560, + "usd": 0.335417, + "source": "spans" + } + ], + "attempt_usd_total": 1.329155, + "llm_calls": 62, + "input_tokens": 502728, + "output_tokens": 123560, + "usd": 0.335417, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-233054-44b004", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3354, + "minutes": 22.48, + "ci_minutes": 0, + "input_tokens": 502728, + "output_tokens": 123560, + "tool_calls": 233, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7015, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 62, + "by_role": { + "moa_member": { + "usd": 0.075862, + "input_tokens": 110787, + "output_tokens": 5244, + "calls": 11 + }, + "reducer": { + "usd": 0.052984, + "input_tokens": 37990, + "output_tokens": 38786, + "calls": 3 + }, + "agent": { + "usd": 0.206571, + "input_tokens": 353951, + "output_tokens": 79530, + "calls": 48 + } + }, + "cache_read_tokens": 1723328, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.61, + "gate": 1.68, + "review": 1343.33, + "report": 0.06 + }, + "time_to_first_result_s": 1348.66 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 120485 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5509.md b/eval/dataset/arms/copilot_moa_cgm/pr5509.md new file mode 100644 index 000000000..66e5ccb6e --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5509.md @@ -0,0 +1,68 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] trtllm_attn.py:88-95,204-216 — _sage_kernel_available() signature probe plus init-time _sage_quantize_fn resolution fail fast with clear RuntimeError when SAGE is requested but FlashInfer lacks sage_attn_sfs / trtllm_sage_attention_quantize. +- [validated] trtllm_attn.py:194-216,296-299 — quantize fn stored on self at init (avoids per-step lru_cache Dynamo breaks); forward_cuda quantizes Q/K/V and passes sage_attn_sfs/num_elts_per_sage_attn_blk only when quant.enabled. +- [validated] trtllm_attn.py:90 — dense TRTLLM path unchanged when HAS_FLASHINFER is false or quant disabled; sage kwargs omitted for older FlashInfer dense compatibility. +- [validated] data.py:1413-1421,1447-1451 — AttnQuantSpec validates dtype/block-size unions; AttentionSpec rejects quant on non TRTLLM/FLASHINFER backends. +- [validated] data.py:1472-1483 — backend_kwargs() serializes quant with defaults (q_block_size=1, k_block_size=16) and optional dtype_vo/flashinfer_backend passthrough. +- [validated] tests/diffusion/attention/test_attention_config.py:13,55-91 and test_trtllm_attn.py:13,50-85,298-321 — pytestmark includes core_model; config/plumbing tests are CPU-safe; GPU SAGE parity test skips on missing kernel/arch. +- [sweep] test_trtllm_attn.py:315 — Missing TRTLLM-GEN kernel RuntimeError branch correctly pytest.skip()s int8 on SM103-only kernels. +- [sweep] flashinfer_attn.py:55-66 vs layer.py:101-109 — backend_kwargs passed from Attention layer but FlashInferAttentionImpl __init__ has no backend_kwargs param; kwargs land in unused **extra_impl_args. + +`vllm_omni/diffusion/data.py:?` [major] — vllm_omni/diffusion/data.py:1447-1450 admits `quant` for FLASHINFER_ATTN, but FlashInferAttentionImpl.__init__ (flashinfer_attn.py:55-66) stores only causal/softmax_scale and absorbs `backend_kwargs` in `**extra_impl_args`; forward_cuda passes only sm_scale/causal/return_lse/custom_mask to single_prefill_with_kv_cache. No FLASHINFER_ATTN reader for dtype_qk/dtype_vo/flashinfer_backend exists anywhere (only data.py references those keys), and AttnQuantSpec has a single global dtype allowlist with no per-backend rejection, so `backend='FLASHINFER_ATTN', quant={'dtype_qk':'int8'}` validates then silently runs dense BF16. Either restrict the allowlist to TRTLLM_ATTN or implement the FLASHINFER_ATTN reader with wrong-set rejection; test_quant_superset_fields_passed_through cannot catch this because it only asserts backend_kwargs() round-trips. (evidence: vllm_omni/diffusion/data.py:1447-1449 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` / `raise ValueError(` / `f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but "`; vllm_omni/diffusion/attention/backends/flashinfer_attn.py:55-66 `def __init__(self, num_heads: int, head_size: int, softmax_scale: float, causal: bool = False, num_kv_heads: int | None = None, prefix: str = "", **extra_impl_args,) -> None:` / `self.causal = causal` / `self.softmax_scale = softmax_scale`; vllm_omni/diffusion/attention/layer.py:109 `backend_kwargs=backend_kwargs,`; docs/user_guide/diffusion/attention_backends.md:185-188 `The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the fields that apply and rejects the rest. `TRTLLM_ATTN` (SAGE) reads `dtype_qk` (`int8` / `fp8_e4m3`), `q_block_size`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`), `dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.`) + +`vllm_omni/diffusion/data.py:1425` [major] — Confirmed: data.py:1425 `return self.dtype_qk is not None or self.dtype_vo is not None` makes dtype_vo-only quant appear enabled, so `AttentionSpec(backend='TRTLLM_ATTN', quant={'dtype_vo': 'fp8_e4m3'})` validates and backend_kwargs() emits a quant block, but trtllm_attn.py:131 `return self.dtype_qk is not None` leaves QuantConfig.enabled False and trtllm_attn.py:296 `if self.quant.enabled:` skips SAGE entirely — the user's quant config is silently ignored and dense BF16 runs. Align enabled semantics per backend or reject dtype_vo-only quant on TRTLLM_ATTN at config time. (evidence: vllm_omni/diffusion/data.py:1425 `return self.dtype_qk is not None or self.dtype_vo is not None`; vllm_omni/diffusion/data.py:1479 `if q.dtype_vo is not None:` and :1480 `quant_kw["dtype_vo"] = q.dtype_vo` (serialized because the shared `if self.quant is not None and self.quant.enabled:` gate is true); vllm_omni/diffusion/attention/backends/trtllm_attn.py:131 `return self.dtype_qk is not None`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:296 `if self.quant.enabled:`.) + +`tests/diffusion/attention/test_attention_config.py:65` [major] — In this PR, data.py accepts quant for FLASHINFER_ATTN and AttentionSpec.backend_kwargs() emits it, but FlashInferAttentionImpl.__init__ (flashinfer_attn.py:55-66) swallows backend_kwargs via **extra_impl_args and never reads, warns, or raises, so FLASHINFER_ATTN+quant is silently ignored — contradicting the doc claim. test_quant_superset_fields_passed_through only asserts dict serialization and cannot catch this. Implement the FLASHINFER_ATTN quant path or reject/warn at validation/init, and add an Attention(role=..., backend=FLASHINFER_ATTN, quant=...) test. (evidence: vllm_omni/diffusion/attention/layer.py:109 ` backend_kwargs=backend_kwargs,` — the serialized quant kwargs are passed to the impl. vllm_omni/diffusion/attention/backends/flashinfer_attn.py:63 ` **extra_impl_args,` and vllm_omni/diffusion/attention/backends/flashinfer_attn.py:65-66 ` self.causal = causal` / ` self.softmax_scale = softmax_scale` — backend_kwargs is captured but never read, warned, or rejected. tests/diffusion/attention/test_attention_config.py:65 ` def test_quant_superset_fields_passed_through(self):` with only dict equality assertions at tests/diffusion/attention/test_attention_config.py:71-73 (`assert bk["dtype_qk"] == "bfloat16"`, `assert bk["dtype_vo"] == "fp8_e4m3"`, `assert bk["flashinfer_backend"] == "trtllm-gen"`), never instantiating the backend.) + +`docs/user_guide/diffusion/attention_backends.md:170` [major] — docs/user_guide/diffusion/attention_backends.md adds the SAGE section at line 170 but in the same file deletes Known Limitations (LTX-2 CUDNN torch.compile crash, #3121), the FA4 note, the mixed-backend/FLASH_ATTN examples, Validation Guidance, and the bench_attention_backends.py reproduction recipe. Line 232 still says 'see Known Limitations above', but that section no longer exists. Restore or move those sections and fix the xref; in particular keep the 'don't treat a faster backend as numerically interchangeable with TORCH_SDPA' checklist while documenting a lossy quantized backend. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` docs/user_guide/diffusion/attention_backends.md:170 `## TRTLLM_ATTN SAGE Quantization`) + +`tests/diffusion/attention/test_trtllm_attn.py:298` [major] — This PR's only real-kernel SAGE test is `test_sage_quant_matches_sdpa` (tests/diffusion/attention/test_trtllm_attn.py:306), which runs `forward_cuda` → `trtllm_ragged_attention_deepseek` with `sage_attn_sfs`/`num_elts_per_sage_attn_blk`. It is gated by `@requires_sage`, whose skipif calls `_has_trtllm_attn()` — that helper returns False unless `torch.cuda.get_device_capability()[0] >= 10` (line 18) — so it never runs on any merge-ready GPU lane: `.buildkite/cuda/test-ready.yml` runs `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` on `mirror_hardwares: l4_1` (sm_89) and the H100 lanes are sm_90, while grep shows no sm_100+/Blackwell lane in `.buildkite`. (Additionally, the `tests/diffusion/*.py` glob does not recurse into `tests/diffusion/attention/`, so that lane does not even collect this file.) Therefore CI green does not exercise the new quantize→kernel behavior. Add an L4-runnable monkeypatch test that stubs `trtllm_ragged_attention_deepseek` and asserts `sage_attn_sfs`/`num_elts_per_sage_attn_blk` are passed when `quant` is set, and either mark `test_sage_quant_matches_sdpa` manual Blackwell-only or add a Blackwell nightly job. (evidence: tests/diffusion/attention/test_trtllm_attn.py:18 ` if torch.cuda.get_device_capability()[0] < 10:` +tests/diffusion/attention/test_trtllm_attn.py:298 `requires_sage = pytest.mark.skipif(` +tests/diffusion/attention/test_trtllm_attn.py:299 ` not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None),` +.buildkite/cuda/test-ready.yml:68 ` - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"` +.buildkite/cuda/test-ready.yml:69 ` mirror_hardwares: l4_1`) + +`vllm_omni/diffusion/data.py:1447` [minor] — AttentionSpec.__post_init__ accepts quant on both TRTLLM_ATTN and FLASHINFER_ATTN but never validates dtype_qk against the backend's actual set (TRTLLM: {int8, fp8_e4m3}; FLASHINFER: {float16, bfloat16}). AttnQuantSpec._VALID_DTYPES is the union of both, so AttentionSpec(backend='TRTLLM_ATTN', quant={'dtype_qk': 'bfloat16'}) parses cleanly and only fails later in TrtllmAttentionImpl.__init__ with RuntimeError (trtllm_attn.py:199-202), while FLASHINFER_ATTN with dtype_qk='int8' is silently ignored (no dtype_qk handling in flashinfer_attn.py). Add per-backend dtype_qk validation here so invalid config raises ValueError at parse time. (evidence: data.py:1410 ` _VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})` — accepts all four dtypes regardless of backend. +data.py:1447 ` if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` — only backend membership is validated. +trtllm_attn.py:199-202 ` if self.quant.dtype_qk not in _QK_QUANT_DTYPES: + raise RuntimeError( + f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got " + f"{self.quant.dtype_qk!r}. FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE."` — TRTLLM rejects wrong dtype only at backend init, proving parse-time acceptance of invalid dtypes.) + +`docs/user_guide/diffusion/attention_backends.md:208` [minor] — docs/user_guide/diffusion/attention_backends.md:208 documents SAGE as requiring FlashInfer ≥ 0.6.16rc1, but the repo's requirements files declare no flashinfer lower bound (`grep flashinfer requirements/` is empty). The only enforcement is `TrtllmAttentionImpl.__init__` (vllm_omni/diffusion/attention/backends/trtllm_attn.py:212-215), which raises RuntimeError when `trtllm_sage_attention_quantize` is unavailable. Add an install-time lower bound for the optional flashinfer dependency, or explicitly document the vLLM/FlashInfer version coupling so the support matrix matches what the repo actually requires. (evidence: docs/user_guide/diffusion/attention_backends.md:208 `**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:214 `"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "`; `grep 'flashinfer' requirements/` -> no matches (requirements/common.txt, cuda.txt, cpu.txt, rocm.txt, etc. contain no flashinfer entry).) + +`docs/user_guide/diffusion/attention_backends.md:188` [minor] — Line 188 says `... flashinfer_backend` (see its section), but this file has no FLASHINFER_ATTN quant section; the closest FLASHINFER_ATTN mentions are the Backend Options table row (line 20) and auto-route bullets. Remove the parenthetical or add the referenced section. (evidence: docs/user_guide/diffusion/attention_backends.md:188 `dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.` — grep of the same file for `FLASHINFER_ATTN` hits only lines 20, 108, 127, 187, 216, 337, none of which is a dedicated FLASHINFER quant section.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/data.py:1447` [minor] A raw checklist notes that the opt-in `AttentionSpec(quant=...)` applies SAGE to every role, including Wan2.2 cross-attn, but PR e2e coverage is Wan2.2 I2V only. Could you either add cross-attn/LTX-2 coverage or document +- `tests/diffusion/attention/test_trtllm_attn.py:50` [minor] tests/diffusion/attention/test_trtllm_attn.py:50-86 adds four CPU-pure tests (test_quant_config_from_backend_kwargs, test_quant_rejects_non_sage_dtype, test_quant_quantize_requires_flashinfer_routine, test_quant_quantize +- `docs/user_guide/diffusion/attention_backends.md:136` [minor] At docs/user_guide/diffusion/attention_backends.md:72 the guide still says 'A backend that needs configuration exposes it as a typed field on the spec. Today the only one is `TRTLLM_ATTN`'s Skip-Softmax'. This PR adds `q +- `vllm_omni/diffusion/attention/backends/trtllm_attn.py:204` [minor] Confirmed at vllm_omni/diffusion/attention/backends/trtllm_attn.py:204. Add a CPU/L4-runnable unit test that monkeypatches `_sage_kernel_available` to False (leaving `_sage_quantize_fn` available) and asserts `_impl(quan + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5509.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5509.trace.json.gz new file mode 100644 index 000000000..26a1439d6 Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5550.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5550.cost.json new file mode 100644 index 000000000..323fd800c --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5550.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1348.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5550/run-20260814-214244-f04fdd", + "recorded_at": "2026-08-14T22:05:10", + "attempts": [ + { + "invocation_id": "inv-56b69761", + "outcome": "run:run-20260814-214244-f04fdd", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-214244-f04fdd", + "llm_calls": 75, + "input_tokens": 553241, + "output_tokens": 102858, + "usd": 0.424319, + "source": "spans" + } + ], + "attempt_usd_total": 0.424319, + "llm_calls": 75, + "input_tokens": 553241, + "output_tokens": 102858, + "usd": 0.424319, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-214244-f04fdd", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4243, + "minutes": 22.43, + "ci_minutes": 0, + "input_tokens": 553241, + "output_tokens": 102858, + "tool_calls": 265, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.743, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 75, + "by_role": { + "moa_member": { + "usd": 0.054308, + "input_tokens": 92825, + "output_tokens": 1241, + "calls": 9 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.341175, + "input_tokens": 438435, + "output_tokens": 80829, + "calls": 63 + }, + "reducer": { + "usd": 0.028836, + "input_tokens": 21981, + "output_tokens": 20788, + "calls": 2 + } + }, + "cache_read_tokens": 5533760, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.32, + "gate": 2.02, + "review": 1340.07, + "report": 0.08 + }, + "time_to_first_result_s": 1345.46 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 36957 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5550.md b/eval/dataset/arms/copilot_moa_cgm/pr5550.md new file mode 100644 index 000000000..f6ca29593 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5550.md @@ -0,0 +1,67 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] tests/diffusion/diffusion_kv/test_metadata.py:37-38 — indexed [0] accesses are test assertions on locally constructed single-element tuples; safe by construction. +- [validated] tests/diffusion/diffusion_kv/test_metadata.py:68,109 — CFG shared/branch context_id assertions index sequences[0] on tuples built with known length 2; safe. +- [validated] tests/diffusion/diffusion_kv/test_worker_contract.py:221 — calls[0][1][0].prepared_layout asserts the req object (1st RPC arg) is forwarded unchanged; matches multiproc_executor.py:544. +- [validated] vllm_omni/diffusion/diffusion_engine.py:199 — scheduler_to_close branch runs when _init_scheduler assigned self.scheduler before initialize() raises; covered by test_scheduler_initialization_failure_closes_scheduler_and_executor. +- [validated] vllm_omni/diffusion/executor/multiproc_executor.py:545 — metadata appended as 4th positional RPC arg only when not None; dense legacy stays 3-tuple per test_dense_request_rpc_keeps_legacy_positional_shape. +- [validated] vllm_omni/diffusion/sched/interface.py:184,192 — validate_new_request_data_identity checks envelope→req and metadata→req; invoked before RPC at multiproc_executor.py:459-460 and in batch/step runners. +- [validated] vllm_omni/diffusion/worker/diffusion_model_runner.py:172 — dense_legacy rejects unexpected metadata; paged_scheduler allows None per W0 contract. +- [validated] vllm_omni/diffusion/worker/diffusion_model_runner.py:175 — metadata.request_id mismatch rejected before _execute_request_list. + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [major] — This diff extends DiffusionModelRunner.execute_model with diffusion_kv_metadata (diffusion_model_runner.py:589-592) and DiffusionWorker.execute_model now forwards it as a keyword when non-None (diffusion_worker.py:460-463). ARDiffusionModelRunner.execute_model still overrides only (self, req, kv_prefetch_job=None) and calls super().execute_model(req, kv_prefetch_job=kv_prefetch_job) at runner.py:243 and runner.py:255, so a metadata-carrying request routed through engine_backend='ar_diffusion' (engine.py:40 sets default_diffusion_model_runner_cls = AR_DIFFUSION_MODEL_RUNNER_CLS) raises TypeError: execute_model() got an unexpected keyword argument 'diffusion_kv_metadata' before base validation runs. Update the override to accept diffusion_kv_metadata: DiffusionKVMetadata | None = None and forward it to both super().execute_model(...) calls (or explicitly reject non-None), and add a regression test using the real ARDiffusionModelRunner. (evidence: vllm_omni/experimental/ar_diffusion/runner.py:240 ` kv_prefetch_job: KVPrefetchJob | None = None,`; runner.py:243 ` return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)`; runner.py:255 ` output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)`; vllm_omni/diffusion/worker/diffusion_worker.py:460-463 ` kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}` / ` if diffusion_kv_metadata is not None:` / ` kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata` / ` output = self.model_runner.execute_model(req, **kwargs)`; vllm_omni/diffusion/worker/diffusion_model_runner.py:589-592 ` def execute_model(` / ` self,` / ` req: OmniDiffusionRequest,` / ` kv_prefetch_job: KVPrefetchJob | None = None,` / ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,`) + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — vllm_omni/diffusion/executor/multiproc_executor.py:462-471 computes `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)` and adds `and not has_diffusion_kv_metadata` to the DLO+AllGather list-RPC branch, so a metadata-bearing wave silently falls back to per-request RPCs with `unique_reply_rank=0`. Please add a one-line rationale for why the broadcast list-RPC path cannot carry per-request metadata, a log line when this fallback occurs, and a test in tests/diffusion/diffusion_kv/test_worker_contract.py with `enable_distributed_layerwise_offload=True`, `dlo_use_allgather=True`, and two metadata-bearing `NewRequestData`s asserting `collective_rpc` is never called with a request list as its first argument. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)`; vllm_omni/diffusion/executor/multiproc_executor.py:471 `and not has_diffusion_kv_metadata`; vllm_omni/diffusion/executor/multiproc_executor.py:545 `args = (req, self.od_config, scheduler_output.kv_prefetch_job)` with subsequent `self.collective_rpc("execute_model", args=args, unique_reply_rank=0, exec_all_ranks=True)` in the fallback loop.) + +`vllm_omni/diffusion/sched/base_scheduler.py:114` [minor] — Confirmed: vllm_omni/diffusion/sched/base_scheduler.py:114 still calls `NewRequestData.from_state(state)` without `diffusion_kv_metadata`, and PR #5550's body frames this as the intended W0 boundary (metadata optional; W1 producer/installer not yet wired). Please confirm a linked W1 issue tracking the Scheduler allocation producer/attachment work — the body references RFC #5244 but does not link a distinct W1 issue number. (evidence: vllm_omni/diffusion/sched/base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — the scheduler constructs each new-request envelope without supplying `diffusion_kv_metadata`, matching the comment's W0-boundary claim.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:613` [minor] — The execute_batch DLO DP branch (multiproc_executor.py:605-613) returns `self.execute_request(scheduler_output)` at line 613, so MiniMax-H3-style DLO DP waves re-enter execute_request and are subject to the new `not has_diffusion_kv_metadata` gate at line 471; when any new_req carries metadata, the DLO AllGather broadcast is skipped in favor of the per-request RPC loop. Please confirm that fallback is intentional and add a DLO+metadata regression for this re-entry path, not just the direct worker-contract call site. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:613 ` return self.execute_request(scheduler_output)` — inside execute_batch's DLO DP branch (lines 605-609), so execute_batch DLO DP waves re-enter execute_request. vllm_omni/diffusion/executor/multiproc_executor.py:471 ` and not has_diffusion_kv_metadata` — part of execute_request's DLO multi-concurrency condition, so when metadata is present the broadcast path is skipped and requests fall through to the per-request RPC loop.) + +`tests/diffusion/diffusion_kv/test_metadata.py:37` [minor] — test_metadata.py only round-trips constructor fields (see lines 37-38, 67-69, 109-110), so it cannot catch transport bugs. The kept transport tests in test_worker_contract.py target worker-contract RPC shape (e.g. lines 213 and 261). Please confirm whether test_metadata.py should also get a MessageQueue/argument-path round-trip or whether worker-contract coverage is the intended sole transport coverage. (evidence: tests/diffusion/diffusion_kv/test_metadata.py:37 ` assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])`; tests/diffusion/diffusion_kv/test_metadata.py:38 ` assert metadata.contexts[0].block_ids == ([7], [11])`; tests/diffusion/diffusion_kv/test_metadata.py:67 ` assert metadata.contexts == (shared_context,)`; tests/diffusion/diffusion_kv/test_metadata.py:109 ` assert metadata.sequences[0].context_ids == (conditional_context.context_id,)`; tests/diffusion/diffusion_kv/test_worker_contract.py:213 ` assert calls == [ + ( + "execute_model", + (req, executor.od_config, None), + 0, + True, + ) + ]`; tests/diffusion/diffusion_kv/test_worker_contract.py:261 ` assert [call[1] for call in calls] == [ + (req_0, executor.od_config, None, metadata_0), + (req_1, executor.od_config, None, metadata_1), + ]`) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:?` [minor] — test_worker_wrapper_only_extends_paged_request_rpc only calls wrapper.execute_model with diffusion_kv_metadata as a kwarg, so it never exercises the live RPC dispatch WorkerProc._execute_rpc -> WorkerWrapperBase.execute_method(method, *args, **kwargs) -> func(*args, **kwargs). A regression that dropped the executor's appended fourth positional tuple member (multiproc_executor.py:544-546) would go undetected. Add a test that drives the real positional shape via wrapper.execute_method('execute_model', req, od_config, None, metadata) and asserts the inner worker receives metadata as the fourth positional argument; note the test's inner Worker mock currently has signature execute_model(self, req, od_config, **kwargs), so it must be changed to the real execute_model(self, req, od_config, kv_prefetch_job=None, diffusion_kv_metadata=None) signature for that assertion to work. (evidence: tests/diffusion/diffusion_kv/test_worker_contract.py:258-259 `wrapper.execute_model(req, od_config)` / `wrapper.execute_model(req, od_config, diffusion_kv_metadata=metadata)` — the test never drives execute_method. diffusion_worker.py:1036 `result = self.worker.execute_method(method, *args, **kwargs)` — live RPC dispatch enters through execute_method. diffusion_worker.py:1442-1443 `func = getattr(self.worker, method)` / `return func(*args, **kwargs)` — positional args are forwarded to the inner method. multiproc_executor.py:546 `args += (new_req.diffusion_kv_metadata,)` — the executor appends metadata as a fourth positional tuple member.) + +`vllm_omni/diffusion/sched/interface.py:192` [minor] — The metadata-vs-forwarded-request branch of validate_new_request_data_identity (vllm_omni/diffusion/sched/interface.py:192) has no test coverage: test_batch_path_rejects_envelope_request_identity_mismatch and test_step_path_rejects_envelope_request_identity_mismatch (tests/diffusion/diffusion_kv/test_worker_contract.py:161-164 and the parallel step test) make request_id differ from req.request_id, so they stop at interface.py:184 and never reach the metadata branch. The only mismatched-metadata test (test_paged_request_rejects_mismatched_metadata_identity, tests/diffusion/diffusion_kv/test_worker_contract.py:116) goes through DiffusionModelRunner.execute_model/_validate_diffusion_kv_metadata and matches 'request mismatch', not 'request identity mismatch'. Add a batch or step test with request_id == req.request_id but diffusion_kv_metadata.request_id different, asserting ValueError with match 'request identity mismatch'. (evidence: vllm_omni/diffusion/sched/interface.py:184 ` if new_req.request_id != forwarded_request_id:`; vllm_omni/diffusion/sched/interface.py:192 ` if metadata is not None and metadata.request_id != forwarded_request_id:`; tests/diffusion/diffusion_kv/test_worker_contract.py:162-164 ` request_id="envelope-id",` / ` req=SimpleNamespace(request_id="actual-request-id"),` / ` diffusion_kv_metadata=make_metadata("envelope-id"),` (envelope branch raises first, so the metadata branch is unreached); tests/diffusion/diffusion_kv/test_worker_contract.py:120 ` with pytest.raises(ValueError, match="request mismatch"):` and :123 ` diffusion_kv_metadata=make_metadata("stale"),` (exercises the runner's different 'request mismatch' validator, not the interface validator's 'request identity mismatch' branch).) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:592` [nit] — The diff extends execute_model with diffusion_kv_metadata in DiffusionModelRunner (diffusion_model_runner.py:592), DiffusionWorker (diffusion_worker.py:426), and WorkerWrapperBase (diffusion_worker.py:1358), but the docstrings do not document the new parameter: DiffusionModelRunner's Args section only lists req (diffusion_model_runner.py:595-596), DiffusionWorker's docstring has no Args section (diffusion_worker.py:428-437), and WorkerWrapperBase's Args section lists only req, od_config, and kv_prefetch_job (diffusion_worker.py:1363-1366). Add a one-line Args entry for diffusion_kv_metadata in DiffusionModelRunner and WorkerWrapperBase, and add an Args section (or at least the parameter) to DiffusionWorker. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:592 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,` and vllm_omni/diffusion/worker/diffusion_model_runner.py:595-596 ` Args: + req: A diffusion request containing a list of prompts to process.`; vllm_omni/diffusion/worker/diffusion_worker.py:426 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,` with no Args section in its docstring; vllm_omni/diffusion/worker/diffusion_worker.py:1358 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,` and vllm_omni/diffusion/worker/diffusion_worker.py:1363-1366 ` Args: + req: Diffusion request. + od_config: OmniDiffusionConfig configuration + kv_prefetch_job: Optional next-request KV prefetch descriptor.`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5550.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5550.trace.json.gz new file mode 100644 index 000000000..05280bf8f Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5610.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5610.cost.json new file mode 100644 index 000000000..8720960e1 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5610.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 780.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5610/run-20260814-214244-eba460", + "recorded_at": "2026-08-14T21:55:42", + "attempts": [ + { + "invocation_id": "inv-753af5e9", + "outcome": "run:run-20260814-214244-eba460", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-214244-eba460", + "llm_calls": 27, + "input_tokens": 286974, + "output_tokens": 42623, + "usd": 0.138168, + "source": "spans" + } + ], + "attempt_usd_total": 0.138168, + "llm_calls": 27, + "input_tokens": 286974, + "output_tokens": 42623, + "usd": 0.138168, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-214244-eba460", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1382, + "minutes": 12.97, + "ci_minutes": 0, + "input_tokens": 286974, + "output_tokens": 42623, + "tool_calls": 203, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4376, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 27, + "by_role": { + "planner": { + "usd": 0.000524, + "input_tokens": 105, + "output_tokens": 400, + "calls": 1 + }, + "reducer": { + "usd": 0.020315, + "input_tokens": 13891, + "output_tokens": 15027, + "calls": 2 + }, + "agent": { + "usd": 0.11733, + "input_tokens": 272978, + "output_tokens": 27196, + "calls": 24 + } + }, + "cache_read_tokens": 511104, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.66, + "gate": 2.2, + "review": 772.09, + "report": 0.07 + }, + "time_to_first_result_s": 777.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 26321 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5610.md b/eval/dataset/arms/copilot_moa_cgm/pr5610.md new file mode 100644 index 000000000..525a642b5 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5610.md @@ -0,0 +1,43 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] docs/design/feature/omni_async_output_materialization.md:1595-1619 gpu_ar_model_runner.py — `_should_use_async_omni_output()` gates match the compatibility table (async scheduling, prefix cache, speculative decoding, async_chunk, routed experts, model opt-in, eager postprocess). +- [validated] docs/design/feature/omni_async_output_materialization.md:49-61 omni_connector_model_runner_mixin.py — `should_accumulate_full_payload_output()` returns False when `async_chunk` is True, so the full-payload `self.requests` branch is unreachable on this feature path. +- [validated] docs/design/feature/omni_async_output_materialization.md:133-153 gpu_ar_model_runner.py:151-239 — `OmniAsyncGPUModelRunnerOutput` background thread, `get_output()` join/exception boundary, and sampled-token finalize via upstream `get_output()` match the architecture section. +- [validated] docs/design/feature/omni_async_output_materialization.md:191-197 omni_connector_model_runner_mixin.py:208,1080-1081,1510-1524 — connector `_lock` and `get_omni_connector_output()` as sole drain consumer align with Live Runner State section. +- [validated] docs/design/feature/omni_async_output_materialization.md:89-96 qwen3_omni.py:133-164 — Thinker/Talker `use_async_omni_output=True`; Talker sets `eager_omni_postprocess_before_async_output` and `omni_pooler_payload_include_hidden=False`. +- [validated] docs/design/feature/omni_async_output_materialization.md:98-100 qwen3_tts_talker.py:351-353 — Qwen3-TTS Talker uses the same async-output + eager-postprocess pattern. +- [validated] docs/design/feature/omni_async_output_materialization.md:356-359 npu_ar_model_runner.py:1273-1328 — NPU builds `OmniModelRunnerOutput` and drains connector output before `AsyncGPUModelRunnerOutput` wrapper (sync Omni materialization). +- [validated] docs/design/feature/omni_async_output_materialization.md:257-275 vllm_omni/deploy/qwen3_omni_moe.yaml:15,59-60 — `async_chunk: true`, stage-2 `async_scheduling: false`, prefix caching disabled on AR stages. + +`docs/design/feature/omni_async_output_materialization.md:89` [minor] — docs/design/feature/omni_async_output_materialization.md:89 — qualify the blanket statement: async output materialization is enabled for both AR stages only in the full multi-stage Qwen3-Omni Instruct pipeline. Thinker-only/Captioner checkpoints resolve to QWEN3_OMNI_THINKER_ONLY_PIPELINE (pipeline.py:107-109), and omni_config.py:1281-1282 forces async_chunk=False for single-stage pipelines, so _should_use_async_omni_output() falls back to synchronous construction despite the Thinker setting use_async_omni_output=True (qwen3_omni.py:133-134). (evidence: docs/design/feature/omni_async_output_materialization.md:89 `For Qwen3-Omni, the feature is enabled for both AR stages:`; vllm_omni/model_executor/models/qwen3_omni/pipeline.py:108 `if not hf_config.enable_audio_output:` / :109 `return QWEN3_OMNI_THINKER_ONLY_PIPELINE`; vllm_omni/config/omni_config.py:1281 `if len(pipeline_cfg.stages) <= 1:` / :1282 `deploy.async_chunk = False`; vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py:133 `if self.model_stage == "thinker":` / :134 `self.use_async_omni_output = True`; docs/design/feature/omni_async_output_materialization.md (Compatibility) `| `async_chunk` is enabled | The feature targets incremental downstream Omni payloads |`) + +`docs/design/feature/omni_async_output_materialization.md:104` [minor] — The new page records async output materialization as +22% req/s over CUDA Graph + async chunk (omni_async_output_materialization.md:111) and states the bundled deployment profile enables it automatically (line 241, not 258-260). docs/design/qwen3_omni_tts_performance_optimization.md:5 lists only Batching, CUDA Graph, and Async Chunk as the 'same set of stacked optimizations', and lines 484-488 describe the default qwen3_tts.yaml 'full optimization stack' as only Batching, CUDA Graph, and Async chunk — no async-output layer. Update that page (or link the new page) so the playbook does not contradict the newly documented default-profile behavior. (evidence: ['docs/design/feature/omni_async_output_materialization.md:111 `| + async output materialization | 11.3 req/s | 631 ms | 0.47 |`', 'docs/design/feature/omni_async_output_materialization.md:241 `bundled deployment profile is enough to turn it on for the supported`', 'docs/design/qwen3_omni_tts_performance_optimization.md:5 `benefit from the same set of stacked optimizations:`', 'docs/design/qwen3_omni_tts_performance_optimization.md:484 `The default config (`qwen3_tts.yaml`) enables the full optimization stack:`', 'docs/design/qwen3_omni_tts_performance_optimization.md:488 `- Async chunk with streaming transport`']) + +`docs/design/feature/omni_async_output_materialization.md:321` [minor] — Confirmed. `prefix_caching.md:44` presents `enable_prefix_caching` as an independent per-stage toggle with no mention that enabling it makes `_should_use_async_omni_output()` return False, silently falling back to synchronous output construction. Add the same incompatibility note next to the `enable_prefix_caching` description in `prefix_caching.md` so operators enabling prefix caching are warned that it disables async Omni output. (evidence: docs/design/feature/prefix_caching.md:44 `Prefix caching in vLLM-Omni currently is only supported on AutoRegressive stages with one kv-cache group. It can be enabled/disabled per-stage via the `enable_prefix_caching` parameter in the model's stage config.`; docs/design/feature/omni_async_output_materialization.md:321-322 `Do not pass `--no-async-chunk` or enable prefix caching when you want this` / `optimization. Either change causes the runner to fall back to synchronous`; vllm_omni/worker/gpu_ar_model_runner.py:1598-1599 `if self.omni_prefix_cache is not None:` / `return False`; vllm_omni/worker/gpu_model_runner.py:170-171 `if self.cache_config.enable_prefix_caching:` / `self.omni_prefix_cache = OmniTensorPrefixCache(`) + +`docs/design/feature/omni_async_output_materialization.md:73` [nit] — The new doc embeds the figure at line 73 as . In GitHub PR/markdown preview this resolves from docs/design/feature/ to repo-root source/architecture, while the SVG is at docs/source/architecture/, so the image does not render there (the correct local relative path would be ../../source/architecture/...). Sibling pages such as async_chunk.md use raw.githubusercontent.com URLs or blocks for architecture assets. Switch this image to the raw raw.githubusercontent.com/vllm-project/vllm-omni/.../docs/source/architecture/... pattern (or a block) for GitHub-preview parity and consistency. (evidence: ['docs/design/feature/omni_async_output_materialization.md:73: src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"', 'docs/design/feature/async_chunk.md:85: ', 'docs/design/feature/async_chunk.md:86: Async Chunk Pipeline Architecture', 'diff: b/docs/source/architecture/qwen3-omni-async-output-step-gap.svg (new file)']) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5610.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5610.trace.json.gz new file mode 100644 index 000000000..1a2636d39 Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5703.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5703.cost.json new file mode 100644 index 000000000..d55368237 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5703.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1365.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5703/run-20260814-215544-12374e", + "recorded_at": "2026-08-14T22:18:27", + "attempts": [ + { + "invocation_id": "inv-94d83d2e", + "outcome": "run:run-20260814-215544-12374e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-215544-12374e", + "llm_calls": 73, + "input_tokens": 314250, + "output_tokens": 111277, + "usd": 0.291744, + "source": "spans" + } + ], + "attempt_usd_total": 0.291744, + "llm_calls": 73, + "input_tokens": 314250, + "output_tokens": 111277, + "usd": 0.291744, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-215544-12374e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2917, + "minutes": 22.71, + "ci_minutes": 0, + "input_tokens": 314250, + "output_tokens": 111277, + "tool_calls": 265, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6831, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 73, + "by_role": { + "moa_member": { + "usd": 0.045336, + "input_tokens": 57384, + "output_tokens": 8627, + "calls": 10 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.213658, + "input_tokens": 240793, + "output_tokens": 76853, + "calls": 60 + }, + "reducer": { + "usd": 0.032751, + "input_tokens": 16073, + "output_tokens": 25797, + "calls": 2 + } + }, + "cache_read_tokens": 2751936, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.88, + "gate": 1.91, + "review": 1356.75, + "report": 0.1 + }, + "time_to_first_result_s": 1362.59 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 31933 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5703.md b/eval/dataset/arms/copilot_moa_cgm/pr5703.md new file mode 100644 index 000000000..369e0fc23 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5703.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [sweep] Read vllm_omni/diffusion/models/minimax_h3/vae.py:117-220 — adds self.device_module=torch.get_device_module() and switches encode_image/encode_video from cuda-only to non-CPU device seeding via device_module.device/manual_seed. +- [validated] vllm_omni/diffusion/models/minimax_h3/vae.py:170,214 — CUDA/ROCm behavior preserved: parameter.device.type "cuda" satisfies both old (=="cuda") and new (!="cpu") guards. +- [validated] vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:545-548,963,1066 — sole production consumers of MiniMaxH3VideoVAE.encode_image/encode_video; signatures unchanged, no caller updates required. +- [sweep] Grep tests/diffusion/models/minimax_h3 — no fork_rng/device_module/KEYFRAME_ENCODE coverage; test_video_vae_encode_uses_configured_parallel_tiling uses object.__new__ with CPU FakeModel so devices=[] and device_module is never exercised. +- [sweep] PR diff contains zero test files; PR body claims audio-path coverage and lists tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py which is absent from tests/diffusion/models/minimax_h3/ (only contract/packing/parallel/e2e/quantization files exist). +- [sweep] Read recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76 — prose says "validated Ref2VA configuration" but export MODEL uses FL2VA partition; conflicts with validation section at :124-127 describing Ref2VA TP4 serving. +- [sweep] Read recipes/README.md:48-49 — index row added for MUSA recipe; table entry consistent with recipe title but inherits the Ref2VA hardware claim. +- [sweep] Grep repo for get_device_module( — only vae.py:120 uses it; no existing helper to reuse; PyTorch docs say no-arg form returns current accelerator, while encode uses parameter.device at call time. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This diff introduces the non-CPU accelerator seeding branch (vae.py:167/211 and 174-176/218-220) and self.device_module = torch.get_device_module() (vae.py:120), but no test exercises it: test_video_vae_encode_uses_configured_parallel_tiling (tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:327-359) builds MiniMaxH3VideoVAE via object.__new__ with a CPU FakeModel and never sets device_module, so devices == [] and the new branch never runs; the diff adds no test file and tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py does not exist. Add a CPU-runnable test that monkeypatches a non-CPU parameter.device and a fake device_module, drives both encode_image and encode_video, and asserts fork_rng receives the device and device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED) is called. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()`; vllm_omni/diffusion/models/minimax_h3/vae.py:167 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:175 `with self.device_module.device(device):`; tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:344 `video_vae = object.__new__(MiniMaxH3VideoVAE)`; tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:350 `video_vae.model = FakeModel()` (FakeModel defined with a CPU torch.nn.Parameter at line 329)) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — This hunk replaces the CUDA-only RNG gate with `parameter.device.type != "cpu"`, so every non-CPU backend enters `fork_rng` and the cached `self.device_module` path. That contradicts the PR body statement that the allowlist is limited to CUDA and MUSA, and routes devices such as Ascend NPU (recipe recipes/MiniMaxAI/MiniMax-H3-NPU.md exists; `MiniMaxH3Pipeline.__init__` constructs `MiniMaxH3VideoVAE(..., device=self.device)` at pipeline_minimax_h3.py:545-547) into an unvalidated `fork_rng`/`manual_seed` path. Gate `devices` to the validated set (`parameter.device.type in {"cuda", "musa"}`), resolve the device module at encode time in both `encode_image` (:170) and `encode_video` (:214), and pass `device_type=parameter.device.type` to `fork_rng`; if NPU/XPU is intended, validate it and correct the PR description instead. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:545-547 `self.video_vae = MiniMaxH3VideoVAE( os.path.join(model_path, "video_vae"), device=self.device, )`) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — The block labeled 'The validated Ref2VA configuration' at line 72 sets `export MODEL="${MODEL_ROOT}/FL2VA"` at line 76, contradicting this file's rule that Ref2VA serving requires MODEL=Ref2VA. Change line 76 to `export MODEL="${MODEL_ROOT}/Ref2VA"` (mirroring MiniMax-H3.md:168 and MiniMax-H3-NPU.md:117), or relabel the block as the FL2VA template and add a separate Ref2VA export. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72 `The validated Ref2VA configuration uses tensor parallelism across four MTT`; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76 `export MODEL="${MODEL_ROOT}/FL2VA"`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — vae.py:120 now calls `self.device_module = torch.get_device_module()` in `MiniMaxH3VideoVAE.__init__`. The AMD lane (`.buildkite/amd/test-amd-ready.yml:19`) runs `tests/diffusion tests/model_executor -m 'core_model and cpu'`, which includes `test_video_vae_keeps_reference_fp32_weights`; that test constructs the VAE with `device=torch.device("cpu")` at tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:319. Thus the CPU/AMD test executes the new constructor call but never enters the changed `device.type != "cpu"` RNG branch, so it cannot validate the MUSA RNG path. `buildkite/vllm-omni-amd-ci` is red; determine whether it is pre-existing on main or starts at 1c6ee28f and, if newly red, bisect this call first. Do not cite the red AMD lane as evidence that the MUSA RNG path passed. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 ` self.device_module = torch.get_device_module()` +tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:319 ` video_vae = vae_module.MiniMaxH3VideoVAE(` +tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` +.buildkite/amd/test-amd-ready.yml:19 ` - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:170` [minor] — vae.py:170 now uses `torch.get_device_module()` to make the RNG seed path device-generic, but `.buildkite/` has no MUSA lane (only amd, common, cuda, intel, npu, release), so the MUSA-specific behavior has no automated regression coverage. Please confirm manual validation is acceptable or that a MUSA CI lane should be added. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; `.buildkite/` directory listing: `amd/ common/ cuda/ intel/ npu/ release/` (no MUSA lane)) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:56` [minor] — Minor/question: `MiniMaxH3AudioVAE.encode_waveform` still calls `_AudioVAEDeterminismContext` (defined at vae.py:56, used at vae.py:318), which toggles only `torch.backends.cuda`/`torch.backends.cudnn` flags and is outside this PR's video-VAE RNG scope. Please confirm this is explicitly accepted out of scope or file a follow-up issue for MUSA audio-VAE determinism. Separately, the PR body's validation command lists `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py`, which does not exist in the tree. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:56 `class _AudioVAEDeterminismContext(AbstractContextManager):`; vae.py:59 `backends.cuda.matmul.allow_tf32,`; vae.py:60 `backends.cudnn.allow_tf32,`; vae.py:318 `with _AudioVAEDeterminismContext():`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [nit] — Both `encode_image` (vae.py:170-176) and `encode_video` (vae.py:214-220) contain the same RNG seeding block: `devices = [parameter.device] if parameter.device.type != "cpu" else []`, then `with self.device_module.device(device):` and `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`. Extract a small shared helper so the device allowlist or device-module-resolution fix cannot drift between the two encode paths. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:175 `with self.device_module.device(device):`; vae.py:176 `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:219 `with self.device_module.device(device):`; vae.py:220 `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` — identical block duplicated in both encode paths.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5703.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5703.trace.json.gz new file mode 100644 index 000000000..4c82ccd4e Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5715.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5715.cost.json new file mode 100644 index 000000000..22ca28d48 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5715.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 309.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5715/run-20260814-220512-7c25eb", + "recorded_at": "2026-08-14T22:10:20", + "attempts": [ + { + "invocation_id": "inv-2cf280aa", + "outcome": "run:run-20260814-220512-7c25eb", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-220512-7c25eb", + "llm_calls": 12, + "input_tokens": 60203, + "output_tokens": 23458, + "usd": 0.052973, + "source": "spans" + } + ], + "attempt_usd_total": 0.052973, + "llm_calls": 12, + "input_tokens": 60203, + "output_tokens": 23458, + "usd": 0.052973, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-220512-7c25eb", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.053, + "minutes": 5.12, + "ci_minutes": 0, + "input_tokens": 60203, + "output_tokens": 23458, + "tool_calls": 37, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2061, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 12, + "by_role": { + "agent": { + "usd": 0.052973, + "input_tokens": 60203, + "output_tokens": 23458, + "calls": 12 + } + }, + "cache_read_tokens": 404224, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.78, + "gate": 1.59, + "review": 301.89, + "report": 0.08 + }, + "time_to_first_result_s": 307.32 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 12571 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5715.md b/eval/dataset/arms/copilot_moa_cgm/pr5715.md new file mode 100644 index 000000000..3e2830a04 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5715.md @@ -0,0 +1,41 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] README.md:19 new 0.26.0 entry links verified: recipes/MiniMaxAI/MiniMax-H3.md and recipes/OpenBMB/MiniCPM-o-4_5.md both exist in the tree. +- [validated] README.md:21 consolidated news entry: repo-wide grep shows the removed standalone v0.14.0/v0.16.0/v0.18.0/v0.20.0/v0.22.0 news lines are gone; only the consolidated line remains (plus an unrelated examples helm tag v0.16.0). +- [validated] vllm-omni-skills removal: repo-wide grep returns no matches, so no stale references remain. +- [validated] docs/configuration/README.md:5 config link: only docs.vllm.ai v0.26.0 configuration URL remains in the repo; the old v0.16.0 URL is gone. +- [validated] docs/getting_started/installation/README.md:4 compatibility note matches existing quickstart guidance ('install the same major & minor version of vLLM and vLLM Omni'). +- [validated] docs/getting_started/installation/gpu/cuda.inc.md:127 BASE_IMAGE=vllm/vllm-openai:v0.26.0 matches docker/Dockerfile.cuda:1 default 'ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0'. +- [validated] cuda.inc.md:5/127 v0.22.1 -> v0.26.0: repo-wide grep for 'v0.22.1' returns no matches. +- [validated] docs/getting_started/installation/npu/npu.inc.md:57-67: old v0.18.0/v0.18.0rc1 pins are gone (grep 'v0.18.0rc1' has no matches), vllm is pinned to v0.26.0, and the broken 'cd /vllm-workspace/vllm-omni' is replaced by 'cd vllm-omni'. + +`README.md:?` [minor] — This consolidated entry folds the two removed February 2026 release lines into the [2026/06] header, so a reader now sees 'Starting with 0.14.0' dated 2026/06 even though the cadence actually began with the 2026/02 0.14.0 release. Date this entry to the cadence start (e.g. [2026/02]) or reword so [2026/06] clearly attaches to 0.22.0 completing the cadence. (evidence: The diff removes '- [2026/02] We released [0.14.0]...' and '- [2026/02] We released [0.16.0]...'; README.md:21 now reads '- [2026/06] Starting with [0.14.0](...)' while the original 0.22.0 line was '- [2026/06] We released [0.22.0]...'.) + +`docs/getting_started/installation/README.md:4` [minor] — This new global compatibility note sits above the hardware-platform index and therefore also governs the MUSA path, but the MUSA tab still pins vLLM-MUSA to v0.18.0-dev. A MUSA reader is told 0.26.x requires the vLLM 0.26.x line and then, one click later, told to build against a 0.18.0-dev base. Since this PR intentionally keeps MUSA out of scope, scope the note to the updated platforms (CUDA/ROCm/NPU) or add an explicit MUSA exception. (evidence: docs/getting_started/installation/gpu/musa.inc.md:31 reads 'git checkout v0.18.0-dev'; PR body states 'Confirm the MUSA and quickstart changes requested for removal in review are absent from the final PR diff.') + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5715.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5715.trace.json.gz new file mode 100644 index 000000000..36c176870 Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5840.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5840.cost.json new file mode 100644 index 000000000..0043514b5 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5840.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1051.9, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5840/run-20260814-221021-2cfb25", + "recorded_at": "2026-08-14T22:27:52", + "attempts": [ + { + "invocation_id": "inv-7c1cc13c", + "outcome": "run:run-20260814-221021-2cfb25", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-221021-2cfb25", + "llm_calls": 58, + "input_tokens": 482800, + "output_tokens": 94210, + "usd": 0.289176, + "source": "spans" + } + ], + "attempt_usd_total": 0.289176, + "llm_calls": 58, + "input_tokens": 482800, + "output_tokens": 94210, + "usd": 0.289176, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-221021-2cfb25", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2892, + "minutes": 17.5, + "ci_minutes": 0, + "input_tokens": 482800, + "output_tokens": 94210, + "tool_calls": 263, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5981, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 58, + "by_role": { + "moa_member": { + "usd": 0.061584, + "input_tokens": 98978, + "output_tokens": 5633, + "calls": 9 + }, + "agent": { + "usd": 0.194, + "input_tokens": 358761, + "output_tokens": 64222, + "calls": 47 + }, + "reducer": { + "usd": 0.033592, + "input_tokens": 25061, + "output_tokens": 24355, + "calls": 2 + } + }, + "cache_read_tokens": 1526848, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.78, + "gate": 1.81, + "review": 1044.11, + "report": 0.06 + }, + "time_to_first_result_s": 1049.75 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 39648 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5840.md b/eval/dataset/arms/copilot_moa_cgm/pr5840.md new file mode 100644 index 000000000..27ae3ddc7 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5840.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] backend.py:76-101 — `enable_minimax_h3_teacache` rejects `ref2va`, allows `fl2va`/`combined`, hooks only `pipeline.transformer`, warns for combined Ref2VA. +- [validated] pipeline_minimax_h3.py:736-739,1498-1506 — combined Ref2VA routes to `transformers_ref`, which never receives TeaCache hooks; matches documented partial caching. +- [validated] extractors.py:1252-1395 vs minimax_h3_transformer.py:1085-1196 — extractor mirrors native forward; `test_extractor_matches_native_forward` asserts logits parity. +- [validated] config.py:96-133 — `DiffusionCacheConfig.rel_l1_thresh=None` defers to `TeaCacheConfig.__post_init__`; MiniMax-H3 default 0.17, others 0.2. +- [validated] test_cache_backends.py:474-532 — backend tests cover generic 0.2 default, H3 0.17 default, combined enable, ref2va rejection. +- [validated] test_teacache_extractors.py:41,569-698 — `TestMiniMaxH3Extractor` is `core_model` CPU-marked; includes cache-hit block-skip test. +- [sweep] extractors.py:1312-1318 — batch-size/shape guards on `x`, `token_tags`, `inverse_indices`; intentional single-batch `[1,S,C]` contract. +- [sweep] extractors.py:1341-1342 — `blocks[0]` used only for TeaCache modulated-input probe, same pattern as other extractors. + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:?` [major] — quality_policy.py:146 (`if quality == "high":`) unconditionally builds a Cache-DiT target, and pipeline_minimax_h3.py:1766 always applies it via `_cache_dit_runtime.prepare(quality_plan.cache_dit)`. On a TeaCache-enabled MiniMax-H3 server this stacks Cache-DiT on the already TeaCache-hooked transformer, contradicting the documented TeaCache/Cache-DiT mutual exclusivity. Gate `quality == "high"` when `_configured_backend == "tea_cache"` (raise or treat as lossless) and add a CPU test. (evidence: vllm_omni/diffusion/models/minimax_h3/quality_policy.py:146 ` if quality == "high":` and quality_policy.py:161-166 ` return MiniMaxH3QualityPlan( + cache_dit=CacheDiTRequestSpec( + installation_key=_cache_installation_key(base_key, force_refresh), + cache_config=_with_force_refresh(base_config, force_refresh), + num_inference_steps=num_inference_steps, + ), + )`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1766 ` self._cache_dit_runtime.prepare(quality_plan.cache_dit)`; vllm_omni/diffusion/cache/teacache/backend.py:127 ` "MiniMaxH3Pipeline": enable_minimax_h3_teacache,`.) + +`vllm_omni/diffusion/cache/teacache/config.py:98` [major] — Serving MiniMax-H3 with `--cache-backend tea_cache` and no `--cache-config` still gets `rel_l1_thresh=0.2`, not the advertised calibrated 0.17: `AsyncOmniEngine._get_default_cache_config('tea_cache')` returns `{"rel_l1_thresh": 0.2}` (async_omni_engine.py:853-856), `_normalize_cache_config` applies it when `cache_config is None` (async_omni_engine.py:881-882), and `_create_default_diffusion_stage_cfg` forwards it as `cache_config` (async_omni_engine.py:955). Because `DiffusionCacheConfig.rel_l1_thresh` is now `float | None = None` and the dict path sets it to 0.2, `TeaCacheConfig.__post_init__` never reaches the `_MODEL_DEFAULT_REL_L1_THRESH['MiniMaxH3DiTModel'] = 0.17` branch. Remove the pinned 0.2 from the tea_cache serve injector (or emit `{"rel_l1_thresh": None}`) and add a regression test that the injector does not pin 0.2. (evidence: vllm_omni/diffusion/cache/teacache/config.py:98 ` "MiniMaxH3DiTModel": 0.17,`; vllm_omni/diffusion/cache/teacache/config.py:129-133 ` threshold = self.rel_l1_thresh` / ` if threshold is None:` / ` threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)` / ` self.rel_l1_thresh = threshold`; vllm_omni/engine/async_omni_engine.py:853-856 ` if cache_backend == "tea_cache":` / ` return {` / ` "rel_l1_thresh": 0.2,` / ` }`; vllm_omni/engine/async_omni_engine.py:881-882 ` if cache_config is None and cache_backend not in (None, "", "none"):` / ` cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`; vllm_omni/diffusion/data.py:422 ` rel_l1_thresh: float | None = None`) + +`serve.py:586` [minor] — vllm_omni/entrypoints/cli/serve.py:586 — the `--cache-config` help still shows `TeaCache: '{"rel_l1_thresh": 0.2}'` as the example. Since this PR makes `rel_l1_thresh` model-specific (None -> 0.17 for MiniMax-H3, 0.2 generic fallback), this example is now stale; update it or remove the hard-coded 0.2. (evidence: vllm_omni/entrypoints/cli/serve.py:586 ` "TeaCache: '{\"rel_l1_thresh\": 0.2}'. "` — `--cache-config` CLI help still advertises a fixed 0.2 TeaCache threshold while the PR defers the threshold to a model-specific default (None in `DiffusionCacheConfig`/`TeaCacheConfig`, with 0.17 for `MiniMaxH3DiTModel` and 0.2 as the generic fallback).) + +`recipes/MiniMaxAI/MiniMax-H3.md:?` [minor] — The Request-scoped quality section (recipes/MiniMaxAI/MiniMax-H3.md:676-689) still presents `quality=high` only as 'H3's conservative Cache-DiT profile' with 'No Cache-DiT startup option is required' and no TeaCache caveat, while the new TeaCache section and Known limitations state TeaCache and Cache-DiT are mutually exclusive per server. Clarify the interaction: on a TeaCache-configured server a `quality=high` request targets Cache-DiT and conflicts with the TeaCache backend (state whether it is rejected or must not be sent). (evidence: recipes/MiniMaxAI/MiniMax-H3.md:684 `-F 'quality=high' # H3's conservative Cache-DiT profile.`; recipes/MiniMaxAI/MiniMax-H3.md:761 `TeaCache and Cache-DiT are mutually exclusive; pick one cache backend per server.`; recipes/MiniMaxAI/MiniMax-H3.md:838 `- TeaCache and Cache-DiT cannot be enabled on the same server.`) + +`tests/diffusion/cache/test_cache_backends.py:490` [minor] — This PR adds `test_minimax_h3_only_enables_fl2va_teacache` and `test_minimax_h3_rejects_ref2va_teacache` to `tests/diffusion/cache/test_cache_backends.py`, which has module-level `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (line 32). The failing `buildkite/vllm-omni-amd-ci` lane runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py` (.buildkite/amd/test-amd-ready.yml:19), so these new tests are collected by that lane. Confirm they pass on AMD or fix them in this PR rather than landing with a red AMD lane. (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; tests/diffusion/cache/test_cache_backends.py:490 ` def test_minimax_h3_only_enables_fl2va_teacache(`; .buildkite/amd/test-amd-ready.yml:19 `"timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:156` [minor] — `test_minimax_h3_t2va_teacache_single_gpu_smoke` at tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:156 passes `cache_config={"rel_l1_thresh": 0.17}`, so the new Omni -> TeaCacheBackend -> TeaCacheConfig.__post_init__ None-resolution path (MiniMaxH3DiTModel default 0.17) is never exercised end-to-end. Consider a sibling run with `cache_config={}` (or no rel_l1_thresh key) and assert generation succeeds. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:156 `cache_config={"rel_l1_thresh": 0.17},`; vllm_omni/diffusion/cache/teacache/config.py:121-126 `if threshold is None: threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)` with `_MODEL_DEFAULT_REL_L1_THRESH = {"MiniMaxH3DiTModel": 0.17}`; tests/diffusion/cache/test_cache_backends.py:499-516 unit test passes `DiffusionCacheConfig()` (threshold None) but mocks `apply_teacache_hook`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143` [minor] — `test_minimax_h3_t2va_teacache_single_gpu_smoke` (tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143) ends at line 186 with `_assert_joint_output(outputs)`, whose body only checks output count, frame shape, audio shape, sample rate, and fps. It never asserts that the `tea_cache` backend was applied or that a cache hit occurred, so a regression that silently disables TeaCache would still pass. Add a TeaCache-specific signal, e.g. capture the `TeaCache applied` log or assert a second-step cache hit. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:186 `_assert_joint_output(outputs)` — final statement of the TeaCache smoke test. tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:20-27 `assert len(outputs) == 1` / `frames = np.asarray(outputs[0].images[0])` / `assert frames.shape == (107, 256, 448, 3)` / `multimodal = outputs[0].multimodal_output` / `assert multimodal is not None` / `assert np.asarray(multimodal["audio"]).shape[1] == 2` / `assert multimodal["audio_sample_rate"] == 32000` / `assert multimodal["fps"] == 24` — helper contains no TeaCache/cache-hit assertion.) + +`.buildkite/amd/test-amd-ready.yml:?` [minor] — `.buildkite/amd/test-amd-ready.yml:19` ignores `tests/diffusion/cache/test_teacache_extractors.py` in the AMD-ready CPU lane, but this PR adds `TestMiniMaxH3Extractor` to that file at `tests/diffusion/cache/test_teacache_extractors.py:570` with `@pytest.mark.cpu` at line 569. The 'Diffusion · Cache Test' lane filters `-m 'core_model and cache and diffusion and not distributed_cuda and L4'`, which does not select these tests (no `cache` mark), so the new H3 extractor is currently uncovered on AMD CI. Add an AMD-blocking lane running `pytest tests/diffusion/cache/test_teacache_extractors.py -k TestMiniMaxH3Extractor -m cpu`, or drop the blanket ignore for that class. (evidence: .buildkite/amd/test-amd-ready.yml:19 `"timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; tests/diffusion/cache/test_teacache_extractors.py:569 `@pytest.mark.cpu`; tests/diffusion/cache/test_teacache_extractors.py:570 `class TestMiniMaxH3Extractor(BaseExtractorTest):`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5840.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5840.trace.json.gz new file mode 100644 index 000000000..b9b8e57eb Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5863.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5863.cost.json new file mode 100644 index 000000000..daf51d779 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5863.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 285.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5863/run-20260814-221035-333b96", + "recorded_at": "2026-08-14T22:15:19", + "attempts": [ + { + "invocation_id": "inv-bc382f94", + "outcome": "run:run-20260814-221035-333b96", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-221035-333b96", + "llm_calls": 12, + "input_tokens": 150527, + "output_tokens": 20700, + "usd": 0.078104, + "source": "spans" + } + ], + "attempt_usd_total": 0.078104, + "llm_calls": 12, + "input_tokens": 150527, + "output_tokens": 20700, + "usd": 0.078104, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-221035-333b96", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0781, + "minutes": 4.73, + "ci_minutes": 0, + "input_tokens": 150527, + "output_tokens": 20700, + "tool_calls": 25, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2062, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 12, + "by_role": { + "agent": { + "usd": 0.078104, + "input_tokens": 150527, + "output_tokens": 20700, + "calls": 12 + } + }, + "cache_read_tokens": 544128, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.49, + "gate": 1.77, + "review": 277.15, + "report": 0.06 + }, + "time_to_first_result_s": 283.45 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 13985 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5863.md b/eval/dataset/arms/copilot_moa_cgm/pr5863.md new file mode 100644 index 000000000..fd99ac74a --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5863.md @@ -0,0 +1,39 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read the full new file recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md (346 lines) in situ. +- [sweep] Cross-checked every serve flag in the diff against vllm_omni/entrypoints/cli/serve.py: --task-type:213, --num-gpus:395, --usp:461, --ring:479, --diffusion-attention-backend:538, --vae-use-tiling:605, --cfg-parallel-size:704, --vae-patch-parallel-size:711, --text-encoder-tp-size:719, --vae-parallel-mode:728, --enable-diffusion-pipeline-profiler:784 all present; --cpunodebind/--membind absent. +- [sweep] Repo-wide grep for cpunodebind/membind matched only the new doc (lines 201/203); `numactl` appears only in docker/Dockerfile.xpu:25 — basis of the single review comment. +- [validated] recipe claim 't2va requires an explicit aspect_ratio' matches vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py `_resolve_minimax_h3_aspect_ratio` ('t2va requires an explicit aspect_ratio'). +- [validated] recipe claim 'ref2va defaults to a 16:9 aspect ratio' matches the same function's None branch returning MINIMAX_H3_SUPPORTED_ASPECT_RATIOS['16:9']. +- [validated] recipe claim that RTX PRO 6000 Blackwell (SM120) does not get the TRTLLM_ATTN datacenter default matches docs/user_guide/diffusion/attention_backends.md:114 ('TRTLLM_ATTN is skipped on workstation Blackwell (sm_120 / sm_121)'). +- [validated] recipe claim 'FP8 cannot be combined with layerwise offload' matches recipes/MiniMaxAI/MiniMax-H3.md Known limitations ('Online FP8 is not compatible with layerwise offload'). +- [sweep] Checked warmup arithmetic (107.58/90.48 = 1.189), scaling (278.55/168.73 = 1.65, 168.73/87.90 = 1.92), memory fit (55.4 + 22.1/usp predicts 60.9 vs 61.07), and headroom (96 - 77.49/66.44/61.07) — all consistent with the table values. + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [minor] — This diff instructs users to pass `--cpunodebind=0 --membind=0` on the four-GPU serve command, but those are `numactl` options, not vLLM-Omni CLI flags. Copying the command as written fails with argparse 'unrecognized arguments'. Wrap the serve invocation with numactl (e.g. `numactl --cpunodebind=0 --membind=0 vllm serve ...`) or remove the flags and note they require a numactl wrapper. (evidence: Repo-wide grep for `cpunodebind`/`membind` matches only this file (lines 201 and 203); `vllm_omni/entrypoints/cli/serve.py`, which registers every Omni serve flag, contains no `--cpunodebind`/`--membind` add_argument (all parallelism/CLI flags live there, e.g. `--vae-patch-parallel-size` at line 711); `numactl` appears only in `docker/Dockerfile.xpu:25`. Diff line: 'Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and'.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5863.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5863.trace.json.gz new file mode 100644 index 000000000..e864701e0 Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5863.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5884.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5884.cost.json new file mode 100644 index 000000000..98074fb2f --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5884.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1381.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5884/run-20260814-221521-fd14df", + "recorded_at": "2026-08-14T22:38:21", + "attempts": [ + { + "invocation_id": "inv-90d41a6a", + "outcome": "run:run-20260814-221521-fd14df", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-221521-fd14df", + "llm_calls": 57, + "input_tokens": 328217, + "output_tokens": 92178, + "usd": 0.276195, + "source": "spans" + } + ], + "attempt_usd_total": 0.276195, + "llm_calls": 57, + "input_tokens": 328217, + "output_tokens": 92178, + "usd": 0.276195, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-221521-fd14df", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2762, + "minutes": 22.99, + "ci_minutes": 0, + "input_tokens": 328217, + "output_tokens": 92178, + "tool_calls": 259, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6792, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 57, + "by_role": { + "moa_member": { + "usd": 0.037556, + "input_tokens": 48138, + "output_tokens": 3071, + "calls": 12 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.206035, + "input_tokens": 263295, + "output_tokens": 63643, + "calls": 41 + }, + "reducer": { + "usd": 0.032604, + "input_tokens": 16784, + "output_tokens": 25464, + "calls": 3 + } + }, + "cache_read_tokens": 2876224, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.21, + "gate": 1.91, + "review": 1374.16, + "report": 0.07 + }, + "time_to_first_result_s": 1379.33 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5884.trace.json.gz", + "trace_bytes": 24117 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5884.md b/eval/dataset/arms/copilot_moa_cgm/pr5884.md new file mode 100644 index 000000000..d5cbeab8c --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5884.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5884, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5dc24e7f6a0c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] backend.py:47-52 — attrgetter resolves Bagel `_dit_modules=['language_model.model']` to `Qwen2MoTModel` at `pipeline_bagel.py:159,254`; old getattr-only discovery returned empty tuple and triggered `ValueError` at enable. +- [validated] backend.py:61-62 — `BlockAdapter.is_cached` gate skips uncached nested modules; Cosmos3 keeps custom enabler at `model_specific.py:781-804` (still targets `pipeline.transformer` only) while summary now safely iterates `['transformer.language_model','transformer']` from `pipeline_cosmos3.py:816`. +- [sweep] backend.py:45 — non-str `_dit_modules` entries are skipped; same effective contract as pre-change filter. +- [sweep] backend.py:51 — `module is not None` drops resolved None intermediates; AttributeError from broken dotted paths is swallowed (same silent-skip pattern as `module_collector.py:109-119`). +- [sweep] backend.py:61 — `BlockAdapter.is_cached` branch is reachable for Cosmos3-like nested-first declarations and is covered by `test_summary_skips_uncached_nested_dit`. +- [validated] tests/diffusion/cache/test_cache_backends.py:139-161 — nested enable/refresh test drives real `CacheDiTBackend.enable/refresh/cache_summary` with `language_model.model`, matching Bagel declaration. +- [validated] tests/diffusion/cache/test_cache_backends.py:32 — `pytestmark = [core_model, cpu]` ensures new tests are CI-selectable (not path-deselected). +- [upstream-verify] cache-dit `BlockAdapter.is_cached` for plain modules uses `getattr(adapter, '_is_cached', False)` (requirements/common.txt pins cache-dit==1.3.0), so the test’s `_is_cached` flags on Mock/SimpleNamespace modules exercise the real gate. + +`vllm_omni/diffusion/cache/cachedit/backend.py:57` [minor] — The docstring no longer matches the implementation: cache_summary now emits Cache-DiT summaries only for declared DiT modules with an active Cache-DiT context (BlockAdapter.is_cached() is True). Update the docstring to state that uncached declared modules are skipped. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:57: ` """Log Cache-DiT statistics for every transformer on the pipeline."""` — vllm_omni/diffusion/cache/cachedit/backend.py:60-63: ` for transformer in transformers:` / ` if not BlockAdapter.is_cached(transformer):` / ` continue` / ` cache_dit.summary(transformer, details=details)`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [minor] — Confirmed: `_dit_module_names` (vllm_omni/diffusion/cache/cachedit/backend.py:39-52) now resolves dotted paths via `attrgetter`, so `CacheDiTBackend.enable` resolves SoulX's `_dit_modules = ["cfm_decoder.model.diff_estimator"]` to DiffLlama, which declares `_cache_dit_adapter_config` (llama.py:115). Since SoulX pipeline class names are not in CUSTOM_DIT_ENABLERS, enable behavior changes from `ValueError: ... no declared DiT modules` to wrapping the nested CFM DiT. The new tests only cover the two-level `language_model.model` path, so add a SoulX nested enable/refresh test or record the smoke that exercised it. (evidence: ['vllm_omni/diffusion/cache/cachedit/backend.py:48 ` module = attrgetter(name)(pipeline)` — dotted `_dit_modules` entries now resolve.', 'vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_base.py:63 ` _dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]` — SoulX declares a three-level dotted DiT path.', 'vllm_omni/diffusion/models/soulx_singer/modules/llama.py:115 ` _cache_dit_adapter_config = CacheDiTAdapterConfig(` — the resolved DiffLlama declares a Cache-DiT adapter config.', 'vllm_omni/diffusion/cache/cachedit/model_specific.py:846 ` CUSTOM_DIT_ENABLERS.update(` with entries only `Wan22Pipeline`, `Wan22I2VPipeline`, `Wan22TI2VPipeline`, `Wan22VACEPipeline`, `Wan22S2VPipeline`, `Cosmos3OmniDiffusersPipeline`, `Cosmos3OmniPipeline`, `Krea2Pipeline` — SoulX is not registered and therefore uses the shared `attrgetter` discovery path.']) + +`tests/diffusion/cache/test_cache_backends.py:32` [minor] — `buildkite/vllm-omni-amd-ci` is failing on this PR while the other checks pass, and the new cache tests live in a module marked `core_model` + `cpu` (tests/diffusion/cache/test_cache_backends.py:32). Please confirm the AMD failure is pre-existing on main and not triggered by these tests before merging. (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; gh_ci_read(5884): `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` with all other checks SUCCESS.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:45` [minor] — This PR fixes Cache-DiT discovery only and intentionally leaves residual flat-getattr callers out of scope: `diffusion_model_runner._compile_transformer` (diffusion_model_runner.py:153), `lora/manager.py:388`, and `registry.py:451` (not 443). Can you confirm those remainders are tracked as a follow-up on issue #5879 so the issue isn't closed while they still exist? (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:49 ` module = attrgetter(name)(pipeline)` — the PR switches Cache-DiT discovery to dotted-path resolution; vllm_omni/diffusion/worker/diffusion_model_runner.py:153 ` model = getattr(self.pipeline, attr_name, None)` — residual flat getattr on `_dit_modules`; vllm_omni/diffusion/lora/manager.py:388 ` component = getattr(self.pipeline, component_name)` — residual flat getattr on declared `_dit_modules`; vllm_omni/diffusion/registry.py:451 ` transformer = getattr(model, attr)` — residual flat getattr in `_apply_sequence_parallel_if_enabled`.) + +`tests/diffusion/cache/test_cache_backends.py:?` [minor] — `test_enable_and_refresh_nested_declared_dit` patches the entire `BlockAdapter` class before calling `cache_summary`, so `BlockAdapter.is_cached(transformer)` returns a truthy `MagicMock` and the new `if not BlockAdapter.is_cached(transformer): continue` guard can never skip. `mock_cache_dit.summary.assert_called_once_with(transformer, details=True)` therefore passes vacuously and cannot catch enable marking a different object than summary inspects (BagelCachedAdapter sets `_is_cached` on `block_adapter.pipe.__class__`). Unpatch/autospec `BlockAdapter.is_cached` for the summary assertion, or move the skip assertion into `test_summary_skips_uncached_nested_dit` after a real enable. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:61 ` if not BlockAdapter.is_cached(transformer):`; tests/diffusion/cache/test_cache_backends.py:137 ` @patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")`; tests/diffusion/cache/test_cache_backends.py:139 ` def test_enable_and_refresh_nested_declared_dit(self, mock_cache_dit, mock_block_adapter):`; tests/diffusion/cache/test_cache_backends.py:154 ` cache_summary(pipeline)`; tests/diffusion/cache/test_cache_backends.py:161 ` mock_cache_dit.summary.assert_called_once_with(transformer, details=True)`; vllm_omni/diffusion/cache/cachedit/model_specific.py:645 ` block_adapter.pipe.__class__._is_cached = True`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5884.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5884.trace.json.gz new file mode 100644 index 000000000..697fc9102 Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5884.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5957.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5957.cost.json new file mode 100644 index 000000000..e14b7d62a --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5957.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1488.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5957/run-20260814-221829-48aad9", + "recorded_at": "2026-08-14T22:43:16", + "attempts": [ + { + "invocation_id": "inv-7a52d3b4", + "outcome": "run:run-20260814-221829-48aad9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-221829-48aad9", + "llm_calls": 68, + "input_tokens": 703622, + "output_tokens": 136779, + "usd": 0.549906, + "source": "spans" + } + ], + "attempt_usd_total": 0.549906, + "llm_calls": 68, + "input_tokens": 703622, + "output_tokens": 136779, + "usd": 0.549906, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-221829-48aad9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5499, + "minutes": 24.78, + "ci_minutes": 0, + "input_tokens": 703622, + "output_tokens": 136779, + "tool_calls": 310, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8346, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 68, + "by_role": { + "moa_member": { + "usd": 0.083186, + "input_tokens": 154970, + "output_tokens": 5606, + "calls": 6 + }, + "agent": { + "usd": 0.417117, + "input_tokens": 446810, + "output_tokens": 111109, + "calls": 60 + }, + "reducer": { + "usd": 0.049602, + "input_tokens": 101842, + "output_tokens": 20064, + "calls": 2 + } + }, + "cache_read_tokens": 7058112, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.59, + "gate": 1.88, + "review": 1479.7, + "report": 0.06 + }, + "time_to_first_result_s": 1486.24 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5957.trace.json.gz", + "trace_bytes": 128387 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5957.md b/eval/dataset/arms/copilot_moa_cgm/pr5957.md new file mode 100644 index 000000000..fec55c12f --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5957.md @@ -0,0 +1,62 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5957, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 19a6e513f28e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness: read talker2s2mel_full_payload and indextts2_talker use_gpt_latent/meta emission; code-only v2.5 uses request.output_token_ids, latent mode keeps codes+latent with strict length checks — matches tests in test_stage_input_processor.py and test_indextts2.py. +- [validated] Native speed: IndexTTS25Adapter sets duration_factor=1.0/speed; serving_speech._uses_native_speed_control skips generic encode speed (serving_speech.py:517-522) — covered by test_tts_adapter.py and test_serving_speech.py. +- [validated] Request-end payload: omni_payload_at_request_end only for indextts2_5 (indextts2_talker.py:171); GPU snapshot via snapshot_mm_payload — test_omni_gpu_model_runner.py and test_mm_outputs_partition.py. +- [sweep] Breaking behavior: stage processor now raises instead of returning None; IndexTTS2 talker still emits meta.use_gpt_latent=True (indextts2_talker.py:453,739) — existing latent path preserved. +- [sweep] Blast radius: qwen2_5_omni XPU enforce_eager flip, Mistral import fallback, optional renderer warmup, num_stale_output_tokens shim — scoped compatibility for vLLM 0.27; test_config_factory.py and test_arg_utils.py cover key paths. +- [sweep] Failure paths: compute_fbank and local bundle loaders now raise instead of zero/hub fallback (preprocess_utils.py) — intentional fail-fast per PR goal; affects IndexTTS2 shared code. +- [sweep] Indexed accesses: stage_configs[0] in end2end.py assumes standard two-stage deploy with stage 0 talker — OK for indextts2/indextts2_5.yaml; deploy tests pin stage_id ordering. +- [sweep] CI gate: buildkite/vllm-omni-amd-ci failing per gate_report; PR touches tests/model_executor heavily — attribution unclear without log. + +`vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:221` [major] — PR body claims 'Official ... Spanish, and Arabic language preprocessing', but `_normalize_with_official_backend` (text_processing_v2_5.py:221) only normalizes zh/en/zhen. `es` depends on the optional, undeclared `nemo_text_processing` package (silently falling back to raw text), and `ar` always falls through to `return text`. The shipped recipe/docs document only zh/en/zhen/ja/yue. Narrow the description/recipe to the supported set, or implement Arabic normalization and declare the `nemo_text_processing` dependency. (evidence: vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:221 `def _normalize_with_official_backend(text: str, lang: str) -> str:`; :222 `if lang in {"zh", "en", "zhen"}:`; :224 `if lang == "es":`; :237 `return text` (NeMo-import fallback); :238 `return text` (all other languages, including 'ar'). pyproject.toml (PR diff): `indextts2 = [` / `"fugashi>=1.2.0",` / `"unidic-lite>=1.0.0",` / `"wetext>=0.0.9; sys_platform != 'linux'",` / `"WeTextProcessing; sys_platform == 'linux'",` / `]` — no `nemo_text_processing`. PR body: 'Official Chinese, English, Japanese, Spanish, and Arabic language preprocessing, plus an additional mixed Chinese/English (`zhen`) frontend mode.') + +`tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:1` [major] — The three new test modules under tests/model_executor/models/indextts2/ — test_configuration_and_pipeline.py, test_enhanced_codec.py, and test_recipe_defaults.py — have no module-level pytestmark, while the model_executor CI lanes run `pytest -sv tests/model_executor -m 'core_model and cpu'` (.buildkite/cuda/test-ready.yml:13 and .buildkite/cuda/test-merge.yml:13). Unmarked modules are therefore deselected in CI and never execute. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` to all three modules. (Drop the deletion hedge: test_recipe_defaults.py's `s2mel_cfm_batch_size == 4` assertion is not duplicated in the marked test_s2mel_v2_5.py.) (evidence: .buildkite/cuda/test-ready.yml:13 ` - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` +tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:1 `# SPDX-License-Identifier: Apache-2.0` (module header only; file contains no `pytestmark` — grep over tests/model_executor/models/indextts2 returns pytestmark in exactly 7 modules and not in this file, test_enhanced_codec.py, or test_recipe_defaults.py) +tests/model_executor/models/indextts2/test_enhanced_codec.py:1 `# SPDX-License-Identifier: Apache-2.0` (no pytestmark) +tests/model_executor/models/indextts2/test_recipe_defaults.py:1 `# SPDX-License-Identifier: Apache-2.0` (no pytestmark) +tests/model_executor/models/indextts2/test_s2mel_v2_5.py:26 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +`.buildkite/amd/test-amd-ready.yml:?` [minor] — The AMD ready gate at `.buildkite/amd/test-amd-ready.yml:19` runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`, which now collects the newly added `tests/model_executor/models/indextts2/test_*` files (all marked `core_model and cpu`), and `buildkite/vllm-omni-amd-ci` is FAILURE while the other vllm-omni lanes pass. Please attach the failing AMD job/step log and confirm whether main exhibits the same failure; fix or xfail only if it is PR-specific. (evidence: .buildkite/amd/test-amd-ready.yml:19 - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml" +tests/model_executor/models/indextts2/test_s2mel_v2_5.py:26 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]) + +`vllm_omni/model_executor/models/indextts2/preprocess_utils.py:282` [minor] — `compute_fbank` now raises `RuntimeError('IndexTTS requires torchaudio Kaldi fbank support')` instead of returning a zero fbank, and its only caller (`indextts2_talker.py:634`, `fbank = compute_fbank(wav_16k, device)`) is not wrapped, so a torchaudio Kaldi failure surfaces as a raw RuntimeError to the user. The fail-fast intent appears deliberate; consider a user-facing error translation only if raw RuntimeError is undesirable. (evidence: preprocess_utils.py:281-282 `except (ImportError, RuntimeError, OSError) as exc:` / ` raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc` — and indextts2_talker.py:634 ` fbank = compute_fbank(wav_16k, device)` with no enclosing try/except in `_preprocess_prefill`.) + +`recipes/IndexTeam/IndexTTS-2_5.md:166` [minor] — The recipe's multilingual walkthrough (recipes/IndexTeam/IndexTTS-2_5.md:166-167) only documents zh/en/zhen/ja/yue, while the PR body claims Spanish and Arabic preprocessing and test_tokenizer_v2_5.py:218 parametrizes `es`/`ar`. Add an es/ar client example to the recipe, or note that the documented five are the common codes and es/ar are exercised only in unit tests. (evidence: recipes/IndexTeam/IndexTTS-2_5.md:166-167 `The client accepts `zh` (Mandarin), `en` (English), `zhen` (mixed Chinese and` / `English), `ja` (Japanese), and `yue` (Cantonese). The following commands use` — tests/model_executor/models/indextts2/test_tokenizer_v2_5.py:218-220 `def test_official_spanish_and_arabic_cases_preserve_expected_text_contract(` ... `("es", ...)` ... `("ar", ...)` — PR body: `Official Chinese, English, Japanese, Spanish, and Arabic language preprocessing`.) + +`vllm_omni/deploy/qwen2_5_omni.yaml:85` [minor] — vllm_omni/deploy/qwen2_5_omni.yaml flips XPU AR stages 0/1 to `enforce_eager: true` (lines 88-93) as a PyTorch 2.13 Dynamo workaround, but its only guard, tests/config/test_config_factory.py:2279, is marked `core_model + cpu` (line 41) and the Intel XPU lane runs `-m "core_model and xpu and B60"` (run-xpu-test.sh:90), so this change is never exercised on XPU hardware; PR E2E is CUDA H200. Add XPU runtime/perf evidence for the eager flip or split the workaround out of the new-model PR. (evidence: vllm_omni/deploy/qwen2_5_omni.yaml:88-93 ` - stage_id: 0\n gpu_memory_utilization: 0.9\n enforce_eager: true\n - stage_id: 1\n gpu_memory_utilization: 0.5\n enforce_eager: true`; tests/config/test_config_factory.py:41 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; tests/config/test_config_factory.py:2279 ` def test_qwen2_5_omni_xpu_uses_eager_ar_stages(self):`; .buildkite/intel/scripts/run-xpu-test.sh:90 ` pytest -v -s -m "core_model and xpu and B60"`) + +`pyproject.toml:91` [minor] — pyproject.toml:91 declares `WeTextProcessing; sys_platform == 'linux'` with no version floor, while the Linux branch at vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:106-122 constructs `tn.chinese.normalizer.Normalizer(remove_interjections=False, remove_erhua=False, overwrite_cache=False)` and `tn.english.normalizer.Normalizer(overwrite_cache=False)`. Those kwargs are version-sensitive, so an unbounded range could resolve to a release that rejects them. Pin a floor consistent with this API (or document why the unbounded range is safe). (evidence: ['pyproject.toml:91 ` "WeTextProcessing; sys_platform == \'linux\'",`', 'vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:118 ` remove_interjections=False,`', 'vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:122 ` return Normalizer(overwrite_cache=False)`']) + +`vllm_omni/engine/stage_init_utils.py:1133` [minor] — The new `_TokenOnlyRenderer(stage_vllm_config, tokenizer=None)` path for `skip_tokenizer_init` stage-0 processors is only covered by a test that monkeypatches `_build_token_only_renderer` to a sentinel, so the real constructor and `render_messages` ValueError are never exercised. Add a test that calls `_build_token_only_renderer` with a minimal vllm_config and asserts `render_messages` raises the expected skip_tokenizer_init error. (evidence: vllm_omni/engine/stage_init_utils.py:1132-1133 `def _build_token_only_renderer(stage_vllm_config: Any) -> BaseRenderer:` / ` return _TokenOnlyRenderer(stage_vllm_config, tokenizer=None)`; vllm_omni/engine/stage_init_utils.py:1126-1128 ` def render_messages(self, messages, params):` / ` raise ValueError(` / ` "Chat messages are unavailable when skip_tokenizer_init=True; submit prompt_token_ids or prompt_embeds"`; tests/engine/test_async_omni_engine_stage_init.py:855-857 ` monkeypatch.setattr(` / ` init_mod,` / ` "_build_token_only_renderer",` / ` lambda _config: token_only_renderer,` — the only test exercising this path substitutes a sentinel and never constructs `_TokenOnlyRenderer`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/worker/gpu_ar_model_runner.py:1972` [minor] In `vllm_omni/worker/gpu_ar_model_runner.py:1972-1974` the deferred path now calls `snapshot_mm_payload(flat_mm)` instead of `build_mm_cpu`. The new tests cover `_should_defer_full_payload_d2h` (gate), `_build_omni_step_ +- `vllm_omni/model_executor/models/indextts2/s2mel/modules/gpt_fast/model.py:276` [minor] `vllm_omni/model_executor/models/indextts2/s2mel/modules/gpt_fast/model.py:276-282` adds a CUDA-only `packed_varlen` branch (`q.device.type == "cuda"` and `_forward_varlen_packed`) for ragged CFM groups, but the new Inde + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5957.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5957.trace.json.gz new file mode 100644 index 000000000..6cb7aec6a Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5976.cost.json b/eval/dataset/arms/copilot_moa_cgm/pr5976.cost.json new file mode 100644 index 000000000..51b14bcb9 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5976.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1837.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_moa_cgm/runs/pr5976/run-20260814-222754-2dcae8", + "recorded_at": "2026-08-14T22:58:29", + "attempts": [ + { + "invocation_id": "inv-e429388b", + "outcome": "run:run-20260814-222754-2dcae8", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-222754-2dcae8", + "llm_calls": 94, + "input_tokens": 801733, + "output_tokens": 184239, + "usd": 0.590277, + "source": "spans" + } + ], + "attempt_usd_total": 0.590277, + "llm_calls": 94, + "input_tokens": 801733, + "output_tokens": 184239, + "usd": 0.590277, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-222754-2dcae8", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5903, + "minutes": 30.58, + "ci_minutes": 0, + "input_tokens": 801733, + "output_tokens": 184239, + "tool_calls": 368, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9323, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 94, + "by_role": { + "moa_member": { + "usd": 0.061414, + "input_tokens": 106747, + "output_tokens": 4465, + "calls": 8 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.464335, + "input_tokens": 597745, + "output_tokens": 145037, + "calls": 82 + }, + "reducer": { + "usd": 0.064528, + "input_tokens": 97241, + "output_tokens": 34737, + "calls": 3 + } + }, + "cache_read_tokens": 5822144, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.07, + "gate": 1.84, + "review": 1827.48, + "report": 0.06 + }, + "time_to_first_result_s": 1834.44 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 70708 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5976.md b/eval/dataset/arms/copilot_moa_cgm/pr5976.md new file mode 100644 index 000000000..9aaf82356 --- /dev/null +++ b/eval/dataset/arms/copilot_moa_cgm/pr5976.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 10 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py:142-146,400-422` + `tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py`: bare-tensor vs capture-tuple contract pinned for non-staged registry runs. +- [validated] `vllm_omni/worker/gpu_generation_model_runner.py:837-845`: accepts `randomize_inputs` but does not forward it (code2wav-safe); `gpu_model_runner.py:1134` forwards to `maybe_randomize_inputs`. +- [validated] `vllm_omni/patch.py:419-507` + `tests/diffusion/test_inductor_divisibility_patch.py`: factorable-divisibility wrapper is sound (polynomial-only, idempotent install). +- [validated] `vllm_omni/engine/stage_engine_startup.py:896-951` + `tests/engine/test_stage_engine_startup_cache_env.py`: replica>0 isolates `VLLM_CACHE_ROOT` and restores parent env. +- [validated] `vllm_omni/diffusion/models/lingbot_video/request_utils.py:299-419` + unit tests: `extra_args.size/width/height` precedence fixed. +- [validated] `vllm_omni/diffusion/distributed/parallel_state.py:526-543,966`: `use_all2all` forwarded only when upstream API accepts kwarg; EP init passes `use_all2all=True`. +- [validated] `vllm_omni/entrypoints/openai/api_server.py:936,1001`: `online_renderer` is constructed before unconditional `warmup()` — upstream 0.27 alignment looks safe. +- [sweep] tests/diffusion/test_compile.py:107/110 blocks[0]: test builds a 1-element ModuleList — safe. qwen2_5_omni_thinker.py:308 videos[0] is in a comment describing the transformers crash, not a new index. + +`vllm_omni/core/sched/omni_ar_scheduler.py:629` [blocker] — Confirmed. vllm_omni/core/sched/omni_ar_scheduler.py:627-629 seeds session.num_stale_output_tokens from num_in_flight_tokens, and when replace_streaming_prompt is set the code calls _replace_streaming_session (omni_ar_scheduler.py:669-671), which adds the same in-flight count again at vllm_omni/core/sched/omni_scheduler_mixin.py:101. No reset of num_in_flight_tokens occurs between the two adds, so the counter becomes 2*in_flight and the drain at the top of update_from_output swallows a valid new-turn frame (or underflows the assert). Seed in only one site and add a CPU test with replace_streaming_prompt + num_in_flight_tokens>0 asserting stale==in_flight. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:627-629: ` in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / ` if in_flight_tokens > 0:` / ` session.num_stale_output_tokens += in_flight_tokens`. vllm_omni/core/sched/omni_ar_scheduler.py:669-671: ` if replace_streaming_prompt:` / ` self._replace_streaming_session(session, update)` / ` return`. vllm_omni/core/sched/omni_scheduler_mixin.py:101: ` session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` — the second add runs on the same request with no reset of num_in_flight_tokens between the two sites.) + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [major] — Replace-path double counting: `_update_request_as_session` seeds `session.num_stale_output_tokens += in_flight_tokens` (vllm_omni/core/sched/omni_ar_scheduler.py:629) and then calls `_replace_streaming_session` (line 670), which seeds the same `num_in_flight_tokens` again (vllm_omni/core/sched/omni_scheduler_mixin.py:105). After the pre-replacement frames drain, the leftover counter swallows the new segment's first valid frames — the exact regression the stale-drain fix was meant to prevent. Remove one of the two seeds (preferably the one in `_replace_streaming_session`) and add a replace-path unit test asserting the counter is seeded exactly once. Note the MiniCPM-o setter is at `vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:954`, not `minicpmo_4_5_omni.py:954`. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:629 ` session.num_stale_output_tokens += in_flight_tokens`; vllm_omni/core/sched/omni_ar_scheduler.py:670 ` self._replace_streaming_session(session, update)`; vllm_omni/core/sched/omni_scheduler_mixin.py:105 ` session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`; vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:954 ` handoff_meta["replace_streaming_prompt"] = True`) + +`vllm_omni/engine/arg_utils.py:?` [major] — Confirmed. This diff removes the 'if arch not in supported_archs' guard, so register_omni_models_to_vllm now unconditionally overwrites the upstream global ModelRegistry for every _OMNI_MODELS entry, and the vllm.general_plugins entry point (pyproject.toml:149) runs this in every plain vLLM subprocess. The PR hardens only qwen2_5 and qwen3 against a missing model_stage; at least ten other registered classes still read vllm_config.model_config.model_stage directly and will AttributeError when their arch is constructed in a non-staged run (verified examples: step_audio2.py:57, voxtral_tts.py:112, ming_tts.py:64, minicpmo_4_5_omni.py:94, covo_audio.py:218, cosyvoice3.py:462, mammoth_moda2.py:890, ming_flash_omni.py:98, omnivoice.py:279, mimo_audio.py:554). Either scope the unconditional override to archs verified safe in non-staged runs, or harden every remaining direct reader and add a non-staged construct test for each. (evidence: vllm_omni/engine/arg_utils.py:116 `ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")` — now inside an unconditional loop over all _OMNI_MODELS items, with the prior `if arch not in supported_archs` guard removed; pyproject.toml:149 `vllm_omni_register_models = "vllm_omni.engine.arg_utils:register_omni_models_to_vllm"` under `[project.entry-points."vllm.general_plugins"]`; step_audio2.py:57 `raw_model_stage = vllm_config.model_config.model_stage`; voxtral_tts.py:112 `self.model_stage = vllm_config.model_config.model_stage`; ming_tts.py:64 `self.model_stage = vllm_config.model_config.model_stage`; minicpmo_4_5_omni.py:94 `self.model_stage = vllm_config.model_config.model_stage` — all direct reads left unhardened by this diff.) + +`vllm_omni/core/sched/omni_ar_scheduler.py:669` [minor] — PR body discloses the remaining seed-TTS WER ~0.39 (build 2957: 0.3916) under 'Known remaining issues' but cites no in-repo tracking issue/ID; please add a link before merge. (evidence: PR #5976 body: 'Qwen3-Omni seed-TTS WER stuck at ~0.39 (baseline 0.27, gate 0.35) — confirmed not resolved by c2b65233 (build 2957: 0.3916, inside the pre-fix range).' and 'The WER regression stays open — tracked under Known remaining issues.' No issue number or link appears in that section.) + +`vllm_omni/entrypoints/openai/api_server.py:1001` [minor] — api_server.py:1001 now calls state.online_renderer.warmup() unconditionally, even when state.openai_serving_chat is None (line 993 when "generate" is not in supported_tasks). OnlineRenderer is always constructed earlier in omni_init_app_state, so this is not a None crash; consider adding an embedding-only smoke test covering this warmup path. (evidence: vllm_omni/entrypoints/openai/api_server.py:993 ` state.openai_serving_chat = OmniOpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None` — vllm_omni/entrypoints/openai/api_server.py:1001 ` state.online_renderer.warmup()`) + +`tests/e2e/accuracy/test_qwen_image.py:38` [minor] — Diff adds SSIM_THRESHOLD_SDPA_FALLBACK = 0.96 and arms it only when kernels-community/flash-attn3 has no build (tests/e2e/accuracy/test_qwen_image.py:262), loosening SSIM from 0.97 to 0.96 on those CI images. Note PSNR_THRESHOLD = 30.0 is already passed unchanged in the same assert_similarity call (line 263), so it remains the hard floor. The fallback backend is printed, but the armed SSIM threshold is not — consider logging which threshold is used so a 0.96 pass cannot be mistaken for FA3-matched 0.97. (evidence: tests/e2e/accuracy/test_qwen_image.py:32 `SSIM_THRESHOLD = 0.97`; tests/e2e/accuracy/test_qwen_image.py:38 `SSIM_THRESHOLD_SDPA_FALLBACK = 0.96`; tests/e2e/accuracy/test_qwen_image.py:39 `PSNR_THRESHOLD = 30.0`; tests/e2e/accuracy/test_qwen_image.py:262 `ssim_threshold=SSIM_THRESHOLD if _flash_attn3_hub_available() else SSIM_THRESHOLD_SDPA_FALLBACK,`; tests/e2e/accuracy/test_qwen_image.py:263 `psnr_threshold=PSNR_THRESHOLD,`) + +`tests/diffusion/quantization/test_quantization_quality.py:525` [minor] — tests/diffusion/quantization/test_quantization_quality.py:525/537 set `enforce_eager=True` on both LPIPS arms, so this quality job no longer runs the compiled fp8 FLUX path that vllm_omni/patch.py's inductor CantSplit wrapper is meant to restore; tests/diffusion/test_inductor_divisibility_patch.py only proves the wrapper on CPU with sympy divisibility and never performs a real compiled forward. Keep eager for LPIPS isolation, but add a compile-on smoke for the fp8 FLUX CantSplit graph so the patch has end-to-end coverage. (evidence: tests/diffusion/quantization/test_quantization_quality.py:525 ` bl_kwargs: dict = {"model": config.baseline_ref(), "enforce_eager": True}`; tests/diffusion/quantization/test_quantization_quality.py:537 ` qt_kwargs: dict = {"model": config.quantized_ref(), "enforce_eager": True}`; tests/diffusion/test_inductor_divisibility_patch.py:20 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` and tests/diffusion/test_inductor_divisibility_patch.py:40 ` assert patch_module._provably_factorable_multiple(numerator, denominator) is expected` — the CantSplit patch's only new test asserts symbolic divisibility on CPU, with no compiled forward of the FLUX fp8 graph.) + +`vllm_omni/benchmarks/patch/patch.py:1781` [minor] — This diff ports upstream probe_request_rate into the patched serve.benchmark (serve.benchmark = benchmark at 1975) but stops the probe only after asyncio.gather succeeds. If gather raises, probe_stop is never set, probe_loop keeps issuing requests, and session.close() at 1971 is skipped. Wrap probe_task cancel and session.close in try/finally so abort/disconnect of the benchmark run cannot leak the probe. (evidence: vllm_omni/benchmarks/patch/patch.py:1545 `probe_request_rate: float = 0.0`; 1712-1714 `if probe_request_rate > 0:` / `probe_task = asyncio.create_task(probe_loop())`; 1778 `outputs: list[MixRequestFuncOutput] = await asyncio.gather(*tasks)`; 1781-1783 `if probe_task is not None:` / `probe_stop.set()` / `await probe_task` with no surrounding finally; 1971 `await session.close()` only on the success path.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:14` [minor] Confirmed: this PR retargets `vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:14` from `from vllm.v1.worker import mamba_utils` to `import vllm.v1.worker.mamba_utils as mamba_utils`, and changes `vllm_omni/ +- `vllm_omni/platforms/xpu/platform.py:139` [minor] vllm_omni/platforms/xpu/platform.py:139 changes XPUOmniPlatform.get_default_ir_op_priority's non-inductor default from ['xpu_kernels', 'native'] to ['vllm_c', 'native']. This is platform-wide (no model scoping) and consu +- `vllm_omni/core/sched/omni_ar_scheduler.py:479` [minor] Confirmed against the PR-time tree: vllm_omni/core/sched/omni_ar_scheduler.py:479 adds `request.async_tokens_to_discard = outstanding_async_tokens`, and line 634 still writes `session.async_tokens_to_discard = 1`. Repo-w +- `vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py:312` [minor] vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py:312 — add a CPU unit test for Qwen2_5OmniVideoProcessorItems.get_processor_data feeding a MediaWithBytes-wrapped (frames, metadata) tuple and asserting + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_moa_cgm/pr5976.trace.json.gz b/eval/dataset/arms/copilot_moa_cgm/pr5976.trace.json.gz new file mode 100644 index 000000000..d1f087823 Binary files /dev/null and b/eval/dataset/arms/copilot_moa_cgm/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_r1/manifest.json b/eval/dataset/arms/copilot_v11_deep_r1/manifest.json new file mode 100644 index 000000000..f019f0e76 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v11_deep_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4804.cost.json b/eval/dataset/arms/copilot_v11_deep_r1/pr4804.cost.json new file mode 100644 index 000000000..85b2cb14c --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1136.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_r1/runs/pr4804/run-20260813-120442-07e181", + "recorded_at": "2026-08-13T12:23:37", + "attempts": [ + { + "invocation_id": "inv-838a12a5", + "outcome": "run:run-20260813-120442-07e181", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-120442-07e181", + "llm_calls": 82, + "input_tokens": 1203030, + "output_tokens": 130757, + "usd": 0.766696, + "source": "spans" + } + ], + "attempt_usd_total": 0.766696, + "llm_calls": 82, + "input_tokens": 1203030, + "output_tokens": 130757, + "usd": 0.766696, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-120442-07e181", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7667, + "minutes": 18.91, + "ci_minutes": 0, + "input_tokens": 1203030, + "output_tokens": 130757, + "tool_calls": 117, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8222, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 82, + "by_role": { + "agent": { + "usd": 0.734896, + "input_tokens": 1150354, + "output_tokens": 114809, + "calls": 80 + }, + "reducer": { + "usd": 0.0318, + "input_tokens": 52676, + "output_tokens": 15948, + "calls": 2 + } + }, + "cache_read_tokens": 11038720, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.37, + "gate": 1.96, + "review": 1128.36, + "report": 0.06 + }, + "time_to_first_result_s": 1134.74 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 55624 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4804.md b/eval/dataset/arms/copilot_v11_deep_r1/pr4804.md new file mode 100644 index 000000000..7878b6bde --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/pr4804.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] read modeling_moss_tts_codec.py fully: stream session acquire/release/reset symmetry checked; on_requests_finished is wired in gpu_generation_model_runner._update_request_states. +- [sweep] verified slot starvation branch: _decode_streaming_batch buffers codes then _finish_stream_request(..., None) pops _stream_pending_codes — buffered audio is discarded on finished-before-slot. +- [sweep] read stage_input_processors/moss_tts.py: raw path keys pending frames by external_req_id; mixin cleanup_finished_request pops both internal/external keys. +- [sweep] grep put_req_chunk consumers: raw_async_chunk reads the mixin's _put_req_chunk (incremented after send_chunk), so initial_chunk_frames threshold logic is consistent. +- [sweep] read modeling_moss_tts_talker.py preprocess: `max_new_frames = info_dict.get("max_new_frames", [-1])[0]` drops the old int/None/tuple normalization. +- [sweep] compared audio_tokenizer.py header vs new audio_tokenizer_v2.py/configuration_moss_audio_tokenizer_v2.py: new files lack the required 'Vendored from @ ' header. +- [sweep] grep qwen3_tts_request_seed: no remaining repo consumers; tts_local_seed only consumed in gpu_model_runner._explicit_talker_seed and produced in serving_speech + tests — seed key rename appears self-consistent. +- [sweep] grep AudioResampler usage: patch.py's new resampler call mirrors existing qwen3_tts/voxcpm2 patterns (AudioResampler(target_sr=...).resample(...)). + +`vllm_omni/v1/worker/gpu_generation_model_runner.py:799` [minor] — `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]` is assigned unconditionally in `GPUGenerationModelRunner._dummy_run` (vllm_omni/worker/gpu_generation_model_runner.py:799), so every generation-model dummy/profile run receives this kwarg. Please confirm it is safe/ignored by all generation models with explicit forward signatures, or gate it to the models that consume it. (evidence: [sweep] read gpu_generation_model_runner._dummy_run: new `seq_token_counts=[num_tokens_padded]` is unconditional for every generation model dummy/profile run; blast radius not checked for models with explicit forward signatures.) + +`buildkite/vllm-omni-npu-ci:0` [minor] — `buildkite/vllm-omni-npu-ci` is failing while the other reported checks (build 3.11/3.12, pre-commit, DCO, `buildkite/vllm-omni`, `buildkite/vllm-omni-amd-ci`, `buildkite/vllm-omni-intel-ci`, docs) pass. The check result alone does not attribute the failure to this diff, so please confirm whether the NPU lane failure pre-exists on main or is introduced by this PR. (evidence: [sweep] CI read: buildkite/vllm-omni-npu-ci is the only failing lane; attribution to this diff unverified (main may also be red).) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:?` [minor] — When `session.acquire()` returns None in `_decode_streaming_batch`, the new frame is buffered via `_append_stream_pending`; if `finished=True`, the code then calls `_finish_stream_request(request_id, session, None)`, which pops `_stream_pending_codes[request_id]` and drops the buffered tail without decoding it (a warning is logged, but the stream is still truncated). The shipped `moss_tts_local.yaml` pairs `codec_stream_slots: 8` with stage-1 `max_num_seqs: 8`, so the default config should not exhaust slots; lower `codec_stream_slots` or stage-1 concurrency above slots will truncate a finished stream. Enforce `codec_stream_slots >= stage-1 max_num_seqs` and fail loudly on exhaustion, or decode the pending sequence once on this finish path instead of dropping it. (evidence: read_file vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py (_decode_streaming_batch and _finish_stream_request) and vllm_omni/deploy/moss_tts_local.yaml (codec_stream_slots=8, stage-1 max_num_seqs=8).) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:?` [minor] — At vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1462 (MossTTSLocalTalkerForGeneration.preprocess), the diff replaces the previous `isinstance`/`int()`/`try-except` normalization with `max_new_frames = info_dict.get("max_new_frames", [-1])[0]`. If `max_new_frames` is present as `None` or a bare `int`, this line raises `TypeError` ('NoneType'/'int' object is not subscriptable), and an empty list raises `IndexError`. In-repo serving_speech.py currently emits `[request.max_new_tokens]` (a non-empty list), so the primary serving path is safe, but the defensive handling for alternate producers/tests has been removed. Restore the old normalization or add a producer-side shape/type assertion. (evidence: read_file vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py (Local preprocess) and the diff hunk showing the removed int/None handling.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:14` [minor] — The two new vendored files (audio_tokenizer_v2.py and configuration_moss_audio_tokenizer_v2.py, both lines 1-13) carry only the Apache license header and module docstring. The existing vendored codec in the same directory (audio_tokenizer.py lines 13-18) carries a provenance header beginning '# Vendored from OpenMOSS-Team/MOSS-Audio-Tokenizer (configuration_moss_audio_tokenizer.py and modeling_moss_audio_tokenizer.py). Simplified for inference-only use: ...'. Add an equivalent provenance header to both new files, ideally including the exact HF revision the code was copied from. (evidence: read_file vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py lines 1-12 (header present) vs the new files' headers in the diff.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_r1/pr4804.trace.json.gz new file mode 100644 index 000000000..366ef421d Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_r1/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4923.cost.json b/eval/dataset/arms/copilot_v11_deep_r1/pr4923.cost.json new file mode 100644 index 000000000..1483c34aa --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1642.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_r1/runs/pr4923/run-20260813-120515-00cc4c", + "recorded_at": "2026-08-13T12:32:30", + "attempts": [ + { + "invocation_id": "inv-414cd709", + "outcome": "run:run-20260813-120515-00cc4c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-120515-00cc4c", + "llm_calls": 102, + "input_tokens": 1144475, + "output_tokens": 204843, + "usd": 0.806164, + "source": "spans" + } + ], + "attempt_usd_total": 0.806164, + "llm_calls": 102, + "input_tokens": 1144475, + "output_tokens": 204843, + "usd": 0.806164, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-120515-00cc4c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8062, + "minutes": 27.26, + "ci_minutes": 0, + "input_tokens": 1144475, + "output_tokens": 204843, + "tool_calls": 204, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9746, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 102, + "by_role": { + "agent": { + "usd": 0.781107, + "input_tokens": 1136468, + "output_tokens": 184061, + "calls": 100 + }, + "reducer": { + "usd": 0.025057, + "input_tokens": 8007, + "output_tokens": 20782, + "calls": 2 + } + }, + "cache_read_tokens": 10067712, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.34, + "gate": 2.88, + "review": 1628.03, + "report": 0.06 + }, + "time_to_first_result_s": 1635.33 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 24025 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4923.md b/eval/dataset/arms/copilot_v11_deep_r1/pr4923.md new file mode 100644 index 000000000..809998fca --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/pr4923.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] read qwen3_tts_talker.py hunk: `talker_mtp_graph_safe=True` + `has_full_cudagraphs()` gating is consistent with the runner wrap decision in gpu_model_runner.py:209 (`if cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe)`). +- [validated] read qwen3_tts_code_predictor_vllm.py + common/qwen3_code_predictor.py `_setup_compile`: with `use_cuda_graphs=False` on GPU the wrapper still runs torch.compile and warmup buckets and skips `_capture_cuda_graphs`, matching the PR's 'GPU torch.compile only' claim. +- [validated] grep `has_full_cudagraphs`: the method is already used pre-existing at gpu_model_runner.py:209, so this diff introduces no new vLLM version floor for that API. +- [sweep] grep `talker_mtp_graph_safe`: only qwen3_tts_talker.py:327 (this diff) and fish_speech_slow_ar.py:198 (pre-existing) set it; no cross-model blast radius beyond Qwen3-TTS. +- [sweep] grep `tts_local_seed`: serving_speech.py:3694/3706 injects it from `request.seed`; the model comment (qwen3_tts_talker.py:334-346) admits it is silently dropped under full cudagraphs — basis for review_comments[0]. +- [sweep] read gpu_ar_model_runner.py `_should_use_async_omni_output`: async output requires `omni_prefix_cache is None`, `async_chunk`, and no spec decode; the yaml disables stage-0 prefix caching and keeps `async_chunk: true`, so the new flags are coherent. +- [sweep] read gpu_ar_model_runner.py `_maybe_run_eager_omni_postprocess_before_async_output`: eager postprocess writes `hidden_states.last` into the buffer before the async snapshot, so next-step talker_mtp inputs stay available. +- [validated] grep `omni_pooler_payload_include_hidden = False` in qwen3_tts_talker.py: set at line 317 and duplicated at line 353 by this diff — basis for review_comments[3]. + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [major] — This hunk disables the per-row generator fast-path whenever talker_mtp is graph-wrapped (full cudagraphs), and the in-file TODO concedes per-request `tts_local_seed` is then non-reproducible and MTP seeding is silently dropped for decode batches > 1. `serving_speech.py:3686-3694` still accepts `request.seed` and injects `tts_local_seed` with no warning, so the API continues to imply the seed is honored. Emit `logger.warning_once` (or return a clear error) when `request.seed` is set for qwen3_tts under full cudagraphs, or document the guarantee change; silently ignoring the seed is a user-visible regression. (evidence: grep tts_local_seed -> serving_speech.py:3694/3706; read qwen3_tts_talker.py:334-346 comment; read gpu_model_runner.py `_init_talker_mtp` (wraps talker_mtp in CUDAGraphWrapper under full cudagraphs) and `_talker_mtp_forward` (generator path + scalar fallback); read tests/worker/test_omni_gpu_model_runner.py:317.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — This hunk declares `self.talker_mtp_graph_safe = True` unconditionally (qwen3_tts_talker.py:327), while the code predictor still sets `use_cuda_graphs=current_omni_platform.is_npu()` (qwen3_tts_code_predictor_vllm.py:39), so on NPU the inner NPU graph capture still runs (common/qwen3_code_predictor.py:532-535). On NPU the only thing preventing the runner from also wrapping talker_mtp in an outer ACLGraphWrapper is the shipped `platforms.npu` `cudagraph_mode: PIECEWISE` override in qwen3_tts.yaml; `gpu_model_runner._init_talker_mtp` wraps talker_mtp whenever full cudagraphs and `talker_mtp_graph_safe` are both true, and `npu_ar_model_runner._capture_talker_mtp_graphs` confirms the ACLGraphWrapper path. A user overriding NPU to FULL cudagraphs would re-enter nested outer ACLGraphWrapper plus inner NPU graph capture. Gate the flag in code (e.g. `talker_mtp_graph_safe = not current_omni_platform.is_npu()`) or assert at the runner level. (evidence: read qwen3_tts.yaml platforms.npu PIECEWISE override; read platforms/npu/worker/npu_ar_model_runner.py `_capture_talker_mtp_graphs` (ACLGraphWrapper + torch.npu.synchronize); read common/qwen3_code_predictor.py `_setup_compile` (`is_npu and use_cuda_graphs -> _capture_npu_graphs`).) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The YAML header (lines 6-8) still says stage 0 uses max_num_batched_tokens=512 for low first-chunk latency while line 48 now sets 32768, and the stage-0 inline comment (lines 41-45) still says 'Prefix caching on by default' while line 46 disables it. Update both comments (or restore the values) so the stated intent matches the config. (evidence: read vllm_omni/deploy/qwen3_tts.yaml header and stage-0 block; diff hunk `max_num_batched_tokens: 512 -> 32768` and `enable_prefix_caching: true -> false`.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — Add a unit test for the mode-dependent gate. qwen3_tts_talker.py now sets talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped (line 355), where talker_mtp_graph_wrapped depends on cudagraph_mode.has_full_cudagraphs(); no test file is changed in this PR, and tests/worker/test_omni_gpu_model_runner.py only sets talker_mtp_accepts_per_row_generators=True directly on a fake model rather than constructing the model-side gate. Construct the talker under FULL vs NONE/PIECEWISE cudagraph_mode and assert False/True respectively, so the gate that drops per-row generators (and makes per-request tts_local_seed non-reproducible under full cudagraphs) is covered in CI. (evidence: grep tests for qwen3_tts -> no changed test; read tests/worker/test_omni_gpu_model_runner.py:198-356 (existing runner tests do not construct the model gate); PR body Test Plan fields empty.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:?` [minor] — The anchor is exact: `use_cuda_graphs=current_omni_platform.is_npu()` at line 39. `OmniPlatform.is_npu()` returns `self._omni_enum == OmniPlatformEnum.NPU`, so this diff disables the wrapper's inner CUDA graphs on every non-NPU platform (CUDA/ROCm/XPU/MUSA/Unspecified) and keeps them only on NPU. CI confirms `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are FAILURE. Please confirm in the PR whether those lanes are also red on main or introduced by this non-NPU flip; if introduced, the platform gating needs explicit non-NPU evidence and scoping beyond the H100 benchmark and the NPU PIECEWISE yaml override. (evidence: gate_report lists buildkite/vllm-omni-npu-ci and buildkite/vllm-omni-amd-ci as failing; read platforms/__init__.py (is_npu False for cuda/rocm/xpu/musa) and platforms/interface.py OmniPlatform.is_npu.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [nit] — Line 353 re-assigns `self.omni_pooler_payload_include_hidden = False`, which is already set to `False` at line 317 with its explanatory comment. The PR-added block should keep only the two new flags (`use_async_omni_output = True` and `eager_omni_postprocess_before_async_output = True`) and drop the duplicate line 353 assignment. (evidence: grep 'omni_pooler_payload_include_hidden = False' in qwen3_tts_talker.py -> lines 317 and 353.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_r1/pr4923.trace.json.gz new file mode 100644 index 000000000..2bc462c5b Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_r1/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4926.cost.json b/eval/dataset/arms/copilot_v11_deep_r1/pr4926.cost.json new file mode 100644 index 000000000..b5aae9da3 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1148.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_r1/runs/pr4926/run-20260813-122340-f64b1f", + "recorded_at": "2026-08-13T12:42:45", + "attempts": [ + { + "invocation_id": "inv-e24e4a33", + "outcome": "run:run-20260813-122340-f64b1f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-122340-f64b1f", + "llm_calls": 79, + "input_tokens": 447093, + "output_tokens": 139238, + "usd": 0.380591, + "source": "spans" + } + ], + "attempt_usd_total": 0.380591, + "llm_calls": 79, + "input_tokens": 447093, + "output_tokens": 139238, + "usd": 0.380591, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-122340-f64b1f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3806, + "minutes": 19.09, + "ci_minutes": 0, + "input_tokens": 447093, + "output_tokens": 139238, + "tool_calls": 136, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6687, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 79, + "by_role": { + "agent": { + "usd": 0.355174, + "input_tokens": 428029, + "output_tokens": 120842, + "calls": 77 + }, + "reducer": { + "usd": 0.025417, + "input_tokens": 19064, + "output_tokens": 18396, + "calls": 2 + } + }, + "cache_read_tokens": 3952384, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.66, + "gate": 1.61, + "review": 1139.89, + "report": 0.06 + }, + "time_to_first_result_s": 1145.22 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 29557 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4926.md b/eval/dataset/arms/copilot_v11_deep_r1/pr4926.md new file mode 100644 index 000000000..3ab1ae12e --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/pr4926.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] Indexed accesses flash_attn_hub.py:47/114/293: all are tuple-first-element unwraps guarded by `isinstance(out, tuple)`; consistent with flash_attn.py:76-77 FA2/FA3 (out, lse) convention. +- [sweep] Branch flash_attn_hub.py:46 (`isinstance(out, tuple)`): safe unwrap, mirrors flash_attn.py. +- [sweep] Branch flash_attn_hub.py:88/267 (`if backend_kwargs`): only logs ignored kwargs, same as flash_attn.py:69-71; no behavior risk. +- [sweep] Branch flash_attn_hub.py:108/287 (`if ... is None and ... is None`): raises only when both funcs absent, so a module with only flash_attn_func leaves varlen=None for the masked path. +- [sweep] Branches flash_attn_hub.py:117/296 (`if attn_func is not None`): piecewise wrapper falls back to varlen; logically safe when init invariant holds. +- [sweep] Branches flash_attn_hub.py:189/205/213 and 368/384/392 (piecewise/mask/dense routing): dense func path is exercised by the added execution test; piecewise and varlen/masked paths are not exercised by any added test. +- [sweep] cuda/platform.py:124-144: fallback order read — kernels missing -> FLASH_ATTN, pre-Hopper FLASH_ATTN_3_HUB -> FLASH_ATTN_HUB; matches docs; only ImportError is caught around the kernels import. +- [sweep] musa/npu/rocm/xpu platform hunks read: each maps HUB -> FLASH_ATTN before enum lookup, so no enum KeyError risk on those platforms. + +`tests/diffusion/attention/test_kernels_hub.py:58` [major] — tests/diffusion/attention/test_kernels_hub.py::test_kernels_hub_execution (lines 56-58, @pytest.mark.core_model + @hardware_test(res={'cuda': 'L4'}, num_cards=1)) is the only behavioral test for the new FLASH_ATTN_HUB/FLASH_ATTN_3_HUB kernels, but no Buildkite lane selects tests/diffusion/attention. The CUDA diffusion lanes in .buildkite/test-ready.yml:64-145 target tests/diffusion/{offloader,cache,batching,distributed,models,*.py} with '-m core_model and cuda', and the recursive tests/diffusion run uses '-m core_model and cpu', so the L4/cuda-marked test is deselected. Add a CUDA/L4 lane for this file and extend it with masked/varlen/piecewise forward coverage (those call self.flash_attn_varlen_func and are untested). (evidence: grep '.buildkite' for 'tests/diffusion/attention' returns no matches; .buildkite/test-ready.yml:64-145 lists the cuda diffusion lanes; tests/diffusion/attention/test_kernels_hub.py:56-58 applies @hardware_test(res={'cuda': 'L4'}) + @pytest.mark.core_model.) + +`vllm_omni/platforms/rocm/platform.py:91` [major] — Buildkite gate reports buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci as FAILURE (verified via gh_ci_read). This diff does touch the ROCm and NPU selector files (rocm/platform.py:91, npu/platform.py:110). Logs are not accessible through current tooling, so whether these failures are pre-existing or caused by the selector changes remains unverified. Please confirm attribution before merge. (evidence: [gate] buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci failing; gh_ci_read exposes status only (no logs), so attribution to this diff cannot be confirmed.) + +`pyproject.toml:?` [minor] — The diff adds runtime `from kernels import get_kernel` and `get_kernel(..., version=1/2/default)` calls in FlashAttentionHubImpl.__init__ (flash_attn_hub.py:92-103) and FlashAttention3HubImpl.__init__ (flash_attn_hub.py:271-282), but pyproject.toml:70 declares kernels only as `kernels<0.15.0` in the dev extra with no lower bound. The docs (attention_backends.md:228) pin kernels==0.14.1, and no runtime requirement exists. An older kernels release may lack get_kernel or the version= kwarg. Add a lower bound (e.g. kernels>=0.14.1,<0.15.0, matching the docs) and consider exposing a runtime extra now that this is a user-facing backend rather than only an accuracy-test dependency. (evidence: pyproject.toml:70 shows only `kernels<0.15.0`; docs/user_guide/diffusion/attention_backends.md pins `kernels==0.14.1`; vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92-103 and 271-282 import `kernels.get_kernel` and call it with `version=`.) + +`tests/diffusion/attention/test_kernels_hub.py:92` [minor] — tests/diffusion/attention/test_kernels_hub.py:92 skips FA3 execution when compute capability major < 9, but test_kernels_hub_execution is only marked for cuda L4 (sm_89, major=8), so FlashAttention3HubImpl.forward can never execute in the PR's declared CI lane. Add an sm_90+ (e.g. H100) lane, or adjust the skip/config, so the FLASH_ATTN_3_HUB path is actually exercised. (evidence: [sweep] FA3 coverage: test_kernels_hub.py:92 skips when major < 9, but the test is gated on L4 (sm_89), so FlashAttention3HubImpl.forward never executes in CI.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [minor] — Confirmed. FlashAttentionHubBackend and FlashAttention3HubBackend return True from supports_attention_mask (lines 55/234), and forward_cuda routes any attn_mask containing a False entry to _forward_varlen_masked, which calls self.flash_attn_varlen_func (line 147; FA3 path mirrored) unconditionally. __init__ only raises when both flash_attn_func and flash_attn_varlen_func are None, so a Hub module that exposes flash_attn_func but not flash_attn_varlen_func passes init and then crashes masked requests with TypeError. sequence_parallel.py:416 gates auto-pad on supports_attention_mask, so SP auto-pad can reach this path. Fix by raising at init when varlen is missing while mask support is advertised (or returning False from supports_attention_mask), and add a masked forward_cuda test since test_kernels_hub.py only uses AttentionMetadata(attn_mask=None). (evidence: read flash_attn_hub.py:105-109 and 147; supports_attention_mask returns True at lines 55/234; vllm_omni/diffusion/hooks/sequence_parallel.py:416 gates auto-pad on supports_attention_mask; tests/diffusion/attention/test_kernels_hub.py never sets attn_mask or full_attn_spans.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96` [minor] — Both Hub impl constructors (flash_attn_hub.py:92-103 and 271-282) import kernels.get_kernel and execute the version=1/2/default chain plus a 'Loading ...' log on every construction. Attention.__init__ (layer.py:94-95) constructs one impl per attention layer, so multi-layer DiTs repeat this per layer unless kernels.get_kernel caches; the log line repeats regardless. Memoize the resolved module (module-level or functools.cache around the version chain) so resolution/logging happens once per process. (evidence: layer.py:94 `self.attention = self.attn_impl_cls(...)` inside Attention.__init__; flash_attn_hub.py:92-103 runs get_kernel unconditionally in __init__.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/platforms/cuda/platform.py:126` [minor] The CUDA selector (cuda/platform.py:124-138) catches only ImportError while probing `kernels`; once the package imports, FlashAttentionHubImpl.__init__ (flash_attn_hub.py:92-105) calls get_kernel with version-1/version-2 +- `vllm_omni/platforms/cuda/platform.py:124` [minor] The CUDA HUB branch at cuda/platform.py:124 routes FLASH_ATTN_HUB whenever `kernels` imports successfully, without the compute-capability gate that line 149 applies to local FLASH_ATTN (sm_80+ else TORCH_SDPA). A pre-Amp + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_r1/pr4926.trace.json.gz new file mode 100644 index 000000000..51bbc3bf8 Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_r1/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4977.cost.json b/eval/dataset/arms/copilot_v11_deep_r1/pr4977.cost.json new file mode 100644 index 000000000..3986123a0 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 824.5, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_r1/runs/pr4977/run-20260813-123233-9b7a40", + "recorded_at": "2026-08-13T12:46:15", + "attempts": [ + { + "invocation_id": "inv-c23aa262", + "outcome": "run:run-20260813-123233-9b7a40", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-123233-9b7a40", + "llm_calls": 71, + "input_tokens": 267604, + "output_tokens": 88577, + "usd": 0.247085, + "source": "spans" + } + ], + "attempt_usd_total": 0.247085, + "llm_calls": 71, + "input_tokens": 267604, + "output_tokens": 88577, + "usd": 0.247085, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-123233-9b7a40", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2471, + "minutes": 13.7, + "ci_minutes": 0, + "input_tokens": 267604, + "output_tokens": 88577, + "tool_calls": 115, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5065, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 71, + "by_role": { + "agent": { + "usd": 0.229324, + "input_tokens": 260861, + "output_tokens": 74117, + "calls": 69 + }, + "reducer": { + "usd": 0.017761, + "input_tokens": 6743, + "output_tokens": 14460, + "calls": 2 + } + }, + "cache_read_tokens": 2866560, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.26, + "gate": 1.58, + "review": 816.83, + "report": 0.06 + }, + "time_to_first_result_s": 821.71 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 19100 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4977.md b/eval/dataset/arms/copilot_v11_deep_r1/pr4977.md new file mode 100644 index 000000000..1150c6b0b --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r1/pr4977.md @@ -0,0 +1,36 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] Correctness of fallback: read flash_attn_hub.py:25-39 — `_load_hub_module` iterates (1, 2, None) and retries on exception exactly like the old per-constructor try/except; `last_error` is always set before the final raise, so `raise RuntimeError(...) from last_error` is safe. +- [sweep] Branch at flash_attn_hub.py:33 `if version is not None:` — both arms reachable (version 1/2 vs None); not dead/redundant. +- [sweep] Branch at flash_attn_hub.py:44 `if repo_id not in _hub_modules:` — both arms reachable (cache miss loads, hit returns cached); not dead/redundant. +- [validated] Concurrency: `_get_hub_module` holds `_hub_lock` across load+store+return (flash_attn_hub.py:42-47); grep shows `_hub_modules` is only accessed inside the lock; no unlocked read/write. +- [sweep] Consumers: `_load_hub_module`/`_get_hub_module` are referenced only inside flash_attn_hub.py (lines 26, 45 and 42, 119, 284 per changed_symbol_consumers); no external callers to leave stale. +- [validated] Public API: `FlashAttentionHubImpl`/`FlashAttention3HubImpl` constructors and backend class names/signatures unchanged; no defaults or protocols shifted. +- [sweep] Blast radius: grep FLASH_ATTN_HUB shows platform fallback in cuda/rocm/musa/npu/xpu platform.py; only CudaOmniPlatform actually returns the hub backend path, while rocm/musa/npu/xpu fall back to local FLASH_ATTN without importing flash_attn_hub, so the changed code path is CUDA-only. +- [validated] Per-layer instantiation confirmed: vllm_omni/diffusion/attention/layer.py:94-110 constructs one `attn_impl_cls` per Attention layer, so the cache genuinely removes repeated `get_kernel` calls. + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [minor] — This diff replaces the per-construction `get_kernel(...)` calls with a process-wide cached `_get_hub_module()` (line 45: `_hub_modules[repo_id] = _load_hub_module(repo_id)`) and centralizes the v1→v2→default fallback in `_load_hub_module()` (lines 26-39), but no CPU test drives the new loading/caching path. The existing `test_kernels_hub_platform_fallback` monkeypatches only `sys.modules['kernels']` and never imports `flash_attn_hub`, while `test_kernels_hub_execution` constructs each Impl once and is `@hardware_test`/L4-gated; grep of `.buildkite/` finds no reference to `tests/diffusion/attention` or `test_kernels_hub`. Add a CPU unit test in `tests/diffusion/attention/test_kernels_hub.py` that monkeypatches `kernels.get_kernel` to fail for `version=1` and `version=2` and succeed for default, then constructs two `FlashAttentionHubImpl`s and asserts `get_kernel` was called exactly once (or asserts `_get_hub_module(repo_id) is _get_hub_module(repo_id)`). + +`.buildkite/test-npu-ready.yaml:21` [minor] — `.buildkite/test-npu-ready.yaml:21` runs only `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`, and `buildkite/vllm-omni-npu-ci` is FAILURE on this head. The test file has no reference to `flash_attn_hub`, so this lane does not directly exercise the changed diffusion attention backend; can you confirm the failure is pre-existing/unrelated or post a link to the failing log? (evidence: CI gate: `buildkite/vllm-omni-npu-ci` is FAILURE; the NPU lane's e2e command (`.buildkite/test-npu-ready.yaml:21`) does not exercise this CUDA-only attention backend, so attribution to this PR cannot be established without main-vs-PR log comparison.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:32` [minor] — The refactor preserves the existing `get_kernel(repo_id, version=version)` (line 32) and `get_kernel(repo_id)` (line 33) calls, while `kernels<0.15.0` (pyproject.toml dev extra) and `kernels==0.14.1` (docs) define the supported surface without a lower bound. Please add or confirm a CI check that exercises the oldest allowed `kernels` version so the version= / no-version fallback stays compatible across the documented range. (evidence: `kernels<0.15.0` (dev extra) plus docs pin `kernels==0.14.1` represents the supported version range for the `get_kernel` calls; the oldest allowed version was not installed locally to verify its `version=` signature.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:34` [minor] — The PR body states the change adds `trust_remote_code=True` for more reliable Hub access, but `_load_hub_module()` (flash_attn_hub.py:33-35) calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` without that kwarg. Update the PR description to drop the claim; if the drop is deliberate, record the compatibility reason. (evidence: Compared pr_context PR body ('adding trust_remote_code=True') with the diff (no trust_remote_code in flash_attn_hub.py lines 30-35) and the commit timeline (9947f414).) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v11_deep_r1/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_r1/pr4977.trace.json.gz new file mode 100644 index 000000000..0691e46be Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_r1/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_r2/manifest.json b/eval/dataset/arms/copilot_v11_deep_r2/manifest.json new file mode 100644 index 000000000..a5930eaaf --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v11_deep_r2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4804.cost.json b/eval/dataset/arms/copilot_v11_deep_r2/pr4804.cost.json new file mode 100644 index 000000000..a9d5eba06 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1404.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_r2/runs/pr4804/run-20260813-124626-0b11b0", + "recorded_at": "2026-08-13T13:09:48", + "attempts": [ + { + "invocation_id": "inv-38e0022c", + "outcome": "run:run-20260813-124626-0b11b0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-124626-0b11b0", + "llm_calls": 77, + "input_tokens": 948038, + "output_tokens": 160886, + "usd": 0.706511, + "source": "spans" + } + ], + "attempt_usd_total": 0.706511, + "llm_calls": 77, + "input_tokens": 948038, + "output_tokens": 160886, + "usd": 0.706511, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-124626-0b11b0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7065, + "minutes": 23.37, + "ci_minutes": 0, + "input_tokens": 948038, + "output_tokens": 160886, + "tool_calls": 130, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8769, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 77, + "by_role": { + "agent": { + "usd": 0.675209, + "input_tokens": 893298, + "output_tokens": 145897, + "calls": 75 + }, + "reducer": { + "usd": 0.031302, + "input_tokens": 54740, + "output_tokens": 14989, + "calls": 2 + } + }, + "cache_read_tokens": 10132096, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.09, + "gate": 1.9, + "review": 1395.89, + "report": 0.06 + }, + "time_to_first_result_s": 1401.92 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 58245 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4804.md b/eval/dataset/arms/copilot_v11_deep_r2/pr4804.md new file mode 100644 index 000000000..50f4d6d78 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/pr4804.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] Seed rename complete: grep for 'qwen3_tts_request_seed' returns zero matches; tts_local_seed is produced in serving_speech.py:3715 and consumed in gpu_model_runner.py:1826; the two changed tests in tests/worker/test_omni_gpu_model_runner.py were updated consistently. +- [validated] Blast radius of generator passing: all four talker_mtp implementations accept **kwargs or generator (qwen3_omni.py:737, qwen3_tts_talker.py:1041, fish_speech_slow_ar.py:653, moss_tts modeling_moss_tts_talker.py:1494), so the now-universal tts_local_seed does not cause a TypeError on seeded requests. +- [validated] Dependency floor: requirements/common.txt:19 pins transformers >= 5.5.3, which includes PreTrainedAudioTokenizerBase imported by audio_tokenizer_v2.py:27; AudioResampler is already used by many in-repo models. +- [validated] seq_token_counts real-run producers exist: gpu_generation_model_runner.py:316 and npu_generation_model_runner.py:299 both set model_kwargs['seq_token_counts'] before forward. +- [sweep] Indexed accesses added in audio_tokenizer_v2.py (cache[0], in_projs[0], projected[0..2], wav_list[0], codes_list[0], nqs[0]) were reviewed in the diff; they are guarded by non-empty checks or fixed tensor shapes. +- [sweep] New branches in seed_tts_eval.py save-audio path are env-gated, OSError is caught and counted, and audio_path is propagated to error and success rows. +- [sweep] Resource lifecycle: on_requests_finished release path exists (modeling_moss_tts_codec.py:564) and payload finished=True releases slots, but the GPU hook is gated on num_scheduled_tokens and the NPU runner never calls it -> finding. +- [sweep] Platform parity: GPU dummy run gained model_kwargs['seq_token_counts'] (gpu_generation_model_runner.py:793-799) but NPUGenerationModelRunner._dummy_run did not -> finding. + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:728` [major] — `_build_codec` now unconditionally tries `MossAudioTokenizerV2Config.from_pretrained` / `MossAudioTokenizerV2Model` and falls back only on exception. Both config classes declare `model_type = "moss-audio-tokenizer"`, and V2Config pops `model_type` and forwards legacy keys, so a legacy v1 `config.json` (e.g. the default `OpenMOSS-Team/MOSS-Audio-Tokenizer`) constructs the v2 module layout instead of triggering the fallback. `load_weights` then tries legacy v1 names (`*.attn.in_proj.*`, `*.ff1.*`, `*.ff2.*`) against v2 params (`self_attn.in_projs.0`/`linear1`/`linear2`); the remap table translates v2 checkpoint names to legacy names, not legacy names to v2, so those weights are skipped and the new strict missing/skipped RuntimeError aborts loading for delay/realtime checkpoints still using the v1 codec. Select the codec class on an explicit version/layout marker rather than bare try/except, and add a legacy-v1 codec load test. (evidence: grep: both `model_type = "moss-audio-tokenizer"`; read audio_tokenizer.py v1 config (permissive kwargs) and configuration_moss_audio_tokenizer_v2.py __init__ (drops model_type, forwards kwargs); read the new strict missing/skipped RuntimeError in load_weights.) + +`vllm_omni/worker/gpu_generation_model_runner.py:81` [major] — Confirmed for GPUGenerationModelRunner: `model.on_requests_finished(...)` at gpu_generation_model_runner.py:81-82 only runs when `async_chunk and num_scheduled_tokens`; the zero-token early return path calls `_update_states` (gpu_model_runner.py:450) which does not invoke the hook, so a zero-token finish/abort can leak MossTTSCodecDecoder stream slots. Correction: do not mirror into `NPUGenerationModelRunner._update_request_states` — no such method exists; NPUGenerationModelRunner has no on_requests_finished hook at all, and NPUARModelRunner already calls the hook unconditionally before its zero-token return (npu_ar_model_runner.py:419-420). (evidence: read gpu_generation_model_runner.py execute_model: hook at lines 79-82 is inside `if self.model_config.async_chunk and num_scheduled_tokens` and is followed by the zero-token early return; read npu_generation_model_runner.py `_update_request_states` (no model hook).) + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:?` [major] — MossTTSCodecDecoder.forward now raises RuntimeError('MossTTS codec requires seq_token_counts...') when seq_token_counts is absent, and the PR patches GPUGenerationModelRunner._dummy_run (gpu_generation_model_runner.py:796-799) but not NPUGenerationModelRunner._dummy_run (npu_generation_model_runner.py:780-782); only the NPU real-run path injects the key (line 299). Any NPU dummy/profile/CUDA-graph warmup reaching MossTTSCodecDecoder.forward with non-empty input_ids will hit the same RuntimeError. Add the same seq_token_counts injection on the NPU dummy path and verify the NPU CI lane. (evidence: read gpu_generation_model_runner.py:793-799 (dummy seq_token_counts added) vs npu_generation_model_runner.py `_dummy_run` (no such line); grep shows seq_token_counts only at line 299 of the NPU file (real run).) + +`vllm_omni/worker/gpu_model_runner.py:1874` [minor] — Confirmed: `_talker_mtp_forward` adds the `talker_mtp_accepts_req_infos` branch at vllm_omni/worker/gpu_model_runner.py:1874 wiring `req_ids`/`req_infos` into the new `MossTTSLocalTalkerForGeneration.talker_mtp`, but the PR's only worker-test changes use `_make_runner`'s `SimpleNamespace(talker_mtp_output_key=...)` model, so the flag is False and neither the branch nor `talker_mtp` is exercised. `grep MossTTSLocal tests/` returns no local MOSS-TTS talker_mtp coverage. Add a CPU unit test that calls `MossTTSLocalTalkerForGeneration.talker_mtp` with a `req_infos` dict and asserts the emit=True -> emit=False stop transition plus the `audio_codes.current` / `audio_codes.accumulated` writes. (evidence: grep tests/ for talker_mtp/make_omni_output/MossTTSLocal (no moss-tts local match); read tests/worker/test_omni_gpu_model_runner.py `_make_runner` (model is a SimpleNamespace without talker_mtp_accepts_req_infos).) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1673` [minor] — Confirmed at line 1673: MossTTSLocalTalkerForGeneration.make_omni_output emits codes.ref = per_req_ref_codes alongside codes.audio, but every stage-0→1 MOSS processor (talker2codec via _extract_audio_codes, talker2codec_delay_async_chunk, and talker2codec_raw_async_chunk) reads only codes_dict['audio']; per_req_ref_codes has no consumer on this path. Drop the ref key/per_req_ref_codes or wire a consumer so the stage-0 payload matches what the codec consumes. (evidence: read stage_input_processors/moss_tts.py: `_extract_audio_codes` uses codes_dict.get('audio') only and talker2codec_raw_async_chunk uses codes_dict.get('audio') only; grep 'ref' in that file returns no matches.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:14` [nit] — At the top of the newly vendored `audio_tokenizer_v2.py`, after the Apache-2.0 license header (line 14 starts `"""PyTorch MossAudioTokenizer model."""`), add a `# Vendored from /` provenance block matching `audio_tokenizer.py`, whose `# Vendored from OpenMOSS-Team/MOSS-Audio-Tokenizer ...` block begins at line 9. This anchors the exact upstream source revision for future diffs and rebases. (evidence: read the top of audio_tokenizer_v2.py (license + docstring, no provenance) and audio_tokenizer.py:9 (has the Vendored-from header).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_r2/pr4804.trace.json.gz new file mode 100644 index 000000000..d2ed451a8 Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_r2/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4923.cost.json b/eval/dataset/arms/copilot_v11_deep_r2/pr4923.cost.json new file mode 100644 index 000000000..f0244e5cc --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1269.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_r2/runs/pr4923/run-20260813-124627-247844", + "recorded_at": "2026-08-13T13:07:34", + "attempts": [ + { + "invocation_id": "inv-ef1cb63e", + "outcome": "run:run-20260813-124627-247844", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-124627-247844", + "llm_calls": 98, + "input_tokens": 772321, + "output_tokens": 150441, + "usd": 0.540588, + "source": "spans" + } + ], + "attempt_usd_total": 0.540588, + "llm_calls": 98, + "input_tokens": 772321, + "output_tokens": 150441, + "usd": 0.540588, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-124627-247844", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5406, + "minutes": 21.12, + "ci_minutes": 0, + "input_tokens": 772321, + "output_tokens": 150441, + "tool_calls": 148, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7732, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 98, + "by_role": { + "agent": { + "usd": 0.499102, + "input_tokens": 756353, + "output_tokens": 116703, + "calls": 95 + }, + "reducer": { + "usd": 0.041485, + "input_tokens": 15968, + "output_tokens": 33738, + "calls": 3 + } + }, + "cache_read_tokens": 6169472, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.16, + "gate": 1.86, + "review": 1261.93, + "report": 0.06 + }, + "time_to_first_result_s": 1266.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 24331 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4923.md b/eval/dataset/arms/copilot_v11_deep_r2/pr4923.md new file mode 100644 index 000000000..dfeba53bd --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/pr4923.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Diff hunks verified in situ: vllm_omni/deploy/qwen3_tts.yaml (lines 35,43,67,95 area) and vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py (324-353) read directly. +- [validated] use_cuda_graphs=current_omni_platform.is_npu() matches qwen3_omni_moe_code_predictor_mtp.py:21; CodePredictorWrapper._setup_compile/_capture_cuda_graphs confirm GPU uses torch.compile only when use_cuda_graphs=False and NPU keeps internal graphs (common/qwen3_code_predictor.py:532,552). +- [validated] talker_mtp_graph_safe=True is consumed by OmniGPUModelRunner._init_talker_mtp (gpu_model_runner.py:208-210) to wrap in the platform full-graph wrapper, and _capture_talker_mtp_graphs (gpu_ar_model_runner.py) captures per-size graphs. +- [sweep] talker_mtp_accepts_per_row_generators consumer found at gpu_model_runner.py:1854: with the new False-under-full-cudagraphs value, seeded multi-row decode batches enter the scalar per-row fallback (gpu_model_runner.py:1852-1883) — basis of the major finding. +- [sweep] use_async_omni_output / eager_omni_postprocess_before_async_output / omni_pooler_payload_include_hidden consumers located in gpu_ar_model_runner.py:1534-1710 and 2020-2077; the async path exists and reads these flags. +- [sweep] Duplicate assignment found: self.omni_pooler_payload_include_hidden=False is set both at ~line 317 (pre-existing, commented) and again at line 353 (added by this diff). +- [sweep] Stale deploy comments: qwen3_tts.yaml header (lines 5-7) still documents max_num_batched_tokens=512 for stage 0 (now 32768), and the inline prefix-cache comment (line 43 area) says 'on by default / disable explicitly' while the value is now false. +- [sweep] CI gate: buildkite/vllm-omni-npu-ci and buildkite/vllm-omni-amd-ci are failing; the diff adds NPU compilation_config.cudagraph_mode=PIECEWISE, so the NPU lane plausibly exercises this change — attribution not verified. + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [major] — Verified: talker.py:350 sets talker_mtp_accepts_per_row_generators=False whenever talker_mtp_graph_safe and cudagraph_mode.has_full_cudagraphs() (talker_mtp_graph_wrapped=True). gpu_model_runner.py:1854's fallback branch tests `not getattr(self.model, "talker_mtp_accepts_per_row_generators", False)` and gpu_model_runner.py:1870 recurses with a single req_id, so under full cudagraphs a seeded decode batch >1 still takes the scalar per-row path — the N-call hot spot this PR targets. Decide the seeded-batch contract: either keep per-row reproducibility and document the N-call cost, or reject tts_local_seed under full cudagraphs and remove the fallback; fix the 'silently dropped' wording accordingly. (evidence: Read qwen3_tts_talker.py:324-350 (new gate and comment) and gpu_model_runner.py:1848-1883 (fallback condition: decode_batch_size > 1 and any seeded and not talker_mtp_accepts_per_row_generators).) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — In vllm_omni/deploy/qwen3_tts.yaml, line 43 is `enable_prefix_caching: false` under the unchanged comment block that still says 'Prefix caching on by default' and 'Disable explicitly if memory headroom is tight'. The file header is likewise stale: it documents Stage 0 `max_num_batched_tokens=512` while stage 0 now sets 32768. Update both comments to match the shipped defaults. (evidence: Read vllm_omni/deploy/qwen3_tts.yaml lines 4-10 and 35-46; the changed line is `enable_prefix_caching: false` under the unchanged 'on by default' comment.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — Confirmed: line 317 already sets `self.omni_pooler_payload_include_hidden = False` with a stage-1 payload rationale comment, and this diff adds the identical assignment again at line 353. The duplicate is dead code, and the PR description's Changes list overstates it as a newly enabled async flag. Remove the line 353 assignment and drop this item from the description. (evidence: read_file qwen3_tts_talker.py shows line 317 `self.omni_pooler_payload_include_hidden = False` (pre-existing) and line 353 (added by this diff); grep confirms both occurrences.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — This diff sets talker_mtp_accepts_per_row_generators = False whenever full cudagraphs are active (talker.py:350). tts_local_seed is plumbed from request.seed (serving_speech.py:3694/3706), and under FULL cudagraphs talker_mtp is graph-wrapped with a single captured RNG stream, so a user-supplied seed is not honored. The shipped qwen3_tts.yaml now uses PIECEWISE (not FULL), so this is not the default deploy path, but when FULL is explicitly configured a runtime warning or rejection should be emitted until runner-side generators are graph-safe. (evidence: grep tts_local_seed shows serving_speech.py:3694 `stage0_params.extra_args["tts_local_seed"] = request.seed` and :3706 default seed; gpu_model_runner.py:1854+ shows the seeded batch/scalar path that the captured graph cannot honor; the diff's own comment at talker.py:334-341 acknowledges the silent drop.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — This diff opts the talker into `use_async_omni_output = True` (qwen3_tts_talker.py:351) while the deploy YAML raises `max_num_seqs` to 64 and disables prefix caching — changes that can shift per-step stream timing. The PR body's serving result reports `Streaming continuity OK rate: 21.88%` with no main-baseline value (the before/after artifacts only cover `talker_mtp` timing). Please rerun the same serving harness on main and on this PR and include before/after streaming-continuity metrics so an async-output/prefix-cache continuity regression can be ruled out. (evidence: PR body test result shows `Streaming continuity OK rate: 21.88%` with no baseline; gpu_ar_model_runner.py:1537 `_should_use_async_omni_output` shows the flag only takes effect in the async_chunk streaming path that this metric measures.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — This diff adds a model-side __init__ branch at qwen3_tts_talker.py:346-350 that flips talker_mtp_accepts_per_row_generators based on cudagraph_mode.has_full_cudagraphs(), but no test covers it. tests/worker/test_omni_gpu_model_runner.py:374 sets the flag directly on a SimpleNamespace fake, and the Qwen3-TTS model tests build the instance via __new__ (bypassing __init__), so the FULL vs NONE/PIECEWISE gating is unexercised. Add a unit test that drives this init logic with cudagraph_mode FULL and NONE/PIECEWISE and asserts the resulting flag value. (evidence: grep tests for talker_mtp shows tests/worker/test_omni_gpu_model_runner.py:356-398 only sets `talker_mtp_accepts_per_row_generators=True` on a SimpleNamespace fake; no test file is touched by this diff.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_r2/pr4923.trace.json.gz new file mode 100644 index 000000000..d8cf28ac5 Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_r2/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4926.cost.json b/eval/dataset/arms/copilot_v11_deep_r2/pr4926.cost.json new file mode 100644 index 000000000..bc2d45cdd --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1860.5, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_r2/runs/pr4926/run-20260813-130951-783171", + "recorded_at": "2026-08-13T13:40:49", + "attempts": [ + { + "invocation_id": "inv-95c41c43", + "outcome": "run:run-20260813-130951-783171", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-130951-783171", + "llm_calls": 117, + "input_tokens": 582597, + "output_tokens": 203494, + "usd": 0.569528, + "source": "spans" + } + ], + "attempt_usd_total": 0.569528, + "llm_calls": 117, + "input_tokens": 582597, + "output_tokens": 203494, + "usd": 0.569528, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-130951-783171", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5695, + "minutes": 30.97, + "ci_minutes": 0, + "input_tokens": 582597, + "output_tokens": 203494, + "tool_calls": 210, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9281, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 117, + "by_role": { + "agent": { + "usd": 0.539063, + "input_tokens": 560541, + "output_tokens": 181244, + "calls": 115 + }, + "reducer": { + "usd": 0.030465, + "input_tokens": 22056, + "output_tokens": 22250, + "calls": 2 + } + }, + "cache_read_tokens": 6977152, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.78, + "gate": 1.6, + "review": 1852.57, + "report": 0.06 + }, + "time_to_first_result_s": 1858.0 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 35479 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4926.md b/eval/dataset/arms/copilot_v11_deep_r2/pr4926.md new file mode 100644 index 000000000..dca986c36 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/pr4926.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] flash_attn_hub.py:47,114,293 indexed accesses `out[0]`: all guarded by `isinstance(out, tuple)`; FA2 returns tensor, FA3 returns (out, lse); first element is the output and `_run_varlen_dense` reshapes afterward. +- [validated] flash_attn_hub.py:108/287 both-funcs-None branch: raises RuntimeError; constructor guarantees at least one of flash_attn_func/flash_attn_varlen_func is loaded. +- [sweep] flash_attn_hub.py:117/296 `attn_func is not None` branch: piecewise wrapper falls back to `_run_varlen_dense` only when varlen func exists; consistent with constructor. +- [sweep] flash_attn_hub.py:88/267 `if backend_kwargs:` branch only warns; benign. +- [sweep] flash_attn_hub.py:189/368 full_attn_spans and 205/384 attention_mask branches mirror local flash_attn.py; read piecewise_attn.py and utils/fa.py; mask contract is 2D bool/int, same as existing backend. +- [sweep] cuda/platform.py hub branches 124/129/134/140/142: read full get_diffusion_attn_backend_cls; found the HUB names skip the `flash_attn_supported`/compute-capability gate (review comment 1). +- [sweep] musa/npu/rocm/xpu platform.py hub fallback branches: each routes HUB names to FLASH_ATTN and then existing checks; xpu lacks a selected-backend FLASH_ATTN support check but that is pre-existing. +- [validated] registry.py enum additions resolve to the new classes; _DiffusionBackendEnumMeta handles unknown names. + +`vllm_omni/platforms/cuda/platform.py:124` [major] — In vllm_omni/platforms/cuda/platform.py:124-127, a successful `importlib.import_module("kernels")` leaves `backend_upper` as FLASH_ATTN_HUB with no compute-capability check. The subsequent local FLASH_ATTN gate (lines 149-158) only runs for `backend_upper == "FLASH_ATTN"`, so FLASH_ATTN_HUB reaches the unconditional enum lookup at line 175. On a pre-Ampere GPU (capability < 8.0) with `kernels` installed, this selects hub FA2 and later fails to load/run instead of falling back like local FLASH_ATTN does to TORCH_SDPA. Gate FLASH_ATTN_HUB on `compute_supported` (or fall back to TORCH_SDPA) before the enum lookup. (evidence: Read vllm_omni/platforms/cuda/platform.py: lines 61-66 (FA requires sm_80+), 124-147 (new HUB branch), 149-158 (local FLASH_ATTN fallback to TORCH_SDPA), 175 (unconditional enum lookup).) + +`tests/diffusion/attention/test_kernels_hub.py:58` [major] — The PR adds `tests/diffusion/attention/test_kernels_hub.py` with `@hardware_test(res={"cuda": "L4"})` on `test_kernels_hub_execution` (line 57, def line 58), but no Buildkite CUDA step collects `tests/diffusion/attention/` (the diffusion steps enumerate offloader/cache/batching/distributed/models and the non-recursive `tests/diffusion/*.py` glob), so the new hub backends have no automated FA2 execution in CI. The CPU step deselects it via `-m 'core_model and cpu'`, and the FA3 branch skips at line 92 (`if major < 9`) on L4 (sm_89), so FA3 would also not execute on that lane. Add a CUDA Buildkite step that collects `tests/diffusion/attention/`, and run the FA3 case on an H100/SM90 queue or change the marker/queue to satisfy the SM90 requirement. (evidence: Read .buildkite/test-ready.yml lines 13, 64, 80, 96, 112, 129, 145; grep tests/diffusion/attention in .buildkite returned no matches; read tests/diffusion/attention/test_kernels_hub.py lines 57 and 92.) + +`tests/diffusion/attention/test_kernels_hub.py:75` [minor] — The execution test only builds AttentionMetadata(attn_mask=None) at test_kernels_hub.py:75 and passes that same object to both Hub implementations, so full_attn_spans remains None and attn_mask never contains a False entry. As a result, the new _forward_varlen_masked and piecewise_attn branches in flash_attn_hub.py's forward_cuda are never executed. Add a masked 2D bool/int case (at least one False) and a mixed causal/full full_attn_spans case, each compared against the SDPA reference, since these varlen/masked and piecewise paths are the most likely to expose Hub-kernel signature or wrapper-shape divergence. (evidence: Read flash_attn_hub.py lines 128-161, 189-202; test file lines 75-86.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:108` [minor] — FlashAttentionHubBackend.supports_attention_mask() returns True (flash_attn_hub.py:55-56), but __init__ only raises when both flash_attn_func and flash_attn_varlen_func are None (lines 108-109). The masked path calls self.flash_attn_varlen_func unconditionally (line 147), so a Hub module exposing only the dense flash_attn_func will crash with TypeError on the first masked request. Require flash_attn_varlen_func at construction or derive supports_attention_mask() from the loaded functions. The same issue applies to FlashAttention3HubImpl (lines 287 and 326). (evidence: Read flash_attn_hub.py lines 108-109, 55-56, 128-147, 205-211.) + +`vllm_omni/platforms/cuda/platform.py:126` [minor] — The CUDA selector gates Hub backends solely on `importlib.import_module("kernels")` (platform.py:126), while `FlashAttentionHubImpl.__init__` and `FlashAttention3HubImpl.__init__` require `from kernels import get_kernel` and call `get_kernel(..., version=...)` (flash_attn_hub.py:92, :271). A `kernels` install that imports but lacks `get_kernel` passes selection and then raises during backend impl construction rather than falling back to FLASH_ATTN as documented. Recommend probing for a callable `kernels.get_kernel` (and version compatibility) in the selector, and adding a lower bound such as `kernels>=0.14.1` (pyproject.toml:70 currently only has `kernels<0.15.0` in dev). (evidence: Read vllm_omni/platforms/cuda/platform.py line 126; vllm_omni/diffusion/attention/backends/flash_attn_hub.py lines 92-103; pyproject.toml line 70; docs/user_guide/diffusion/attention_backends.md install section.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:108` [minor] — The `FlashAttentionHubImpl.__init__` retry chain (flash_attn_hub.py:95-108) catches failures for `version=1` and `version=2`, but the final default `get_kernel('kernels-community/flash-attn2')` at line 103 has no try/except; the same pattern repeats for `FlashAttention3HubImpl`. The CUDA selector (`cuda/platform.py:124-139`) only falls back on `ImportError` of the `kernels` package, so with `kernels` installed but Hub artifacts uncached and `HF_HUB_OFFLINE=1`, model-load raises instead of falling back. Either probe kernel availability at selection time or document the offline/uncached behavior. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:80-108; vllm_omni/platforms/cuda/platform.py:124-139; vllm_omni/diffusion/attention/layer.py:100-110.) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/attention/test_kernels_hub.py:70` [minor] In test_kernels_hub_execution (tests/diffusion/attention/test_kernels_hub.py:69-71), k = q.clone() and v = q.clone() make q/k/v identical, so the SDPA-vs-hub comparison cannot detect q/k/v operand swaps (any permutation +- `vllm_omni/diffusion/attention/layer.py:100` [minor] Could you confirm whether `kernels.get_kernel` caches internally across calls? Each `Attention` module constructs one impl via `self.attn_impl_cls(...)` (layer.py:95), and `FlashAttentionHubImpl.__init__` / `FlashAttenti +- `pyproject.toml:70` [minor] pyproject.toml:70 declares `kernels<0.15.0` only in the `dev` extra (no lower bound, no runtime optional extra), yet the new FLASH_ATTN_HUB/FLASH_ATTN_3_HUB backends import `kernels` at runtime and the docs instruct `pip +- `vllm_omni/platforms/cuda/platform.py:141` [minor] `vllm_omni/platforms/cuda/platform.py:141` gates `FLASH_ATTN_3_HUB` with `compute_capability.major >= 9`, so Blackwell sm_100/103/120/121 (and future major 11/12 variants) pass the gate whenever `kernels` is installed. T + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_r2/pr4926.trace.json.gz new file mode 100644 index 000000000..135e57dae Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_r2/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4977.cost.json b/eval/dataset/arms/copilot_v11_deep_r2/pr4977.cost.json new file mode 100644 index 000000000..cb8bf6722 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 752.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_r2/runs/pr4977/run-20260813-130737-da7225", + "recorded_at": "2026-08-13T13:20:07", + "attempts": [ + { + "invocation_id": "inv-099e8be4", + "outcome": "run:run-20260813-130737-da7225", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-130737-da7225", + "llm_calls": 35, + "input_tokens": 168357, + "output_tokens": 74745, + "usd": 0.152822, + "source": "spans" + } + ], + "attempt_usd_total": 0.152822, + "llm_calls": 35, + "input_tokens": 168357, + "output_tokens": 74745, + "usd": 0.152822, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-130737-da7225", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1528, + "minutes": 12.49, + "ci_minutes": 0, + "input_tokens": 168357, + "output_tokens": 74745, + "tool_calls": 60, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4354, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 35, + "by_role": { + "agent": { + "usd": 0.134975, + "input_tokens": 161649, + "output_tokens": 60199, + "calls": 33 + }, + "reducer": { + "usd": 0.017846, + "input_tokens": 6708, + "output_tokens": 14546, + "calls": 2 + } + }, + "cache_read_tokens": 931328, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.33, + "gate": 1.6, + "review": 744.22, + "report": 0.06 + }, + "time_to_first_result_s": 749.19 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 15467 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4977.md b/eval/dataset/arms/copilot_v11_deep_r2/pr4977.md new file mode 100644 index 000000000..7b9d9b83d --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_r2/pr4977.md @@ -0,0 +1,34 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] Correctness: read vllm_omni/diffusion/attention/backends/flash_attn_hub.py fully; fallback loop `for version in (1, 2, None)` at line 31 preserves v1→v2→default; the `if version is not None` branch (line 33) has all three arms reachable (1, 2, and the no-version default); total failure raises RuntimeError chained `from last_error` (line 39). +- [validated] Cache/concurrency: `_get_hub_module` (lines 42-46) checks membership inside `_hub_lock`, so both the miss (load+store) and hit (return cached) arms occur; the context manager releases the lock on exception, and a failed load is not cached so the next construction retries. +- [validated] No breaking API: `_load_hub_module`/`_get_hub_module` are new private helpers; grep shows every reference is inside flash_attn_hub.py (lines 26, 42, 45, 119, 284). No external callers or stale consumers exist. +- [sweep] Diff hunk locations 1/19/116/281 were all read in the full file; hunk at lines 19-46 adds the module cache and helpers, hunks at 116-119 and 281-284 replace the inline loads with the cached helper. +- [sweep] Tests: read tests/diffusion/attention/test_kernels_hub.py — `test_kernels_hub_platform_fallback` (lines 19-55) only monkeypatches sys.modules['kernels'] and calls CudaOmniPlatform.get_diffusion_attn_backend_cls, never importing flash_attn_hub or constructing an impl; `test_kernels_hub_execution` (lines 58-105) constructs each impl once and asserts only numerics, with the FA3 branch skipped on sm_89. No test asserts single-load caching or the v1→v2→default order. +- [validated] Dependency compatibility: `kernels` is not declared in pyproject.toml; docs pin `kernels==0.14.1` (docs/user_guide/diffusion/attention_backends.md:228). The diff introduces no new `get_kernel` kwarg (same `version=1`/`version=2`/no-arg calls as before), so no new lower bound is required; dropping `trust_remote_code` avoids unknown-kwarg risk on the pinned version. +- [sweep] Scope: diff touches only flash_attn_hub.py, matching the stated purpose, but the PR description's claim of adding `trust_remote_code=True` is not delivered by the final diff (commit 9947f414 'Drop trust_remote_code from hub kernel loading') — stale description. +- [validated] Blast radius: shared code affects only CUDA FLASH_ATTN_HUB/FLASH_ATTN_3_HUB; read vllm_omni/platforms/npu/platform.py — NPU falls back to local FLASH_ATTN and never imports this module (same platform-level fallback exists on ROCm/MUSA/XPU). + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:27` [minor] — PR body still says this PR adds `trust_remote_code=True`, but `_load_hub_module` only calls `get_kernel(repo_id, version=...)` or `get_kernel(repo_id)` with no `trust_remote_code` kwarg. Remove that claim from the Purpose or note the reason it was dropped. (evidence: PR description in pr_context; commit timeline lists 9947f414 'Drop trust_remote_code from hub kernel loading'; docs/user_guide/diffusion/attention_backends.md:228 pins `kernels==0.14.1`.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:1` [minor] — The buildkite/vllm-omni-npu-ci gate is failing. The changed module vllm_omni/diffusion/attention/backends/flash_attn_hub.py is never imported on the NPU path — NPUOmniPlatform.get_diffusion_attn_backend_cls explicitly falls back to FLASH_ATTN for FLASH_ATTN_HUB/FLASH_ATTN_3_HUB and returns backend path strings without importing the module. This diff therefore cannot cause the NPU failure; please confirm it is pre-existing on main before merge. (evidence: gate_report lists buildkite/vllm-omni-npu-ci FAILURE; since this diff is CUDA-only and the NPU platform never imports flash_attn_hub.py (verified), the failure is not attributable here — ask whether it is pre-existing on main.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [nit] — This diff adds module-level `_hub_modules`/`_hub_lock` and `_load_hub_module`/`_get_hub_module`; the cache is process-wide and never invalidated, and failed loads are deliberately not cached because `_get_hub_module` assigns only after `_load_hub_module` returns (line 44-45). Add a short docstring to `_get_hub_module` (line 42) documenting that `_hub_lock` serializes initialization and that failures are not cached, so future maintainers do not accidentally cache failures. (evidence: Read vllm_omni/diffusion/attention/backends/flash_attn_hub.py lines 19-46.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v11_deep_r2/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_r2/pr4977.trace.json.gz new file mode 100644 index 000000000..5bf61db89 Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_r2/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_val/manifest.json b/eval/dataset/arms/copilot_v11_deep_val/manifest.json new file mode 100644 index 000000000..4da07c263 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v11_deep_val", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4810.cost.json b/eval/dataset/arms/copilot_v11_deep_val/pr4810.cost.json new file mode 100644 index 000000000..8b81e8d2d --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4810.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1089.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_val/runs/pr4810/run-20260813-134520-993dd9", + "recorded_at": "2026-08-13T14:03:26", + "attempts": [ + { + "invocation_id": "inv-bd7c06dd", + "outcome": "run:run-20260813-134520-993dd9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-134520-993dd9", + "llm_calls": 84, + "input_tokens": 545604, + "output_tokens": 117169, + "usd": 0.461226, + "source": "spans" + } + ], + "attempt_usd_total": 0.461226, + "llm_calls": 84, + "input_tokens": 545604, + "output_tokens": 117169, + "usd": 0.461226, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-134520-993dd9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4612, + "minutes": 18.11, + "ci_minutes": 0, + "input_tokens": 545604, + "output_tokens": 117169, + "tool_calls": 131, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6875, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 84, + "by_role": { + "agent": { + "usd": 0.445147, + "input_tokens": 533662, + "output_tokens": 105514, + "calls": 82 + }, + "reducer": { + "usd": 0.016079, + "input_tokens": 11942, + "output_tokens": 11655, + "calls": 2 + } + }, + "cache_read_tokens": 6852864, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.95, + "gate": 1.7, + "review": 1080.93, + "report": 0.06 + }, + "time_to_first_result_s": 1086.64 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 24449 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4810.md b/eval/dataset/arms/copilot_v11_deep_val/pr4810.md new file mode 100644 index 000000000..afd9b60ea --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4810.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read new branch arms at mammoth_moda2.py:486/488 and mimo_audio_llm.py:1156/1158 in the tree — both `quant_config is not None` and `cache_scale_mapper is not None` arms are reachable (None quant_config skips, base mapper returning None skips); no dead/redundant branch found. +- [sweep] Non-test files touched and their coverage: hunyuan_image3.py and qwen2_old.py are exercised by test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale; mammoth_moda2.py and mimo_audio_llm.py are exercised by test_direct_custom_loaders_apply_cache_scale_mapper. +- [sweep] Read all diff hunk locations in situ: tests/model_executor/models/test_kv_cache_scale_mapper.py (full), hunyuan_image3.py around 222, mammoth_moda2.py around 483, mimo_audio_llm.py around 1153, qwen2_old.py around 335. +- [sweep] Repo-wide grep `get_cache_scale` => exactly one production survivor: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238; grep `"get_cache_scale"` in vllm_omni => no getattr-style survivors. +- [validated] HunyuanModel and Qwen2Model have no standalone entries in registry.py, so relying on the outer AutoWeightsLoader for those two loaders is consistent with the PR's reasoning. +- [validated] HunyuanImage3ForConditionalGeneration.load_weights uses AutoWeightsLoader (hunyuan_image3.py:2178); Qwen2ForCausalLM.load_weights uses AutoWeightsLoader (qwen2_old.py end). +- [validated] MiMoAudioLLMForConditionalGeneration.load_weights is the top-level custom loader and iterates weights directly (no AutoWeightsLoader), so the added mapper application there is required. +- [validated] MammothModa2Qwen2ForCausalLM is registered standalone in registry.py:92, so its direct-load path needs the mapper application even if the nested AR path may already map the names. + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:?` [blocker] — The diff removes the `get_cache_scale` branch in the four AR/custom loaders, but `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` still calls the removed API (`if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`); a repo-wide grep confirms it is the only production survivor. Quantized HunyuanImage3 DiT checkpoints reaching this path will hit `AttributeError` once `quant_config` no longer exposes `get_cache_scale`. Migrate this branch to `get_cache_scale_mapper()` (or remove it if the loader is reached via an outer `AutoWeightsLoader`) and add `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py` to `_STALE_API_FILES` in `tests/model_executor/models/test_kv_cache_scale_mapper.py:15` so the guard test fails until this code is fixed. (evidence: grep 'get_cache_scale' across the repo shows the surviving call only at this file:2238; read_file of the load_weights body confirms the branch is intact and still reads `params_dict[scale_name]` / `loaded_weight[0]`.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:12` [minor] — Confirmed against the PR-time tree: the new test at tests/model_executor/models/test_kv_cache_scale_mapper.py:12 is marked `core_model` + `cpu`, and both failing merge lanes run `tests/model_executor -m 'core_model and cpu'` (.buildkite/test-merge.yml:13 and .buildkite/test-amd-merge.yml:32), so this test is selected there. Confirm from the AMD/NPU lane logs whether the failures pre-exist on base or are caused by this test. (evidence: gate_report lists the two failing lanes; grep of .buildkite/test-merge.yml:13 and test-amd-merge.yml:32 shows the `core_model and cpu` model_executor commands that select this test.) + +`setup.py:1` [minor] — setup.py install_requires is assembled by get_install_requires() from requirements/{device}.txt, and no vllm version constraint is declared there (grep matched only a comment in requirements/common.txt). The PR now depends on quant_config.get_cache_scale_mapper() in mammoth_moda2.py:487 and mimo_audio_llm.py:1157 and removes the old get_cache_scale fallback in hunyuan_image3, so if vLLM < 0.23.0rc2 is still supported the loaders would fail on older quant_configs; otherwise an explicit vllm floor should be declared. (evidence: [sweep] Dependency check: setup.py/requirements do not declare a vllm lower bound; PR states the API landed in vLLM 0.23.0rc2 and env reports vLLM 0.24.0, so no lower-bound gap is checkable from this repo.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:144` [minor] — The diff deletes the `loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]` scalarization from MammothModa2 and MiMo-Audio load_weights, but `test_direct_custom_loaders_apply_cache_scale_mapper` (line 144) only feeds a 0-dim `torch.ones(())`. A 1-element safetensors scale is shape `[1]`, so the un-scalarized mapper path is never exercised by the regression test. Parametrize with `torch.ones(())` and `torch.ones(1)` (or otherwise assert shape-`[1]` handling) to prove the new path is shape-safe. (evidence: Diff of mammoth_moda2.py and mimo_audio_llm.py deletes the `loaded_weight[0]` normalization; both new direct-loader tests call load_weights with `torch.ones(())` only.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:112` [nit] — At lines 98-114, `test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale` calls `HunyuanModel.load_weights` and `Qwen2Model.load_weights` directly with `_MAPPED_SCALE_NAME`; it never instantiates `AutoWeightsLoader`, so it does not test the PR's claim that the outer loader maps `.k_proj.output_scale` to `.attn.k_scale` before delegation. Drive the parent loaders (`HunyuanImage3ForConditionalGeneration.load_weights` / `Qwen2ForCausalLM.load_weights`) with a real quant config, or rename the test to state it verifies delegated loaders accept already-mapped names. (evidence: Test lines 98-114 call the inner class load_weights directly; hunyuan_image3.py:2178 and qwen2_old.py's Qwen2ForCausalLM.load_weights both construct AutoWeightsLoader.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_val/pr4810.trace.json.gz new file mode 100644 index 000000000..e2d0b5506 Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_val/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4816.cost.json b/eval/dataset/arms/copilot_v11_deep_val/pr4816.cost.json new file mode 100644 index 000000000..90404d600 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4816.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 935.6, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_val/runs/pr4816/run-20260813-140328-27c7da", + "recorded_at": "2026-08-13T14:19:02", + "attempts": [ + { + "invocation_id": "inv-8e2ed7b2", + "outcome": "run:run-20260813-140328-27c7da", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-140328-27c7da", + "llm_calls": 44, + "input_tokens": 507301, + "output_tokens": 74874, + "usd": 0.297442, + "source": "spans" + } + ], + "attempt_usd_total": 0.297442, + "llm_calls": 44, + "input_tokens": 507301, + "output_tokens": 74874, + "usd": 0.297442, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-140328-27c7da", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2974, + "minutes": 15.56, + "ci_minutes": 0, + "input_tokens": 507301, + "output_tokens": 74874, + "tool_calls": 74, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5668, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "planner": { + "usd": 0.000515, + "input_tokens": 87, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.274234, + "input_tokens": 500102, + "output_tokens": 55622, + "calls": 41 + }, + "reducer": { + "usd": 0.022692, + "input_tokens": 7112, + "output_tokens": 18852, + "calls": 2 + } + }, + "cache_read_tokens": 2892928, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.26, + "gate": 1.87, + "review": 928.46, + "report": 0.07 + }, + "time_to_first_result_s": 933.63 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 14821 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4816.md b/eval/dataset/arms/copilot_v11_deep_val/pr4816.md new file mode 100644 index 000000000..a859c277c --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4816.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] grep 'openai_serving_tokenization' repo-wide -> 0 matches; rename is complete across api_server.py:754,1017,1157,1248,1286,1329 and test_serving_speech.py:2533,2546,2615. +- [sweep] grep 'serving_tokenization' repo-wide -> only api_server.py producers/consumers and the three test assignments; no stale consumers left. +- [validated] api_server.py omni_init_app_state pure-diffusion branch (line 754) and LLM branch (line 1017) both set `serving_tokenization`; no other state attribute uses the old name. +- [sweep] read create_chat_completion (1157), create_speech (1248), create_speech_batch (1286), create_audio_generate (1329): all None-handler branches consistently use `getattr(..., "serving_tokenization", None)`. +- [sweep] read _create_speech_error_json_response (api_server.py:369-376): calls `base(raw_request).create_error_response(...)`; with pure-diffusion `state.serving_tokenization = None`, upstream `base()` returns None -> AttributeError -> 500 (residual defect). +- [sweep] read _patch_api_server_base (test_serving_speech.py:2339-2353): patches `api_server_module.base`, so list_voices/upload_voice/delete_voice tests never execute the real `base()` that reads `state.serving_tokenization`; the fixed path lacks regression coverage. +- [validated] The three modified tests (2533/2546/2615) set `state.serving_tokenization = fake_base`, matching the production getattr; without the rename the handler-None branch would raise HTTPException and fail the test, so the test changes are required and correct. +- [sweep] grep 'instrumentator' and 'from vllm.entrypoints.serve.instrumentator.basic import base' -> only api_server.py:67 imports `base`; no other in-repo consumer of the renamed attribute. + +`vllm_omni/entrypoints/openai/api_server.py:754` [major] — Line 754 changes the pure-diffusion app-state slot from `state.openai_serving_tokenization = None` to `state.serving_tokenization = None`. Upstream `base(raw_request)` forwards to `tokenization()`, which returns `request.app.state.serving_tokenization`; in pure-diffusion mode that is now `None`, so `_create_speech_error_json_response` still executes `base(raw_request).create_error_response(...)` at api_server.py:376 and raises `AttributeError: 'NoneType' object has no attribute 'create_error_response'` whenever a speech error reaches the helper (e.g. `upload_voice` raising `ValueError` for undecodable audio). Provide a non-None error responder in pure-diffusion mode or make the helper None-safe (fall back to a locally built `ErrorResponse`/`HTTPException`). (evidence: Read api_server.py:369-376 and 1374-1542; read serving_speech.py `OmniOpenAIServingSpeech.for_diffusion` and `upload_voice` (ValueError on decode failure). PR description quotes upstream `tokenization(request)` returning `request.app.state.serving_tokenization`.) + +`setup.py:69` [minor] — setup.py's install_requires is derived only from requirements/{device}.txt, and a grep of requirements/*.txt shows no vllm version pin. Since vllm_omni/entrypoints/openai/api_server.py already imports post-refactor vllm modules (`vllm.entrypoints.serve.tokenize.serving.ServingTokenization` at line 69 and `vllm.entrypoints.serve.instrumentator.basic.base` at line 67), consider declaring the minimum supported vllm version explicitly in the packaging requirements to make the dependency contract clear. (evidence: setup.py + requirements/*.txt -> no vllm version pin declared, so the oldest supported vllm cannot be checked; api_server.py already imports `vllm.entrypoints.serve.tokenize.serving.ServingTokenization` and `serve.instrumentator.basic.base` (post-refactor modules), implying the repo already requires the new vllm layout.) + +`vllm_omni/entrypoints/openai/api_server.py:66` [minor] — At vllm_omni/entrypoints/openai/api_server.py:65-67 the comment says to 'Keep a fallback for older/newer upstream layouts during rebase windows', but line 67 is a direct `from vllm.entrypoints.serve.instrumentator.basic import base` with no try/except or legacy-path fallback. This is pre-existing and the import block is untouched by this diff; maintainers may update or remove the stale comment separately. (evidence: Stale import comment at api_server.py:65-67 claims 'Keep a fallback for older/newer upstream layouts' but the import is direct and no fallback exists (pre-existing, imports untouched by this diff).) + +`gate_report:1` [minor] — Gate report shows buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, and buildkite/vllm-omni-npu-ci failing. The diff only renames `openai_serving_tokenization` to `serving_tokenization` in api_server.py and CPU/mock tests, so it does not plausibly affect those hardware lanes. Please confirm these are pre-existing/expected and tracked separately so they do not block merge. (evidence: gate_report: amd/intel/npu lanes failing; this rename in api_server.py + CPU/mock test does not plausibly touch those hardware lanes, so no attribution comment added.) + +`tests/entrypoints/openai_api/test_serving_speech.py:?` [minor] — Lines 2533/2546/2615 only rename the app-state attribute in the three `without_handler` tests, which call create_speech/create_speech_batch/create_audio_generate and therefore only hit the direct `getattr(raw_request.app.state, "serving_tokenization", None)` branch (api_server.py:1248/1286/1329). The voice endpoint error path goes through `_create_speech_error_json_response` -> `base(raw_request)` (api_server.py:369-376) and is covered only by tests that mock `api_server_module.base` via `_patch_api_server_base` (test_serving_speech.py:2339), so the real `base()` that reads `state.serving_tokenization` is never exercised. Consider a regression test that sets `raw_request.app.state.serving_tokenization` to a fake exposing `create_error_response`, does NOT patch `api_server_module.base`, and drives upload_voice/delete_voice/list_voices into an error branch. (evidence: Read _patch_api_server_base at test_serving_speech.py:2339-2353 and api_server.py:1374-1542; the modified tests call create_speech/create_speech_batch/create_audio_generate, none of which invoke api_server_module.base.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_val/pr4816.trace.json.gz new file mode 100644 index 000000000..f1f518116 Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_val/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4825.cost.json b/eval/dataset/arms/copilot_v11_deep_val/pr4825.cost.json new file mode 100644 index 000000000..6ce12584c --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4825.cost.json @@ -0,0 +1,115 @@ +{ + "wall_s": 999.0, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_val/runs/pr4825/run-20260813-141905-e75827", + "recorded_at": "2026-08-13T14:35:41", + "attempts": [ + { + "invocation_id": "inv-df91e568", + "outcome": "run:run-20260813-141905-e75827", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-141359-6be7ec", + "llm_calls": 58, + "input_tokens": 297025, + "output_tokens": 89846, + "usd": 0.254925, + "source": "spans" + }, + { + "run_dir": "run-20260813-141905-e75827", + "llm_calls": 60, + "input_tokens": 272835, + "output_tokens": 115896, + "usd": 0.288982, + "source": "spans" + } + ], + "attempt_usd_total": 0.543907, + "llm_calls": 60, + "input_tokens": 272835, + "output_tokens": 115896, + "usd": 0.288982, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-141905-e75827", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.289, + "minutes": 16.61, + "ci_minutes": 0, + "input_tokens": 272835, + "output_tokens": 115896, + "tool_calls": 106, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5822, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 60, + "by_role": { + "agent": { + "usd": 0.269363, + "input_tokens": 265021, + "output_tokens": 100010, + "calls": 58 + }, + "reducer": { + "usd": 0.019619, + "input_tokens": 7814, + "output_tokens": 15886, + "calls": 2 + } + }, + "cache_read_tokens": 3252992, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.89, + "gate": 1.63, + "review": 991.02, + "report": 0.06 + }, + "time_to_first_result_s": 996.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 37222 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4825.md b/eval/dataset/arms/copilot_v11_deep_val/pr4825.md new file mode 100644 index 000000000..62e2f56ab --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4825.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4825, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac3a2ce5a9b9) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read both diff hunks in situ: manager.py default_components (lines 368-383) adds "unet"; test adds test_lora_manager_discovers_unet_component (lines 578-619). +- [sweep] Branch tests/diffusion/lora/test_lora_manager.py:587 `if isinstance(layer, _FakeLinearBase):` — both arms reachable: container `down_block` returns layer unchanged, `proj` (FakeLinearBase) wraps; not dead. +- [sweep] Non-test file manager.py covered by the new test test_lora_manager_discovers_unet_component; the file's pytestmark (core_model+cpu) selects it, and the changed tuple is driven through `_replace_layers_with_lora`. +- [validated] manager.py:380 — adding unet matches StableDiffusionXLPipeline.unet (pipeline_sdxl.py:110); grep `self.unet =` finds no other built-in top-level unet (only a nested rmvpe unet at soulx_singer/modules/preprocess/rmvpe.py:236). +- [validated] registry.py:427 already includes "unet" in transformer_attrs, so treating unet as a denoiser component is consistent with the existing SP path. +- [validated] SDXL UNet layers are LoRA-replaceable: to_qkv QKVParallelLinear, to_q/to_k/to_v ColumnParallelLinear, ff.geglu.proj MergedColumnParallelLinear, to_out/out_proj RowParallelLinear (sdxl_unet.py:121-177); from_layer_diffusion wraps these classes (lora/utils.py). +- [validated] Packed to_qkv sublayer mapping is available: SDXL load_weights sets stacked_params_mapping (sdxl_unet.py:827-832) and DiffusionWorker calls load_model before init_lora_manager (diffusion_worker.py:230-232,338-348). +- [sweep] _lora_components consumers: only SenseNovaU1Pipeline (pipeline_sensenova_u1.py:543) defines it as ["language_model"]; no overlap with the new unet default, so no double-scan today. + +`vllm_omni/diffusion/lora/manager.py:380` [major] — This diff adds 'unet' to default_components (manager.py:380), so _replace_layers_with_lora now walks SDXL's UNet, but standard SDXL LoRA target_modules 'ff.net.0.proj'/'ff.net.2' never match the repo module names 'ff.geglu.proj'/'ff.out_proj': _match_target_modules (utils.py:20) does literal suffix matching with no remap, and the only ff.net->geglu/out_proj remap is in SDXLUNet2DConditionModel.load_weights (sdxl_unet.py). FF layers are therefore never wrapped, and their deltas are silently dropped. Please apply the same diffusers->internal name normalization in the LoRA matching path (or normalize SDXL target_modules before matching) and add a regression test using an SDXL FF target name. (evidence: Read vllm_omni/diffusion/lora/utils.py `_match_target_modules` (literal suffix match, no remap); read vllm_omni/diffusion/models/sdxl/sdxl_unet.py:835-848 (ff.net.* remap only in load_weights); grepped vllm_omni/diffusion/lora for `ff.net`/`geglu` (no matches).) + +`tests/diffusion/lora/test_lora_manager.py:580` [minor] — Confirmed: tests/diffusion/lora/test_lora_manager.py:19 sets pytestmark=[core_model, cpu]; test_lora_manager_discovers_unet_component is at line 580. .buildkite/test-amd-merge.yml:21 runs `pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py`, so the new test is selected in the AMD lane. gh_ci_read(4825) reports buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE while buildkite/vllm-omni passes; grep found no tests/diffusion command in the NPU pipeline files. Please confirm whether the AMD failure is pre-existing on main or introduced here. (evidence: gh_ci_read(4825) reports buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE while buildkite/vllm-omni passes; read .buildkite/test-amd-merge.yml:21 and grepped the NPU pipeline files for tests/diffusion.) + +`vllm_omni/diffusion/lora/manager.py:380` [minor] — SDXLTransformer2DModel.proj_in/proj_out (vllm_omni/diffusion/models/sdxl/sdxl_unet.py:281,296) are plain nn.Linear, which from_layer_diffusion cannot wrap, so any SDXL LoRA adapter targeting proj_in/proj_out will be silently skipped now that 'unet' is a default scanned component in _replace_layers_with_lora. Consider documenting or guarding this unsupported-target case. (evidence: Residual gap: SDXL LoRA path applies attention q/k/v (and to_qkv via packed mapping), but diffusers-style target names `ff.net.0.proj`/`ff.net.2` won't match `ff.geglu.proj`/`ff.out_proj` (sdxl_unet.py:835-848 remaps only inside load_weights); proj_in/proj_out are nn.Linear (sdxl_unet.py:281,296) and unwrappable by from_layer_diffusion. Verified from files read; documented as a minor comment.) + +`tests/diffusion/lora/test_lora_manager.py:616` [minor] — This diff adds test_lora_manager_discovers_unet_component (tests/diffusion/lora/test_lora_manager.py:580), but the synthetic leaf 'pipeline.unet.down_block.proj' and peft_helper without target_modules never exercise the SDXL-specific matching this new 'unet' default depends on: _matches_target returns True when target_modules is unset (manager.py:356), so the packed/sublayer branch at manager.py:405-419 (e.g. SDXL self-attention 'to_qkv') is skipped, and no real SDXL module names ('attn1.to_out.0', 'attn2.to_q', 'ff.geglu.proj'/'ff.out_proj') are asserted. Please add an SDXL-shaped UNet test with explicit target_modules (e.g. ['to_q','to_k','to_v','to_out.0']) and assert _lora_modules contains both '...attn1.to_qkv' (packed path) and '...attn2.to_q/to_k/to_v', so a naming regression cannot silently drop SDXL layers again. (evidence: Read tests/diffusion/lora/test_lora_manager.py (new test) and vllm_omni/diffusion/lora/manager.py lines 400-419.) + +`vllm_omni/diffusion/lora/manager.py:370` [nit] — Reword the comment above default_components: 'unet' is now a new default component, so _lora_components is not the only opt-in path and 'other pipelines unchanged' no longer holds. Suggested wording: 'Default denoising components (including unet for SDXL-style pipelines) plus any pipeline opt-ins via _lora_components.' (evidence: grep `self.unet =` in vllm_omni: only pipeline_sdxl.py:110 (plus nested rmvpe.py:236).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_val/pr4825.trace.json.gz new file mode 100644 index 000000000..2b315777d Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_val/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4837.cost.json b/eval/dataset/arms/copilot_v11_deep_val/pr4837.cost.json new file mode 100644 index 000000000..70c7d5275 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4837.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1041.2, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_val/runs/pr4837/run-20260813-134521-66bf63", + "recorded_at": "2026-08-13T14:02:40", + "attempts": [ + { + "invocation_id": "inv-914f48c1", + "outcome": "run:run-20260813-134521-66bf63", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-134521-66bf63", + "llm_calls": 69, + "input_tokens": 358964, + "output_tokens": 107182, + "usd": 0.347113, + "source": "spans" + } + ], + "attempt_usd_total": 0.347113, + "llm_calls": 69, + "input_tokens": 358964, + "output_tokens": 107182, + "usd": 0.347113, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-134521-66bf63", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3471, + "minutes": 17.32, + "ci_minutes": 0, + "input_tokens": 358964, + "output_tokens": 107182, + "tool_calls": 121, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6224, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 69, + "by_role": { + "agent": { + "usd": 0.329313, + "input_tokens": 350855, + "output_tokens": 93022, + "calls": 67 + }, + "reducer": { + "usd": 0.0178, + "input_tokens": 8109, + "output_tokens": 14160, + "calls": 2 + } + }, + "cache_read_tokens": 4899712, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.93, + "gate": 1.66, + "review": 1033.57, + "report": 0.07 + }, + "time_to_first_result_s": 1039.21 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 20549 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4837.md b/eval/dataset/arms/copilot_v11_deep_val/pr4837.md new file mode 100644 index 000000000..459db6ebb --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4837.md @@ -0,0 +1,36 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4837, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] Read both diff hunks in situ: pipeline_ming_imagegen.py:446-472 and orchestrator.py:1287-1301; both match the provided diff. +- [validated] orchestrator.py:1290 unconditional unwrap matches StagePool.submit_initial/submit_update (stage_pool.py:932-944 and 1001-1020), which both raise ValueError for list diffusion prompts; the prior already_submitted gate left the initial-submit path broken. +- [validated] thinker2imagegen (ming_flash_omni.py:346-450) returns a list of length 0 or 1, so len(diffusion_prompt) == 1 is exactly the produced case; issue #4839 confirms Hunyuan ar2diffusion is also list[dict] with a single AR output. +- [sweep] Multi-element diffusion lists remain rejected by design (stage_pool error text points to submitting independent requests); this PR does not change that contract and no current bridge produces >1 for these stages. +- [validated] pipeline_ming_imagegen.py:462-472: ZImagePipeline.forward returns DiffusionOutput(output=tensor) (pipeline_z_image.py:729) and DiffusionOutput has an .output field (data.py:1186), so raw = outputs.output + return outputs is type-correct; the removed outputs[0] indexing was a TypeError on DiffusionOutput. +- [sweep] pipeline_ming_imagegen.py:449 calls super().forward(z_req, prompt=None, height=..., width=..., num_inference_steps=..., guidance_scale=..., prompt_embeds=..., negative_prompt_embeds=..., generator=..., output_type='pt', return_dict=True), but ZImagePipeline.forward is def forward(self, req) (pipeline_z_image.py:413); the unexpected 'prompt' kwarg raises TypeError before the new output handling. +- [sweep] _ZPipelineSamplingParams (pipeline_ming_imagegen.py:58-76) lacks latents, cfg_normalize, extra_args, and output_type, all read unconditionally by ZImagePipeline.forward at pipeline_z_image.py:464-471; dropping the extra kwargs would still fail on these attributes. +- [sweep] DiffusionModelRunner._execute_request_list invokes self.pipeline.forward(batch) (diffusion_model_runner.py:462), so MingImagePipeline.forward is the real entry point; there is no alternate route that accepts those kwargs. + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:?` [blocker] — Confirmed at head 78cc4dcbb161. pipeline_ming_imagegen.py:449-464 calls super().forward(z_req, prompt=None, height=..., width=..., num_inference_steps=..., guidance_scale=..., prompt_embeds=..., negative_prompt_embeds=..., generator=..., output_type="pt", return_dict=True), but ZImagePipeline.forward is `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput` (pipeline_z_image.py:413, no **kwargs). Every request raises TypeError: forward() got an unexpected keyword argument 'prompt' before raw=outputs.output / return outputs can execute. Dropping the kwargs is not sufficient: ZImagePipeline.forward reads req.sampling_params.latents, .cfg_normalize, .extra_args, and .output_type, none of which exist on _ZPipelineSamplingParams (lines 62-76), and it calls self.encode_prompt -> _encode_prompt -> self.tokenizer/self.text_encoder while Ming sets both to None. Rework MingImagePipeline.forward to the single-request super().forward(z_req) contract (route cap_feats/negative_cap_feats through an overridden encode_prompt, or stop delegating to ZImagePipeline.forward) and re-run serve+curl at the merged head. (evidence: grep 'def forward' vllm_omni/diffusion/models/z_image/pipeline_z_image.py -> only line 413 `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:`; read pipeline_ming_imagegen.py:449-475 (the diff hunk) and pipeline_z_image.py:405-720 (fields accessed from req.sampling_params and encode_prompt via self.text_encoder); gh_pr_view(#4837) shows the test was recorded at vLLM-Omni commit 5c51782c, before the merge commit 78cc4dcb.) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449` [major] — The merged head still crashes in MingImagePipeline.forward. Line 449 calls `super().forward(z_req, prompt=None, height=..., width=..., num_inference_steps=..., guidance_scale=..., prompt_embeds=..., negative_prompt_embeds=..., generator=..., output_type='pt', return_dict=True)`, but the inherited `ZImagePipeline.forward(self, req: DiffusionRequestBatch)` (vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413) accepts only `req`, so this raises `TypeError: forward() got an unexpected keyword argument 'prompt'`. The orchestrator fix lets the request reach this line, so the #4839 ValueError is only moved, not resolved. Please fix this forward call (and add a regression test) or narrow the PR scope before closing #4839. (evidence: scope — description says 'Resolves #4839', but the merged head only moves the user-visible error: the orchestrator ValueError is fixed while the Ming pipeline still crashes (see review comment), so the linked issue is not actually resolved.) + +`vllm_omni/engine/orchestrator.py:?` [minor] — The diff removes the `already_submitted` guard at orchestrator.py:1290, so the initial diffusion submit path now unwraps a single-element `diffusion_prompt` list; StagePool.submit_initial and submit_update both reject list diffusion prompts, and the existing bridge test only covers the LLM path. Add a diffusion-stage fake returning `[{"prompt": "", "extra": {"thinker_hidden_states": ...}}]` and assert `submit_initial` receives the unwrapped dict, plus an `already_submitted=True` case for `submit_update`. (evidence: read tests/engine/test_orchestrator_stage_input_bridge.py and vllm_omni/engine/stage_pool.py:932-1020 (both diffusion submit paths raise ValueError on list); issue #4839 comment asks for a Hunyuan before/after test.) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449` [minor] — CI at this head: buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are FAILURE while buildkite/vllm-omni is SUCCESS. These lanes may be pre-existing on main; lane logs are needed before attributing them to this diff. (evidence: CI — gh_ci_read(#4837): buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE, buildkite/vllm-omni SUCCESS; this review cannot attribute the red lanes to the diff vs pre-existing on main.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_val/pr4837.trace.json.gz new file mode 100644 index 000000000..454deb3f8 Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_val/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4893.cost.json b/eval/dataset/arms/copilot_v11_deep_val/pr4893.cost.json new file mode 100644 index 000000000..496e4f17a --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4893.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1826.4, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v11_deep_val/runs/pr4893/run-20260813-140243-05689e", + "recorded_at": "2026-08-13T14:33:07", + "attempts": [ + { + "invocation_id": "inv-d5d960f7", + "outcome": "run:run-20260813-140243-05689e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-140243-05689e", + "llm_calls": 112, + "input_tokens": 876701, + "output_tokens": 174096, + "usd": 0.635074, + "source": "spans" + } + ], + "attempt_usd_total": 0.635074, + "llm_calls": 112, + "input_tokens": 876701, + "output_tokens": 174096, + "usd": 0.635074, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-140243-05689e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6351, + "minutes": 30.4, + "ci_minutes": 0, + "input_tokens": 876701, + "output_tokens": 174096, + "tool_calls": 183, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9494, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 112, + "by_role": { + "agent": { + "usd": 0.611439, + "input_tokens": 865363, + "output_tokens": 155424, + "calls": 110 + }, + "reducer": { + "usd": 0.023635, + "input_tokens": 11338, + "output_tokens": 18672, + "calls": 2 + } + }, + "cache_read_tokens": 7661440, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.05, + "gate": 1.65, + "review": 1817.1, + "report": 0.06 + }, + "time_to_first_result_s": 1823.85 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 27194 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4893.md b/eval/dataset/arms/copilot_v11_deep_val/pr4893.md new file mode 100644 index 000000000..d722d1881 --- /dev/null +++ b/eval/dataset/arms/copilot_v11_deep_val/pr4893.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4893, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] hunyuan_fused_moe.py:91 indexed access hidden_states.shape[0] — read file; hidden_states is a torch.Tensor from kwargs['hidden_states']/args[0] (args guarded by `if hidden_states is None and args`), so shape[0] is a Python int; no unsafe first-element assumption. +- [sweep] tests/diffusion/distributed/test_expert_parallel_layout.py:24 `if reduce_scatter is not None` — read file; reachable only when fake passes reduce_scatter; correctly avoids setting attribute otherwise. +- [sweep] parallel_state.py:868 `if use_moe_parallel_mapping` — read file; both arms reachable (enable_expert_parallel on/off); EP-on creates vLLM PCP companion; layout test asserts `_PCP is not _SP` and world_size/local_group. +- [sweep] parallel_state.py:875 `else` — read file; non-MoE branch keeps diffusion tensor coordinator; covered by test_cfg_parallel_keeps_diffusion_dp_without_ep (which asserts `vllm_parallel_state._DP is parallel_state._DP`). +- [sweep] parallel_state.py:882 `if use_moe_parallel_mapping` — read file; EP-on builds vLLM DP companion from cfg-dp ranks; layout test asserts `_DP is not _DP` and world_size 4. +- [sweep] hunyuan_fused_moe.py:31/34/38/47 new branches — read file; unavailable-context and existing-metadata early returns are correct, existing-metadata is unit-tested; single-rank/None-DP return is reachable but untested; None-gather failure path raises RuntimeError. +- [validated] parallel_state.py:868-905 — rank layouts preserved: test_expert_parallel_layout.py asserts PCP local_group [0,2], DP [0,8,16,24], EP 16-rank group, and vllm_group_names == ['vllm_pcp','vllm_tp','vllm_dp','vllm_ep']. +- [validated] hunyuan_fused_moe.py:31-50 — _set_forward_context_dp_metadata correctly gathers int token counts via all_gather_object on dp_group.cpu_group and constructs _vllm_fc.DPMetadata; unit test asserts [1024, 1025]. + +`pyproject.toml:1` [minor] — In-repo package metadata does not declare any vLLM version floor: pyproject.toml uses dynamic dependencies, setup.py builds install_requires from requirements/.txt, and requirements/ contains no vllm constraint. The patch newly depends on vLLM-provided `init_model_parallel_group(..., use_device_communicator=True)` (parallel_state.py) and `_vllm_fc.DPMetadata` (hunyuan_fused_moe.py), so the 0.24 compatibility assumption is only enforced by external pins (e.g. Docker). Consider an explicit vllm>=0.24 floor/pin or documented check so older vLLM installations fail fast. (evidence: [sweep] Dependency floor: pyproject declares dependencies dynamically and no vllm pin exists in requirements/; vllm version comes from external Docker pins (docker/Dockerfile.xpu v0.24.0). The new `use_device_communicator=True` kwarg and `_vllm_fc.DPMetadata` are 0.24 APIs, so an older vLLM floor is assumed rather than enforced in-repo — recorded as an assumption, not a posted comment.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:38` [minor] — In `_set_forward_context_dp_metadata`, the line-38 early return for `dp_group is None` / `world_size <= 1` is reachable but untested: `TestSetForwardContextDPMetadata` only covers `world_size == 2` and the existing-metadata path. Please add a unit test for the None/single-rank return, or state explicitly that it is intentionally out of scope. (evidence: [sweep] hunyuan_fused_moe.py:31/34/38/47 new branches — read file; unavailable-context and existing-metadata early returns are correct, existing-metadata is unit-tested; single-rank/None-DP return is reachable but untested; None-gather failure path raises RuntimeError.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:91` [minor] — Verified: `_set_forward_context_dp_metadata` is called unconditionally at hunyuan_fused_moe.py:91 from `_num_tokens_pre_hook`. In `initialize_model_parallel`, `vllm_parallel_state._DP = _DP` is set to the diffusion DP coordinator before the `if use_moe_parallel_mapping:` block and is only replaced with a vLLM GroupCoordinator inside that block. The helper guards only on `dp_group.world_size > 1` (hunyuan_fused_moe.py:38-46) and uses `dp_group.cpu_group`, which the diffusion GroupCoordinator also exposes. Therefore in a non-EP (`enable_expert_parallel=False`) DP>1 run, every FusedMoE forward performs a blocking CPU `all_gather_object` over the diffusion DP group and writes `ForwardContext.dp_metadata` that the non-EP path does not consume. Gate the call on the MoE-EP mapping (e.g. only when `enable_expert_parallel` is true, or `_vllm_ps._DP is not parallel_state._DP`) and add a non-EP DP>1 test. (evidence: Read vllm_omni/diffusion/distributed/parallel_state.py:800-816 (non-EP sets vllm_parallel_state._DP = _DP) and hunyuan_fused_moe.py:38-50 (guard checks only world_size and uses dp_group.cpu_group).) + +`vllm_omni/diffusion/distributed/parallel_state.py:526` [minor] — Verified: `init_vllm_model_parallel_group` (parallel_state.py:515-527) forwards `use_device_communicator=True` to vLLM and is used for `_PCP`/`_TP`/`_DP`/`_EP` whenever `use_moe_parallel_mapping` is true in `initialize_model_parallel`. `DiffusionWorker.init_device` calls `initialize_model_parallel` at diffusion_worker.py:306 without platform gating, so MoE-EP runs on AMD/NPU exercise this path. CI shows buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE while buildkite/vllm-omni is SUCCESS. Please confirm whether the AMD/NPU failures are pre-existing on main or caused by this new coordinator path, and fix or explicitly acknowledge before merge. (evidence: gh_ci_read(4893) shows amd-ci and npu-ci FAILURE and vllm-omni SUCCESS; parallel_state.py:515-527 introduces the new call; diffusion_worker.py:306 is the shared call site.) + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:?` [minor] — The added `TestSetForwardContextDPMetadata` tests call `_set_forward_context_dp_metadata(1024)` directly against a MagicMock `_DP` (test_hunyuan_fused_moe.py:74 and :89), so they verify only the helper body, not the production pre-hook wiring registered in `HunyuanFusedMoEDefault.__new__` (hunyuan_fused_moe.py:87-91). If `__new__` stopped registering `_num_tokens_pre_hook` or stopped invoking `_set_forward_context_dp_metadata`, these tests would still pass. Consider an integration-style test that constructs `HunyuanFusedMoEDefault` with a mocked upstream `FusedMoE`, fires the registered pre-hook, and asserts the metadata helper is invoked with `hidden_states.shape[0]`. (evidence: Read tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:53-93 — no test exercises HunyuanFusedMoEDefault.__new__'s hook registration; the helper is called directly at line 74.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v11_deep_val/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v11_deep_val/pr4893.trace.json.gz new file mode 100644 index 000000000..ef4fe88f9 Binary files /dev/null and b/eval/dataset/arms/copilot_v11_deep_val/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/manifest.json b/eval/dataset/arms/copilot_v12_hybrid_val/manifest.json new file mode 100644 index 000000000..86f786216 --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v12_hybrid_val", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 5, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.cost.json b/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.cost.json new file mode 100644 index 000000000..a8ebe41be --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1183.4, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v12_hybrid_val/runs/pr4810/run-20260813-143840-e4a76f", + "recorded_at": "2026-08-13T14:58:19", + "attempts": [ + { + "invocation_id": "inv-e7dd8f8e", + "outcome": "run:run-20260813-143840-e4a76f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-143840-e4a76f", + "llm_calls": 171, + "input_tokens": 1671254, + "output_tokens": 250249, + "usd": 1.211413, + "source": "spans" + } + ], + "attempt_usd_total": 1.211413, + "llm_calls": 171, + "input_tokens": 1671254, + "output_tokens": 250249, + "usd": 1.211413, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-143840-e4a76f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2114, + "minutes": 19.66, + "ci_minutes": 0, + "input_tokens": 1671254, + "output_tokens": 250249, + "tool_calls": 316, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9795, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 171, + "by_role": { + "agent": { + "usd": 1.184222, + "input_tokens": 1654547, + "output_tokens": 229662, + "calls": 169 + }, + "reducer": { + "usd": 0.027191, + "input_tokens": 16707, + "output_tokens": 20587, + "calls": 2 + } + }, + "cache_read_tokens": 17959296, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.82, + "gate": 2.23, + "review": 1173.32, + "report": 0.06 + }, + "time_to_first_result_s": 1179.41 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 36777 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.md b/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.md new file mode 100644 index 000000000..8b6208b16 --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] tests/model_executor/models/test_kv_cache_scale_mapper.py:78-82 — the source-grep test only checks the 4 AR/old loader files; the 5th call site vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 still calls the removed API and is deliberately delegated to PR #4808 per issue #4809 comment. +- [validated] vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py:2178 — HunyuanImage3ForConditionalGeneration.load_weights uses AutoWeightsLoader, so the removed manual branch is covered by the outer mapper. +- [validated] vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py:443-448 — Qwen2ForCausalLM.load_weights uses AutoWeightsLoader, so Qwen2Model's removed guarded fallback is redundant and correct to drop. +- [validated] installed vllm/model_executor/models/utils.py:407-416 — AutoWeightsLoader.load_weights finds a module with quant_config and merges quant_config.get_cache_scale_mapper() into the applied mapper before dispatching to submodule load_weights. +- [sweep] vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:810 → MammothModa2ARForConditionalGeneration inherits Qwen2_5_VLForConditionalGeneration.load_weights (AutoWeightsLoader) → dispatches to MammothModa2Qwen2ForCausalLM.load_weights; the inner get_cache_scale_mapper().apply() at line 487 runs after the outer AutoWeightsLoader already applied the same mapper. +- [validated] installed vllm/model_executor/layers/quantization/fp8.py:223-232 and base_config.py:195-223 — the cache-scale mappers are regex/suffix renames with a `(? AutoWeightsLoader and already applies the quant-config cache-scale mapper before dispatching to this delegated language_model loader. Standard mappers are idempotent so this is not a correctness break. Drop the block and classify this as a delegated loader in tests, or document the specific direct-call path that requires it. (evidence: Read mammoth_moda2.py lines 476-500 and 810; confirmed MammothModa2ARForConditionalGeneration inherits Qwen2_5_VLForConditionalGeneration.load_weights which uses AutoWeightsLoader; read installed utils.py:407-416.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:112` [minor] — In `test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale`, both parametrizations call `HunyuanModel.load_weights` / `Qwen2Model.load_weights` directly with the already-remapped `_MAPPED_SCALE_NAME` (line 112), so this only proves the inner loaders accept a mapped name. The PR's hunyuan_image3.py and qwen2_old.py changes delete the old `get_cache_scale` branch and rely on the outer `AutoWeightsLoader` (hunyuan_image3.py:2178, qwen2_old.py:443-447) to apply `get_cache_scale_mapper()` before delegation, so a regression in that mapping step would not be caught. Add a sibling case that drives the unmapped `_SOURCE_SCALE_NAME` through the outer loader class and asserts the mapped scale is loaded. (evidence: Read the new test file (delegated test lines 99-115) and installed vLLM AutoWeightsLoader.load_weights, which applies the quant-config mapper before `_load_module`.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:12` [minor] — tests/model_executor/models/test_kv_cache_scale_mapper.py:14 sets pytestmark = [pytest.mark.core_model, pytest.mark.cpu], so the new tests are collected by the AMD merge step at .buildkite/test-amd-merge.yml:32 (`pytest -sv tests/model_executor -m 'core_model and cpu'`). CI currently shows buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE while buildkite/vllm-omni passes. Confirm the AMD/NPU failures are pre-existing on main and not introduced by this diff (or fix the introduced break) before merging. (evidence: gate_report: FAILING CHECKS buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci; .buildkite/test-amd-merge.yml:32 runs tests/model_executor with `-m 'core_model and cpu'`; pyproject.toml:236 registers `core_model`.) + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:486` [minor] — The changed paths remove the legacy get_cache_scale() KV-cache-scale handling from HunyuanImage3 AR, MammothModa2 AR, MiMo-Audio, and Qwen2.5-Omni qwen2_old. The only added validation (tests/model_executor/models/test_kv_cache_scale_mapper.py) is marked pytest.mark.cpu and uses fake models with a fake quant config/mapper, so it does not exercise a real FP8/compressed-tensors checkpoint load. Please add or point to a real FP8/compressed-tensors checkpoint load covering these loaders before merge. (evidence: [sweep] blast radius: changed paths affect quantized checkpoints for HunyuanImage3 AR, MammothModa2 AR, MiMo-Audio, and Qwen2.5-Omni old Qwen2; PR validation is CPU fakes only, no real FP8/compressed-tensors checkpoint load shown.) + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:487` [minor] — The new load_weights path calls self.quant_config.get_cache_scale_mapper() (line 487). pyproject declares dependencies dynamically and setup.py does not pin vLLM, so the minimum supported vLLM is not visible there. Please confirm the declared minimum vLLM version includes QuantizationConfig.get_cache_scale_mapper(), or add an explicit floor. (evidence: [sweep] dependency floor for get_cache_scale_mapper not fully verified against pyproject lower bound; PR env pins vLLM 0.24.0 and the API landed in v0.23.0rc2 per the description.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:81` [minor] — The guard at line 81 (`assert ".get_cache_scale(" not in source`) does not cover the removed qwen2_old.py pattern `get_cache_scale = getattr(self.quant_config, "get_cache_scale", None)` + `get_cache_scale(name)`, since that text contains no `.get_cache_scale(`. The qwen2_old behavior test would also not catch its reintroduction because _QuantConfigWithoutOldCacheScale lacks get_cache_scale, so the old alias returns None and falls through. Tighten the assertion to require that `get_cache_scale` appears only as part of `get_cache_scale_mapper` rather than relying on the behavior test. (evidence: qwen2_old.py removed lines in the diff show the old `getattr(self.quant_config, "get_cache_scale", None)` pattern; test line 81 checks only the literal `.get_cache_scale(` substring.) + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:?` [nit] — In MammothModa2Qwen2ForCausalLM.load_weights the new call is at line 487 (line 486 is the guard `if self.quant_config is not None:`). ComponentQuantizationConfig (vllm_omni/quantization/component_config.py:75) indeed does not override get_cache_scale_mapper; if MammothModa2 can be constructed with ComponentQuantizationConfig, confirm the inherited QuantizationConfig.get_cache_scale_mapper returns the expected FP8 mapper or add an override/guard. (evidence: component_config.py does not override get_cache_scale_mapper.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.trace.json.gz new file mode 100644 index 000000000..8c16724ef Binary files /dev/null and b/eval/dataset/arms/copilot_v12_hybrid_val/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.cost.json b/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.cost.json new file mode 100644 index 000000000..a3b5705d6 --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 857.7, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v12_hybrid_val/runs/pr4816/run-20260813-143839-f8a305", + "recorded_at": "2026-08-13T14:52:53", + "attempts": [ + { + "invocation_id": "inv-ab69fa61", + "outcome": "run:run-20260813-143839-f8a305", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-143839-f8a305", + "llm_calls": 54, + "input_tokens": 515696, + "output_tokens": 93525, + "usd": 0.325478, + "source": "spans" + } + ], + "attempt_usd_total": 0.325478, + "llm_calls": 54, + "input_tokens": 515696, + "output_tokens": 93525, + "usd": 0.325478, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-143839-f8a305", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3255, + "minutes": 14.24, + "ci_minutes": 0, + "input_tokens": 515696, + "output_tokens": 93525, + "tool_calls": 83, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5539, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 54, + "by_role": { + "planner": { + "usd": 0.000515, + "input_tokens": 87, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.303846, + "input_tokens": 506940, + "output_tokens": 76091, + "calls": 51 + }, + "reducer": { + "usd": 0.021116, + "input_tokens": 8669, + "output_tokens": 17034, + "calls": 2 + } + }, + "cache_read_tokens": 3087488, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.24, + "gate": 1.66, + "review": 849.25, + "report": 0.07 + }, + "time_to_first_result_s": 854.19 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 17449 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.md b/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.md new file mode 100644 index 000000000..1659537e2 --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.md @@ -0,0 +1,36 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] Grepped repo for `openai_serving_tokenization`: 0 matches — no stale references to the old app-state key remain. +- [sweep] Grepped repo for `serving_tokenization`: 9 matches, all in the two changed files (api_server.py:754,1017,1157,1248,1286,1329; test_serving_speech.py:2533,2546,2615). +- [validated] Read api_server.py hunks in situ (751-754 pure-diffusion None; 1014-1022 ServingTokenization; 1154-1157, 1245-1248, 1283-1286, 1326-1329 getattr fallbacks) — all renamed consistently. +- [validated] Read test_serving_speech.py changed hunks (2530-2533, 2543-2546, 2612-2615) — they set state.serving_tokenization and exercise the 404 getattr branches. +- [sweep] Read `_create_speech_error_json_response` (api_server.py:369-379): it still calls `base(raw_request).create_error_response(...)` with no None guard. +- [sweep] Read serving_speech.py for_diffusion (416-433) and upload_voice (1214+): pure diffusion installs a real speech handler, and undecodable audio raises ValueError, so `_create_speech_error_json_response` is reachable with state.serving_tokenization=None. +- [sweep] Checked changed_symbol_consumers for omni_init_app_state/create_chat_completion/create_speech/create_speech_batch/create_audio_generate: no stale in-repo consumer of the old attr remains. +- [sweep] Checked pyproject.toml/requirements: vllm is not declared as a repo dependency, so no lower-bound compatibility check for the new upstream base/ServingTokenization API is possible from repo metadata. + +`vllm_omni/entrypoints/openai/api_server.py:754` [major] — In pure-diffusion mode `state.serving_tokenization = None` (api_server.py:754) while `state.openai_serving_speech` is still installed via `OmniOpenAIServingSpeech.for_diffusion`, so `Omnispeech()` is non-None and the handler-is-None branches don't run. `upload_voice`'s ValueError and both-`audio_sample`+`speaker_embedding` branches call `_create_speech_error_json_response` (api_server.py:376), which calls `base(raw_request).create_error_response(...)` with no None guard. Since upstream `base()` returns `request.app.state.serving_tokenization` (None here), this raises AttributeError and surfaces as 500 instead of the intended 400. Guard the `base()` result in `_create_speech_error_json_response` and add a pure-diffusion regression test for upload_voice validation errors. (evidence: Read api_server.py:369-379 and 744-756; read serving_speech.py:416-433 and 1214+; repo grep shows these are the only `serving_tokenization` references.) + +`tests/entrypoints/openai_api/test_serving_speech.py:?` [minor] — The diff updates the 404 tests to set `raw_request.app.state.serving_tokenization = fake_base` instead of `openai_serving_tokenization` (test_serving_speech.py:2533). Those tests only exercise the handler-is-None fallback in create_speech/create_speech_batch/create_audio_generate; the reported 500 originates in `_create_speech_error_json_response` -> `base(raw_request)`, and existing voice tests patch `api_server_module.base` through `_patch_api_server_base` (test_serving_speech.py:2339-2353), so no test fails on the real `base()` path. Add an unmocked regression test that initializes app state via the changed `omni_init_app_state` path (so the pre-fix state lacks `serving_tokenization`) and calls `upload_voice` with undecodable audio or both `audio_sample`+`speaker_embedding`, asserting a 400 OpenAI ErrorResponse rather than 500; also cover `serving_tokenization=None` on the pure-diffusion path. (evidence: Read test_serving_speech.py:2340-2351 (_patch_api_server_base mocks base) and 2514-2615 (api_server tests); read api_server.py:369-379.) + +`pyproject.toml:1` [minor] — vllm-omni's packaging metadata (pyproject.toml with dynamic deps, setup.py platform requirement routing, and requirements/*.txt) does not declare or pin vllm, so the rename to upstream serving_tokenization/ServingTokenization has no repo-metadata lower-bound guard. Consider adding a minimum vllm version pin or a CI compatibility check for the upstream version this change requires. (evidence: Checked pyproject.toml/requirements: vllm is not declared as a repo dependency, so no lower-bound compatibility check for the new upstream base/ServingTokenization API is possible from repo metadata.) + +`vllm_omni/entrypoints/openai/api_server.py:754` [minor] — Confirmed against gh_ci_read for PR 4816: buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, and buildkite/vllm-omni-npu-ci are all FAILURE, while buildkite/vllm-omni and the GitHub checks pass. The lane logs are not readable with the tools available here, so this rename cannot be exonerated or blamed from CI state alone. Minor ask: confirm these three lanes are pre-existing on main or attach the failing logs before merge. (evidence: gate_report lists 3 failing Buildkite lanes (amd/intel/npu); cannot read lane logs from this environment, so attribution to this PR is recorded only as a minor ask.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.trace.json.gz new file mode 100644 index 000000000..deb9f2c74 Binary files /dev/null and b/eval/dataset/arms/copilot_v12_hybrid_val/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.cost.json b/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.cost.json new file mode 100644 index 000000000..1c72c686b --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 859.9, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v12_hybrid_val/runs/pr4825/run-20260813-143838-2c0697", + "recorded_at": "2026-08-13T14:52:56", + "attempts": [ + { + "invocation_id": "inv-104fe864", + "outcome": "run:run-20260813-143838-2c0697", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-143838-2c0697", + "llm_calls": 115, + "input_tokens": 735264, + "output_tokens": 154650, + "usd": 0.546098, + "source": "spans" + } + ], + "attempt_usd_total": 0.546098, + "llm_calls": 115, + "input_tokens": 735264, + "output_tokens": 154650, + "usd": 0.546098, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-143838-2c0697", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5461, + "minutes": 14.3, + "ci_minutes": 0, + "input_tokens": 735264, + "output_tokens": 154650, + "tool_calls": 192, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6478, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 115, + "by_role": { + "agent": { + "usd": 0.526845, + "input_tokens": 724016, + "output_tokens": 139939, + "calls": 113 + }, + "reducer": { + "usd": 0.019254, + "input_tokens": 11248, + "output_tokens": 14711, + "calls": 2 + } + }, + "cache_read_tokens": 6572672, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.28, + "gate": 1.61, + "review": 852.88, + "report": 0.07 + }, + "time_to_first_result_s": 857.81 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 28905 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.md b/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.md new file mode 100644 index 000000000..3df210f48 --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4825, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac3a2ce5a9b9) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read manager.py hunk in situ (offset 300, lines 350-430): default_components tuple now includes `unet`; loop guards `hasattr`/`isinstance(nn.Module)` unchanged. +- [sweep] Read test hunk in situ (offset 520, lines 575-620): test_lora_manager_discovers_unet_component builds synthetic `pipeline.unet.down_block.proj` and asserts replacement; peft_helper has only `r`, no target_modules. +- [sweep] Branch `if isinstance(layer, _FakeLinearBase):` (test line 587): both arms reachable — container modules (`unet`, `down_block`) return `layer` unchanged, `proj` returns `_DummyBaseLayerWithLoRA`; not dead/redundant. +- [validated] tests/diffusion/lora/test_lora_manager.py:617-619: new test asserts `unet.down_block.proj` in manager._lora_modules and that the tree module is actually replaced (not just recorded). +- [validated] vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py:46,110: SDXL pipeline declares `_dit_modules=["unet"]` and assigns `self.unet = SDXLUNet2DConditionModel`. +- [sweep] Blast radius: grep `self.unet =` across vllm_omni shows only sdxl/pipeline_sdxl.py top-level; soulx_singer rmvpe.py:236 `self.unet` is nested inside a preprocess submodule, not a top-level pipeline attr; flux2_klein/ovis_image hits are docstrings. +- [sweep] `_lora_components` consumers: only sensenova_u1 pipeline_sensenova_u1.py:543 sets it (`["language_model"]`); adding `unet` to defaults does not conflict with the opt-in list. +- [sweep] DiffusionLoRAManager consumers: diffusion_worker.py:342, ltx2 pipelines; none depend on the exact default_components tuple, so no stale caller. + +`vllm_omni/diffusion/models/sdxl/sdxl_unet.py:?` [major] — Confirmed: adding `unet` to manager.py default_components means SDXL adapters are now scanned, but SDXLUNet2DConditionModel.load_weights (sdxl_unet.py:835-845) remaps diffusers `.ff.net.0.proj.` -> `.ff.geglu.proj.` and `.ff.net.2.` -> `.ff.out_proj.` while DiffusionLoRAManager._replace_layers_with_lora (manager.py:340-418) matches raw full module names and _get_lora_weights (manager.py:487-510) does no ff remap; the packed fallback only derives from SDXL's attn1.to_qkv stacked mapping. SDXL LoRAs targeting ff.net.0.proj/ff.net.2 therefore silently skip those FF layers. Add the same normalization (or explicit reject/log) plus a round-trip FF-target test. (evidence: Read vllm_omni/diffusion/lora/manager.py (target matching uses the tree module name with no ff remap) and vllm_omni/diffusion/models/sdxl/sdxl_unet.py:835-848 (load_weights remaps ff.net.0.proj and ff.net.2).) + +`vllm_omni/diffusion/lora/manager.py:380` [minor] — This diff adds "unet" to default_components (manager.py:374-380) so SDXL's denoiser is scanned, but there is still no `to_out.0` -> `to_out` normalization anywhere in vllm_omni/diffusion/lora/. SDXL stores attention outputs as nn.ModuleList (sdxl_unet.py:151,175), so live paths are `...attn1.to_out.0` / `...attn2.to_out.0`, while common SDXL adapters target `to_out.0`. Since `_expected_lora_modules` is derived from vLLM get_supported_lora_modules before replacement (manager.py:85-88) and activation falls back to the last name segment (`0`), those weights can be silently skipped. Please state why the attempted normalization was reverted and confirm with a real SDXL adapter target-name CPU test that `to_out`/`ff` weights actually load and activate. (evidence: Read manager.py:85-88,374-380; sdxl_unet.py:151,175,243,256; pr_context commit timeline 632efb42 and ac3a2ce5.) + +`vllm_omni/diffusion/lora/manager.py:?` [minor] — Anchor verified: manager.py:380 adds "unet" to default_components, and tests/diffusion/lora/test_lora_manager.py:19 marks the file core_model+cpu, so the new test_lora_manager_discovers_unet_component runs in AMD's tests/diffusion -m 'core_model and cpu' lane (.buildkite/test-amd-merge.yml:21/44). gh_ci_read(4825) confirms buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are FAILURE while buildkite/vllm-omni is SUCCESS. Correction: .buildkite/test-npu-ready.yaml only runs a TTS -m 'core_model' test, not the diffusion core_model+cpu lane, so the NPU failure cannot come from this new test; still confirm whether the AMD and NPU failures are pre-existing on main and link both failing job logs before merge. (evidence: gh_ci_read(4825) returned FAILURE for buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci and SUCCESS for buildkite/vllm-omni; .buildkite/test-merge.yml and test-amd-merge.yml run tests/diffusion with -m 'core_model and cpu'.) + +`vllm_omni/diffusion/lora/manager.py:380` [minor] — Confirmed: manager.py:380 adds "unet" to the shared default_components tuple that _replace_layers_with_lora iterates for every pipeline, while manager.py:370 frames _lora_components as the opt-in channel for pipeline-specific components. Grep shows StableDiffusionXLPipeline (pipeline_sdxl.py:110) is the only pipeline with a top-level self.unet, and SenseNovaU1Pipeline already models pipeline-scoped components via _lora_components (pipeline_sensenova_u1.py:543). Prefer StableDiffusionXLPipeline._lora_components = ['unet'] over widening the shared default, or document 'unet' as a universal denoiser name. (evidence: Read manager.py:368-382; grep self.unet across vllm_omni; grep _lora_components across repo; read diffusion_worker.py:342.) + +`tests/diffusion/lora/test_lora_manager.py:600` [minor] — The new test verifies that _replace_layers_with_lora scans the 'unet' component, but it does not exercise the real SDXL replacement path: peft_helper has no target_modules (manager.py:351 returns True for every module), and the only replaceable layer is a dummy _FakeLinearBase. SDXLUNet2DConditionModel's actual QKVParallelLinear/MergedColumnParallelLinear/RowParallelLinear layers (sdxl_unet.py:129-177, routed via utils.py:58) have no committed test (grep tests/ for SDXL returns nothing). Consider a test using real SDXL layer classes and a target_modules list, or commit lora_sdxl_test.py as an integration test; run pytest -m 'core_model and cpu' tests/diffusion/lora/test_lora_manager.py. (evidence: tests/diffusion/lora/test_lora_manager.py:580-619; vllm_omni/diffusion/lora/manager.py:351-355; vllm_omni/diffusion/lora/utils.py:58-78; vllm_omni/diffusion/models/sdxl/sdxl_unet.py:121-177; grep tests/ for SDXL symbols returned no matches.) + +`vllm_omni/diffusion/lora/manager.py:85` [nit] — The raw review notes that vLLM `get_supported_lora_modules` returns leaf-name suffixes, so an SDXL `to_out` module stored as `nn.ModuleList` contributes suffix `0` to `_expected_lora_modules`; the downstream vLLM checkpoint-filtering effect cannot be verified in-repo. Since `check_unexpected_modules` raises when a checkpoint module suffix is not in `expected_lora_modules` (manager.py:85-88), please confirm that `to_out.0` checkpoint weights are not rejected or dropped by this leaf-suffix behavior, or scope a follow-up to verify the real SDXL checkpoint path. (evidence: vLLM get_supported_lora_modules returns leaf suffixes, so SDXL `to_out` (nn.ModuleList -> leaf `to_out.0`) contributes suffix `0` to `_expected_lora_modules`; downstream vLLM checkpoint filtering effect cannot be verified in-repo, so this stays out of review_comments.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.trace.json.gz new file mode 100644 index 000000000..8185d31d7 Binary files /dev/null and b/eval/dataset/arms/copilot_v12_hybrid_val/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.cost.json b/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.cost.json new file mode 100644 index 000000000..7f8092f29 --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1401.0, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v12_hybrid_val/runs/pr4837/run-20260813-143838-cd70a6", + "recorded_at": "2026-08-13T15:01:57", + "attempts": [ + { + "invocation_id": "inv-accbf38a", + "outcome": "run:run-20260813-143838-cd70a6", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-143838-cd70a6", + "llm_calls": 154, + "input_tokens": 1240621, + "output_tokens": 229002, + "usd": 0.892332, + "source": "spans" + } + ], + "attempt_usd_total": 0.892332, + "llm_calls": 154, + "input_tokens": 1240621, + "output_tokens": 229002, + "usd": 0.892332, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-143838-cd70a6", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8923, + "minutes": 23.31, + "ci_minutes": 0, + "input_tokens": 1240621, + "output_tokens": 229002, + "tool_calls": 242, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9443, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 154, + "by_role": { + "agent": { + "usd": 0.849007, + "input_tokens": 1224200, + "output_tokens": 193703, + "calls": 151 + }, + "reducer": { + "usd": 0.043325, + "input_tokens": 16421, + "output_tokens": 35299, + "calls": 3 + } + }, + "cache_read_tokens": 11313408, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.74, + "gate": 2.1, + "review": 1392.55, + "report": 0.07 + }, + "time_to_first_result_s": 1398.43 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 28683 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.md b/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.md new file mode 100644 index 000000000..19cf5914e --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4837, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [sweep] orchestrator.py:1290 `if len(diffusion_prompt) == 1:` — read in-situ (offset 49079 window includes line 1290); branch now reachable on both already_submitted True and False; no dead/redundant arm. +- [sweep] pipeline_ming_imagegen.py:446/462/472 — read in-situ; `raw = outputs.output` and `return outputs` verified against ZImagePipeline.forward returning `DiffusionOutput(output=image, stage_durations=...)`. +- [validated] orchestrator.py:1275-1297: `submit_initial`/`submit_update` both reject list prompts for diffusion stages (`StagePool.submit_initial/submit_update` raise ValueError), so unwrapping single-element lists before both submit paths is required for Ming's thinker2imagegen which returns `[{"prompt": "", "extra": ...}]`. +- [validated] stage_pool.py:932-1010: `submit_initial` and `submit_update` both raise ValueError when `isinstance(request, list)` for stage_type == 'diffusion'; this confirms the changed hunk addresses a real crash path. +- [validated] ming_flash_omni.py:346-433: `thinker2imagegen` returns `[{"prompt": "", "extra": extra}]` (always a list of one dict), so the unwrap is required for Ming imagegen. +- [sweep] Non-test files touched (pipeline_ming_imagegen.py, orchestrator.py) — no test files were changed in this diff; grep for `MingImagePipeline` in tests only found config_factory assertions unrelated to forward()/orchestrator behavior. +- [sweep] Changed symbol `forward` consumers — grep across repo; only MingImagePipeline overrides ZImagePipeline.forward with super().forward; registry references model class and post-process funcs, no other forward consumer affected. +- [sweep] Changed symbol `_forward_to_next_stage` consumers — tests/engine/test_orchestrator_stage_input_bridge.py and tests/engine/test_orchestrator_kv_sender_info.py call it; no assertions cover the diffusion list-prompt unwrap branch. + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449` [blocker] — Confirmed blocker: MingImagePipeline.forward (vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449) calls `super().forward(z_req, prompt=None, height=..., width=..., num_inference_steps=..., guidance_scale=..., prompt_embeds=..., negative_prompt_embeds=..., generator=..., output_type="pt", return_dict=True)`, but ZImagePipeline.forward is `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput` (vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413) with no **kwargs, so this raises TypeError before reaching `raw = outputs.output`. Additionally, the `_ZPipelineSamplingParams` shim (pipeline_ming_imagegen.py:40-52) lacks `latents`, `cfg_normalize`, `extra_args`, and `output_type`, which ZImagePipeline.forward reads directly (pipeline_z_image.py:464,466,467,471); removing the kwargs alone is insufficient. Align the call with the req-only contract and populate the shim, then re-run the serve+curl imagegen test. (evidence: Read pipeline_z_image.py:413 (forward(self, req) only) and pipeline_ming_imagegen.py:40-52 (_ZPipelineSamplingParams fields); grep confirms a single `def forward` in pipeline_z_image.py.) + +`vllm_omni/engine/orchestrator.py:?` [major] — Removing `already_submitted and` correctly makes the one-element unwrap apply to initial submits too; without it, Ming's thinker2imagegen result `[{"prompt": "", "extra": extra}]` reaches StagePool.submit_initial as a list, which raises ValueError on diffusion stages. Please add a regression test that drives `_forward_to_next_stage` with `next_pool.stage_type == 'diffusion'`, `already_submitted=False`, and a `[{...}]` processor result, asserting `submit_initial` receives a dict. (evidence: Read vllm_omni/engine/stage_pool.py:932-1010 and vllm_omni/model_executor/stage_input_processors/ming_flash_omni.py:346-433; grep confirmed no existing test exercises the diffusion list-prompt unwrap.) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:475` [minor] — Confirmed: `return outputs` at pipeline_ming_imagegen.py:475 changes MingImagePipeline.forward to return the parent DiffusionOutput instead of a rebuilt DiffusionOutput(output=raw), and this path has no automated coverage. grep tests/ finds no `pipeline_ming_imagegen` references, and `MingImagePipeline` appears only in tests/test_config_factory.py as registry/model_arch assertions; e2e ming tests use ci/ming_flash_omni_thinker_only.yaml and ci/ming_flash_omni.yaml, not the ming_flash_omni_image deploy. Please add a CPU/mock unit test stubbing ZImagePipeline.forward and asserting the same DiffusionOutput (including stage_durations) is returned. (evidence: grep tests/ for `pipeline_ming_imagegen` and `MingImagePipeline`; read tests/e2e/offline_inference/test_ming_flash_omni_expansion.py and tests/e2e/online_serving/test_ming_flash_omni_expansion.py.) + +`vllm_omni/engine/orchestrator.py:1290` [minor] — Confirmed: orchestrator.py:1290 now performs the singleton unwrap regardless of `already_submitted`, so the change affects every diffusion stage whose `custom_process_input_func` returns a single-element list on initial submit (hunyuan_image3, glm_image, mammoth_moda2, ming_flash_omni, etc.), while the PR only changes Ming's pipeline. Please confirm the other bridge functions return single-dict lists, or add coverage for those paths. (evidence: [sweep] blast radius: grep for `custom_process_input_func` shows the singleton unwrap now applies repo-wide on initial submit to every diffusion stage with a bridge func (hunyuan_image3, glm_image, mammoth_moda2, ming_flash_omni, etc.), but the PR validates only Ming.) + +`vllm_omni/engine/orchestrator.py:1293` [minor] — The else branch (`custom_process_input_func is None`) assigns `diffusion_prompt = req_state.prompt` with no list unwrap, while `StagePool.submit_initial`/`submit_update` raise ValueError for diffusion list prompts. Confirm no diffusion-stage config reaches this branch with `req_state.prompt` still a list. (evidence: [sweep] orchestrator.py:1293 else branch `diffusion_prompt = req_state.prompt` is not covered by the unwrap; if `req_state.prompt` is a list for a diffusion stage the submit paths would still reject it (not verified that any config reaches this branch with a list).) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449` [minor] — The head has failing Buildkite lanes (`vllm-omni-amd-ci` and `vllm-omni-npu-ci`). The raw CI attribution to main vs this PR is unavailable; can you confirm attribution and hold the PR if these lanes are failing because of this diff? (evidence: [sweep] CI: gh_ci_read shows buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE; attribution to main cannot be determined from the available evidence.) + +`vllm_omni/engine/orchestrator.py:?` [nit] — The unwrap at orchestrator.py:1290-1291 now applies to both submit_initial and submit_update for len(diffusion_prompt) == 1. When a diffusion-stage custom_process_input_func returns len > 1, the list still reaches StagePool.submit_initial/submit_update and raises the generic ValueError('Diffusion list-prompt batch requests are no longer supported...') without request id or bridge context; empty lists are already handled above with a terminal error. Consider a logger.warning or an explicit RuntimeError naming next_client.custom_process_input_func and req_id before submission so a multi-request bridge result is distinguishable from a malformed single request. (evidence: Read vllm_omni/engine/orchestrator.py:1268-1298 and vllm_omni/engine/stage_pool.py:932-1010; the only current guard is the empty-list terminal error, so len>1 reaches the StagePool ValueError with no request/bridge context.) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:?` [nit] — Returning the parent `outputs` object is safe today: ZImagePipeline.forward is annotated and returns `DiffusionOutput(output=image, stage_durations=stage_durations)`, and `DiffusionOutput` has no `images` field, so the removed fallbacks were unreachable. The `raw = outputs.output` + `isinstance(raw, torch.Tensor)` check still fires before the return, so a non-tensor `.output` is caught; the only non-self-enforcing part is that the returned object itself is a `DiffusionOutput`. As defensive polish, consider asserting `isinstance(outputs, DiffusionOutput)` before `return outputs` to make the pipeline's advertised 'one DiffusionOutput with .output tensor' invariant explicit. (evidence: Read vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413-774 (forward returns DiffusionOutput(output=image, stage_durations=...)); read vllm_omni/diffusion/data.py:1180 (DiffusionOutput has no `images` field).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.trace.json.gz new file mode 100644 index 000000000..ecf946f65 Binary files /dev/null and b/eval/dataset/arms/copilot_v12_hybrid_val/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.cost.json b/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.cost.json new file mode 100644 index 000000000..df2910979 --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1049.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v12_hybrid_val/runs/pr4893/run-20260813-143838-b50730", + "recorded_at": "2026-08-13T14:56:05", + "attempts": [ + { + "invocation_id": "inv-15fdc3ba", + "outcome": "run:run-20260813-143838-b50730", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-143838-b50730", + "llm_calls": 145, + "input_tokens": 1028116, + "output_tokens": 191408, + "usd": 0.758265, + "source": "spans" + } + ], + "attempt_usd_total": 0.758265, + "llm_calls": 145, + "input_tokens": 1028116, + "output_tokens": 191408, + "usd": 0.758265, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-143838-b50730", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7583, + "minutes": 17.44, + "ci_minutes": 0, + "input_tokens": 1028116, + "output_tokens": 191408, + "tool_calls": 259, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7909, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 145, + "by_role": { + "agent": { + "usd": 0.730707, + "input_tokens": 1013285, + "output_tokens": 170027, + "calls": 143 + }, + "reducer": { + "usd": 0.027558, + "input_tokens": 14831, + "output_tokens": 21381, + "calls": 2 + } + }, + "cache_read_tokens": 10004608, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.9, + "gate": 1.59, + "review": 1040.71, + "report": 0.1 + }, + "time_to_first_result_s": 1046.25 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 34328 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.md b/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.md new file mode 100644 index 000000000..e5f45da05 --- /dev/null +++ b/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4893, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] hunyuan_fused_moe.py:91 indexed access `hidden_states.shape[0]`: read the full pre-hook; `hidden_states` is resolved from kwargs/args and None-guarded before use, so the access is safe. +- [sweep] test_expert_parallel_layout.py:24 `if reduce_scatter is not None:`: read the _FakeGroup constructor; branch only adds the attribute when the fake supplies it, reachable and harmless. +- [sweep] parallel_state.py:868/875/882 branches: read initialize_model_parallel in full; MoE-vs-non-MoE arms are both reachable and the else branch preserves the old diffusion GroupCoordinator for `_TP`. +- [sweep] hunyuan_fused_moe.py:31/34/38/47 branches: read; each guards unavailable context, pre-existing dp_metadata, single-rank DP, and missing gather entries — no dead or redundant arm found. +- [sweep] changed_symbol_consumers: grep for init_vllm_model_parallel_group and _set_forward_context_dp_metadata shows only the new definition/call sites and tests; no stale caller found. +- [validated] init_vllm_model_parallel_group matches installed vLLM 0.26 `init_model_parallel_group(group_ranks, local_rank, backend, use_message_queue_broadcaster=False, group_name=None, use_device_communicator=True)`; the call is API-compatible on this install. +- [validated] destroy_model_parallel (parallel_state.py:919-945) destroys vllm `_DP`/`_PCP`/`_TP`/`_EP` when distinct from the diffusion groups, so the new companion coordinators have a teardown path. +- [validated] non-MoE path is covered by test_cfg_parallel_keeps_diffusion_dp_without_ep, which asserts `vllm_parallel_state._DP is parallel_state._DP` and `_PCP/_EP is None`. + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:?` [major] — The diff adds `_set_forward_context_dp_metadata`, called from `_num_tokens_pre_hook` for every HunyuanImage3 FusedMoE forward, and it unconditionally calls `torch.distributed.all_gather_object` over `_vllm_ps._DP` whenever `dp_group.world_size > 1`. In the CFG-parallel AR-KV-reuse path, `_maybe_run_negative_cfg_prefill` runs the uncond prefill only on `cfg_rank == 1` (early return for the other rank), while the tp2_cfgp2 vLLM DP group spans both CFG ranks; the skipped rank does not enter the same blocking collective, so this can hang or gather mismatched negative-prefill/main-forward token counts into `dp_metadata`. Gate the gather out of the rank-asymmetric prefill, or skip it when the CFG-parallel uncond path runs on only a subset of the DP group, and add a regression for AR-KV-reuse + cfg-parallel. (evidence: Read hunyuan_fused_moe.py lines 29-50 (pre-hook call at line 91, unconditional gather at 42) and hunyuan_image3_transformer.py `_maybe_run_negative_cfg_prefill` (early return `if cfg_parallel_ready and cfg_rank != 1`); parallel_state.py maps vLLM _DP to the cfg-dp ranks in the MoE mapping, so for tp2_cfgp2 the DP group spans both CFG ranks.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:?` [minor] — `_set_forward_context_dp_metadata` builds `DPMetadata` directly instead of via `DPMetadata.make`, bypassing the upstream `num_tokens_across_dp_cpu[dp_rank] == batchsize` invariant. Downstream consumers depend on per-rank token counts: `routed_experts_capturer.py` slices with `num_tokens_dp[self.dp_rank]`, and `deepep_v2.py` sizes the dispatch buffer from `num_tokens_across_dp_cpu.max()`. `all_gather_object` ordering is normally by rank, but if it ever diverges the failure is silent. Add `assert gathered_num_tokens[dp_group.rank_in_group] == int(num_tokens)` before assigning `dp_metadata`. (evidence: Read /usr/local/lib/python3.12/dist-packages/vllm/forward_context.py:73-99 for the upstream assert and grepped num_tokens_across_dp_cpu consumers in vllm/model_executor/layers/fused_moe (deepep_v2.py:136-138, routed_experts_capturer.py:139-144).) + +`vllm_omni/diffusion/distributed/parallel_state.py:?` [minor] — Confirmed: `init_vllm_model_parallel_group` (parallel_state.py:526) unconditionally passes `use_device_communicator=True`, and the MoE+EP path now builds vLLM `_PCP`/`_TP`/`_DP`/`_EP` through it. gh_ci_read(4893) shows NPU and AMD CI failing while CUDA/intel pass, and `hunyuan_image3_dit.yaml` enables `enable_expert_parallel: true` for NPU/XPU. Please confirm whether the NPU/AMD lanes are already red on main; if this PR introduced them, verify `GroupCoordinator(use_device_communicator=True)` on Ascend/ROCm or gate it by platform. (evidence: gh_ci_read(4893) shows buildkite/vllm-omni-npu-ci and buildkite/vllm-omni-amd-ci FAILURE while buildkite/vllm-omni passes; the diff adds `use_device_communicator=True` at vllm_omni/diffusion/distributed/parallel_state.py:526 and the PR body only reports CUDA smoke coverage.) + +`vllm_omni/diffusion/distributed/parallel_state.py:902` [minor] — The shared use_moe_parallel_mapping remap (enable_expert_parallel && od_config.is_moe) now builds vllm_parallel_state._EP as a vLLM GroupCoordinator, and NPU Hunyuan builds its MC2 group from the same layout via get_expert_parallel_group_ranks() (vllm_omni/platforms/npu/models/hunyuan_fused_moe.py:85). The PR adds coverage only for the CUDA DiT HunyuanImage3 fused-MoE path and a gloo layout test. Please add NPU MC2 validation for this path, or explicitly scope/gate the remap so the NPU platform is not exposed without coverage. (evidence: [blast-radius] use_moe_parallel_mapping is gated on enable_expert_parallel && od_config.is_moe in shared parallel_state.py; consumers include NPU Hunyuan (`vllm_omni/platforms/npu/models/hunyuan_fused_moe.py` builds MC2 from `get_expert_parallel_group_ranks()`), but PR validates only CUDA DiT HunyuanImage3.) + +`vllm_omni/diffusion/distributed/parallel_state.py:526` [minor] — parallel_state.py:526 calls vllm_parallel_state.init_model_parallel_group(..., group_name=group_name, use_device_communicator=True). No pinned/minimum vLLM version is declared in pyproject.toml, setup.py, or requirements/*.txt, so the 0.24 `init_model_parallel_group` signature and `GroupCoordinator.device_communicator`/`reduce_scatter` dependency is unpinned and undocumented — please pin or document the minimum vLLM version, or verify the exact 0.24 signature. (evidence: [sweep] dependency compatibility: pyproject/setup.py/requirements carry no pinned vllm version; I could not read the exact vLLM 0.24 source, so 0.24 API support for `use_device_communicator`/`group_name` rests on the PR's smoke evidence plus the 0.26 source I read.) + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:?` [minor] — TestSetForwardContextDPMetadata.test_sets_dp_metadata_from_vllm_dp_group mocks torch.distributed.all_gather_object with a side effect that fabricates `[obj, obj + 1]` and then asserts `num_tokens_across_dp_cpu.tolist() == [1024, 1025]` (line 77). Because the fake supplies those exact values, the assertion cannot catch ordering, dtype, or rank mismatches in the production gather at hunyuan_fused_moe.py:41-49. Fill the output from the actual input (e.g. `[obj] * len(output)`), assert the call is made with `int(num_tokens)` and `group=fake_group.cpu_group`, and derive order-sensitive assertions from the fake's input rather than hardcoded values. (evidence: Read tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:56-76 and vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:39-50; the production helper supplies only `int(num_tokens)` and constructs `DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64))`.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:38` [minor] — `_set_forward_context_dp_metadata` is gated only on `dp_group.world_size > 1`, but `vllm_parallel_state._DP` remains the diffusion DP coordinator (`_DP`) whenever `enable_expert_parallel=false` (the MoE remap only runs inside `use_moe_parallel_mapping`). In that non-EP HunyuanImage3 config with `data_parallel_size > 1`, every FusedMoE forward now performs a blocking CPU `torch.distributed.all_gather_object` over the diffusion DP `cpu_group` and writes `forward_context.dp_metadata` that the all2all dispatcher only consumes under expert-parallel mapping. Scope the gather to the EP mapping (e.g. only when `vllm_parallel_state._EP` is initialized), or prove no extra collective is introduced for DP>1 non-EP. (evidence: Read parallel_state.py lines 800-890 (unconditional `vllm_parallel_state._DP = _DP` at 809, MoE-only remap later) and hunyuan_fused_moe.py lines 38-50; grep for `use_moe_parallel_mapping` confirms the remap is EP/MoE-only.) + +`vllm_omni/diffusion/distributed/parallel_state.py:882` [minor] — Confirmed: at vllm_omni/diffusion/distributed/parallel_state.py:882 the branch is now `if use_moe_parallel_mapping:` (previously also required `cfg_parallel_size > 1`). With MoE+EP and `cfg_parallel_size == 1`, `data_parallel_size > 1`, `rank_generator.get_ranks("cfg-dp")` yields groups of world_size == data_parallel_size, so `vllm_parallel_state._DP` is replaced by a vLLM GroupCoordinator and `_set_forward_context_dp_metadata` runs `all_gather_object` because `world_size > 1`. `tests/diffusion/distributed/test_expert_parallel_layout.py` only exercises EP with cfg=2/dp=2; no test covers EP with cfg=1/dp=2. Please add a layout test (tensor_parallel_size=2, data_parallel_size=2, cfg_parallel_size=1, enable_expert_parallel=True, asserting `vllm_parallel_state._DP.world_size == 2` and the gather) or document this layout as unsupported. (evidence: Read tests/diffusion/distributed/test_expert_parallel_layout.py (only cfg=2,dp=2 and non-EP cfg=2,dp=2 cases); grep of deploy YAMLs for `enable_expert_parallel: true` shows hunyuan_image3_dit.yaml and hunyuan_image_3_moe.yaml all use data_parallel_size=1; PR body smoke evidence only covers tp2_cfgp2/tp2_sp2.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.trace.json.gz new file mode 100644 index 000000000..e2a54ea4a Binary files /dev/null and b/eval/dataset/arms/copilot_v12_hybrid_val/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/manifest.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/manifest.json new file mode 100644 index 000000000..5f44f0d56 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v13_ds_wave2_r2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.cost.json new file mode 100644 index 000000000..0b4869a8d --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1139.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5509/run-20260814-145729-1003f8", + "recorded_at": "2026-08-14T15:16:25", + "attempts": [ + { + "invocation_id": "inv-09359366", + "outcome": "run:run-20260814-145729-1003f8", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-145729-1003f8", + "llm_calls": 130, + "input_tokens": 838828, + "output_tokens": 200550, + "usd": 0.682905, + "source": "spans" + } + ], + "attempt_usd_total": 0.682905, + "llm_calls": 130, + "input_tokens": 838828, + "output_tokens": 200550, + "usd": 0.682905, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-145729-1003f8", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6829, + "minutes": 18.94, + "ci_minutes": 0, + "input_tokens": 838828, + "output_tokens": 200550, + "tool_calls": 246, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7919, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 130, + "by_role": { + "agent": { + "usd": 0.656485, + "input_tokens": 814297, + "output_tokens": 182584, + "calls": 128 + }, + "reducer": { + "usd": 0.026421, + "input_tokens": 24531, + "output_tokens": 17966, + "calls": 2 + } + }, + "cache_read_tokens": 8733952, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.51, + "gate": 1.66, + "review": 1131.13, + "report": 0.1 + }, + "time_to_first_result_s": 1136.35 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 43253 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.md new file mode 100644 index 000000000..9a40a1b16 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] read vllm_omni/diffusion/attention/backends/trtllm_attn.py fully; branch `if not HAS_FLASHINFER:` (line 90) returns False before the signature probe — safe. +- [validated] trtllm_attn.py:198-215: init gates quant on dtype_qk membership in _QK_QUANT_DTYPES, then on _sage_kernel_available(), then on _sage_quantize_fn(); all three raise clear RuntimeErrors. +- [validated] trtllm_attn.py:296-299: forward_cuda quantize path runs only when `self.quant.enabled`; dense path receives no SAGE kwargs so older FlashInfer builds stay compatible. +- [validated] data.py:1415/1418: AttnQuantSpec validates dtype_qk/dtype_vo against {float16,bfloat16,int8,fp8_e4m3} and block sizes {1,4,16}. +- [sweep] data.py:1447: quant is accepted for TRTLLM_ATTN and FLASHINFER_ATTN; FLASHINFER_ATTN acceptance is a finding because the backend never reads backend_kwargs. +- [validated] data.py:1455 `_coerce` handles None/pass-through/Mapping/TypeError for both skip_softmax and quant. +- [validated] data.py:1464-1482 backend_kwargs serializes skip and quant; dtype_vo/flashinfer_backend pass-through is ignored by TRTLLM_ATTN (finding). +- [validated] data.py:1553 spec_keys now includes 'quant', so per_role nested flattening accepts quant blocks. + +`vllm_omni/diffusion/data.py:1447` [major] — AttentionSpec now accepts and serializes `quant` for FLASHINFER_ATTN (data.py:1447, 1472-1482), and Attention.__init__ forwards it via backend_kwargs (layer.py:109), but FlashInferAttentionImpl.__init__ (flashinfer_attn.py:55-66) only has **extra_impl_args and never reads backend_kwargs or any quant/SAGE path, so a FLASHINFER_ATTN quant config silently runs dense attention. Reject `quant` for FLASHINFER_ATTN (including its documented float16/bfloat16 dtype_qk set) or implement a real consumer, remove the doc claim at docs/user_guide/diffusion/attention_backends.md, and change test_quant_superset_fields_passed_through to assert backend consumption, not dict round-trip. (evidence: vllm_omni/diffusion/data.py:1447 ` if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` — FLASHINFER_ATTN quant is accepted. vllm_omni/diffusion/attention/layer.py:109 ` backend_kwargs=backend_kwargs,` — the serialized quant dict is passed to the backend impl constructor. vllm_omni/diffusion/attention/backends/flashinfer_attn.py:55 ` def __init__(` ... vllm_omni/diffusion/attention/backends/flashinfer_attn.py:63 ` **extra_impl_args,` ... vllm_omni/diffusion/attention/backends/flashinfer_attn.py:65 ` self.causal = causal` ... vllm_omni/diffusion/attention/backends/flashinfer_attn.py:66 ` self.softmax_scale = softmax_scale` — no backend_kwargs parameter and no quant handling; the dict is absorbed and ignored.) + +`vllm_omni/diffusion/attention/layer.py:?` [major] — This diff wires `TRTLLM_ATTN` + `quant` through `TrtllmAttentionImpl.forward_cuda` (`if self.quant.enabled: q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(...)` at trtllm_attn.py:296), but `Attention._run_ring_attention` (layer.py:320-327) only guards `skip_softmax` and then calls ring kernels (ring.py:133/157) that bypass the backend impl entirely. With `ring_degree > 1`, a `TRTLLM_ATTN` + `quant` config is therefore silently run dense instead of raising like `skip_softmax`. Add the analogous `getattr(self.attention, "quant", None).enabled` guard so this new config can never be silently dropped. (evidence: vllm_omni/diffusion/attention/layer.py:320-324 ` skip = getattr(self.attention, "skip", None)` / ` if skip is not None and getattr(skip, "configured", False):` / ` raise NotImplementedError(` / ` "the ring path bypasses the backend, so the skip config would be silently ignored. "`; vllm_omni/diffusion/attention/layer.py:319 ` def _run_ring_attention(self, query, key, value, attn_metadata):`; vllm_omni/diffusion/attention/parallel/ring.py:133 ` return ring_pytorch_attn_func(`; vllm_omni/diffusion/attention/parallel/ring.py:157 ` return ring_flash_attn_func(`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:296 ` if self.quant.enabled:`) + +`tests/diffusion/attention/test_trtllm_attn.py:?` [major] — The new QuantConfig validation/error paths are not gated by CI: tests/diffusion/attention/test_trtllm_attn.py is module-marked core_model,diffusion,cuda (line 13); the merge CPU lane runs `pytest -sv tests/diffusion -m 'core_model and cpu'` (.buildkite/cuda/test-merge.yml:18), which deselects cuda-marked modules; and the CUDA lane only globs top-level `tests/diffusion/*.py` (.buildkite/cuda/test-ready.yml:68), never entering tests/diffusion/attention/. Consequently test_quant_config_from_backend_kwargs, test_quant_rejects_non_sage_dtype, test_quant_quantize_requires_flashinfer_routine, and test_quant_quantize_calls_routine_and_shapes_sfs never run in CI. Move them to a cpu-marked file or add a tests/diffusion/attention CUDA lane. (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]`; tests/diffusion/attention/test_trtllm_attn.py:44 `def test_quant_config_from_backend_kwargs():`; tests/diffusion/attention/test_trtllm_attn.py:53 `def test_quant_rejects_non_sage_dtype():`; tests/diffusion/attention/test_trtllm_attn.py:57 `def test_quant_quantize_requires_flashinfer_routine(monkeypatch):`; tests/diffusion/attention/test_trtllm_attn.py:65 `def test_quant_quantize_calls_routine_and_shapes_sfs():`; .buildkite/cuda/test-merge.yml:18 `- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/cuda/test-ready.yml:68 `- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"`) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — docs/user_guide/diffusion/attention_backends.md:232 still says 'see Known Limitations above', but this PR deletes the `## Known Limitations` section (plus `## Validation Guidance` and `## Reproducing the Benchmark Table`) and the PR body only covers SAGE quant config. Restore those sections, or move them in a separate docs change, and fix the line 232 cross-reference. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — the only remaining occurrence of 'Known Limitations' in the file; the referenced `## Known Limitations` section was deleted by this PR, so the cross-reference is dangling.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:124` [minor] — QuantConfig.from_backend_kwargs (trtllm_attn.py:121-126) reads only dtype_qk/q_block_size/k_block_size, while AttentionSpec.backend_kwargs() (data.py:1473-1483) serializes dtype_vo and flashinfer_backend for any quant-enabled spec, and AttentionSpec.__post_init__ permits quant on TRTLLM_ATTN. So TRTLLM_ATTN with quant={'dtype_qk':'int8','dtype_vo':'float16'} parses and silently ignores dtype_vo, while docs/user_guide/diffusion/attention_backends.md:186 claims each backend 'rejects the rest'. Either reject unsupported quant keys for TRTLLM_ATTN at spec/impl time or correct the doc to say unsupported keys are ignored. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:122-126 `q = (backend_kwargs or {}).get("quant") or {}` / `return cls(` / `dtype_qk=q.get("dtype_qk"),` / `q_block_size=int(q.get("q_block_size", 1) or 1),` / `k_block_size=int(q.get("k_block_size", 16) or 16),` — reads only dtype_qk/q_block_size/k_block_size, never dtype_vo/flashinfer_backend; vllm_omni/diffusion/data.py:1479-1482 `if q.dtype_vo is not None:` / `quant_kw["dtype_vo"] = q.dtype_vo` / `if q.flashinfer_backend is not None:` / `quant_kw["flashinfer_backend"] = q.flashinfer_backend` — emitted for any quant-enabled spec, and AttentionSpec.__post_init__ allows quant on TRTLLM_ATTN; docs/user_guide/diffusion/attention_backends.md:186 `fields that apply and rejects the rest.`) + +`tests/diffusion/attention/test_trtllm_attn.py:321` [minor] — In tests/diffusion/attention/test_trtllm_attn.py:321, test_sage_quant_matches_sdpa accepts rel < 0.1, while the same file's dense comparisons use 0.01 (lines 295 and 368) and skip-tiny uses 0.02 (line 334). The test is also gated by requires_sage (lines 298-300) on _has_trtllm_attn() (device capability >= 10, line 19) plus a live _sage_quantize_fn(), so it is skipped in ordinary CI. Tighten the SAGE tolerance to the measured per-dtype bound or document why 10% is acceptable, and keep the Missing TRTLLM-GEN kernel skip scoped to the arch-dependent int8/SM100-only case rather than both parametrized dtypes. (evidence: tests/diffusion/attention/test_trtllm_attn.py:321 `assert rel < 0.1, f"SAGE({dtype_qk}) rel err {rel:.4f} too high"`; tests/diffusion/attention/test_trtllm_attn.py:295 `assert rel < 0.01, f"BF16 dense rel err {rel:.4f} too high"`; tests/diffusion/attention/test_trtllm_attn.py:334 `assert rel < 0.02, f"skip(tiny) rel err {rel:.4f} should be ~dense"`; tests/diffusion/attention/test_trtllm_attn.py:368 `assert rel < 0.01, f"GQA rel err {rel:.4f} vs SDPA(GQA) too high"`; tests/diffusion/attention/test_trtllm_attn.py:19 `if torch.cuda.get_device_capability()[0] < 10:`; tests/diffusion/attention/test_trtllm_attn.py:298-300 `requires_sage = pytest.mark.skipif( not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None), reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)", )`) + +`vllm_omni/diffusion/data.py:1559` [minor] — data.py:1559: the per-role flattening error still says only `backend/skip_softmax` even though `quant` was just added to `spec_keys`; include `quant` in the message so nested per-role config users get accurate guidance. (evidence: vllm_omni/diffusion/data.py:1559 `f"Invalid per_role entry for role {role!r}: cannot mix backend/skip_softmax with nested role keys."` — error text names only `backend/skip_softmax` and omits the newly added `quant` spec key.) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — This diff adds `quant` (AttnQuantSpec) as a second typed field on `AttentionSpec` (vllm_omni/diffusion/data.py:1434), but the user guide still says 'Today the only one is TRTLLM_ATTN's Skip-Softmax' (docs/user_guide/diffusion/attention_backends.md:72-73) and docs/design/module/dit_module.md:543 repeats 'Today the only such field is `skip_softmax`'. Update both paragraphs to list `skip_softmax` and `quant` so the docs match the new config surface. (evidence: vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None` — second typed field added to `AttentionSpec`. docs/user_guide/diffusion/attention_backends.md:72-73 "A backend that needs configuration exposes it as a typed field on the spec. Today the only" / "one is `TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):". docs/design/module/dit_module.md:543 "Today the only such field is `skip_softmax` (TRTLLM_ATTN), so e.g. `target_sparsity` can be set without changing model code.") + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/attention/backends/trtllm_attn.py:199` [minor] Confirmed. TrtllmAttentionImpl.__init__ (trtllm_attn.py:198-211) gates SAGE only on `self.quant.dtype_qk not in _QK_QUANT_DTYPES`, `_sage_kernel_available()` (a signature check for `sage_attn_sfs`), and `_sage_quantize_f + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.trace.json.gz new file mode 100644 index 000000000..5ceed0dcb Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.cost.json new file mode 100644 index 000000000..7e3c6dff0 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1070.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5550/run-20260814-145729-0e2105", + "recorded_at": "2026-08-14T15:15:16", + "attempts": [ + { + "invocation_id": "inv-a15d4382", + "outcome": "run:run-20260814-145729-0e2105", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-145729-0e2105", + "llm_calls": 113, + "input_tokens": 873341, + "output_tokens": 198064, + "usd": 0.777717, + "source": "spans" + } + ], + "attempt_usd_total": 0.777717, + "llm_calls": 113, + "input_tokens": 873341, + "output_tokens": 198064, + "usd": 0.777717, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-145729-0e2105", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7777, + "minutes": 17.78, + "ci_minutes": 0, + "input_tokens": 873341, + "output_tokens": 198064, + "tool_calls": 226, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8045, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 113, + "by_role": { + "agent": { + "usd": 0.755586, + "input_tokens": 848954, + "output_tokens": 183962, + "calls": 111 + }, + "reducer": { + "usd": 0.022131, + "input_tokens": 24387, + "output_tokens": 14102, + "calls": 2 + } + }, + "cache_read_tokens": 12001664, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.77, + "gate": 2.05, + "review": 1060.91, + "report": 0.06 + }, + "time_to_first_result_s": 1066.77 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 39745 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.md new file mode 100644 index 000000000..ac953f7c4 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.md @@ -0,0 +1,64 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] tests/diffusion/diffusion_kv/test_metadata.py:37,38,68,109 indexed accesses — read file; they index tuples constructed immediately above with known cardinality (single/two-element); safe. +- [sweep] tests/diffusion/diffusion_kv/test_worker_contract.py:221 calls[0][1][0] — read test + make_executor; dense path records args=(req, od_config, None), so [1][0] is req; safe. +- [sweep] branch vllm_omni/diffusion/diffusion_engine.py:199 scheduler_to_close None guard — read __init__/_init_scheduler; self.scheduler is assigned before initialize(), so the close reaches the scheduler on initialize failure and the None guard covers constructor failures. +- [sweep] branch vllm_omni/diffusion/executor/multiproc_executor.py:545 metadata append — read execute_request; metadata appended as 4th positional only when non-None; dense legacy keeps (req, od_config, kv_prefetch_job). +- [sweep] branches vllm_omni/diffusion/sched/interface.py:184,192 identity checks — read validate_new_request_data_identity; envelope vs forwarded req and metadata vs forwarded req both raise ValueError. +- [sweep] branches vllm_omni/diffusion/worker/diffusion_model_runner.py:172,175 mode/metadata checks — read _validate_diffusion_kv_metadata; only PAGED_SCHEDULER accepts metadata; metadata.request_id must equal req.request_id. +- [sweep] branches vllm_omni/diffusion/worker/diffusion_worker.py:463,1372 metadata kwargs — read both; diffusion_kv_metadata forwarded only when non-None, preserving dense legacy model-runner signature. +- [validated] all six non-test touched files read in situ: diffusion_engine.py, diffusion_kv/metadata.py, executor/multiproc_executor.py, sched/interface.py, worker/diffusion_model_runner.py, worker/diffusion_worker.py. + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [major] — DiffusionWorker.execute_model now forwards diffusion_kv_metadata to the runner via **kwargs whenever it is non-None (diffusion_worker.py:459-462), and the base DiffusionModelRunner.execute_model accepts it (diffusion_model_runner.py:592). The AR override ARDiffusionModelRunner.execute_model (vllm_omni/experimental/ar_diffusion/runner.py:237-241) still only accepts (self, req, kv_prefetch_job=None), so a metadata-bearing request on an AR engine raises TypeError before any KV install/validation. Add the diffusion_kv_metadata kwarg to the override and pass it through to both super().execute_model(...) calls (or reject it explicitly), and extend the existing parity regression (tests/diffusion/ar_diffusion/test_engine_routing.py:141) to assert the new parameter. (evidence: vllm_omni/experimental/ar_diffusion/runner.py:237-241 ` def execute_model( + self, + req: OmniDiffusionRequest, + kv_prefetch_job: KVPrefetchJob | None = None, + ) -> DiffusionOutput:`; vllm_omni/diffusion/worker/diffusion_model_runner.py:592 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,`; vllm_omni/diffusion/worker/diffusion_worker.py:459-462 ` kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job} + if diffusion_kv_metadata is not None: + kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata + output = self.model_runner.execute_model(req, **kwargs)`) + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — This diff adds `and not has_diffusion_kv_metadata` at vllm_omni/diffusion/executor/multiproc_executor.py:471 to the DLO+AllGather DP broadcast condition, with `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)` at :462. When multiple requests carry metadata under `enable_distributed_layerwise_offload`/`dlo_use_allgather`, control falls through to the serial loop at :543-551, which calls `collective_rpc('execute_model', args=(req, od_config, kv_prefetch_job[, metadata]), unique_reply_rank=0, exec_all_ranks=True)`; because `collective_rpc` sets `execute_all_ranks=True` and the worker sets `should_execute = exec_all_ranks or ...`, every rank executes the same single request, so DP multi-request concurrency is silently lost. W0 does not produce metadata, but W1's multi-request DLO wave will hit this. Consider warning or raising NotImplementedError for the unsupported combination (or implementing a metadata-aware DP path) and add a unit test for a metadata-bearing multi-request DLO+AllGather wave. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)`; :471 `and not has_diffusion_kv_metadata`; :544 `args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)`; :550 `unique_reply_rank=0,`; :551 `exec_all_ranks=True,`; vllm_omni/diffusion/worker/diffusion_worker.py:997 `should_execute = exec_all_ranks or output_rank is None or output_rank == self.gpu_id`) + +`vllm_omni/diffusion/executor/multiproc_executor.py:?` [minor] — This diff adds pre-RPC identity validation only in MultiprocDiffusionExecutor.execute_request (multiproc_executor.py:458-460). The fused execute_batch (multiproc_executor.py:615-620) and execute_step (multiproc_executor.py:651-656) paths still forward scheduler_output via collective_rpc without that loop, and DiffusionWorker.execute_model_batch (diffusion_worker.py:487) / execute_stepwise (diffusion_worker.py:503) activate LoRA before the runner re-validates, so an invalid envelope reaches worker state mutation and surfaces as a wrapped worker error. Add the same validate_new_request_data_identity loop at the top of execute_batch and execute_step before collective_rpc (or before LoRA activation in DiffusionWorker) so all three RPC paths reject invalid envelopes before side effects. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:458-460 ` for new_req in new_reqs: + validate_new_request_data_identity(new_req)` ; vllm_omni/diffusion/executor/multiproc_executor.py:615-620 ` result = self.collective_rpc( + "execute_model_batch", + args=(scheduler_output, self.od_config), + unique_reply_rank=0, + exec_all_ranks=True, + )` (no validation loop before) ; vllm_omni/diffusion/executor/multiproc_executor.py:651-656 ` result = self.collective_rpc( + "execute_stepwise", + args=(scheduler_output,), + unique_reply_rank=0, + exec_all_ranks=True, + )` (no validation loop before) ; vllm_omni/diffusion/worker/diffusion_worker.py:487 ` self.lora_manager.set_active_adapter(sp.lora_request, sp.lora_scale)` before delegating to model_runner.execute_model_batch ; vllm_omni/diffusion/worker/diffusion_worker.py:503 ` self._activate_step_lora(scheduler_output)` before delegating to model_runner.execute_stepwise.) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:?` [minor] — The guard at diffusion_model_runner.py:172 (`if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:`) and the mismatch check at :174 mean PAGED_SCHEDULER accepts a non-None matching DiffusionKVMetadata and proceeds to _execute_request_list. The paged model-runner tests only exercise absent metadata (test_paged_request_without_metadata_reaches_request_forward) and mismatched metadata (test_paged_request_rejects_mismatched_metadata_identity), plus dense-mode rejection, so the positive acceptance branch is uncovered on execute_model, execute_model_batch, and execute_stepwise. Add a PAGED_SCHEDULER test that passes diffusion_kv_metadata=make_metadata(req.request_id) and asserts _execute_request_list is called once, with batch and stepwise variants. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:172 ` if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:` — PAGED_SCHEDULER skips this rejection. vllm_omni/diffusion/worker/diffusion_model_runner.py:174 ` if metadata is not None and metadata.request_id != request_id:` — only mismatched metadata is rejected. tests/diffusion/diffusion_kv/test_worker_contract.py:96 `def test_paged_request_without_metadata_reaches_request_forward() -> None:` — calls runner.execute_model(req) with no metadata. tests/diffusion/diffusion_kv/test_worker_contract.py:117 `def test_paged_request_rejects_mismatched_metadata_identity() -> None:` — calls runner.execute_model(SimpleNamespace(request_id="req-0"), diffusion_kv_metadata=make_metadata("stale")); no test passes a non-None matching metadata (grep for matching_metadata/diffusion_kv_metadata in tests returns no such paged positive case).) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:?` [nit] — The metadata-id check in `_validate_diffusion_kv_metadata` (diffusion_model_runner.py:175-179) is dead on the batch/step paths: `execute_model_batch` (line 637) and `execute_stepwise` (line 751) call `validate_new_request_data_identity(new_req)` first, which already raises when `metadata.request_id != req.request_id` (interface.py:192-194). The duplicate check also emits a different message (`Diffusion KV metadata request mismatch` vs `Diffusion request identity mismatch`). Consider dropping the id re-check on those two paths or unify the identity validator/message so the contract has one source of truth. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:175 ` if metadata is not None and metadata.request_id != request_id:` — duplicate metadata-id check inside `_validate_diffusion_kv_metadata`. vllm_omni/diffusion/sched/interface.py:192 ` if metadata is not None and metadata.request_id != forwarded_request_id:` — this raises `ValueError("Diffusion request identity mismatch: ...")` before the runner re-check. vllm_omni/diffusion/worker/diffusion_model_runner.py:637 ` validate_new_request_data_identity(new_req)` and :751 ` validate_new_request_data_identity(new_req)` — called first in both loops, followed by `_validate_diffusion_kv_metadata(request_id=new_req.req.request_id, metadata=new_req.diffusion_kv_metadata)` at :638-640 and :752-754, where `request_id` equals the already-validated `forwarded_request_id`.) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:?` [nit] — The diff adds `diffusion_kv_metadata: DiffusionKVMetadata | None = None` to `DiffusionModelRunner.execute_model` (diffusion_model_runner.py:592), `DiffusionWorker.execute_model` (diffusion_worker.py:426), and `WorkerWrapperBase.execute_model` (diffusion_worker.py:1358), but none of the three docstrings document it. Add a brief entry at each boundary (rejected for dense_legacy when non-None, identity-validated against `request_id`, forwarded only when non-None) so the Worker/Runner RPC contract stays self-describing. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:592 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,` — new parameter; its docstring Args block (lines 597-598) reads ` Args:` / ` req: A diffusion request containing a list of prompts to process.` with no diffusion_kv_metadata entry. vllm_omni/diffusion/worker/diffusion_worker.py:426 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,` in `DiffusionWorker.execute_model`, whose docstring (line 428) ` """Execute a forward pass by delegating to the model runner.` enumerates no parameters. vllm_omni/diffusion/worker/diffusion_worker.py:1358 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,` in `WorkerWrapperBase.execute_model`, whose Args block (lines 1364-1366) reads ` req: Diffusion request.` / ` od_config: OmniDiffusionConfig configuration` / ` kv_prefetch_job: Optional next-request KV prefetch descriptor.` and omits the new parameter.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.trace.json.gz new file mode 100644 index 000000000..cda547846 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.cost.json new file mode 100644 index 000000000..4fb53193e --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1551.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5610/run-20260814-145729-f98a48", + "recorded_at": "2026-08-14T15:23:17", + "attempts": [ + { + "invocation_id": "inv-9dbd0354", + "outcome": "run:run-20260814-145729-f98a48", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-145729-f98a48", + "llm_calls": 96, + "input_tokens": 644889, + "output_tokens": 150798, + "usd": 0.619999, + "source": "spans" + } + ], + "attempt_usd_total": 0.619999, + "llm_calls": 96, + "input_tokens": 644889, + "output_tokens": 150798, + "usd": 0.619999, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-145729-f98a48", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.62, + "minutes": 25.8, + "ci_minutes": 0, + "input_tokens": 644889, + "output_tokens": 150798, + "tool_calls": 206, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8795, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 96, + "by_role": { + "planner": { + "usd": 0.000524, + "input_tokens": 105, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.597062, + "input_tokens": 631651, + "output_tokens": 133277, + "calls": 93 + }, + "reducer": { + "usd": 0.022414, + "input_tokens": 13133, + "output_tokens": 17121, + "calls": 2 + } + }, + "cache_read_tokens": 10370432, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.7, + "gate": 2.47, + "review": 1541.83, + "report": 0.06 + }, + "time_to_first_result_s": 1548.09 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 30660 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.md new file mode 100644 index 000000000..d4b086ae0 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] docs/.nav.yml:118 — read in situ; the new `design/feature/omni_async_output_materialization.md` entry is correctly placed after `async_chunk.md`. A duplicate `design/feature/vae_parallel.md` exists elsewhere in the same section, but it is pre-existing (the diff adds only one line) so it is not this PR's regression and was not flagged. +- [sweep] docs/design/index.md:10 — read in situ; `[Async Omni Output Materialization](feature/omni_async_output_materialization.md)` was added and the target file exists. +- [sweep] docs/design/feature/omni_async_output_materialization.md:1 — read the full new doc in the tree; every technical paragraph checked against live code. +- [sweep] docs/source/architecture/qwen3-omni-async-output-step-gap.svg:1 — reviewed via the diff; well-formed SVG (role/aria labels, markers defined in defs) and the figure agrees with the doc's 2.8 ms -> 41 us claim. +- [validated] vllm_omni/worker/gpu_ar_model_runner.py:1595-1620 — `_should_use_async_omni_output()` checks use_async_scheduling, omni_prefix_cache, speculative_config, async_chunk, enable_return_routed_experts, use_async_omni_output, and has_postprocess/eager_omni_postprocess_before_async_output; these match the doc's Compatibility-and-Fallbacks table exactly. +- [validated] vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py:109,134,162-163 — thinker sets use_async_omni_output=True (has_postprocess=False); talker sets use_async_omni_output=True + eager_omni_postprocess_before_async_output=True + omni_pooler_payload_include_hidden=False, matching the doc's Stage Behavior table. +- [validated] vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:351-353 — Qwen3-TTS talker sets use_async_omni_output=True, eager_omni_postprocess_before_async_output=True, omni_pooler_payload_include_hidden=False, matching the doc's Qwen3-TTS section. +- [validated] vllm_omni/platforms/npu/worker/npu_ar_model_runner.py sample_tokens — constructs OmniModelRunnerOutput and calls get_omni_connector_output() before wrapping in AsyncGPUModelRunnerOutput, so Ascend NPU stays synchronous as documented. + +`docs/source/architecture/qwen3-omni-async-output-step-gap.svg:138` [minor] — The SVG presents the 2.8 ms -> 41 µs Talker step-gap claim with no benchmark-config lock or evidence-grade label of its own. The accompanying markdown does attribute the figure to the Qwen3-Omni optimization blog and defers full methodology to the blog/PR #4476, but that deferral is not an inline scope lock. Please either add an evidence-grade label (e.g., 'workload-aligned only') or a short config lock (model snapshot, GPU, resolution, steps, warmup, seed, compile/eager) to the figure caption; severity stays minor. (evidence: docs/source/architecture/qwen3-omni-async-output-step-gap.svg:138 `2.8 ms → 41 µs` — the inline step-gap claim with no benchmark-config lock; docs/design/feature/omni_async_output_materialization.md:120 `for the complete benchmark methodology and results.` — the doc defers methodology to the blog instead of providing a scope lock; docs/design/feature/omni_async_output_materialization.md:82 `Adapted from the` — explicit attribution to the Qwen3-Omni optimization blog.) + +`docs/design/feature/omni_async_output_materialization.md:219` [minor] — Rephrase this paragraph to avoid implying connector I/O is synchronous with model execution. The mixin's `_recv_loop`/`_save_loop` threads do the actual I/O; on the model thread, `execute_model` only registers/drains via `register_chunk_recv`, `recv_full_payload_inputs`, and `flush_full_payload_outputs` (vllm_omni/worker/gpu_ar_model_runner.py:1042-1048). Suggested wording: 'Input-side connector registration and drain stay on the model-thread path in `execute_model`; the actual receive/send I/O continues to run on the connector mixin's background threads, and those results affect scheduler-visible request state.' (evidence: docs/design/feature/omni_async_output_materialization.md:219-222 "Input-side connector operations remain synchronous with model execution. In particular, receiving inputs and flushing previously completed connector outputs are not deferred because they affect scheduler-visible request state." ; docs/design/feature/omni_async_output_materialization.md:195-196 "Receive-side connector state written by the background receiver is coordinated by the connector mixin's `_lock`." ; vllm_omni/worker/omni_connector_model_runner_mixin.py:216-227 "# Start background threads only when there's a connector / self._recv_thread: threading.Thread | None = None / self._save_thread: threading.Thread | None = None / if self._omni_connector is not None: / self._recv_thread = threading.Thread( / target=self._recv_loop, / daemon=True, / name=\"omni-mixin-recv\", / ) / self._recv_thread.start() / self._save_thread = threading.Thread( / target=self._save_loop, / daemon=True, / name=\"omni-mixin-save\", / ) / self._save_thread.start()" ; vllm_omni/worker/gpu_ar_model_runner.py:1042-1048 "if hasattr(self, \"_omni_connector\"): / for request in getattr(scheduler_output, \"pending_input_registrations\", []): / self.register_chunk_recv(request) / self.recv_full_payload_inputs(scheduler_output) / if self._pending_full_payload_send: / flush_ids = set(getattr(scheduler_output, \"finished_req_ids\", set())) / flush_ids.update({rid for rid in self._pending_full_payload_send if rid not in self.requests}) / if flush_ids: / self.flush_full_payload_outputs(flush_ids)") + +`docs/design/feature/omni_async_output_materialization.md:64` [nit] — The note at line 64 says XPU/MUSA `may enter the asynchronous path when its other guards pass`. That is only half the story: the dedicated-CUDA-stream/pinned-buffer overlap in Output Lifecycle steps 3-4 is CUDA-only — `_clone_cuda_tensor_payload` gates on `value.device.type == "cuda"` (gpu_ar_model_runner.py:73), and `_snapshot_tensor_payload_to_cpu_async` / `OmniAsyncGPUModelRunnerOutput.__init__` call `torch.cuda.current_stream()` unconditionally (:135, :183). XPU/MUSA tensors report `xpu`/`musa` (platforms/xpu/platform.py:86, platforms/musa/platform.py:133). Add one sentence stating that the dedicated-CUDA-stream/pinned-buffer overlap applies only to CUDA/ROCm so readers do not attribute that mechanism to XPU/MUSA. (evidence: docs/design/feature/omni_async_output_materialization.md:64 `backends may enter the asynchronous path when its other guards pass. XPU`; vllm_omni/worker/gpu_ar_model_runner.py:73 ` if value.device.type == "cuda":`; vllm_omni/worker/gpu_ar_model_runner.py:135 ` source_stream = torch.cuda.current_stream()`; vllm_omni/worker/gpu_ar_model_runner.py:183 ` default_stream = torch.cuda.current_stream()`; vllm_omni/platforms/xpu/platform.py:86 ` return torch.device("xpu")`; vllm_omni/platforms/musa/platform.py:133 ` return torch.device("musa")`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.trace.json.gz new file mode 100644 index 000000000..f1931c935 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.cost.json new file mode 100644 index 000000000..2085aa3c1 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1237.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5703/run-20260814-145729-085226", + "recorded_at": "2026-08-14T15:18:03", + "attempts": [ + { + "invocation_id": "inv-f6a2c421", + "outcome": "run:run-20260814-145729-085226", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-145729-085226", + "llm_calls": 136, + "input_tokens": 545189, + "output_tokens": 197533, + "usd": 0.570137, + "source": "spans" + } + ], + "attempt_usd_total": 0.570137, + "llm_calls": 136, + "input_tokens": 545189, + "output_tokens": 197533, + "usd": 0.570137, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-145729-085226", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5701, + "minutes": 20.57, + "ci_minutes": 0, + "input_tokens": 545189, + "output_tokens": 197533, + "tool_calls": 259, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7763, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 136, + "by_role": { + "agent": { + "usd": 0.550527, + "input_tokens": 530690, + "output_tokens": 183296, + "calls": 134 + }, + "reducer": { + "usd": 0.01961, + "input_tokens": 14499, + "output_tokens": 14237, + "calls": 2 + } + }, + "cache_read_tokens": 7616640, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.4, + "gate": 1.64, + "review": 1228.96, + "report": 0.06 + }, + "time_to_first_result_s": 1234.04 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 35196 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.md new file mode 100644 index 000000000..9d29de6f1 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.md @@ -0,0 +1,52 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] Read full diff: recipes/MiniMaxAI/MiniMax-H3-MUSA.md (new, 147 lines), recipes/README.md:45-50, vllm_omni/diffusion/models/minimax_h3/vae.py:117-224. +- [validated] vae.py:117-120 adds `self.device_module = torch.get_device_module()` in __init__; repo-wide grep shows it is the only use of get_device_module. +- [validated] vae.py:163-182 and 206-229 mirror each other: fork_rng(devices=[param.device]) + torch.default_generator.manual_seed(42) + per-device device_module.manual_seed(42); encode_image still restores parallel_tiling/dtype in finally and encode_video restores dtype. +- [sweep] Blast radius of changed predicate: `parameter.device.type != "cpu"` now covers CUDA, ROCm, XPU, NPU, and MUSA, while the PR body says the allowlist is limited to CUDA and MUSA — mismatch found. +- [sweep] NPU platform check: vllm_omni/platforms/npu/platform.py uses torch.npu.device_count/synchronize/mem_get_info/amp.autocast, but repo-wide grep finds no torch.npu.manual_seed or torch.npu.device usage to substantiate NPU support for the new path. +- [sweep] test_minimax_h3_vae.py does not exist: list_dir tests/diffusion/models/minimax_h3 shows only contract/e2e/packing/parallel/quantization/quantization_quality; grep 'minimax_h3_vae' in tests returned no matches. +- [validated] Existing VAE contract test constructs MiniMaxH3VideoVAE via object.__new__ with a CPU FakeModel (test_minimax_h3_contract.py:344-366), so the new device_module/fork_rng path is never exercised in-tree. +- [sweep] Changed-symbol consumers: pipeline_minimax_h3.py:963 (_encode_visual_condition, rank==0 only) and 1066 (_encode_video_conditions, rank==0 or distributed) were read; broadcasts are unchanged, so no consumer contract is broken by the RNG change. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [major] — PR adds device-aware RNG (`vae.py:120,175-176,219-220`) but no test exercises the non-CPU branch: `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py` named in the PR validation command does not exist (dir contains only contract/e2e/packing/parallel/quantization/quantization_quality), and the existing VAE encode test (`test_video_vae_encode_uses_configured_parallel_tiling`, test_minimax_h3_contract.py:327) constructs `video_vae = object.__new__(MiniMaxH3VideoVAE)` at line 344 with a CPU parameter, so `devices == []` and the branch is skipped. Add a mock test driving `encode_video` with a non-CPU first parameter that asserts `self.device_module.manual_seed` is invoked and RNG state restored, and fix the validation command. (evidence: vae.py:120 ` self.device_module = torch.get_device_module()`; vae.py:170 ` devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:175-176 ` with self.device_module.device(device):` / ` self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; test_minimax_h3_contract.py:327 `def test_video_vae_encode_uses_configured_parallel_tiling():`; test_minimax_h3_contract.py:344 ` video_vae = object.__new__(MiniMaxH3VideoVAE)`; tests/diffusion/models/minimax_h3 directory listing: `__init__.py`, `test_minimax_h3_contract.py`, `test_minimax_h3_e2e.py`, `test_minimax_h3_packing.py`, `test_minimax_h3_parallel.py`, `test_minimax_h3_quantization.py`, `test_minimax_h3_quantization_quality.py` (no `test_minimax_h3_vae.py`); grep `test_minimax_h3_vae` under tests returned no matches.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — vae.py:170 and vae.py:214 replace the CUDA-only filter with `parameter.device.type != "cpu"`, so every non-CPU accelerator (NPU, XPU, MPS, privateuse1) now enters `torch.random.fork_rng(devices=[parameter.device])` plus `self.device_module.device(device)`/`self.device_module.manual_seed(...)` (vae.py:172-176 and 216-220). This contradicts the PR body claim that the allowlist is limited to CUDA and MUSA; MiniMax-H3-NPU.md is an in-tree deployment that reaches this path. Either restrict to an explicit allowlist (e.g. `parameter.device.type in {"cuda","musa"}` or a resolved device-module type check) or update the claim with NPU/XPU RNG evidence. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:175-176 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; PR body: 'The device allowlist is limited to CUDA and MUSA. Other accelerator RNG semantics are not inferred from mocked interfaces.') + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72 labels the following block 'The validated Ref2VA configuration', but line 76 exports MODEL="${MODEL_ROOT}/FL2VA". Lines 69-70 state one server loads one partition and that Ref2VA serving requires MODEL=Ref2VA, so the block as written serves the wrong partition. Change line 76 to export MODEL="${MODEL_ROOT}/Ref2VA" or move the block under the T2VA/FL2VA profile. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:69 `One server loads one checkpoint partition. Set `MODEL` to `FL2VA` for T2VA`; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:70 `and FL2VA, or to `Ref2VA` for Ref2VA.`; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72 `The validated Ref2VA configuration uses tensor parallelism across four MTT`; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76 `export MODEL="${MODEL_ROOT}/FL2VA"`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — CI shows buildkite/vllm-omni-amd-ci failing while buildkite/vllm-omni, intel-ci, and npu-ci pass. The diff does touch shared VAE init/RNG code (vae.py:120 adds torch.get_device_module(); encode_image/encode_video replace torch.cuda with self.device_module). Before merge, please confirm whether the AMD gate is pre-existing on main or attributable to this change; note I could not find any MiniMax H3 test in .buildkite/amd, so the AMD lane's direct construction of MiniMaxH3VideoVAE is not yet verified. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()`; vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:173 `with self.device_module.device(device):`; CI check `buildkite/vllm-omni-amd-ci` state=FAILURE) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — vllm_omni/diffusion/models/minimax_h3/vae.py:120 uses `torch.get_device_module()`, but no requirements file pins a torch lower bound, so the supported torch floor for this API is not verifiable from the requirements. Please document the minimum torch version or add an explicit lower bound. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 ` self.device_module = torch.get_device_module()` +requirements/common.txt:21 `torchsde>=0.2.6` +requirements/npu.txt:3 `torchaudio==2.10.0` +requirements/musa.txt:3 `torchada>=0.1.52` +Repo-wide grep for `torch[<>=!~]` matched no explicit torch version constraint in any requirements file.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:64` [minor] — recipes/MiniMaxAI/MiniMax-H3-MUSA.md:64 and :111 describe the torchaudio->soundfile->ffmpeg fallback, but this PR changes only recipes/README.md, the new recipe, and vllm_omni/diffusion/models/minimax_h3/vae.py. The PR body claims it 'adds coverage for the torchaudio, soundfile, and ffmpeg audio-loading paths', yet no audio-loading code or test is in the diff and the fallback in vllm_omni/diffusion/models/minimax_h3/reference_video.py is unchanged (grep under tests/diffusion/models/minimax_h3 finds no soundfile/ffmpeg/torchaudio/load_audio_file references). Either remove the coverage claim from the PR body or add an actual audio-path regression test. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:64 "When torchaudio cannot load TorchCodec, vLLM-Omni automatically falls back to +soundfile. Formats that libsndfile cannot read are demuxed through ffmpeg."; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:111 "- the soundfile fallback preserves stereo channel layout, FP32 samples, and + the native sample rate when torchaudio/TorchCodec is unavailable."; vllm_omni/diffusion/models/minimax_h3/reference_video.py:480 "torchaudio 2.6+ routes ``load`` through TorchCodec, whose wheels do not load"; PR body: "adds coverage for the torchaudio, soundfile, and ffmpeg audio-loading paths" while gh_pr_view files = [recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/README.md, vllm_omni/diffusion/models/minimax_h3/vae.py] and grep of tests/diffusion/models/minimax_h3 for soundfile|ffmpeg|torchaudio|load_audio_file returns no matches.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.trace.json.gz new file mode 100644 index 000000000..c8d113e6f Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.cost.json new file mode 100644 index 000000000..1b38a6817 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 255.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5715/run-20260814-145729-047b65", + "recorded_at": "2026-08-14T15:01:41", + "attempts": [ + { + "invocation_id": "inv-f1593474", + "outcome": "run:run-20260814-145729-047b65", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-145729-047b65", + "llm_calls": 12, + "input_tokens": 71509, + "output_tokens": 14398, + "usd": 0.046467, + "source": "spans" + } + ], + "attempt_usd_total": 0.046467, + "llm_calls": 12, + "input_tokens": 71509, + "output_tokens": 14398, + "usd": 0.046467, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-145729-047b65", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0465, + "minutes": 4.19, + "ci_minutes": 0, + "input_tokens": 71509, + "output_tokens": 14398, + "tool_calls": 36, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.1748, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 12, + "by_role": { + "agent": { + "usd": 0.046467, + "input_tokens": 71509, + "output_tokens": 14398, + "calls": 12 + } + }, + "cache_read_tokens": 419328, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.46, + "gate": 1.62, + "review": 246.41, + "report": 0.07 + }, + "time_to_first_result_s": 251.53 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 12478 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.md new file mode 100644 index 000000000..1d74457a1 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] README.md:19 release note links recipes/MiniMaxAI/MiniMax-H3.md and recipes/OpenBMB/MiniCPM-o-4_5.md; both recipe files exist and describe the claimed features. +- [validated] README.md:20 (0.24.0 entry) and README.md:23 (0.22.0 consolidation) keep release references consistent; no other docs reference the removed v0.18.0/v0.16.0 news entries. +- [validated] README.md:54 full-duplex realtime entry is a plain feature-list bullet with '(experimental)' suffix; no false capability claim found. +- [validated] README.md:59-61 model list examples (MiniCPM-o 4.5 in omni; MiniMax H3 in diffusion) match recipes/ and docs/models/supported_models.md:47. +- [validated] docs/README.md:56,61 mirror the README model-list edits; content is a duplicate of README as expected. +- [validated] docs/configuration/README.md:5 vLLM 0.26 configuration link is the only docs.vllm.ai link; no stale v0.16.0/v0.22.x configuration links remain in docs. +- [validated] docs/getting_started/installation/README.md:4 compatibility note matches version.py warn_if_misaligned_vllm_version major/minor check. +- [validated] docs/getting_started/installation/gpu/cuda.inc.md:8 and :126-127 pins match docker/Dockerfile.cuda:1 ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0; no stale v0.22.1 base-image refs remain. + +`docs/getting_started/installation/npu/npu.inc.md:62` [minor] — This hunk pins the NPU latest-main path to a vllm-ascend `releases/v0.26.0rc` prerelease branch. The surrounding release-installation section recommends the v0.26.0 Docker image, so a first-time reader could reasonably expect the from-source path to pin a final release ref rather than an rc. If `releases/v0.26.0rc` is the only published branch for the 0.26 line, add a one-line note ('this is the current release candidate branch; swap to releases/v0.26.0 once available') so the transient prerelease pin doesn't silently instruct users onto a branch that may move or disappear. (evidence: docs/getting_started/installation/npu/npu.inc.md:62 uses `releases/v0.26.0rc`, while the Docker-image comments in the same file use `v0.26.0` (`quay.io/atlas-ci/vllm-ascend:v0.26.0`, `v0.26.0-a3`); the PR body only says 'vLLM-Ascend releases/v0.26.0rc' was verified, not that a final 0.26 branch exists.) + +`docs/getting_started/installation/gpu/cuda.inc.md:?` [nit] — This hunk upgrades the CUDA base-image example to v0.26.0, but the build-from-source block above it (lines 73-85, which the diff does NOT touch) runs `cd vllm`, then `python use_existing_torch.py`, `uv pip install ...`, and `uv pip install --no-build-isolation --editable .` without ever `cd ..` before the install instructions end. A reader copy-pasting the sequence ends up outside the repo directory for any subsequent step. Since this PR is the dedicated 0.26 installation-doc pass and already edits this exact file, adding a trailing `cd ..` (or otherwise normalizing the build-from-source block) belongs here. (evidence: docs/getting_started/installation/gpu/cuda.inc.md:73-85 in the PR-time tree: `git clone https://github.com/vllm-project/vllm.git` ... `cd vllm` ... `uv pip install --no-build-isolation --editable .` with no matching `cd ..`; compare the NPU file this PR also edits, which adds `cd ..` after each clone (docs/getting_started/installation/npu/npu.inc.md:58,61).) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.trace.json.gz new file mode 100644 index 000000000..079ae5036 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.cost.json new file mode 100644 index 000000000..cfea18a40 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1447.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5840/run-20260814-150143-4bd217", + "recorded_at": "2026-08-14T15:25:48", + "attempts": [ + { + "invocation_id": "inv-b6977985", + "outcome": "run:run-20260814-150143-4bd217", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-150143-4bd217", + "llm_calls": 147, + "input_tokens": 1487146, + "output_tokens": 240299, + "usd": 1.107991, + "source": "spans" + } + ], + "attempt_usd_total": 1.107991, + "llm_calls": 147, + "input_tokens": 1487146, + "output_tokens": 240299, + "usd": 1.107991, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-150143-4bd217", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.108, + "minutes": 24.08, + "ci_minutes": 0, + "input_tokens": 1487146, + "output_tokens": 240299, + "tool_calls": 243, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0288, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 147, + "by_role": { + "agent": { + "usd": 1.074351, + "input_tokens": 1459558, + "output_tokens": 216520, + "calls": 145 + }, + "reducer": { + "usd": 0.03364, + "input_tokens": 27588, + "output_tokens": 23779, + "calls": 2 + } + }, + "cache_read_tokens": 16375296, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.77, + "gate": 1.67, + "review": 1438.25, + "report": 0.06 + }, + "time_to_first_result_s": 1444.74 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 45567 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.md new file mode 100644 index 000000000..143aaaa04 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.md @@ -0,0 +1,50 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] tests/diffusion/cache/test_teacache_extractors.py:629-632,656,675,697-698 indexed accesses: read the file + diff; sample_inputs['img_pos_info'/'audio_pos_info'] keys and block[0] are guaranteed by _minimax_h3_sample_kwargs and arch num_layers=1 (read minimax_h3_transformer.py). +- [sweep] vllm_omni/diffusion/cache/teacache/extractors.py:1312,1315,1317 shape/length checks mirror the native forward's identical guards (read minimax_h3_transformer.py:1288-1296). +- [sweep] extractors.py:1341-1342 module.blocks[0].adaln_proj/norm1 exist on MiniMaxH3DiTBlock (read minimax_h3_transformer.py MiniMaxH3DiTBlock.__init__). +- [sweep] extractors.py:1379-1384 mask branches: video update_mask has a length guard; update_audio_mask lacks one, but this duplicates native forward behavior (read minimax_h3_transformer.py:1387-1394), so no new regression. +- [sweep] backend.py:79/83/93 partition branches: MiniMaxH3Pipeline always sets self.partition in __init__ (read pipeline_minimax_h3.py:572), so ref2va/fl2va/combined arms are reachable and the unsupported-partition fallback catches None/other. +- [sweep] config.py:129/131 None/<=0 branches: __post_init__ resolves model default or 0.2 then rejects non-positive values (read config.py). +- [sweep] hook.py:236 rel_l1_thresh assert: TeaCacheConfig.__post_init__ guarantees non-None, so the assert is defensive and sound (read hook.py + config.py). +- [sweep] test branches test_cache_backends.py:506-507, test_teacache_extractors.py:451/453/590, test_minimax_h3_e2e.py:150: read; configured_threshold None/0.2 parametrization is sound and the device_count<1 skip is appropriate. + +`vllm_omni/engine/async_omni_engine.py:?` [major] — The diff makes DiffusionCacheConfig.rel_l1_thresh default to None (data.py) and TeaCacheConfig.__post_init__ resolve None to the MiniMax-H3-specific 0.17 (vllm_omni/diffusion/cache/teacache/config.py:96-98, 129-134), but AsyncOmniEngine._get_default_cache_config("tea_cache") still returns {"rel_l1_thresh": 0.2} (async_omni_engine.py:853-856) and _normalize_cache_config injects that dict when cache_config is omitted (async_omni_engine.py:881-882). Consequently `vllm serve ... --cache-backend tea_cache` without `--cache-config`, and the offline Omni default path, force 0.2 and make the MiniMax-H3 model-specific default unreachable. Change the omitted-config path for tea_cache to defer rel_l1_thresh (return {} or omit the key) so None reaches TeaCacheConfig, and add an AsyncOmniEngine/Omni-layer regression test that cache_backend='tea_cache', cache_config=None resolves to 0.17 for MiniMax-H3. (evidence: vllm_omni/engine/async_omni_engine.py:852-856 `if cache_backend == "tea_cache":` / ` return {` / ` "rel_l1_thresh": 0.2,` / ` }` — the omitted-config path hardcodes 0.2; vllm_omni/engine/async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"):` / ` cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)` — injects that dict whenever cache_config is omitted; vllm_omni/diffusion/cache/teacache/config.py:96-98 `_DEFAULT_REL_L1_THRESH = 0.2` / `_MODEL_DEFAULT_REL_L1_THRESH = {` / ` "MiniMaxH3DiTModel": 0.17,` — the intended MiniMax-H3 default that the engine path bypasses.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143` [minor] — The new `test_minimax_h3_t2va_teacache_single_gpu_smoke` (tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143) lives in a file with `pytestmark = [local_model, cuda, diffusion, parallel]`, while CI's diffusion jobs select tests with `-m 'core_model and cpu'` (test-merge.yml) or `-m 'core_model and cuda'` (test-ready.yml). Since this file has neither `core_model` nor `cpu`, the E2E TeaCache path is never selected by CI, leaving only the CPU unit tests as automated TeaCache coverage for MiniMax-H3. Add a CI-selectable regression or explicitly record this E2E (and any `ulysses_degree>1` TeaCache run) as manual-only in the PR description with the exact command already run. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:8-12 `pytestmark = [ pytest.mark.local_model, pytest.mark.cuda, pytest.mark.diffusion, pytest.mark.parallel, ]`; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143 `def test_minimax_h3_t2va_teacache_single_gpu_smoke():`; .buildkite/cuda/test-merge.yml:18 `"timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/cuda/test-ready.yml:65 `"timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level \"core_model\""`) + +`recipes/MiniMaxAI/MiniMax-H3.md:762` [minor] — recipes/MiniMaxAI/MiniMax-H3.md:762 documents the MiniMax-H3 default and examples as rel_l1_thresh=0.17, and the code registers that default (_MODEL_DEFAULT_REL_L1_THRESH['MiniMaxH3DiTModel'] = 0.17). However, PR #5840's online smoke server command passes --cache-config '{"rel_l1_thresh":0.2}' and its server evidence logs 'TeaCache applied with rel_l1_thresh=0.2, transformer_class=MiniMaxH3DiTModel', so the online HTTP path does not exercise the documented 0.17 default (0.17 is covered by the offline sweep and the single-GPU E2E smoke test, not by the online smoke). Re-run the online smoke with 0.17 or without cache-config, or narrow the recipe sentence. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:762 `The model-specific default and examples use `rel_l1_thresh=0.17`, which`; PR #5840 body online smoke server: `--cache-config '{"rel_l1_thresh":0.2}'`; PR #5840 body server evidence: `TeaCache applied with rel_l1_thresh=0.2, transformer_class=MiniMaxH3DiTModel`; vllm_omni/diffusion/cache/teacache/config.py:97-99 `_MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, }`.) + +`vllm_omni/diffusion/cache/teacache/extractors.py:1352` [minor] — The diff adds `extract_minimax_h3_context`, whose `run_transformer_blocks` shards `decoder_input`/`rope_freqs` through `module.sp_prepare` and gathers `hidden` through `module.sp_gather` (vllm_omni/diffusion/cache/teacache/extractors.py:1352,1368), and the feature table now marks MiniMax-H3 TeaCache supported. The TeaCache E2E smoke sets `DiffusionParallelConfig(ulysses_degree=1)` (tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:157), so SP+TeaCache parity is not exercised there. Add an E2E smoke with `ulysses_degree>1` plus `cache_backend="tea_cache"` (skipped below the required GPU count), or mark the combination unverified in the docs/feature table. (evidence: vllm_omni/diffusion/cache/teacache/extractors.py:1352 ` hidden, block_rope, block_combined = module.sp_prepare(` +vllm_omni/diffusion/cache/teacache/extractors.py:1368 ` hidden = module.sp_gather(hidden)` +tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:157 ` parallel_config=DiffusionParallelConfig(ulysses_degree=1),`) + +`vllm_omni/entrypoints/cli/serve.py:?` [minor] — The PR changes DiffusionCacheConfig.rel_l1_thresh from 0.2 to None (vllm_omni/diffusion/data.py:422) and registers {"MiniMaxH3DiTModel": 0.17} with a 0.2 generic fallback (vllm_omni/diffusion/cache/teacache/config.py:96-98,130). The untouched CLI help at vllm_omni/entrypoints/cli/serve.py:586 and docs/user_guide/diffusion/cache_acceleration/teacache.md:122 still present 0.2 as the default without mentioning the None-deferral or the MiniMax-H3 override. Update both to state that an unset value defers to the model-specific default (MiniMax-H3: 0.17) and falls back to 0.2 otherwise. (evidence: vllm_omni/entrypoints/cli/serve.py:586 `"TeaCache: '{\"rel_l1_thresh\": 0.2}'. "`; docs/user_guide/diffusion/cache_acceleration/teacache.md:122 `| `rel_l1_thresh` | float | `0.2` | Similarity threshold for cache reuse. Lower values prioritize quality (less caching), higher values prioritize speed (more caching). Suggested range: 0.1-0.8 |`; vllm_omni/diffusion/data.py:422 `rel_l1_thresh: float | None = None`; vllm_omni/diffusion/cache/teacache/config.py:96 `_DEFAULT_REL_L1_THRESH = 0.2`; vllm_omni/diffusion/cache/teacache/config.py:98 `"MiniMaxH3DiTModel": 0.17,`; vllm_omni/diffusion/cache/teacache/config.py:130 `threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)`) + +`tests/diffusion/cache/test_teacache_extractors.py:?` [minor] — The new `test_teacache_reuses_h3_block_residual` proves caching only with `rel_l1_thresh=0.3` (tests/diffusion/cache/test_teacache_extractors.py:688). The MiniMax-H3 coefficient constant term is `2.173782527946167e-01` (vllm_omni/diffusion/cache/teacache/config.py:92), so identical inputs (relative L1 = 0) rescale to ~0.217 and would NOT cache at the shipped MiniMax-H3 default `rel_l1_thresh=0.17`. The E2E smoke sets `cache_config={"rel_l1_thresh": 0.17}` (test_minimax_h3_e2e.py:156) with `num_inference_steps=2` (line 170) but only calls `_assert_joint_output(outputs)` (line 186), which checks shapes/metadata. Please add/extend a test that feeds representative consecutive-step inputs through the 0.17 config and asserts at least one block skip, or make the E2E smoke assert a cache hit, so a 0.17-never-caches regression cannot pass CI. (evidence: vllm_omni/diffusion/cache/teacache/config.py:92 ` 2.173782527946167e-01,` (constant term of MiniMaxH3DiTModel coefficients). tests/diffusion/cache/test_teacache_extractors.py:688 ` rel_l1_thresh=0.3,`. vllm_omni/diffusion/cache/teacache/hook.py:234 ` if state.accumulated_rel_l1_distance < rel_l1_thresh:` (cache only when below threshold). tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:156 ` cache_config={"rel_l1_thresh": 0.17},` and tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:186 ` _assert_joint_output(outputs)` (no cache-hit assertion).) + +`.buildkite/amd/test-amd-ready.yml:?` [minor] — .buildkite/amd/test-amd-ready.yml:19 runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py ...`, and tests/diffusion/cache/test_cache_backends.py:32 is `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, so this PR's new TeaCache backend tests run in the failing `buildkite/vllm-omni-amd-ci` lane. Verify whether the failure reproduces on this PR head or is pre-existing on main before merging; the red state alone does not prove cause. (evidence: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.trace.json.gz new file mode 100644 index 000000000..51b49091e Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5863.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5863.cost.json new file mode 100644 index 000000000..e624257f2 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5863.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1515.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5863/run-20260814-151519-ae62d9", + "recorded_at": "2026-08-14T15:40:31", + "attempts": [ + { + "invocation_id": "inv-6055e342", + "outcome": "run:run-20260814-151519-ae62d9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-151519-ae62d9", + "llm_calls": 86, + "input_tokens": 844690, + "output_tokens": 147657, + "usd": 0.574822, + "source": "spans" + } + ], + "attempt_usd_total": 0.574822, + "llm_calls": 86, + "input_tokens": 844690, + "output_tokens": 147657, + "usd": 0.574822, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-151519-ae62d9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5748, + "minutes": 25.21, + "ci_minutes": 0, + "input_tokens": 844690, + "output_tokens": 147657, + "tool_calls": 164, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8517, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 86, + "by_role": { + "agent": { + "usd": 0.554636, + "input_tokens": 833629, + "output_tokens": 132053, + "calls": 84 + }, + "reducer": { + "usd": 0.020185, + "input_tokens": 11061, + "output_tokens": 15604, + "calls": 2 + } + }, + "cache_read_tokens": 6827136, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.09, + "gate": 1.72, + "review": 1506.58, + "report": 0.06 + }, + "time_to_first_result_s": 1512.43 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 28294 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5863.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5863.md new file mode 100644 index 000000000..0d82d65b7 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5863.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read the entire new file recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md (346 lines) in-tree; it is a pure docs addition with no code/test changes (diff hunk line 1). +- [validated] All flags used by the recipe exist in vllm_omni/entrypoints/cli/serve.py: --task-type (213), --usp (461), --ring (479), --diffusion-attention-backend (538), --vae-use-tiling (605), --vae-patch-parallel-size (711), --vae-parallel-mode (728), --enable-diffusion-pipeline-profiler (784), plus --num-gpus/--tensor-parallel-size/--text-encoder-tp-size/--quantization. +- [validated] Attention-backend claim correct: platform.py get_diffusion_attn_backend_cls only defaults TRTLLM_ATTN when compute_capability.major==10 (datacenter Blackwell); sm_120 (major 12) falls to CUDNN_ATTN (platform.py:189-226). 'SM120 / TRTLLM_ATTN default does not apply' is accurate. +- [validated] 'H3 is CFG-distilled, --cfg-parallel-size must remain 1' is enforced: MiniMaxH3Pipeline.__init__ raises ValueError when cfg_parallel_size != 1 (pipeline_minimax_h3.py). +- [validated] text_encoder_tp_size 2/4/8 valid: must divide 64 attn heads and 8 KV heads and not exceed DiT group (pipeline_minimax_h3.py __init__); DiT group = TP x USP x ring (parallel_state.py dit_parallel_size), so 2/4/8 are valid for the doc's configs. +- [validated] VAE patch parallelism must be 1 or full DiT group size (minimax_h3/vae.py:167); the doc's VPP 2/4/8 each equal TP x USP (2/4/8), so all configs are valid. +- [validated] Rank grouping claims correct: RankGenerator order 'tp-sp-pp-cfg-dp' yields TP groups consecutive and SP/Ulysses groups strided by TP (parallel_state.py); for TP2/USP4 the Ulysses groups are (0,2,4,6)/(1,3,5,7) and TP pairs (0,1),(2,3),(4,5),(6,7) as the doc states. +- [validated] 't2va requires an explicit aspect_ratio even when width/height are supplied' matches _resolve_minimax_h3_aspect_ratio, which raises 't2va requires an explicit aspect_ratio' when value is None (pipeline_minimax_h3.py:329-374); ref2va defaults to 16:9. + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [major] — In `recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md` lines 201-203, `--cpunodebind` and `--membind` are listed next to `CUDA_VISIBLE_DEVICES` and `--port` as if they are `vllm serve` flags, but they are `numactl` options and are not registered by `vllm_omni/entrypoints/cli/serve.py` (repo-wide grep finds these tokens only in this recipe; `numactl` appears only in `docker/Dockerfile.xpu:25` as an installed package). Rewrite the two-server instructions to invoke numactl explicitly, e.g. `CUDA_VISIBLE_DEVICES=0,2,1,3 numactl --cpunodebind=0 --membind=0 vllm serve ... --port 8091` (mirrored for the Ref2VA server), or at minimum state that these are `numactl` arguments, so the command does not fail with unrecognized-argument errors. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201 `Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and` ; recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:203 ``--cpunodebind=1 --membind=1`, and `--port 8092` for the Ref2VA server.` ; vllm_omni/entrypoints/cli/serve.py:561 `omni_config_group.add_argument( + "--text-encoder-tp-size",` (serve CLI registers diffusion flags such as --num-gpus/--usp/--ring/--text-encoder-tp-size/--vae-patch-parallel-size but never --cpunodebind/--membind); docker/Dockerfile.xpu:25 ` numactl \` (only repo-wide numactl match).) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:?` [minor] — The Ref2VA example uses curl's `-F "field= 0)`, default `guidance_scale=3.0` at pipeline_soulx_singer_base.py:317/356) = two forward passes per step. Per the repo's own invariant (model_specific.py:818), the flag must match the per-step forward count, so this path can desync cond/uncond parity and silently degrade audio. Set `has_separate_cfg=True` (or add a paired-CFG guard) for SoulXSinger with a nested-path enable/refresh test, or scope the change so SoulXSinger is not enabled. (evidence: backend.py:48 `module = attrgetter(name)(pipeline)` (this diff) — dotted `_dit_modules` now resolve. pipeline_soulx_singer_base.py:63 `_dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]`. llama.py:115 `_cache_dit_adapter_config = CacheDiTAdapterConfig(` with only `block_forward_patterns={"layers": ForwardPattern.Pattern_3}`. config.py:29 `has_separate_cfg: bool = False`. pipeline_soulx_singer_base.py:317 `do_true_cfg=(cfg > 0),` and :356 `guidance_scale = sampling_params.guidance_scale if sampling_params.guidance_scale_provided else 3.0`. model_specific.py:818 `cache-dit tells cond/uncond apart purely by transformer-forward parity, so the flag must match the actual per-step forward count — True only for the CFG (Raw) path.`) + +`tests/diffusion/cache/test_cache_backends.py:?` [minor] — In `test_enable_and_refresh_nested_declared_dit` (tests/diffusion/cache/test_cache_backends.py:138-162), `@patch('vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter')` makes the new guard `if not BlockAdapter.is_cached(transformer): continue` (backend.py:61) a truthy Mock, so `mock_cache_dit.summary.assert_called_once_with(transformer, details=True)` passes even if the real enable/refresh path never marked the transformer cached. Set `transformer._is_cached = True` and drop the `BlockAdapter` patch for the final `cache_summary` step (or assert the flag explicitly), and add an enable-driven test for the `['transformer.language_model', 'transformer']` pair asserting only `transformer` is summarized. (evidence: tests/diffusion/cache/test_cache_backends.py:138-139 ` @patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter") + @patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit") + def test_enable_and_refresh_nested_declared_dit(self, mock_cache_dit, mock_block_adapter):`; tests/diffusion/cache/test_cache_backends.py:157-158 ` cache_summary(pipeline) + mock_cache_dit.enable_cache.assert_called_once()`; tests/diffusion/cache/test_cache_backends.py:163 ` mock_cache_dit.summary.assert_called_once_with(transformer, details=True)`; vllm_omni/diffusion/cache/cachedit/backend.py:61 ` if not BlockAdapter.is_cached(transformer):`) + +`tests/diffusion/cache/test_cache_backends.py:?` [minor] — `.buildkite/amd/test-amd-ready.yml:19` runs `pytest -sv tests/diffusion ...` with only `test_teacache_extractors.py` ignored, so it executes `tests/diffusion/cache/test_cache_backends.py`; the PR adds tests there at lines 121 and 139. CI shows `buildkite/vllm-omni-amd-ci` FAILURE while `buildkite/vllm-omni` passes. Determine whether the AMD red is pre-existing on main or introduced by this branch before merging. (evidence: tests/diffusion/cache/test_cache_backends.py:121 ` def test_summary_skips_uncached_nested_dit(self, mock_cache_dit):`; tests/diffusion/cache/test_cache_backends.py:139 ` def test_enable_and_refresh_nested_declared_dit(self, mock_cache_dit, mock_block_adapter):`; .buildkite/amd/test-amd-ready.yml:19 ` - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`) + +`tests/diffusion/cache/test_cache_backends.py:?` [minor] — The new nested-DiT tests at tests/diffusion/cache/test_cache_backends.py:139 use `SimpleNamespace`/`Mock` and do not instantiate the real `BagelPipeline` whose `_dit_modules=["language_model.model"]` was the failing discovery path; they validate attrgetter mechanics but not the real Bagel discovery. Please name and link the exact regression run: `pytest -sv tests/e2e/online_serving/test_bagel_expansion.py -k single_card_cache_dit --run-level core_model` (PR body currently only reports 'Bagel full-model E2E: 1 passed' without the case). (evidence: tests/diffusion/cache/test_cache_backends.py:139 `def test_enable_and_refresh_nested_declared_dit(self, mock_cache_dit, mock_block_adapter):`; tests/diffusion/cache/test_cache_backends.py:145-147 `pipeline = SimpleNamespace(_dit_modules=["language_model.model"], language_model=SimpleNamespace(model=transformer))`; vllm_omni/diffusion/models/bagel/pipeline_bagel.py:159 `_dit_modules: ClassVar[list[str]] = ["language_model.model"]` (real declaration, not instantiated by the test); PR 5884 body Test Result: `Bagel full-model E2E: 1 passed in 212.18s, including valid 512x512 response validation.` (no exact case named); issue 5879 title: `tests/e2e/online_serving/test_bagel_expansion.py::test_bagel[single_card_cache_dit], ValueError: Pipeline BagelPipeline has no declared DiT modules for Cache-DiT`.) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:?` [minor] — In `diffusion_model_runner.py`, `_compile_transformer` still resolves each `_dit_modules` entry with `getattr(self.pipeline, attr_name, None)` and returns on None, so dotted declarations such as Bagel `language_model.model` and SoulXSinger `cfm_decoder.model.diff_estimator` are silently skipped for torch.compile when `enforce_eager=false`. Since backend.py now uses `attrgetter`, align `_compile_transformer` (or the `_dit_modules` resolution) and add a nested-path compile regression test; otherwise explicitly document that nested `_dit_modules` are not compile targets. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:283 `transformer_attrs = getattr(self.pipeline, "_dit_modules", None)`; vllm_omni/diffusion/worker/diffusion_model_runner.py:153-154 `model = getattr(self.pipeline, attr_name, None)` / `if model is None:` / ` return`; vllm_omni/diffusion/models/bagel/pipeline_bagel.py:159 `_dit_modules: ClassVar[list[str]] = ["language_model.model"]`; vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_base.py:63 `_dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [nit] — Update the docstring at backend.py:57 to state that cache_summary only summarizes cached transformers; optionally warn when transformers resolve but none are cached, since that path is currently a silent no-op. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:57 ` """Log Cache-DiT statistics for every transformer on the pipeline."""` +vllm_omni/diffusion/cache/cachedit/backend.py:61 ` if not BlockAdapter.is_cached(transformer):` +vllm_omni/diffusion/cache/cachedit/backend.py:62 ` continue`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5884.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5884.trace.json.gz new file mode 100644 index 000000000..92368de51 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5884.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.cost.json new file mode 100644 index 000000000..cebedd561 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1652.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5957/run-20260814-151805-a06d81", + "recorded_at": "2026-08-14T15:45:36", + "attempts": [ + { + "invocation_id": "inv-c50cbd38", + "outcome": "run:run-20260814-151805-a06d81", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-151805-a06d81", + "llm_calls": 167, + "input_tokens": 1698801, + "output_tokens": 278668, + "usd": 1.463945, + "source": "spans" + } + ], + "attempt_usd_total": 1.463945, + "llm_calls": 167, + "input_tokens": 1698801, + "output_tokens": 278668, + "usd": 1.463945, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-151805-a06d81", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.4639, + "minutes": 27.51, + "ci_minutes": 0, + "input_tokens": 1698801, + "output_tokens": 278668, + "tool_calls": 256, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1906, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 167, + "by_role": { + "agent": { + "usd": 1.377648, + "input_tokens": 1502152, + "output_tokens": 248541, + "calls": 164 + }, + "reducer": { + "usd": 0.086297, + "input_tokens": 196649, + "output_tokens": 30127, + "calls": 3 + } + }, + "cache_read_tokens": 25879040, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.91, + "gate": 1.7, + "review": 1643.57, + "report": 0.06 + }, + "time_to_first_result_s": 1650.24 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5957.trace.json.gz", + "trace_bytes": 134530 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.md new file mode 100644 index 000000000..1fc02675a --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.md @@ -0,0 +1,60 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5957, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 19a6e513f28e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] read central changed file vllm_omni/model_executor/stage_input_processors/indextts2.py in full — verified code-only path (output_token_ids→mel_codes), latent-path mismatch raise, seed passthrough, and fail-fast conditioning checks. +- [sweep] read vllm_omni/entrypoints/openai/tts_adapters/indextts2.py — verified IndexTTS25Adapter.native_speed_control=True, validate() bounds [0.5,2.0], duration_factor=1.0/speed. +- [sweep] read vllm_omni/model_executor/models/indextts2/indextts2_s2mel_decoder.py (offset 400) — verified _duration_factors, _target_lengths, _run_cfm_groups, _validate_payload_policy/contract, _sample_cfm_noise. +- [sweep] read vllm_omni/model_executor/models/indextts2/indextts2_talker.py — verified omni_payload_at_request_end=indextts2_5, omni_pooler_payload_include_hidden=use_gpt_latent, duration_factor emitted in prefill meta and validated. +- [sweep] grepped talker2s2mel_full_payload/custom_process_next_stage_input_func consumers — producer is indextts2 pipeline stages 0; connector mixin and chunk_transfer_adapter both call the hook. +- [sweep] read connector error paths: vllm_omni/worker/omni_connector_model_runner_mixin.py _build_custom_process_payload tail and vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py _send_single_request — both catch Exception and return None/empty payload. +- [sweep] read benchmarks/tts/bench_tts.py — verified --request-seed injects extra_body.seed; --served-model-name, --num-warmups, and '--' passthrough strip logic. +- [sweep] read vllm_omni/engine/arg_utils.py _patch_empty_hf_config — verified model-scoped exception fallback only when model_type=='indextts2_5' and local dir lacks config.json. + +`vllm_omni/model_executor/stage_input_processors/indextts2.py:?` [major] — This diff converts the full-payload builder's silent `return None` paths into fail-fast `ValueError`s (`stage_input_processors/indextts2.py:242` and the `meta.use_gpt_latent`, mel-codes, and conditioning guards). Those raises do not reach the client in the serving path: `chunk_transfer_adapter._send_single_request` wraps `custom_process_next_stage_input_func` in `except Exception` -> `logger.error(...)` and substitutes `OmniPayloadStruct()` for a None result, while `omni_connector_model_runner_mixin._build_custom_process_payload` logs `custom_process_stage_input_func failed for chunk` and returns None, which `send_full_payload_outputs` drops via `if payload is None: continue`. Either fail the request with the ValueError at the API boundary or document that malformed payloads are log-and-drop; otherwise the advertised fail-fast contract is undelivered. (evidence: ['vllm_omni/model_executor/stage_input_processors/indextts2.py:242 `raise ValueError(f"IndexTTS code-only payload has no completed output_token_ids for req={rid}")`', 'vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py `except Exception as e:` / `logger.error(f"Failed to use custom_process_input_func for payload extraction: {e}")` followed by `if payload_data is None:` -> `payload_data = OmniPayloadStruct()`', 'vllm_omni/worker/omni_connector_model_runner_mixin.py:1927 `logger.exception("custom_process_stage_input_func failed for chunk %s", request_id)` and `send_full_payload_outputs`: `if payload is None:` -> `continue`']) + +`tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:16` [major] — The three new test files — test_configuration_and_pipeline.py, test_enhanced_codec.py, and test_recipe_defaults.py — have no module-level `pytestmark`, so every CI lane using `-m 'core_model and cpu'` deselects them and they never execute. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` to each file, matching the other 7 files in tests/model_executor/models/indextts2. (evidence: ['.buildkite/cuda/test-ready.yml:13 `- "timeout 20m pytest -sv tests/model_executor -m \'core_model and cpu\' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`', 'tests/model_executor/models/indextts2/test_recipe_defaults.py:1-14 `# SPDX-License-Identifier: Apache-2.0\\n# SPDX-FileCopyrightText: Copyright contributors to the vLLM project\\n\\nfrom pathlib import Path\\n\\nimport yaml\\n\\n\\ndef test_indextts25_default_recipe_enables_validated_cfm_batching():\\n recipe_path = Path(__file__).parents[4] / "vllm_omni" / "deploy" / "indextts2_5.yaml"\\n recipe = yaml.safe_load(recipe_path.read_text(encoding="utf-8"))\\n stage1 = next(stage for stage in recipe["stages"] if stage["stage_id"] == 1)\\n\\n assert stage1["hf_overrides"]["s2mel_cfm_batch_size"] == 4`', 'tests/model_executor/models/indextts2/test_enhanced_codec.py:1-8 `# SPDX-License-Identifier: Apache-2.0\\n# SPDX-FileCopyrightText: Copyright contributors to the vLLM project\\n\\nimport torch\\n\\nfrom vllm_omni.model_executor.models.indextts2 import codec as codec_module\\nfrom vllm_omni.model_executor.models.indextts2 import preprocess_utils\\nfrom vllm_omni.model_executor.models.indextts2.codec import EnhancedCodec` (read in full; no pytestmark assignment in the file)', 'tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:1-16 `# SPDX-License-Identifier: Apache-2.0\\n# SPDX-FileCopyrightText: Copyright contributors to the vLLM project\\n\\nfrom pathlib import Path\\n\\nimport yaml\\n\\nfrom vllm_omni.config.pipeline_registry import OMNI_PIPELINES\\nfrom vllm_omni.model_executor.models.indextts2.configuration_indextts2 import (\\n IndexTTS2Config,\\n IndexTTS25Config,\\n)\\nfrom vllm_omni.model_executor.models.indextts2.pipeline import INDEXTTS25_PIPELINE\\n\\n\\ndef test_indextts25_defaults_are_distinct_from_v2():` (read in full; no pytestmark assignment in the file)']) + +`.buildkite/amd/test-amd-ready.yml:19` [minor] — `buildkite/vllm-omni-amd-ci` is FAILURE while the other checks pass. `.buildkite/amd/test-amd-ready.yml:19` runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`, and every new `tests/model_executor/models/indextts2/test_*.py` sets `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, so these tests execute in the failing lane. Attribution (pre-existing main vs this PR) is still undetermined; please link the main-branch comparison run. This is an attribution question, not a confirmed code defect, so severity should be minor. (evidence: `.buildkite/amd/test-amd-ready.yml:19` `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; `tests/model_executor/models/indextts2/test_s2mel_v2_5.py:26` `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; `gh_ci_read(5957)` returns `{"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` while `buildkite/vllm-omni`, `buildkite/vllm-omni-intel-ci`, and `buildkite/vllm-omni-npu-ci` are SUCCESS.) + +`vllm_omni/engine/stage_init_utils.py:?` [minor] — The new branch at vllm_omni/engine/stage_init_utils.py:1140 installs `_TokenOnlyRenderer` for every `skip_tokenizer_init` stage, not only IndexTTS 2.5. IndexTTS-2 (indextts2/pipeline.py:29 stage-0 extras; also :46) and PersonaPlex (personaplex.yaml:42, :69) also set `skip_tokenizer_init: true` and therefore now route chat-message inputs to `render_messages`, which raises ValueError (stage_init_utils.py:1125-1128). These models submit prompt_token_ids so serving should be unaffected, but this is a repo-wide common-path flip — add/name regression coverage that the existing skip_tokenizer_init models still serve. (evidence: vllm_omni/engine/stage_init_utils.py:1140 `if bool(getattr(stage_vllm_config.model_config, "skip_tokenizer_init", False)):` ; vllm_omni/engine/stage_init_utils.py:1126-1128 `def render_messages(self, messages, params): raise ValueError("Chat messages are unavailable when skip_tokenizer_init=True; submit prompt_token_ids or prompt_embeds")` ; vllm_omni/model_executor/models/indextts2/pipeline.py:29 `extras={"skip_tokenizer_init": True, "tokenizer": "gpt2"},` ; vllm_omni/deploy/personaplex.yaml:42 `skip_tokenizer_init: true` and :69 `skip_tokenizer_init: true`) + +`pyproject.toml:?` [minor] — The `indextts2` extra (pyproject.toml:87-91) omits `torchaudio`, but the IndexTTS prefill path now hard-fails without it: `compute_fbank` (preprocess_utils.py:282) raises `RuntimeError("IndexTTS requires torchaudio Kaldi fbank support")` instead of the previous zero-fbank fallback, and it is called on the required ref-audio path at indextts2_talker.py:634. `torchaudio` is declared only in requirements/npu.txt:3, so `pip install 'vllm-omni[indextts2]'` does not install it. Add `torchaudio` to this extra (or the core deps) so the documented install does not fail on the first request. (evidence: pyproject.toml:87-91 `indextts2 = [\n "fugashi>=1.2.0",\n "unidic-lite>=1.0.0",\n "wetext>=0.0.9; sys_platform != 'linux'",\n "WeTextProcessing; sys_platform == 'linux'",\n]` — no torchaudio. vllm_omni/model_executor/models/indextts2/preprocess_utils.py:282 `raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc`. vllm_omni/model_executor/models/indextts2/indextts2_talker.py:634 `fbank = compute_fbank(wav_16k, device)`. requirements/npu.txt:3 `torchaudio==2.10.0` (the only torchaudio declaration among pyproject.toml/setup.py/requirements).) + +`vllm_omni/deploy/qwen2_5_omni.yaml:?` [minor] — This hunk flips Qwen2.5-Omni XPU stages 0/1 from enforce_eager: false to enforce_eager: true (vllm_omni/deploy/qwen2_5_omni.yaml:90 and :93) as a PyTorch 2.13 XPU Dynamo workaround, but the only added coverage is tests/config/test_config_factory.py:2279, a config-merge assertion that the override resolves to True. The PR Test Plan's E2E environment is 1x NVIDIA H200 (CUDA) and no XPU lane is cited as actually running qwen2_5_omni after the flip. Since this changes execution mode for every Qwen2.5-Omni-on-XPU deployment, either name the Intel XPU lane/run that exercised it or split this platform change out of the new-model PR. (evidence: ['vllm_omni/deploy/qwen2_5_omni.yaml:90 ` enforce_eager: true` (xpu stage_id 0; diff flipped from false)', 'vllm_omni/deploy/qwen2_5_omni.yaml:93 ` enforce_eager: true` (xpu stage_id 1; diff flipped from false)', 'vllm_omni/deploy/qwen2_5_omni.yaml:86-87 ` # PyTorch 2.13 XPU Dynamo raises duplicate-handler assertions while` / ` # compiling both Qwen2.5 AR stages. Keep them eager until that toolchain`', 'tests/config/test_config_factory.py:2279 ` def test_qwen2_5_omni_xpu_uses_eager_ar_stages(self):` with `assert deploy.stages[0].enforce_eager is True` / `assert deploy.stages[1].enforce_eager is True` — config-merge only', 'PR #5957 body Test Plan: `Environment used for E2E validation:` `- 1x NVIDIA H200` ... `- CUDA driver compatibility: 12.8` — no XPU lane cited']) + +`vllm_omni/model_executor/models/indextts2/preprocess_utils.py:282` [minor] — Confirmed: preprocess_utils.py:282 replaces the old `return torch.zeros(1, 100, 80, device=device)` fallback with `raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc`. The caller indextts2_talker.py:634 (`fbank = compute_fbank(wav_16k, device)`) sits in the shared `IndexTTS2TalkerForConditionalGeneration._preprocess_prefill` and is not gated by model_type, so the behavior change also hits existing IndexTTS-2.0 voice cloning. No test covers the missing-Kaldi path. Please confirm the 2.0 change is intended and add a unit test for the missing-torchaudio-kaldi path. (evidence: preprocess_utils.py:276-282 ` except (ImportError, RuntimeError, OSError) as exc:\n raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc`; indextts2_talker.py:632-635 ` campplus = load_campplus(self.model_path, device)\n fbank = compute_fbank(wav_16k, device)\n with torch.no_grad():\n style = campplus(fbank) # [1, 192]` — compute_fbank is called unconditionally in the shared talker prefill, with no `model_type == "indextts2_5"` gate.) + +`vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py:251` [minor] — `tiktoken` is used at vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py:251 (`import tiktoken`) and :270-276 (`tiktoken.Encoding(name=..., explicit_n_vocab=..., pat_str=_PATTERN, mergeable_ranks=..., special_tokens=...)`), but it is not declared in `pyproject.toml`'s `indextts2` extra (only fugashi, unidic-lite, wetext, WeTextProcessing) nor anywhere in `requirements/`. Declare it with a lower bound supporting those `Encoding` kwargs, otherwise `pip install 'vllm-omni[indextts2]'` can fail with ImportError on this path. (evidence: vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py:251 ` import tiktoken`; vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py:270-276 ` return tiktoken.Encoding( + name=Path(vocab_path).name, + explicit_n_vocab=explicit_n_vocab, + pat_str=_PATTERN, + mergeable_ranks=ranks, + special_tokens=special_tokens, + )`; pyproject.toml `[project.optional-dependencies]` indextts2 block lists only `fugashi>=1.2.0`, `unidic-lite>=1.0.0`, `wetext>=0.0.9; sys_platform != 'linux'`, `WeTextProcessing; sys_platform == 'linux'` — no tiktoken.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:224` [minor] The PR body promises 'Official Chinese, English, Japanese, Spanish, and Arabic language preprocessing', but `_normalize_with_official_backend` (text_processing_v2_5.py:222) only has a real number-expansion backend for zh + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.trace.json.gz new file mode 100644 index 000000000..e68b7a62e Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.cost.json b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.cost.json new file mode 100644 index 000000000..988295f4e --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2079.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_ds_wave2_r2/runs/pr5976/run-20260814-152319-04aa17", + "recorded_at": "2026-08-14T15:57:57", + "attempts": [ + { + "invocation_id": "inv-6f289089", + "outcome": "run:run-20260814-152319-04aa17", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-152319-04aa17", + "llm_calls": 166, + "input_tokens": 1301222, + "output_tokens": 281987, + "usd": 1.167115, + "source": "spans" + } + ], + "attempt_usd_total": 1.167115, + "llm_calls": 166, + "input_tokens": 1301222, + "output_tokens": 281987, + "usd": 1.167115, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-152319-04aa17", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.1671, + "minutes": 34.62, + "ci_minutes": 0, + "input_tokens": 1301222, + "output_tokens": 281987, + "tool_calls": 311, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2036, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 166, + "by_role": { + "agent": { + "usd": 1.132537, + "input_tokens": 1250728, + "output_tokens": 262978, + "calls": 164 + }, + "reducer": { + "usd": 0.034578, + "input_tokens": 50494, + "output_tokens": 19009, + "calls": 2 + } + }, + "cache_read_tokens": 18725888, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.49, + "gate": 1.69, + "review": 2069.97, + "report": 0.06 + }, + "time_to_first_result_s": 2077.19 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 72705 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.md b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.md new file mode 100644 index 000000000..bdeb1884f --- /dev/null +++ b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read vllm_omni/core/sched/omni_ar_scheduler.py:320-680 and vllm_omni/core/sched/omni_scheduler_mixin.py:88-140 — stale drain/subtraction and both discard sites verified in situ; found the double seed on the replace path. +- [sweep] Grep for _replace_streaming_session and replace_streaming_prompt: AR calls mixin via omni_ar_scheduler.py:670 only after pre-seeding at line 629; generation scheduler calls mixin directly at omni_generation_scheduler.py:604 (single seed, correct). +- [sweep] Grep repo for num_in_flight_tokens: producers/consumers are omni_ar_scheduler.py, omni_generation_scheduler.py, omni_scheduler_mixin.py, plus test doubles; no other discard site found. +- [validated] Read tests/core/sched/test_omni_ar_scheduler_stale_drain.py:49-223 — tests drive _replace_streaming_session directly, never through OmniARScheduler._update_request_as_session, so they cannot observe the double seed. +- [validated] Read tests/core/sched/test_omni_ar_scheduler_streaming.py:220-340 — existing streaming tests cover _replace_streaming_session but do not set num_in_flight_tokens on the replace path. +- [sweep] Read vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py:125-430 and qwen3_omni_moe_thinker.py:1480-1535 — bare-tensor vs tuple return contract matches the new tests; IntermediateTensors pass-through is handled. +- [sweep] Read vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py forward and __init__ model_stage default — non-staged default to thinker is consistent with the existing thinker branch. +- [sweep] Grep FusedMoEFactory/FusedMoE imports: all changed import sites updated; upstream symbol availability not verifiable from this tree (no installed vLLM), recorded as an assumption. + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [major] — Confirmed: _update_request_as_session seeds `session.num_stale_output_tokens += in_flight_tokens` (vllm_omni/core/sched/omni_ar_scheduler.py:629) and then, when `replace_streaming_prompt` is true, calls `self._replace_streaming_session(session, update)` (line 670), whose vllm_omni/core/sched/omni_scheduler_mixin.py:105 adds `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` again. No reset/decrement of `num_in_flight_tokens` or `num_stale_output_tokens` occurs between the two writes, so the replace path double-counts every in-flight token and can swallow the new segment's first valid frame(s) or trip `assert request.num_stale_output_tokens >= 0`. Seed exactly once on this path and cover it with an `additional_information={"meta": {"replace_streaming_prompt": True}}` test. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:628-629: `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `session.num_stale_output_tokens += in_flight_tokens`; vllm_omni/core/sched/omni_ar_scheduler.py:669-670: `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)`; vllm_omni/core/sched/omni_scheduler_mixin.py:105: `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`. Between line 629 and the call at line 670 there is no assignment or decrement of `session.num_in_flight_tokens` or `session.num_stale_output_tokens` (the only intervening branch that returns early is the chunk_transfer_adapter receives_chunks path).) + +`vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py:86` [major] — qwen2_5_omni.py:86 now defaults model_stage to "thinker" so a plain `vllm serve Qwen2.5-Omni` constructs this class, but the thinker branch of forward() still returns an OmniOutput NamedTuple (qwen2_5_omni.py:325-326). Stock vLLM's GPUModelRunner indexes the raw forward output via hidden_states[logit_indices] during _dummy_run/profile_run, so the plain run hits the same `TypeError: only integer tensors of a single element can be converted to an index` that the Qwen3 fix in this PR prevents. Make the plain-run thinker forward return a bare tensor (keep OmniOutput for the staged contract) and add a forward-contract test analogous to tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py. (evidence: vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py:86 `self.model_stage = getattr(vllm_config.model_config, "model_stage", None) or "thinker"`; vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py:325-326 `# Text-only path` / `return OmniOutput(` `text_hidden_states=(text_hidden_states.reshape(-1, text_hidden_states.shape[-1])),` `multimodal_outputs=None,`; vllm_omni/model_executor/models/output_templates.py:9 `class OmniOutput(NamedTuple):`; vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:748 `# NOTE: gpu_model_runner._dummy_run performs hidden_states[logit_indices] after forward.`) + +`vllm_omni/platforms/xpu/platform.py:139` [minor] — vllm_omni/platforms/xpu/platform.py:139 changes the XPU non-inductor default from `xpu_kernels` to `vllm_c` for every XPU model through `current_omni_platform.get_default_ir_op_priority` (vllm_omni/diffusion/worker/diffusion_worker.py:184). The only direct IR-op-priority platform test is CUDA-only (tests/platforms/test_cuda_ir_op_priority.py:34), and buildkite/vllm-omni-intel-ci is failing. Add an XPU analog asserting the new `vllm_c` default and the inductor-native branch, state whether the Intel failure is pre-existing or introduced by this flip, and scope the default to the validated family if vllm_c is not validated for all XPU. (evidence: vllm_omni/platforms/xpu/platform.py:139 `default = ["native"] if using_inductor else ["vllm_c", "native"]`; vllm_omni/diffusion/worker/diffusion_worker.py:184 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`; tests/platforms/test_cuda_ir_op_priority.py:34 `priority = CudaOmniPlatform.get_default_ir_op_priority(_vllm_config(mode=mode))`) + +`vllm_omni/engine/arg_utils.py:116` [minor] — arg_utils.py:116 now unconditionally registers every _OMNI_MODELS arch into upstream's global ModelRegistry after removing the `if arch not in supported_archs:` guard. Only Qwen2.5-Omni, Qwen3-Omni (and glm_tts, which the original comment omitted) use the `getattr(vllm_config.model_config, "model_stage", ...)` default; the other registered classes still read `vllm_config.model_config.model_stage` directly (voxtral_tts.py:112, minicpmo_4_5_omni.py:94, step_audio2.py:57, ming_flash_omni.py:98, omnivoice.py:279, ming_tts.py:64, cosyvoice3.py:462, covo_audio.py:218, mammoth_moda2.py:890, mimo_audio.py:554). Enumerate which of these archs upstream vLLM 0.27 also registers, and harden each with the same getattr/default pattern or scope the unconditional overwrite to the verified set. (evidence: vllm_omni/engine/arg_utils.py:116 ` ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")` — unconditional registration after the diff removed `if arch not in supported_archs:`; vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py:112 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py:94 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/step_audio2/step_audio2.py:57 ` raw_model_stage = vllm_config.model_config.model_stage`; hardened comparison vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py:144 ` self.model_stage = getattr(vllm_config.model_config, "model_stage", None) or "thinker"`; additional hardened case omitted by the original comment vllm_omni/model_executor/models/glm_tts/glm_tts.py:698 ` self.model_stage = getattr(vllm_config.model_config, "model_stage", "glm_tts")`.) + +`vllm_omni/entrypoints/api_server.py:1001` [minor] — File is vllm_omni/entrypoints/openai/api_server.py (not vllm_omni/entrypoints/api_server.py), line 1001. `state.online_renderer.warmup()` is now unconditional (previously gated by `if state.openai_serving_chat is not None`), so it runs for non-'generate' supported-task sets such as TTS-only servers; `register_omni_models_to_vllm` is likewise unconditional now. Since upstream `OnlineRenderer.warmup` is outside this repo, this remains a scoping/verification ask: a TTS-only server smoke test would confirm the non-generate startup path. (evidence: vllm_omni/entrypoints/openai/api_server.py:993 ` state.openai_serving_chat = OmniOpenAIServingChat(**_chat_kwargs) if "generate" in supported_tasks else None`; vllm_omni/entrypoints/openai/api_server.py:1001 ` state.online_renderer.warmup()`; vllm_omni/engine/arg_utils.py:116 ` ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")`) + +`vllm_omni/core/sched/omni_ar_scheduler.py:491` [minor] — The new stale-drain tests always mock `_update_request_with_output` to return `([42], False)` (test_omni_ar_scheduler_stale_drain.py:93), so the AR scheduler's segment-stop seed site in `update_from_output` (`if stopped:` block entering at omni_ar_scheduler.py:454, seeding `num_stale_output_tokens` at line 491) is never reached. Add a case with `stopped=True` or otherwise drive that branch, or document why it is safe to leave unverified. (evidence: tests/core/sched/test_omni_ar_scheduler_stale_drain.py:93 `sched._update_request_with_output.return_value = ([42], False)` — the mock returns stopped=False in every new stale-drain test; vllm_omni/core/sched/omni_ar_scheduler.py:454 `if stopped:` and vllm_omni/core/sched/omni_ar_scheduler.py:491 `request.num_stale_output_tokens += request.num_in_flight_tokens` — the only AR-scheduler segment-stop seed site, unreached while stopped stays False.) + +`vllm_omni/entrypoints/api_server.py:559` [minor] — PureDiffusionLauncherAdapter (vllm_omni/entrypoints/utils.py:910) is only referenced from production code — imported at vllm_omni/entrypoints/openai/api_server.py:155 and instantiated at line 559 — and has no direct test under tests/. Since it is a forwarding adapter with __getattr__ recursion risk, a direct unit test or documented pure-diffusion e2e teardown coverage is worth adding. Coverage ask, not a demonstrated bug. (evidence: vllm_omni/entrypoints/openai/api_server.py:559 `app.state.engine_client = PureDiffusionLauncherAdapter(`; vllm_omni/entrypoints/utils.py:910 `class PureDiffusionLauncherAdapter:`; `grep -r 'PureDiffusionLauncherAdapter' tests/` returns no matches.) + +`vllm_omni/core/sched/omni_generation_scheduler.py:385` [minor] — The same `num_stale_output_tokens` drain is added to `OmniGenerationScheduler.update_from_output` (vllm_omni/core/sched/omni_generation_scheduler.py:385) and seeded via `OmniSchedulerMixin._replace_streaming_session`, but the only new stale-drain regression test (tests/core/sched/test_omni_ar_scheduler_stale_drain.py) imports and drives `OmniARScheduler` exclusively. `StageExecutionType.LLM_GENERATION` resolves to `OmniGenerationScheduler` (vllm_omni/config/stage_config.py:198-199), and TTS/codec stages (qwen3_tts, moss_tts, higgs_audio_v3, voxtral_tts) declare that type, so the drain on the scheduler those models actually use is unverified. Existing generation-scheduler tests call `update_from_output` but do not seed or assert `num_stale_output_tokens`. Add an `OmniGenerationScheduler` stale-drain test driving the real `update_from_output` plus a `_replace_streaming_session` seed case, or document why AR-only coverage is sufficient. (evidence: vllm_omni/core/sched/omni_generation_scheduler.py:385 ` if request is not None and request.num_stale_output_tokens > 0:`; tests/core/sched/test_omni_ar_scheduler_stale_drain.py:42 `from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler`; vllm_omni/config/stage_config.py:198 ` if execution_type == StageExecutionType.LLM_GENERATION:`; vllm_omni/config/stage_config.py:199 ` return OmniGenerationScheduler`; vllm_omni/model_executor/models/qwen3_tts/pipeline.py:39 ` execution_type=StageExecutionType.LLM_GENERATION,`; vllm_omni/model_executor/models/moss_tts/pipeline.py:43 ` execution_type=StageExecutionType.LLM_GENERATION,`; vllm_omni/model_executor/models/higgs_audio_v3/pipeline.py:51 ` execution_type=StageExecutionType.LLM_GENERATION,`; vllm_omni/model_executor/models/voxtral_tts/pipeline.py:37 ` execution_type=StageExecutionType.LLM_GENERATION,`) + +**Additional observations (beyond the comment budget):** +- `tests/e2e/accuracy/test_qwen_image.py:84` [minor] `_flash_attn3_hub_available()` (tests/e2e/accuracy/test_qwen_image.py:84) catches bare `Exception`, so any kernels import/resolution failure — not just the expected missing-build-variant — sets the flag false and at line + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.trace.json.gz b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.trace.json.gz new file mode 100644 index 000000000..dba3ff33c Binary files /dev/null and b/eval/dataset/arms/copilot_v13_ds_wave2_r2/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/manifest.json b/eval/dataset/arms/copilot_v13_mimo_wave2/manifest.json new file mode 100644 index 000000000..1dbc20cc4 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v13_mimo_wave2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.cost.json new file mode 100644 index 000000000..3f3092eac --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1746.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5509/run-20260814-095608-5fe6fa", + "recorded_at": "2026-08-14T10:25:12", + "attempts": [ + { + "invocation_id": "inv-451ad041", + "outcome": "run:run-20260814-095608-5fe6fa", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-095608-5fe6fa", + "llm_calls": 93, + "input_tokens": 592345, + "output_tokens": 62130, + "usd": 0.442203, + "source": "spans" + } + ], + "attempt_usd_total": 0.442203, + "llm_calls": 93, + "input_tokens": 592345, + "output_tokens": 62130, + "usd": 0.442203, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-095608-5fe6fa", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4422, + "minutes": 29.07, + "ci_minutes": 0, + "input_tokens": 592345, + "output_tokens": 62130, + "tool_calls": 131, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8449, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 93, + "by_role": { + "agent": { + "usd": 0.420955, + "input_tokens": 573744, + "output_tokens": 51603, + "calls": 91 + }, + "reducer": { + "usd": 0.021248, + "input_tokens": 18601, + "output_tokens": 10527, + "calls": 2 + } + }, + "cache_read_tokens": 4225728, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.41, + "gate": 2.0, + "review": 1738.39, + "report": 0.06 + }, + "time_to_first_result_s": 1743.85 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 34876 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.md new file mode 100644 index 000000000..c96d0637a --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 7 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] trtllm_attn.py:85-107 — _sage_kernel_available() and _sage_quantize_fn() are lru_cache(maxsize=1) with no cache-busting; permanent for process lifetime. By design for perf but undocumented. +- [sweep] trtllm_attn.py:133-141 — QuantConfig.quantize() correctly unpacks 6-tuple from flashinfer, repackages SFS as (q_sfs, k_sfs, None, v_sfs) and block sizes as (q_block_size, k_block_size, 0, 1). The None/0 for V is correct (per-channel, no block). No bug. +- [sweep] trtllm_attn.py:191-215 — Triple-guard init (dtype check → kernel check → quantize fn check) prevents silent fallback. All three raise RuntimeError with actionable messages. ✓ +- [sweep] trtllm_attn.py:289-301 — sage_kwargs only injected when quant.enabled, keeping dense path compatible with older FlashInfer builds. Correct. +- [sweep] data.py:1415-1418 — AttnQuantSpec validates dtype and block sizes against frozensets. The _VALID_DTYPES union covers both TRTLLM_ATTN (int8/fp8) and FLASHINFER_ATTN (float16/bfloat16) use cases. ✓ +- [sweep] data.py:1447 — quant allowed only for TRTLLM_ATTN and FLASHINFER_ATTN backends; other backends raise ValueError. ✓ +- [sweep] data.py:1455 — _coerce() generalized from skip_softmax-only to any cls. Clean refactor. +- [sweep] data.py:1464-1481 — backend_kwargs() correctly serializes quant dict with dtype_qk, q_block_size, k_block_size, and optional dtype_vo/flashinfer_backend. ✓ + +`docs/user_guide/diffusion/attention_backends.md:214` [major] — The diff removes five doc sections unrelated to the SAGE addition and not mentioned in the PR description: (1) 'Known Limitations' including the LTX-2.0 CUDNN_ATTN torch.compile workaround (#3121) and FA4 status, (2) 'Mixed backends across roles' example, (3) 'Compare against FlashAttention' example, (4) 'Validation Guidance' (warns users not to assume numerical interchangeability of backends), (5) 'Reproducing the Benchmark Table' (with bench_attention_backends.py). These contain actionable user information — especially the LTX-2.0 workaround and the numerical interchangeability warning. Additionally, line 232 still references the deleted section: 'see Known Limitations above'. Restore these sections or split into a separate PR with rationale. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — dangling reference to the deleted Known Limitations section. The diff at @@ -180,22 +222,6 @@ removes the Known Limitations block (LTX-2.0 crash workaround, FA4 status), and @@ -353,64 +379,3 @@ removes 'Mixed backends across roles', 'Compare against FlashAttention', 'Validation Guidance', and 'Reproducing the Benchmark Table'. None appear elsewhere in the file or diff.) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — The doc at attention_backends.md:179 states FLASHINFER_ATTN 'reads' dtype_qk (float16/bfloat16), dtype_vo, and flashinfer_backend from the quant block, and AttentionSpec.__post_init__ (data.py:1447) validates quant for FLASHINFER_ATTN in its allowlist. However, FlashInferAttentionImpl.__init__ (flashinfer_attn.py:72) accepts only **extra_impl_args which silently discards all kwargs — the quant dict is serialized via backend_kwargs() but never consumed at runtime. Clarify in the doc that FLASHINFER_ATTN quant support is planned/pending, or remove FLASHINFER_ATTN from the validation allowlist until the backend wires up these fields. (evidence: flashinfer_attn.py:FlashInferAttentionImpl.__init__ — accepts no backend_kwargs parameter and stores no quant config. data.py:1447 — validation allows quant for FLASHINFER_ATTN: `if self.quant is not None and self.backend.upper() not in ('TRTLLM_ATTN', 'FLASHINFER_ATTN')`.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:85` [minor] — Minor: `_sage_kernel_available()` and `_sage_quantize_fn()` use `@functools.lru_cache(maxsize=1)` with no `cache_clear()`, making their results permanent for process lifetime. This is correct by design — they probe hardware/library capabilities that cannot change mid-process. `__init__` already documents the Dynamo rationale for `_sage_quantize_fn`; a brief inline note on `_sage_kernel_available()` would complete the picture. (evidence: trtllm_attn.py:85 `@functools.lru_cache(maxsize=1)` / trtllm_attn.py:86 `def _sage_kernel_available() -> bool:` — no `cache_clear()` call exists anywhere in the file; trtllm_attn.py:95 `@functools.lru_cache(maxsize=1)` / trtllm_attn.py:96 `def _sage_quantize_fn():` — same permanent caching pattern) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:89` [minor] — The signature inspection for 'sage_attn_sfs' in _sage_kernel_available() (trtllm_attn.py:89) is fragile if FlashInfer renames the parameter. Although guarded by try/except (returning False on failure, so safe degradation), consider adding a brief comment noting this coupling for future readers. (evidence: trtllm_attn.py:89-95 `def _sage_kernel_available() -> bool: ... return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters` — string-based signature inspection guarded by `except (TypeError, ValueError): return False`) + +`vllm_omni/diffusion/data.py:1410` [minor] — AttnQuantSpec._VALID_DTYPES (data.py:1410) includes float16/bfloat16 alongside int8/fp8_e4m3. The shared spec validates these for all consumers, but TRTLLM_ATTN narrows to int8/fp8_e4m3 at init (trtllm_attn.py:133). This two-layer design is intentional — FLASHINFER_ATTN legitimately uses float16/bfloat16 — but undocumented at the _VALID_DTYPES declaration. Adding a brief comment there would prevent future confusion. (evidence: data.py:1410 `_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})` — shared spec allows float16/bfloat16; trtllm_attn.py:125 `_QK_QUANT_DTYPES = {"int8": torch.int8, "fp8_e4m3": torch.float8_e4m3fn}` + trtllm_attn.py:133 `if self.quant.dtype_qk not in _QK_QUANT_DTYPES: raise RuntimeError(... "FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE")` — backend rejects the same values at init) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — The diff removes the entire '## Known Limitations' section (LTX-2.0 CUDNN_ATTN crash workaround, FA4 not-yet-integrated note) but the bullet '- **Workaround for known issues**: see Known Limitations above.' at line 232 still references it. Delete the bullet or replace with specific workarounds now that the section is gone. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — the `## Known Limitations` heading no longer exists in the file (confirmed via grep returning no matches for `^## Known Limitations`), so this cross-reference points to a deleted section.) + +`vllm_omni/diffusion/data.py:1423` [minor] — AttnQuantSpec.enabled (data.py) returns True when only dtype_vo is set (dtype_qk=None), causing AttentionSpec.backend_kwargs() to serialize a quant dict even though TRTLLM_ATTN's internal QuantConfig.enabled is False (it only checks dtype_qk). The quant dict is harmlessly ignored downstream, but serializing unnecessary fields may confuse future consumers or other backends that inspect the kwargs. Consider gating the quant serialization on dtype_qk being non-None for TRTLLM-relevant fields, or adding a clarifying comment about the two-level enabled check. (evidence: data.py:1429-1430 `@property + def enabled(self) -> bool: + return self.dtype_qk is not None or self.dtype_vo is not None` — AttnQuantSpec.enabled is True with dtype_qk=None. data.py:1472-1473 `if self.quant is not None and self.quant.enabled:` — backend_kwargs serializes quant dict when AttnQuantSpec.enabled is True. trtllm_attn.py:133-135 `@property + def enabled(self) -> bool: + return self.dtype_qk is not None` — QuantConfig.enabled is False when dtype_qk=None, so SAGE is skipped.) + +`docs/user_guide/diffusion/attention_backends.md:?` [nit] — The text at attention_backends.md:187-188 says FLASHINFER_ATTN reads `dtype_qk` (float16 / bfloat16), `dtype_vo`, and `flashinfer_backend` '(see its section)' but no FLASHINFER_ATTN quant subsection exists in the document. Either add a brief FLASHINFER_ATTN quant config note or remove the dangling '(see its section)' parenthetical. (evidence: docs/user_guide/diffusion/attention_backends.md:187-188 `dtype_qk`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`), `dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.` — the only subsections with quant config detail are 'TRTLLM_ATTN Backend and Skip-Softmax' and 'TRTLLM_ATTN SAGE Quantization'; no FLASHINFER_ATTN quant subsection follows.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.trace.json.gz new file mode 100644 index 000000000..5d4420b7d Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.cost.json new file mode 100644 index 000000000..71d27f810 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2017.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5550/run-20260814-095609-f1467f", + "recorded_at": "2026-08-14T10:29:43", + "attempts": [ + { + "invocation_id": "inv-0337b128", + "outcome": "run:run-20260814-095609-f1467f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-095609-f1467f", + "llm_calls": 111, + "input_tokens": 704034, + "output_tokens": 82671, + "usd": 0.621288, + "source": "spans" + } + ], + "attempt_usd_total": 0.621288, + "llm_calls": 111, + "input_tokens": 704034, + "output_tokens": 82671, + "usd": 0.621288, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-095609-f1467f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6213, + "minutes": 33.58, + "ci_minutes": 0, + "input_tokens": 704034, + "output_tokens": 82671, + "tool_calls": 173, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9832, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 111, + "by_role": { + "agent": { + "usd": 0.606091, + "input_tokens": 686324, + "output_tokens": 76256, + "calls": 109 + }, + "reducer": { + "usd": 0.015197, + "input_tokens": 17710, + "output_tokens": 6415, + "calls": 2 + } + }, + "cache_read_tokens": 7403904, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.43, + "gate": 1.86, + "review": 2009.09, + "report": 0.06 + }, + "time_to_first_result_s": 2014.43 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 33421 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.md new file mode 100644 index 000000000..4be726ca1 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.md @@ -0,0 +1,52 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] Engine init cleanup (diffusion_engine.py:191-207): Handler uses getattr for both scheduler and executor — safe even if attributes don't exist. self.executor is set before try block, so cleanup reaches real executor. Test verifies both close() and shutdown() called once. +- [validated] validate_new_request_data_identity (interface.py:180-197): Two-part check — envelope ID vs forwarded ID AND metadata ID vs forwarded ID — both necessary because metadata can independently carry a wrong request_id. +- [validated] _validate_diffusion_kv_metadata (diffusion_model_runner.py:163-177): Correctly rejects metadata only in non-PAGED_SCHEDULER modes. PAGED_WORKER_LOCAL rejected by parse_diffusion_kv_cache_mode before reaching this point. +- [validated] DP multi-concurrency gate (multiproc_executor.py:468-471): `not has_diffusion_kv_metadata` correctly prevents ALL requests from using DP optimization when any has metadata. Conservative but correct for W0 where no metadata producer exists yet. +- [validated] RPC arg chain consistency: args (req, od_config, kv_prefetch_job[, metadata]) forwarded consistently through executor→worker→runner at multiproc_executor.py:543-546, diffusion_worker.py:462-464, diffusion_model_runner.py:592. +- [validated] NewRequestData.from_state (base_scheduler.py:114): Existing callers use default diffusion_kv_metadata=None — no breaking change. +- [validated] execute_model_batch/execute_stepwise validation loops (diffusion_model_runner.py:635-640, 749-754): Correctly validate identity and metadata before model work, even inside grad_context — safe since validation only reads fields. +- [validated] test_metadata.py: all 3 tests exercise metadata construction with tuple[list[int],...] block_ids, CFG shared contexts, and branch-specific contexts. Assertions match dataclass structure. + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — multiproc_executor.py:471: the `not has_diffusion_kv_metadata` guard in the DP multi-concurrency condition disables the allgather broadcast for ALL requests in a schedule when ANY request carries metadata. Currently unreachable (W0 produces no metadata), but once W1 metadata producers exist, mixed-metadata multi-request schedules will silently fall through to per-request RPCs. Consider adding a brief comment noting this is intentional — allgather cannot carry per-request metadata, so the whole batch must downgrade. (evidence: multiproc_executor.py:471 `and not has_diffusion_kv_metadata` — this condition gates the DLO allgather batch path (lines 469-475). When True (metadata present), execution falls through to the single-request RPC loop starting at line 541 (`for new_req in new_reqs:`). No scheduler code in this PR produces DiffusionKVMetadata, so the path is unreachable until W1.) + +`metadata.py:1` [minor] — The metadata dataclasses (DiffusionKVContextMetadata, DiffusionKVSequenceMetadata, DiffusionKVMetadata) are plain (not frozen) and contain mutable lists inside tuples. Acceptable for W0 internal DTOs, but consider freezing or documenting the immutability contract for W1. (evidence: vllm_omni/diffusion/diffusion_kv/metadata.py:22 `block_ids: tuple[list[int], ...]` — mutable list inside tuple on DiffusionKVContextMetadata; vllm_omni/diffusion/diffusion_kv/metadata.py:39 `block_ids: tuple[list[int], ...]` — same pattern on DiffusionKVSequenceMetadata; all three classes decorated with plain `@dataclass` (lines 18, 31, 48), not `@dataclass(frozen=True)`) + +`tests/diffusion/diffusion_kv/test_worker_contract.py:?` [minor] — tests/diffusion/diffusion_kv/test_worker_contract.py has no test for the PAGED_SCHEDULER + valid-metadata happy path. diffusion_model_runner.py:172 `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:` silently passes when cache_mode is PAGED_SCHEDULER and metadata is supplied, but every existing PAGED_SCHEDULER test either omits metadata or uses a mismatched request_id. Add a test calling runner.execute_model(req, diffusion_kv_metadata=make_metadata("req-0")) with a PAGED_SCHEDULER runner to assert _execute_request_list is called, validating the accept-metadata contract. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:172 `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:` — the 'pass' branch for PAGED_SCHEDULER+metadata is not exercised by any test; all PAGED_SCHEDULER tests in tests/diffusion/diffusion_kv/test_worker_contract.py use metadata=None or mismatched identity.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:?` [nit] — WorkerWrapperBase.execute_model now accepts diffusion_kv_metadata: DiffusionKVMetadata | None, but the Args section of its docstring only documents req, od_config, and kv_prefetch_job. Add: diffusion_kv_metadata – Optional scheduler allocation snapshot; forwarded only when not None. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py — WorkerWrapperBase.execute_model signature: + def execute_model( + self, + req: OmniDiffusionRequest, + od_config: OmniDiffusionConfig, + kv_prefetch_job: KVPrefetchJob | None = None, + diffusion_kv_metadata: DiffusionKVMetadata | None = None, + ) -> DiffusionOutput: +docstring Args section lists only: req, od_config, kv_prefetch_job — diffusion_kv_metadata is absent.) + +`vllm_omni/diffusion/diffusion_engine.py:?` [nit] — diffusion_engine.py:197: the comment `close() cannot be used` is ambiguous — the very next line calls `scheduler_to_close.close()`. Clarify to `self.close() cannot be used` so the reader understands only the engine's own `close()` is forbidden here (it requires runtime sync state like `stop_event`, `_out_streams`, and `worker_thread` that may not be initialised yet), while the scheduler's direct `close()` is the intended cleanup path. (evidence: vllm_omni/diffusion/diffusion_engine.py:197-198 ```python # close() cannot be used because runtime synchronization state may not exist yet. scheduler_to_close = getattr(self, "scheduler", None) ``` followed by vllm_omni/diffusion/diffusion_engine.py:200 ```python scheduler_to_close.close() ``` — the bare `close()` in the comment could be read as referring to this `.close()` call rather than `self.close()`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.trace.json.gz new file mode 100644 index 000000000..d657ed4e5 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.cost.json new file mode 100644 index 000000000..01d7782ce --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1278.3, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5610/run-20260814-095610-cbda3c", + "recorded_at": "2026-08-14T10:17:24", + "attempts": [ + { + "invocation_id": "inv-0963d896", + "outcome": "run:run-20260814-095610-cbda3c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-095610-cbda3c", + "llm_calls": 46, + "input_tokens": 317284, + "output_tokens": 17125, + "usd": 0.236732, + "source": "spans" + } + ], + "attempt_usd_total": 0.236732, + "llm_calls": 46, + "input_tokens": 317284, + "output_tokens": 17125, + "usd": 0.236732, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-095610-cbda3c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2367, + "minutes": 21.24, + "ci_minutes": 0, + "input_tokens": 317284, + "output_tokens": 17125, + "tool_calls": 78, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6326, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 46, + "by_role": { + "planner": { + "usd": 0.001292, + "input_tokens": 2092, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.22681, + "input_tokens": 303567, + "output_tokens": 13480, + "calls": 43 + }, + "reducer": { + "usd": 0.00863, + "input_tokens": 11625, + "output_tokens": 3245, + "calls": 2 + } + }, + "cache_read_tokens": 2905920, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.46, + "gate": 2.06, + "review": 1267.74, + "report": 0.06 + }, + "time_to_first_result_s": 1274.31 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 20781 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.md new file mode 100644 index 000000000..bb3736a16 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.md @@ -0,0 +1,36 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] docs/.nav.yml: new entry omni_async_output_materialization.md correctly inserted between async_chunk.md and vae_parallel.md (line 121 in post-diff state) +- [validated] docs/design/index.md: new entry correctly placed after Ray-based Execution in the Feature Design section +- [sweep] docs/design/feature/omni_async_output_materialization.md: SVG image reference ../../../source/architecture/qwen3-omni-async-output-step-gap.svg resolves to correct file in docs/source/architecture/ +- [sweep] docs/design/feature/omni_async_output_materialization.md: relative link [Async Chunk](async_chunk.md) resolves correctly — both files in docs/design/feature/ +- [validated] Guard conditions table (doc lines 336-345) matches _should_use_async_omni_output() at gpu_ar_model_runner.py:1595-1617 — all 7 runtime checks documented +- [sweep] Guard conditions table omits the routed-expert guard: code checks enable_return_routed_experts (gpu_ar_model_runner.py:1704) but doc doesn't list it +- [validated] Talker stage behavior table matches qwen3_omni.py:162-164 (use_async_omni_output=True, eager_omni_postprocess_before_async_output=True, omni_pooler_payload_include_hidden=False) +- [validated] Qwen3-TTS Talker flags match qwen3_tts_talker.py:351-353 + +The sole candidate (0) is dropped: it suggests adding a routed-expert compatibility row that already exists verbatim in the diff's table at line 342 of the new documentation file. No other issues were raised. + +**Verdict:** APPROVE + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.trace.json.gz new file mode 100644 index 000000000..564a3d44f Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.cost.json new file mode 100644 index 000000000..0d27d5162 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2035.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5703/run-20260814-095608-068a60", + "recorded_at": "2026-08-14T10:30:01", + "attempts": [ + { + "invocation_id": "inv-f091559c", + "outcome": "run:run-20260814-095608-068a60", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-095608-068a60", + "llm_calls": 109, + "input_tokens": 341979, + "output_tokens": 69401, + "usd": 0.327311, + "source": "spans" + } + ], + "attempt_usd_total": 0.327311, + "llm_calls": 109, + "input_tokens": 341979, + "output_tokens": 69401, + "usd": 0.327311, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-095608-068a60", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3273, + "minutes": 33.89, + "ci_minutes": 0, + "input_tokens": 341979, + "output_tokens": 69401, + "tool_calls": 170, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8443, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 109, + "by_role": { + "agent": { + "usd": 0.31069, + "input_tokens": 329979, + "output_tokens": 60542, + "calls": 107 + }, + "reducer": { + "usd": 0.016621, + "input_tokens": 12000, + "output_tokens": 8859, + "calls": 2 + } + }, + "cache_read_tokens": 3155904, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.48, + "gate": 1.64, + "review": 2027.9, + "report": 0.08 + }, + "time_to_first_result_s": 2033.07 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 28785 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.md new file mode 100644 index 000000000..c78c7597a --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] vae.py:120 — `self.device_module = torch.get_device_module()` is stored once in `__init__`; on CUDA returns `torch.cuda`, on MUSA returns `torch.musa`, on CPU returns `torch.cpu`. Module cached at construction time is safe because the VAE is not moved between platforms at runtime. +- [validated] vae.py:171,215 — `parameter.device.type != "cpu"` is semantically equivalent to `== "cuda"` on CUDA and ROCm (both report `device.type == "cuda"`); on MUSA (`device.type == "musa"`) the old code excluded the device from RNG seeding while the new code includes it. This is the intended fix. +- [validated] vae.py:175-176,219-220 — `self.device_module.device(device)` and `self.device_module.manual_seed(seed)` replace `torch.cuda.device(device)` and `torch.cuda.manual_seed(seed)`. The API is identical across `torch.cuda`, `torch.musa`, and `torch.hip` modules. +- [validated] vae.py:170,214 — `torch.random.fork_rng(devices=devices)` saves/restores both CPU and device RNG state on context exit. When `devices=[]` (CPU-only), only the CPU generator is forked. Behavior unchanged on CPU path. +- [validated] test_minimax_h3_contract.py:353 — `test_video_vae_encode_uses_configured_parallel_tiling` uses `object.__new__` and never sets `device_module`, but the test runs on CPU where `devices=[]` so the `for` loop body is never reached. Test is safe. +- [validated] test_minimax_h3_contract.py:263 — `test_video_vae_keeps_reference_fp32_weights` uses the real constructor with `device=torch.device("cpu")`; `torch.get_device_module()` returns `torch.cpu` on CPU-only machines. Test is safe. +- [validated] recipes/README.md:48-49 — New MUSA recipe row correctly added in alphabetical order after MiniMax-H3.md row. Description updated for both rows. +- [validated] recipes/MiniMaxAI/MiniMax-H3-MUSA.md — Full recipe covers prerequisites, checkpoint partitioning, serve command, MUSA validation evidence, and known limitations. Cross-references the base recipe correctly. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — vae.py:120 caches `self.device_module = torch.get_device_module()` without a device argument, so it resolves to the global default device at init time rather than the actual parameter device. Because the constructor at line 116 already moves parameters to the explicit `device` argument, using `torch.get_device_module(next(self.parameters()).device)` would tie the cached module to the real parameter device at zero additional cost and eliminate any dependence on the global default matching the target. (evidence: vae.py:120 `self.device_module = torch.get_device_module()` — no device argument; vae.py:116 `self.remote.eval().to(device=device, dtype=torch.float32)` — parameters already on target device before caching; vae.py:175 `with self.device_module.device(device):` and vae.py:176 `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` — consumers depend on cached module being correct backend) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [minor] — recipes/MiniMaxAI/MiniMax-H3-MUSA.md line 72 introduces 'The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs' but the code block starting at line 76 sets `export MODEL="${MODEL_ROOT}/FL2VA"`. To match the described Ref2VA configuration, change the MODEL value to `${MODEL_ROOT}/Ref2VA`. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72 'The validated Ref2VA configuration uses tensor parallelism across four MTT S5000 GPUs' immediately followed by recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76 `export MODEL="${MODEL_ROOT}/FL2VA"` — the MODEL variable contradicts the prose describing the Ref2VA configuration) + +`buildkite/vllm-omni-amd-ci:0` [minor] — The `buildkite/vllm-omni-amd-ci` pipeline is currently red. The diff's change from `device.type == "cuda"` to `!= "cpu"` is functionally equivalent on ROCm (ROCm reports `device.type == "cuda"` and `torch.get_device_module()` returns `torch.cuda`), so this failure is almost certainly pre-existing on main. Verify the failure exists on main before merging so it is not inherited as a false-positive blocker by subsequent PRs. (evidence: CI: `buildkite/vllm-omni-amd-ci` is red. The diff changes `device.type == "cuda"` to `!= "cpu"`, which on ROCm (where `device.type` is still `"cuda"` and `torch.get_device_module()` returns `torch.cuda`) produces identical runtime behavior. The failure is very likely pre-existing on main, not introduced by this PR.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:?` [minor] — test_minimax_h3_contract.py:335 (`test_video_vae_encode_uses_configured_parallel_tiling`) constructs MiniMaxH3VideoVAE via `object.__new__` without setting `device_module`. On CPU, `parameter.device.type != "cpu"` is False so `devices=[]` and the `self.device_module.device(device)` / `self.device_module.manual_seed()` lines added by this PR are never reached — the test passes but provides zero coverage of the new device-agnostic RNG path. Adding `video_vae.device_module = torch` prevents an AttributeError on future GPU CI but still doesn't exercise the loop body on CPU; true coverage requires the parameter to live on a non-CPU device or explicit mocking of the device context manager. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:343-348 `video_vae = object.__new__(MiniMaxH3VideoVAE)` / `torch.nn.Module.__init__(video_vae)` / sets `model` and `config_dict` but no `device_module`; vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []` → empty list on CPU; vae.py:216-217 `for device in devices:` / `with self.device_module.device(device):` → loop body dead) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [nit] — Nit: the PR description states 'The device allowlist is limited to CUDA and MUSA' but vae.py:170 and vae.py:214 gate RNG on `parameter.device.type != "cpu"`, which accepts any non-CPU accelerator (XPU, NPU, ROCm, etc.). The code follows a repo-wide device-agnostic convention, so this appears intentional — consider updating the description to note the code is device-agnostic while validation was scoped to CUDA and MUSA. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` and vae.py:214 identical — accepts any non-CPU device. PR body: 'The device allowlist is limited to CUDA and MUSA. Other accelerator RNG semantics are not inferred from mocked interfaces.') + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [nit] — vae.py:60-86: `_AudioVAEDeterminismContext` hardcodes CUDA-specific `torch.backends.cuda.*` calls (matmul.allow_tf32, flash_sdp_enabled, enable_flash_sdp, etc.) used by `encode_waveform()` at vae.py:319. If the audio VAE runs on MUSA, these calls may not resolve. Out-of-scope for this PR which only makes the Video VAE device-aware, but relevant to the broader MUSA porting effort. (evidence: vae.py:62-78: `backends.cuda.matmul.allow_tf32`, `backends.cuda.flash_sdp_enabled()`, `backends.cuda.mem_efficient_sdp_enabled()`, `backends.cuda.math_sdp_enabled()`, `backends.cuda.enable_flash_sdp(False)`, `backends.cuda.enable_mem_efficient_sdp(False)`, `backends.cuda.enable_math_sdp(True)` — all hardcoded CUDA APIs in `_AudioVAEDeterminismContext.__enter__`; vae.py:319 `with _AudioVAEDeterminismContext():` — sole consumer in `encode_waveform()`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.trace.json.gz new file mode 100644 index 000000000..e7eb0604a Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.cost.json new file mode 100644 index 000000000..0b7e55d5c --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2202.9, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5715/run-20260814-095609-f01d3c", + "recorded_at": "2026-08-14T10:32:49", + "attempts": [ + { + "invocation_id": "inv-a73ef96e", + "outcome": "run:run-20260814-095609-f01d3c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-095609-f01d3c", + "llm_calls": 86, + "input_tokens": 148240, + "output_tokens": 45346, + "usd": 0.188771, + "source": "spans" + } + ], + "attempt_usd_total": 0.188771, + "llm_calls": 86, + "input_tokens": 148240, + "output_tokens": 45346, + "usd": 0.188771, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-095609-f01d3c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1888, + "minutes": 36.66, + "ci_minutes": 0, + "input_tokens": 148240, + "output_tokens": 45346, + "tool_calls": 144, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7943, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 86, + "by_role": { + "agent": { + "usd": 0.182165, + "input_tokens": 146770, + "output_tokens": 40995, + "calls": 85 + }, + "reducer": { + "usd": 0.006606, + "input_tokens": 1470, + "output_tokens": 4351, + "calls": 1 + } + }, + "cache_read_tokens": 2097216, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.31, + "gate": 1.61, + "review": 2194.38, + "report": 0.06 + }, + "time_to_first_result_s": 2199.36 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 17813 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.md new file mode 100644 index 000000000..7eda09a7d --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.md @@ -0,0 +1,45 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] README.md news consolidation: old entries for 0.14.0–0.22.0 correctly distilled into a single 2026/06 entry; vllm-omni-skills entry removed; 0.24.0 and 0.26.0 entries retained as-is. +- [validated] README.md and docs/README.md: full-duplex feature, MiniCPM-o 4.5, MiniMax H3 additions match PR description and are consistent between root and docs copies. +- [validated] docs/configuration/README.md: upstream vLLM link updated to v0.26.0; consistent with all other v0.26 pins. +- [validated] docs/getting_started/installation/README.md: compatibility admonition states 0.26.x ↔ 0.26.x; consistent with CUDA/NPU docs. +- [validated] docs/getting_started/installation/gpu/cuda.inc.md: pip install, Docker base image, git checkout, and prebuilt wheel URL all pinned to v0.26.0; no stale v0.22.0 references remain. +- [validated] docs/getting_started/installation/npu/npu.inc.md: NPU source-build section updated with vLLM v0.26.0 and vLLM-Ascend releases/v0.26.0rc; added cd/.. navigation; pre-built images section (untouched by this PR) already at v0.26.0 tags. +- [sweep] No stale version references found in docs/ tree — grep for v0.22 across docs/ returns zero hits. +- [sweep] MUSA and quickstart files confirmed untouched by the diff. + +`docs/getting_started/installation/npu/npu.inc.md:0` [minor] — Pre-existing duplicate --device/-v flags in npu.inc.md pre-built images section: 7 flags (--device /dev/davinci_manager, --device /dev/devmm_svm, --device /dev/hisi_hdc, -v /usr/local/dcmi, -v /usr/local/bin/npu-smi, -v /usr/local/Ascend/driver/lib64/, -v /usr/local/Ascend/driver/version.info) appear twice consecutively in the docker run block. Outside this PR's diff scope; no regression introduced. (evidence: docs/getting_started/installation/npu/npu.inc.md — in the pre-built-images docker run block, the following 7 flags appear first at ~lines 75-81 and are repeated verbatim at ~lines 82-88: + --device /dev/davinci_manager \ + --device /dev/devmm_svm \ + --device /dev/hisi_hdc \ + -v /usr/local/dcmi:/usr/local/dcmi \ + -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \ + -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \ + -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \) + +`docs/getting_started/installation/npu/npu.inc.md:0` [nit] — The assumption that the vLLM-Ascend branch 'releases/v0.26.0rc' exists upstream is critical for the documentation. Can we confirm this branch is stable and available before merging? (evidence: The vLLM-Ascend branch 'releases/v0.26.0rc' exists upstream (verified as claimed in PR test plan).) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.trace.json.gz new file mode 100644 index 000000000..7f70f7ccc Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.cost.json new file mode 100644 index 000000000..19b5b91d4 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1833.3, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5840/run-20260814-101727-e26481", + "recorded_at": "2026-08-14T10:47:58", + "attempts": [ + { + "invocation_id": "inv-d7b45228", + "outcome": "run:run-20260814-101727-e26481", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-101727-e26481", + "llm_calls": 101, + "input_tokens": 698454, + "output_tokens": 82661, + "usd": 0.544713, + "source": "spans" + } + ], + "attempt_usd_total": 0.544713, + "llm_calls": 101, + "input_tokens": 698454, + "output_tokens": 82661, + "usd": 0.544713, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-101727-e26481", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5447, + "minutes": 30.5, + "ci_minutes": 0, + "input_tokens": 698454, + "output_tokens": 82661, + "tool_calls": 154, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.911, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 101, + "by_role": { + "agent": { + "usd": 0.528915, + "input_tokens": 675691, + "output_tokens": 77080, + "calls": 99 + }, + "reducer": { + "usd": 0.015798, + "input_tokens": 22763, + "output_tokens": 5581, + "calls": 2 + } + }, + "cache_read_tokens": 5272256, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.04, + "gate": 1.65, + "review": 1824.22, + "report": 0.07 + }, + "time_to_first_result_s": 1829.96 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 37387 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.md new file mode 100644 index 000000000..f81b6d4f9 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.md @@ -0,0 +1,50 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] backend.py:78-100 — enable_minimax_h3_teacache correctly rejects ref2va, accepts fl2va/combined, applies hook only to pipeline.transformer (not transformers_ref for combined). +- [validated] config.py:127-133 — TeaCacheConfig.__post_init__ resolves None rel_l1_thresh to _MODEL_DEFAULT_REL_L1_THRESH['MiniMaxH3DiTModel']=0.17 or _DEFAULT_REL_L1_THRESH=0.2. Verified all consumers pass through this resolution. +- [validated] data.py:422 — DiffusionCacheConfig.rel_l1_thresh changed from float=0.2 to float|None=None. No consumer reads the raw None as a float; all flow through TeaCacheConfig.__post_init__. +- [validated] extractors.py:1252-1397 — extract_minimax_h3_context faithfully mirrors MiniMaxH3DiTModel.forward: same kwargs parsing, shape validation, _embed, combined_indices, sp_prepare/block loop/sp_gather, final_layer, index_select, update_mask. Verified by test_extractor_matches_native_forward. +- [validated] hook.py:235 — assert rel_l1_thresh is not None is correct because TeaCacheConfig.__post_init__ always resolves None before the hook runs. +- [validated] test_cache_backends.py:473-545 — Tests cover generic default (0.2), MiniMax-H3 model-specific default (0.17), custom threshold override, and ref2va rejection. +- [validated] test_teacache_extractors.py:430-708 — Tests cover shape, callable, correctness (assert_close vs native forward), packed layout forwarding, TeaCache cache-hit reuse, and invalid module error. +- [validated] test_minimax_h3_e2e.py:22 — Frame count change (39→107) matches explicit duration=4.0 parameter (4.0s × 24fps ≈ 107 frames). New TeaCache e2e smoke test added. + +`vllm_omni/engine/async_omni_engine.py:?` [major] — _get_default_cache_config at vllm_omni/engine/async_omni_engine.py:855 returns `{"rel_l1_thresh": 0.2}` as the tea_cache default. When this dict is passed to DiffusionCacheConfig, rel_l1_thresh becomes 0.2 (not None), which TeaCacheConfig.__post_init__ takes as a user-supplied value and never resolves to the model-specific default. For MiniMax-H3, this means the async engine always uses 0.2 instead of the calibrated 0.17 — overriding the model-specific default that config.py:91-92 was added to provide. Return `{}` (or omit rel_l1_thresh) so the None default propagates through to __post_init__ and the model-specific value is used. (evidence: vllm_omni/engine/async_omni_engine.py:855: returns {"rel_l1_thresh": 0.2} as the tea_cache default config; vllm_omni/diffusion/cache/teacache/config.py:129-133: __post_init__ resolves None to model-specific default) + +`vllm_omni/diffusion/cache/teacache/backend.py:93` [minor] — The warning at vllm_omni/diffusion/cache/teacache/backend.py:93-96 says 'Ref2VA requests on this combined pipeline will run without TeaCache', but Ref2VA uses a completely separate MiniMaxH3DiTModel instance (`transformers_ref`) that never receives the hook — the two partitions share no transformer. Consider rewording to: 'TeaCache is enabled on the FL2VA transformer only; Ref2VA uses a separate uncached transformer (transformers_ref).' (evidence: backend.py:90 `apply_teacache_hook(pipeline.transformer, teacache_config)` — hook applied only to FL2VA transformer; pipeline_minimax_h3.py:639 `self.transformers_ref = MiniMaxH3DiTModel(od_config, quant_config=transformer_quant_config)` — Ref2VA is a separate DiT instance; pipeline_minimax_h3.py:737-738 `if task == "ref2va" and hasattr(self, "transformers_ref"): return self.transformers_ref` — ref2va routes to the separate uncached transformer.) + +`vllm_omni/diffusion/cache/teacache/extractors.py:?` [minor] — extractors.py:~1341-1342 hardcodes `blocks[0].adaln_proj` and `blocks[0].norm1` to compute the modulated_input used as the TeaCache cache key. This is correct — TeaCache only caches the first block's residual, so only block 0's modulation is needed for the similarity signal. Consider adding a brief comment explaining this invariant, matching the style used in other extractors (e.g., extract_zimage_context and extract_longcat_context both annotate the equivalent blocks[0] usage point). (evidence: extractors.py:1341 `shift_msa, scale_msa, *_ = module.blocks[0].adaln_proj(t_emb)` and extractors.py:1342 `modulated_hidden = module.blocks[0].norm1(decoder_input)` — the only blocks[0] reference in extract_minimax_h3_context, computing modulated_input for the TeaCache cache key) + +`buildkite/vllm-omni-amd-ci:?` [minor] — The gate report shows 1 failing check: buildkite/vllm-omni-amd-ci. The diff exclusively touches NVIDIA TeaCache paths (no AMD/ROCm-specific code). Please confirm this failure is pre-existing on main and not introduced by this PR. (evidence: CI read result: {"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"} — the only failing check; all other 8 checks (including buildkite/vllm-omni, pre-commit, DCO, build 3.11/3.12, intel-ci, npu-ci, docs) are SUCCESS. Changed files list from PR view contains zero AMD/ROCm paths: only docs/, recipes/, tests/diffusion/cache/, tests/diffusion/models/minimax_h3/, vllm_omni/diffusion/cache/teacache/, and vllm_omni/diffusion/data.py.) + +`docs/user_guide/diffusion/cache_acceleration/teacache.md:?` [minor] — docs/user_guide/diffusion/cache_acceleration/teacache.md:118 Configuration Parameters table shows `rel_l1_thresh` type as `float` and default as `0.2`. This PR changes both `DiffusionCacheConfig.rel_l1_thresh` and `TeaCacheConfig.rel_l1_thresh` from `float = 0.2` to `float | None = None`, with None resolving to model-specific defaults (0.17 for MiniMaxH3DiTModel, 0.2 fallback via `_DEFAULT_REL_L1_THRESH`). The doc type and default should be updated to `float | None` and note the model-specific resolution. (evidence: vllm_omni/diffusion/cache/teacache/config.py:91-92 `_MODEL_DEFAULT_REL_L1_THRESH = {"MiniMaxH3DiTModel": 0.17}`; config.py:125 `rel_l1_thresh: float | None = None`; config.py:129-131 resolves None via `threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)`. Meanwhile docs/user_guide/diffusion/cache_acceleration/teacache.md:118 still shows `| rel_l1_thresh | float | 0.2 |`.) + +`vllm_omni/entrypoints/cli/serve.py:?` [minor] — serve.py:586 help-text example `"rel_l1_thresh": 0.2` is misleading after this PR. With DiffusionCacheConfig.rel_l1_thresh now defaulting to None, TeaCacheConfig.__post_init__ resolves to _MODEL_DEFAULT_REL_L1_THRESH (0.17 for MiniMaxH3DiTModel) or _DEFAULT_REL_L1_THRESH (0.2 for others). The help string should note the model-specific resolution rather than implying 0.2 is universal. (evidence: ['vllm_omni/entrypoints/cli/serve.py:586 `"TeaCache: \'{\\"rel_l1_thresh\\": 0.2}\'. "` — hard-coded 0.2 example in --cache-config help', 'vllm_omni/diffusion/cache/teacache/config.py:129 `_DEFAULT_REL_L1_THRESH = 0.2` and config.py:130 `_MODEL_DEFAULT_REL_L1_THRESH = {"MiniMaxH3DiTModel": 0.17}`', 'vllm_omni/diffusion/cache/teacache/config.py:139 `threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)` — model-specific resolution when rel_l1_thresh is None']) + +`vllm_omni/diffusion/cache/teacache/backend.py:?` [nit] — Nit: the recipe's TeaCache online-serving section only shows FL2VA serving; adding a note or example showing combined serving with `--cache-backend tea_cache` would clarify that no extra config is needed — but the intro paragraph already states 'In combined serving, FL2VA requests use TeaCache while Ref2VA requests run uncached', so this is minor polish. (evidence: backend.py:89-96 `apply_teacache_hook(pipeline.transformer, teacache_config)` followed by `if partition == "combined": logger.warning("TeaCache is enabled only for MiniMax-H3 FL2VA; Ref2VA requests on this combined pipeline will run without TeaCache")` — the hook is applied only to pipeline.transformer (FL2VA's DiT), auto-dispatching by partition at request time.) + +`vllm_omni/diffusion/cache/teacache/hook.py:?` [nit] — hook.py:235 uses `assert rel_l1_thresh is not None` which is stripped in Python -O mode. Since TeaCacheConfig.__post_init__ always resolves None to a concrete float today, this is not a bug, but a RuntimeError or comment documenting the invariant would be more production-safe. (evidence: vllm_omni/diffusion/cache/teacache/hook.py:235: `assert rel_l1_thresh is not None`; vllm_omni/diffusion/cache/teacache/config.py:119-122: __post_init__ resolves None to concrete float via `_MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)` then assigns `self.rel_l1_thresh = threshold`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.trace.json.gz new file mode 100644 index 000000000..56d9f897c Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.cost.json new file mode 100644 index 000000000..7507f44d1 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 409.3, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5863/run-20260814-102515-bbf73f", + "recorded_at": "2026-08-14T10:32:01", + "attempts": [ + { + "invocation_id": "inv-b5a53523", + "outcome": "run:run-20260814-102515-bbf73f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-102515-bbf73f", + "llm_calls": 8, + "input_tokens": 36890, + "output_tokens": 6953, + "usd": 0.030347, + "source": "spans" + } + ], + "attempt_usd_total": 0.030347, + "llm_calls": 8, + "input_tokens": 36890, + "output_tokens": 6953, + "usd": 0.030347, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-102515-bbf73f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0303, + "minutes": 6.77, + "ci_minutes": 0, + "input_tokens": 36890, + "output_tokens": 6953, + "tool_calls": 16, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2404, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 8, + "by_role": { + "agent": { + "usd": 0.030347, + "input_tokens": 36890, + "output_tokens": 6953, + "calls": 8 + } + }, + "cache_read_tokens": 220032, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.06, + "gate": 1.64, + "review": 400.38, + "report": 0.08 + }, + "time_to_first_result_s": 406.13 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 13048 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.md new file mode 100644 index 000000000..682c73094 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.md @@ -0,0 +1,36 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md: attention backend CUDNN_ATTN is correct for SM120 — vllm_omni/platforms/cuda/platform.py:193 requires major==10 for TRTLLM_ATTN default, so RTX PRO 6000 (sm_120) cannot use it; code routes to CUDNN_ATTN via the Blackwell path at platform.py:222 +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md: cfg_parallel_size must be 1 — vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:551-552 raises ValueError if cfg_parallel_size != 1 +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md: text-encoder-tp-size=8 valid for 8 GPUs — base recipe MiniMax-H3.md confirms valid values are 1,2,4,8 on 8 GPUs (must divide Qwen3-VL 64 heads) +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md: memory formula arithmetic — 55.4+22.1/1=77.5≈77.49, 55.4+22.1/2=66.45≈66.44, 55.4+22.1/4=60.925≈61.07; all within 0.3% of measured +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md: scaling arithmetic — 278.55/168.73=1.65x, 168.73/87.90=1.92x, 5.396/2.791=1.93x, 2.791/1.798=1.55x all match stated values +- [validated] recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md: --enable-diffusion-pipeline-profiler exists in codebase — vllm_omni/entrypoints/cli/serve.py:784 + +`recipes/README.md:?` [minor] — Add a row for the new recipe to the Available Recipes table. The existing MiniMax-H3 variants (base and MUSA) are both listed here; the new RTX PRO 6000 recipe should follow suit. Something like: `| ["MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md"](./MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md) | T2VA on RTX PRO 6000 Blackwell | 2×/4×/8× RTX PRO 6000 (96 GiB) |` (evidence: The README table at recipes/README.md lists `MiniMaxAI/MiniMax-H3.md` (line 48) and `MiniMaxAI/MiniMax-H3-MUSA.md` (line 49) but not the new RTX PRO 6000 recipe.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:1` [minor] — Add a cross-reference to the base guide for full API reference, input matrix, and task details. The MUSA variant opens with `This recipe adapts MiniMax-H3.md for MUSA environments` and the NPU variant opens with `This recipe adapts MiniMax-H3.md for Ascend NPU environments`. This recipe should similarly reference MiniMax-H3.md for FL2VA/Ref2VA details, media limits, and the HTTP API contract that it does not reproduce (e.g. frame_indices, short_edge, start_time_seconds). (evidence: MiniMax-H3-MUSA.md:14 opens with `This recipe adapts [MiniMax-H3.md](MiniMax-H3.md) for MUSA environments.` and MiniMax-H3-NPU.md:15 opens with `This recipe adapts [MiniMax-H3.md](MiniMax-H3.md) for Ascend NPU environments.` The new recipe has no such reference.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.trace.json.gz new file mode 100644 index 000000000..112d0c829 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5863.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.cost.json new file mode 100644 index 000000000..f3d69371e --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1363.1, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5884/run-20260814-102946-566f09", + "recorded_at": "2026-08-14T10:52:27", + "attempts": [ + { + "invocation_id": "inv-31fb4cfe", + "outcome": "run:run-20260814-102946-566f09", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-102946-566f09", + "llm_calls": 72, + "input_tokens": 308930, + "output_tokens": 62029, + "usd": 0.268116, + "source": "spans" + } + ], + "attempt_usd_total": 0.268116, + "llm_calls": 72, + "input_tokens": 308930, + "output_tokens": 62029, + "usd": 0.268116, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-102946-566f09", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2681, + "minutes": 22.67, + "ci_minutes": 0, + "input_tokens": 308930, + "output_tokens": 62029, + "tool_calls": 102, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6703, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 72, + "by_role": { + "agent": { + "usd": 0.250967, + "input_tokens": 300689, + "output_tokens": 51853, + "calls": 70 + }, + "reducer": { + "usd": 0.017149, + "input_tokens": 8241, + "output_tokens": 10176, + "calls": 2 + } + }, + "cache_read_tokens": 2089984, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.59, + "gate": 1.5, + "review": 1354.9, + "report": 0.07 + }, + "time_to_first_result_s": 1360.03 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5884.trace.json.gz", + "trace_bytes": 22667 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.md new file mode 100644 index 000000000..b59f8536f --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5884, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5dc24e7f6a0c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] backend.py:42-55 _dit_module_names — replaced getattr with attrgetter; correctly resolves dotted paths, catches AttributeError for missing attrs, filters None modules. CONSISTENT with module_collector.py:110 pattern. +- [sweep] backend.py:59 cache_summary — uses attrgetter to resolve names, now also filters via BlockAdapter.is_cached to skip uncached modules. Semantic behavior change: modules without active Cache-DiT context are no longer summarized. +- [sweep] backend.py:78 _make_pipeline_transformer_getter — uses attrgetter in closure. Correctly propagates dotted name resolution to enable/refresh/summary paths. +- [validated] backend.py:39 — getattr(pipeline, '_dit_modules', None) is correct; this reads the _dit_modules list attribute itself, not individual entries. +- [validated] Bagel pipeline (bagel/pipeline_bagel.py:159) declares `_dit_modules = ['language_model.model']` — dotted path that was broken before this fix. +- [validated] Cosmos3 (cosmos3/pipeline_cosmos3.py:816) declares `_dit_modules = ['transformer.language_model', 'transformer']` — mixed dotted and direct. +- [validated] SoulXSinger (soulx_singer/pipeline_soulx_singer_base.py:63) declares `_dit_modules = ['cfm_decoder.model.diff_estimator']` — triple-dot path, also fixed by this PR. +- [validated] test_enable_and_refresh_nested_declared_dit (line 139) — correctly sets up pipeline with SimpleNamespace('language_model' -> SimpleNamespace('model' -> transformer_mock)); BlockAdapter is mocked so the enable/refresh flow uses mock paths; cache_summary assertion verifies summary is called. + +`tests/diffusion/cache/test_cache_backends.py:121` [minor] — test_summary_skips_uncached_nested_dit patches only cache_dit, leaving BlockAdapter.is_cached (backend.py:61) as a real call on Mock objects. The test's _is_cached filter logic is coupled to the external library's behavior rather than a controlled mock. Patch BlockAdapter like the sibling test at line 139 does. (evidence: tests/diffusion/cache/test_cache_backends.py:120 `def test_summary_skips_uncached_nested_dit(self, mock_cache_dit):` — only cache_dit is patched; vllm_omni/diffusion/cache/cachedit/backend.py:61 `if not BlockAdapter.is_cached(transformer): continue` — real BlockAdapter called on Mock objects without mocking. The sibling test at line 139 shows the correct pattern with both `@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")` and `@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")`.) + +`lora/manager.py:382` [minor] — The PR fixes dotted path resolution in the cache-dit backend (`cachedit/backend.py`) by switching from `getattr` to `operator.attrgetter`, but `vllm_omni/diffusion/lora/manager.py:382-389` still reads `_dit_modules` and iterates with `hasattr(self.pipeline, component_name)` + `getattr(self.pipeline, component_name)`, which fails for dotted paths (e.g. `"language_model.model"` in Bagel, `"transformer.language_model"` in Cosmos3). For Cosmos3 the flat `"transformer"` entry in `default_components` provides a fallback, but the nested sub-module is silently skipped. Pre-existing, out of PR scope — worth tracking as a follow-up. (evidence: vllm_omni/diffusion/lora/manager.py:382 `declared_components = tuple(getattr(self.pipeline, "_dit_modules", ()) or ())` — reads dotted paths into component_names, then vllm_omni/diffusion/lora/manager.py:387-388 `if not hasattr(self.pipeline, component_name): continue` / `component = getattr(self.pipeline, component_name)` — hasattr/getattr treat dotted strings as a single attribute name, so 'language_model.model' or 'transformer.language_model' always returns False.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:59` [minor] — The semantic behavior change in cache_summary to skip uncached modules (backend.py:59) is real and affects all pipelines using _dit_modules. However, the supporting analysis is inaccurate: soulx_singer also uses a dotted path ('cfm_decoder.model.diff_estimator') — not just Bagel and Cosmos3. The PR already adds test_summary_skips_uncached_nested_dit and test_enable_and_refresh_nested_declared_dit covering the is_cached filter and dotted path resolution. The suggestion to add pipeline-specific integration tests is reasonable polish but not required by the scope of this PR. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:57-59 — new is_cached filter: + for transformer in transformers: + if not BlockAdapter.is_cached(transformer): + continue + +soulx_singer/pipeline_soulx_singer_base.py:63 — dotted path contradicts 'only Bagel and Cosmos3': + _dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]) + +`buildkite/vllm-omni-amd-ci:1` [minor] — The CI check `buildkite/vllm-omni-amd-ci` is failing. Confirm whether this is pre-existing or a regression from this change, since the PR touches diffusion cache code used by diffusion pipelines. (evidence: gh_ci_read for PR 5884 returns {"name": "buildkite/vllm-omni-amd-ci", "state": "FAILURE"}; list_dir('buildkite') returns FileNotFoundError; grep for 'vllm-omni-amd-ci' across the repo returns zero matches — this is a CI pipeline name, not a file in the repo.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [minor] — The new `is_cached` guard in `cache_summary` (backend.py) skips uncached modules via `continue`, but the only downstream log — `if not transformers:` warning — only fires on an empty list. When `_dit_module_names` resolves names whose modules all fail `is_cached`, `transformers` is non-empty, the loop body never executes, and no log is emitted. Consider adding a `logger.debug(...)` after the for-loop when `transformers` is non-empty but `cache_dit.summary` was never called, so operators can distinguish 'no DiT declared' from 'DiT declared but not cached'. (evidence: backend.py:63-70: `transformers = [attrgetter(name)(pipeline) for name in _dit_module_names(pipeline)]` followed by `for transformer in transformers: if not BlockAdapter.is_cached(transformer): continue` — when all transformers are skipped, `transformers` is non-empty so `if not transformers:` at line 69 evaluates False and no warning is logged.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [minor] — The new resolution loop in `_dit_module_names` (backend.py lines 48-52) uses `attrgetter(name)(pipeline)` and appends `name` to `resolved_names` when the result `is not None`. Because `attrgetter` can traverse dotted paths (e.g., `language_model.model`), a mis-declared entry like `language_model` — where `pipeline.language_model` is a non-Module container (SimpleNamespace, dict) — would resolve successfully, pass the `is not None` guard, and propagate to `enable_cache_for_dit` / `BlockAdapter` which expect `nn.Module`. Consider tightening the guard to `if isinstance(module, nn.Module):` (with an early `logger.warning` for skipped non-Modules) to fail early with a clear message rather than producing a confusing downstream type error. (evidence: backend.py:48 `module = attrgetter(name)(pipeline)` — resolves full dotted path with no type check; backend.py:51 `if module is not None:` — sole guard before appending; backend.py:78 `return cast(nn.Module, attrgetter(name)(pipeline))` — `cast` is a runtime no-op; backend.py:133 `cache_dit.enable_cache(cache_target, ...)` — downstream consumer expects nn.Module but receives whatever attrgetter resolved.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [nit] — This PR replaces `getattr` with `attrgetter` in `_dit_module_names` (line 49), `cache_summary` (line 60), and `_make_pipeline_transformer_getter` (line 82), but `_default_get_pipeline_transformer` at line 73 still uses `getattr(pipeline, "transformer")`. Since the argument is a hardcoded string this is functionally equivalent, but for consistency with the PR's use of `attrgetter` for dotted-path resolution, the line could be changed to `return cast(nn.Module, attrgetter("transformer")(pipeline))`. (evidence: backend.py:73 `return cast(nn.Module, getattr(pipeline, "transformer"))` — uses getattr while lines 49, 60, 82 all use attrgetter; functionally equivalent but inconsistent with PR's stated pattern) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.trace.json.gz new file mode 100644 index 000000000..b3370984d Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5884.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.cost.json new file mode 100644 index 000000000..eeb93faaa --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2055.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5957/run-20260814-103003-f332d6", + "recorded_at": "2026-08-14T11:04:17", + "attempts": [ + { + "invocation_id": "inv-b5d6c31a", + "outcome": "run:run-20260814-103003-f332d6", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-103003-f332d6", + "llm_calls": 104, + "input_tokens": 1489326, + "output_tokens": 57083, + "usd": 0.994242, + "source": "spans" + } + ], + "attempt_usd_total": 0.994242, + "llm_calls": 104, + "input_tokens": 1489326, + "output_tokens": 57083, + "usd": 0.994242, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-103003-f332d6", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9942, + "minutes": 34.22, + "ci_minutes": 0, + "input_tokens": 1489326, + "output_tokens": 57083, + "tool_calls": 149, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1389, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 104, + "by_role": { + "agent": { + "usd": 0.9505, + "input_tokens": 1395896, + "output_tokens": 49209, + "calls": 102 + }, + "reducer": { + "usd": 0.043742, + "input_tokens": 93430, + "output_tokens": 7874, + "calls": 2 + } + }, + "cache_read_tokens": 11230336, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.52, + "gate": 1.7, + "review": 2046.85, + "report": 0.07 + }, + "time_to_first_result_s": 2053.13 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5957.trace.json.gz", + "trace_bytes": 124711 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.md new file mode 100644 index 000000000..7a4f93eee --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5957, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 19a6e513f28e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] v2.5 talker intentionally excluded from _OMNI_CONNECTOR_INIT_ARCHS (gpu_ar_model_runner.py) — verified omni_payload_at_request_end=True and code-only path via output_token_ids means no connector accumulation needed +- [validated] _patch_empty_hf_config (arg_utils.py:254) correctly model-scoped to indextts2_5 with lexists check for broken symlinks +- [validated] WaveNet _logical_reflect_in_layer (wavenet.py:73) correctly computes reflect positions; ragged_min_length gate prevents short-row corruption +- [validated] packed_varlen attention (gpt_fast/model.py:275) guarded by device, dtype, and backend capability checks +- [validated] _strip_stop_token and _build_s2mel_additional_information changed from warning+None to ValueError — fail-fast is correct for v2.5 code-only payload contract +- [validated] _FULL_PAYLOAD_INPUT_STAGES (omni_scheduling_coordinator.py:70) correctly includes IndexTTS25S2MelDecoder consumer stage +- [validated] IndexTTS25Adapter.validate() (indextts2.py adapter) correctly validates speed 0.5-2.0 at adapter level after protocol relaxation +- [validated] streaming speed validation removal (protocol/audio.py) is intentional — streaming speech test (test_serving_speech.py) verifies native speed path for indextts2_5 model_type + +`buildkite/vllm-omni-amd-ci:1` [major] — The AMD CI check is failing per gate_report. Since the diff touches XPU/CUDA shared code paths, can you confirm whether this failure is pre-existing or introduced by this PR? If introduced, what's the plan to fix it before merge? (evidence: CI: buildkite/vllm-omni-amd-ci failing per gate_report; diff touches XPU/CUDA shared code paths) + +`tests/model_executor/models/indextts2/test_recipe_defaults.py:1` [major] — tests/model_executor/models/indextts2/test_recipe_defaults.py imports yaml and Path but has no pytestmark. CI's '-m core_model and cpu' filter skips this test. Add pytestmark = [pytest.mark.core_model, pytest.mark.cpu]. Peer file test_configuration_and_pipeline.py and test_enhanced_codec.py in the same directory also lack pytestmark. (evidence: test_recipe_defaults.py:1-14 — entire file has no pytestmark declaration; all 7 peer files in tests/model_executor/models/indextts2/ (test_s2mel_v2_5.py, test_talker_v2_5.py, test_tokenizer_v2_5.py, test_prompt_utils.py, test_dit_cuda_graph.py, test_stage_input_processor.py, test_wavenet_ragged.py) declare `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` at module level) + +`vllm_omni/entrypoints/openai/serving_speech.py:3642` [minor] — There's no dedicated test verifying the 400 response for non-native-speed models on the streaming+speed path after the guard at serving_speech.py:3642. Can we add a test to ensure this error case is covered? (evidence: Streaming speed validation coverage gap — the old protocol-level rejection (speed != 1.0 with stream=True) was removed; the serving-layer replacement at line 3642 is the new guard. No dedicated test verifies the 400 response for non-native-speed models on streaming+speed path.) + +`tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:1` [minor] — tests/model_executor/models/indextts2/test_configuration_and_pipeline.py is a new file that omits the module-level `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` declaration. CI selects tests via `-m 'core_model and cpu'` (.buildkite/cuda/test-ready.yml:13), so all 5 tests in this file are silently skipped. Every peer file in the same directory (test_s2mel_v2_5.py, test_talker_v2_5.py, test_prompt_utils.py, test_dit_cuda_graph.py, test_stage_input_processor.py, test_wavenet_ragged.py, test_tokenizer_v2_5.py) declares the marks. Add `import pytest` and `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` after the SPDX header. (evidence: tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:1-11 — file begins with SPDX header then `from pathlib import Path` / `import yaml` / model imports with no `import pytest` or `pytestmark` declaration. Every peer file in the same directory declares `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (e.g. test_s2mel_v2_5.py:26, test_talker_v2_5.py:17). CI filter `.buildkite/cuda/test-ready.yml:13` runs `pytest -sv tests/model_executor -m 'core_model and cpu'`.) + +`tests/model_executor/models/indextts2/test_enhanced_codec.py:1` [minor] — test_enhanced_codec.py is the only new test file in tests/model_executor/models/indextts2/ without a pytestmark; all 7 sibling files added in this PR declare `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. Add the same pytestmark for consistency. Separately, test_semantic_codec_loader_selects_enhanced_checkpoint replaces EnhancedCodec with a FakeCodec and then asserts isinstance(codec, FakeCodec), making that assertion tautological — the test still validates the loader path/device/dtype arguments, but the isinstance line adds no independent signal. (evidence: test_enhanced_codec.py has no `pytestmark` line anywhere in the file (lines 1-84); every other file in the same directory added by this PR does: + +test_prompt_utils.py:15: `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` +test_dit_cuda_graph.py:12: `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` +test_s2mel_v2_5.py:26: `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` +test_stage_input_processor.py:14: `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` +test_wavenet_ragged.py:14: `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` +test_tokenizer_v2_5.py:23: `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` +test_talker_v2_5.py:17: `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` + +test_enhanced_codec.py:36-38: `monkeypatch.setattr(codec_module, "EnhancedCodec", lambda **kwargs: FakeCodec())` then line 58: `assert isinstance(codec, FakeCodec)` — tautological because codec is the FakeCodec injected 22 lines above) + +`vllm_omni/engine/arg_utils.py:?` [minor] — vllm_omni/engine/arg_utils.py:260 uses a model-scoped guard `if model_type != 'indextts2_5' or not os.path.isdir(self.model)` to handle the official IndexTTS 2.5 bundle's missing HF config.json. This is intentionally narrow. Existing inline comments at lines 257–259 already document the rationale. Consider adding a module-level note or registry-driven pattern so future native-bundle models without config.json don't need another hardcoded exception. (evidence: arg_utils.py:260 `if model_type != "indextts2_5" or not os.path.isdir(self.model):` — the indextts2_5 string literal is the sole hard-coded model check in the except branch. Existing models (CosyVoice3, PersonaPlex) ship an empty config.json so `PretrainedConfig.get_config_dict` succeeds at line 253 and they never reach this branch.) + +`examples/online_serving/text_to_speech/README.md:?` [nit] — The opening paragraph of the 'IndexTTS-2 and IndexTTS-2.5' section (examples/online_serving/text_to_speech/README.md:143) leads with v2.5-specific architecture details (EnhancedCodec, code-only Stage 0→1 transfer) before the shared pipeline description. The offline README (examples/offline_inference/text_to_speech/README.md:440) leads with the shared 'Both versions use a 2-stage TTS pipeline…' sentence before differentiating. Consider restructuring the online serving paragraph to lead with the shared description first, then v2.5-specific details, for consistency. (evidence: examples/online_serving/text_to_speech/README.md:143 — '2-stage TTS at 22.05 kHz. Requests use `ref_audio` for voice cloning, or an uploaded audio `voice` from `/v1/audio/voices`. IndexTTS-2.5 uses a multilingual tokenizer, CAMPPlus speaker projection, EnhancedCodec, and code-only Stage 0→1 transfer by default.' vs examples/offline_inference/text_to_speech/README.md:440 — 'Both versions use a 2-stage TTS pipeline and produce 22.05 kHz mono audio. Stage 0 is a GPT AR talker.') + +`.buildkite/intel/scripts/run-xpu-test.sh:37` [nit] — The diff adds build_vllm_base() (run-xpu-test.sh:37) using a subshell `( )` with a trap. The subshell scope ensures the trap fires on function exit even on failure, correctly scoping the temp-directory cleanup per invocation. (evidence: run-xpu-test.sh:37-40 `build_vllm_base() ( + local vllm_source_dir + vllm_source_dir=$(mktemp -d) + trap 'rm -rf "${vllm_source_dir}"' EXIT` — subshell function with EXIT trap fires on function exit (success or failure), not script exit) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.trace.json.gz new file mode 100644 index 000000000..ea01326b8 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.cost.json b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.cost.json new file mode 100644 index 000000000..d200df6d3 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2342.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_mimo_wave2/runs/pr5976/run-20260814-103205-f17c99", + "recorded_at": "2026-08-14T11:11:04", + "attempts": [ + { + "invocation_id": "inv-606ce33f", + "outcome": "run:run-20260814-103205-f17c99", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-103205-f17c99", + "llm_calls": 83, + "input_tokens": 919705, + "output_tokens": 97352, + "usd": 0.627572, + "source": "spans" + } + ], + "attempt_usd_total": 0.627572, + "llm_calls": 83, + "input_tokens": 919705, + "output_tokens": 97352, + "usd": 0.627572, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-103205-f17c99", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6276, + "minutes": 38.97, + "ci_minutes": 0, + "input_tokens": 919705, + "output_tokens": 97352, + "tool_calls": 122, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0474, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 83, + "by_role": { + "agent": { + "usd": 0.558215, + "input_tokens": 831438, + "output_tokens": 69891, + "calls": 80 + }, + "reducer": { + "usd": 0.069357, + "input_tokens": 88267, + "output_tokens": 27461, + "calls": 3 + } + }, + "cache_read_tokens": 4839488, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.22, + "gate": 1.88, + "review": 2330.88, + "report": 0.06 + }, + "time_to_first_result_s": 2338.04 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 63927 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.md b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.md new file mode 100644 index 000000000..3b4fc4cbc --- /dev/null +++ b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.md @@ -0,0 +1,58 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] stale-drain fix correctness: omni_ar_scheduler.py:336-339 seeds from num_stale_output_tokens>0, drains by num_tokens_scheduled; omni_scheduler_mixin.py:105 seeds from num_in_flight_tokens; omni_ar_scheduler.py:491 seeds at segment stop. All three seed sites use same units as the drain. Tests in test_omni_ar_scheduler_stale_drain.py exercise exact-drain, two-in-flight, spec-placeholder-asymmetry, and in-flight-prefill scenarios. +- [validated] Qwen3-Omni forward contract: qwen3_omni.py:166-172 - thinker forward returns bare tensor when no capture (is_staged_run=False), tuple when capture requested. qwen3_omni_moe_thinker.py:1512-1519 - thinker returns bare tensor when capture_layer_indices is None. qwen3_omni.py:515-518 - make_omni_output tolerates both shapes. Verified via test_qwen3_omni_forward_contract.py covering 6 scenarios. +- [validated] parallel_state use_all2all: parallel_state.py:535-540 - inspects vllm signature for use_all2all param before forwarding. Tests in test_expert_parallel_layout.py cover both modern (use_all2all accepted) and legacy (TypeError-avoiding) paths. +- [validated] FusedMoE→FusedMoEFactory rename: hunyuan_image3.py:32, fused_moe.py:78, modeling_bailing_moe_v2.py:335 - all three consumer sites updated. Tests in test_fused_moe.py updated to match. +- [validated] _replace_streaming_session stale seeding: omni_scheduler_mixin.py:90-106 - clears output token ids, sets num_stale_output_tokens from num_in_flight_tokens, clears placeholders/spec_token_ids. Called by both AR (_update_request_as_session → replace_streaming_prompt path) and generation (_update_request_as_session → direct call) schedulers. +- [validated] PureDiffusionLauncherAdapter: entrypoints/utils.py:926-943 - __getattr__ delegates to wrapped engine_client; vllm_config property returns SimpleNamespace(shutdown_timeout=...). get_vllm_config() on AsyncOmni still returns None (pure-diffusion detection preserved). +- [validated] OnlineRenderer.warmup(): api_server.py:955 sets state.online_renderer, :1001 calls state.online_renderer.warmup(). Correct migration from removed OpenAIServingChat.warmup(). +- [validated] _scoped_replica_compile_cache_env thread safety: stage_engine_startup.py:926 uses _COMPILE_CACHE_ENV_LOCK; tests verify replica 0 keeps shared cache, additional replicas get isolated dirs, env restored on exit. + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [blocker] — Double-seed confirmed. Fix: remove the pre-seed block from _update_request_as_session and seed only in (a) _replace_streaming_session for the replace path and (b) the else-branch before super()._update_request_as_session for the extend-prompt path. (evidence: omni_ar_scheduler.py:628 `session.num_stale_output_tokens += in_flight_tokens` — first seed, reads num_in_flight_tokens. omni_scheduler_mixin.py:104 `session.num_stale_output_tokens += int(getattr(session, 'num_in_flight_tokens', 0) or 0)` — second seed via _replace_streaming_session, same num_in_flight_tokens value since nothing resets it between the two. omni_ar_scheduler.py:638 `self._replace_streaming_session(session, update)` — the call that triggers the second seed when replace_streaming_prompt=True.) + +`vllm_omni/engine/stage_runtime.py:?` [minor] — This wrapper omits stage_id/replica_id kwargs, so the new per-replica compile-cache isolation (_scoped_replica_compile_cache_env) is a no-op here. The critical path through launch_stage_replica (stage_engine_startup.py:1147) does pass them, so engine-core spawns are covered, but acquire_device_locks and other setup steps inside this scope run with the shared cache. If two replicas of the same stage both hit this wrapper simultaneously, they race on VLLM_CACHE_ROOT during lock acquisition. Consider forwarding stage_id/replica_id to this wrapper as well, or document the intentional scoping gap. (evidence: stage_runtime.py:306-314: `def _scoped_spawn_device_env(self, physical_devices: str | None) -> Iterator[None]: ... with scoped_spawn_device_env(physical_devices, self._spawn_device_lock,): yield` — no stage_id/replica_id passed, unlike stage_engine_startup.py:1147 `scoped_spawn_device_env(stage_visible_devices, spawn_device_lock, stage_id=stage_id, replica_id=replica_id)` which has both.) + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [minor] — Dead write-only field: `request.async_tokens_to_discard` is assigned at omni_ar_scheduler.py:479 (segment-stop) and :634 (session-replacement) but never read by any production code after the stale-drain introduction. Upstream vLLM 0.27 removed the consumption; only test assertions (test_omni_ar_scheduler_streaming.py:124,221,243) still reference it. Consider removing both writes and the remaining test assertions to reduce confusion. (evidence: omni_ar_scheduler.py:479 `request.async_tokens_to_discard = outstanding_async_tokens` — written but never read in production. omni_ar_scheduler.py:634 `session.async_tokens_to_discard = 1` — also write-only. Only consumers are test assertions in tests/core/sched/test_omni_ar_scheduler_streaming.py:124,221,243.) + +`vllm_omni/core/sched/omni_ar_scheduler.py:?` [minor] — Dead code: vLLM 0.27 removed `async_tokens_to_discard` from the upstream scheduler (commit a0c092ee72). The field is assigned at omni_ar_scheduler.py:479 (segment stop) and omni_ar_scheduler.py:634 (streaming session replacement) but nothing in vLLM 0.27 or omni reads it — the active replacement mechanism is `num_stale_output_tokens`. Remove both assignments and their associated comments. (Note: the draft cited 'mixin:105' as a set site, but the mixin only has a comment at line 100; both assignments are in omni_ar_scheduler.py.) (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:479 `request.async_tokens_to_discard = outstanding_async_tokens` — set but never read; vllm_omni/core/sched/omni_ar_scheduler.py:634 `session.async_tokens_to_discard = 1` — set but never read; full-repo grep confirms zero read/consume sites for `async_tokens_to_discard` outside comments and test assertions of the set value.) + +`vllm_omni/engine/stage_engine_startup.py:940` [minor] — The `base` path and `isolated` computation sit outside `_COMPILE_CACHE_ENV_LOCK`, so a concurrent thread that already holds the lock (and has transiently set `VLLM_CACHE_ROOT` to its own isolated sub-path) can be read as `base` by a racing thread, producing a nested path. Move the `base`/`isolated` computation inside the lock, or snapshot `base` from the env once before any thread enters and pass it as a parameter. (evidence: stage_engine_startup.py:932 `base = os.environ.get("VLLM_CACHE_ROOT") or os.path.expanduser("~/.cache/vllm")` — outside the lock +stage_engine_startup.py:933 `isolated = os.path.join(base, f"omni_stage{stage_id if stage_id is not None else 'x'}_replica{replica_id}")` — derived from unprotected `base` +stage_engine_startup.py:934 `with _COMPILE_CACHE_ENV_LOCK:` — lock acquired two lines later, after `base` is already stale) + +`vllm_omni/core/sched/omni_scheduler_mixin.py:?` [nit] — This line runs unconditionally inside _replace_streaming_session for both AR and generation schedulers. For the generation scheduler, _update_request_as_session at omni_generation_scheduler.py delegates entirely to _replace_streaming_session, and num_in_flight_tokens is always 0 (generation one-shot scheduling never accumulates it; only AR's schedule() sets it via allocate_slots). The `or 0` and `int()` guards are correct but could be simplified since Request already initializes num_in_flight_tokens to 0 — consider noting this is a no-op for generation in a brief comment to save the next reader the cross-reference. (evidence: vllm_omni/core/sched/omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` — runs unconditionally in _replace_streaming_session for both schedulers +vllm_omni/core/sched/omni_generation_scheduler.py:610-612 `def _update_request_as_session(self, session, update): self._replace_streaming_session(session, update)` — generation delegates entirely to the mixin +vllm_omni/core/sched/omni_generation_scheduler.py:376 `request.num_in_flight_tokens -= num_tokens_scheduled` — only decrement site in generation path; no increment site exists, so the value is always 0 when _replace_streaming_session runs) + +`.buildkite/cuda/test-merge.yml:?` [nit] — This is a valid polish nit — the inline comment at test-merge.yml:142-147 only cites build 2956 but the issue was observed across multiple builds. Consider tightening to just reference the timeout constraint rather than a single build number, since the hardware requirement is a structural property (2 GPUs for parallel Whisper + model) not specific to one build. (evidence: .buildkite/cuda/test-merge.yml:142-147 `# h100_2, not h100_1: advanced_model asserts transcribe every audio +# response with Whisper, and tests/helpers/media.py only uses an +# accelerator for Whisper when a spare GPU exists. On a single card it +# grades on CPU: ~8-10 min per clip vs ~30 s, which alone overruns the +# 50-minute job timeout (build 2956).` — single build number cited; the 2-GPU requirement is structural) + +`vllm_omni/core/sched/omni_scheduler_mixin.py:105` [nit] — The `int(getattr(..., 0) or 0)` pattern at line 105 is defensive but unnecessary: `num_in_flight_tokens` is always an int on `Request` in vLLM 0.27 and is read directly (`request.num_in_flight_tokens`) at the AR scheduler's segment-stop site (omni_ar_scheduler.py:490-491). A plain `session.num_in_flight_tokens` would suffice and match that convention. (evidence: omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` — defensive getattr wrapper; vs omni_ar_scheduler.py:490-491 `if request.num_in_flight_tokens > 0: request.num_stale_output_tokens += request.num_in_flight_tokens` — direct attribute access on the same Request type in the same PR) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.trace.json.gz b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.trace.json.gz new file mode 100644 index 000000000..3ab01e9e4 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_mimo_wave2/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.cost.json new file mode 100644 index 000000000..1ea3eb893 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 2105.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5884/run-20260814-222452-9df050", + "recorded_at": "2026-08-14T22:59:55", + "attempts": [ + { + "invocation_id": "inv-9c470518", + "outcome": "run:run-20260814-222452-9df050", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-222452-9df050", + "llm_calls": 107, + "input_tokens": 798430, + "output_tokens": 151498, + "usd": 0.574693, + "source": "spans" + } + ], + "attempt_usd_total": 0.574693, + "llm_calls": 107, + "input_tokens": 798430, + "output_tokens": 151498, + "usd": 0.574693, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-222452-9df050", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5747, + "minutes": 35.04, + "ci_minutes": 0, + "input_tokens": 798430, + "output_tokens": 151498, + "tool_calls": 365, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9797, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 107, + "by_role": { + "moa_member": { + "usd": 0.04916, + "input_tokens": 74109, + "output_tokens": 4824, + "calls": 10 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.497673, + "input_tokens": 712405, + "output_tokens": 124303, + "calls": 94 + }, + "reducer": { + "usd": 0.02786, + "input_tokens": 11916, + "output_tokens": 22371, + "calls": 2 + } + }, + "cache_read_tokens": 6715904, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.0, + "gate": 1.73, + "review": 2096.8, + "report": 0.06 + }, + "time_to_first_result_s": 2102.59 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5884.trace.json.gz", + "trace_bytes": 31183 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.md b/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.md new file mode 100644 index 000000000..b75e3b992 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.md @@ -0,0 +1,56 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5884, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5dc24e7f6a0c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] vllm_omni/diffusion/cache/cachedit/backend.py:37-53 — `_dit_module_names` uses `attrgetter` with AttributeError skip, aligned with `ModuleDiscovery._collect_modules` (module_collector.py:108-111). +- [sweep] backend.py:45 `if not isinstance(name, str)` — only str entries processed; non-str skipped (same as pre-change). +- [sweep] backend.py:51 `if module is not None` — None intermediates skipped via attrgetter AttributeError or None check. +- [sweep] backend.py:61 `if not BlockAdapter.is_cached(transformer)` — verified on real `nn.Linear`: returns `False`; Cosmos3 nested UND path skipped in summary only. +- [validated] Bagel `_dit_modules = ["language_model.model"]` (pipeline_bagel.py:159) now resolves to `Qwen2MoTModel` with `_cache_dit_adapter_config` (bagel_transformer.py:890-895); generic `CacheDiTBackend.enable` path (no custom enabler). +- [validated] Cosmos3 uses `CUSTOM_DIT_ENABLERS["Cosmos3OmniDiffusersPipeline"]` (model_specific.py:781-804); generic multi-entry enable loop not used for Cosmos3. +- [validated] tests/diffusion/cache/test_cache_backends.py:139-161 — nested enable/refresh drives real `CacheDiTBackend.enable` entry point with `language_model.model` path. +- [validated] tests/diffusion/cache/test_cache_backends.py:121-135 — `test_summary_skips_uncached_nested_dit` exercises real `BlockAdapter.is_cached` (not patched), verifies only cached transformer summarized. + +`vllm_omni/diffusion/cache/cachedit/backend.py:49` [minor] — This diff now resolves dotted `_dit_modules` paths with `attrgetter` and silently skips paths that raise `AttributeError` (backend.py:47-50). No cache-backend test exercises the broken-path branch: the new tests only cover resolvable nested paths (`transformer.language_model`, `language_model.model`). Add a `TestCacheDiTBackend` test using a non-Mock pipeline (e.g. `SimpleNamespace(_dit_modules=["nonexistent.model"])`) and assert `backend.enable(pipeline)` raises `ValueError` matching 'no declared DiT modules', mirroring `test_skips_missing_intermediate` in the offloader tests. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:47-50 ` try:\n module = attrgetter(name)(pipeline)\n except AttributeError:\n continue`; vllm_omni/diffusion/cache/cachedit/backend.py:252 ` raise ValueError(f"Pipeline {pipeline_name} has no declared DiT modules for Cache-DiT")`; tests/diffusion/offloader/test_module_collector.py:241 ` def test_skips_missing_intermediate(self):`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:57` [minor] — backend.py:57 docstring still says 'Log Cache-DiT statistics for every transformer on the pipeline', but the changed body now does `if not BlockAdapter.is_cached(transformer): continue` before `cache_dit.summary(...)`; uncached nested modules such as Cosmos3's `transformer.language_model` are skipped. Update the docstring to say summaries are logged only for DiT modules with an active Cache-DiT context. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:57 ` """Log Cache-DiT statistics for every transformer on the pipeline."""` +vllm_omni/diffusion/cache/cachedit/backend.py:60-63 ` for transformer in transformers: + if not BlockAdapter.is_cached(transformer): + continue + cache_dit.summary(transformer, details=details)` +vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:816 ` _dit_modules: ClassVar[list[str]] = ["transformer.language_model", "transformer"]`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [minor] — Confirmed: `_dit_module_names` now resolves dotted paths (backend.py:48), so `CacheDiTBackend.enable()`'s generic loop (backend.py:239) iterates every entry of Cosmos3's `_dit_modules = ["transformer.language_model", "transformer"]` (pipeline_cosmos3.py:816). Production Cosmos3 is saved only by the class-name `CUSTOM_DIT_ENABLERS` short-circuit (backend.py:232-236; model_specific.py:853-854). If the generic loop ever runs on that dual-entry list (an unregistered subclass or a copycat model), the inner `Cosmos3LanguageModel` has no `_cache_dit_adapter_config`, so `enable_cache_for_dit` raises ValueError (backend.py:156-163) on the inner name before `transformer` is wrapped. Please add an `enable()` test using a pipeline class not in `CUSTOM_DIT_ENABLERS` with `_dit_modules=["transformer.language_model", "transformer"]`, and have `enable()` skip dotted names nested inside another listed DiT (mirroring `PipelineModules.outermost_dits()`). (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:48 ` module = attrgetter(name)(pipeline)`; vllm_omni/diffusion/cache/cachedit/backend.py:239 ` for name in _dit_module_names(pipeline):`; vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:816 ` _dit_modules: ClassVar[list[str]] = ["transformer.language_model", "transformer"]`; vllm_omni/diffusion/cache/cachedit/backend.py:232-234 ` pipeline_name = type(pipeline).__name__` / ` custom_enabler = CUSTOM_DIT_ENABLERS.get(pipeline_name)` / ` if custom_enabler is not None:`; vllm_omni/diffusion/cache/cachedit/model_specific.py:853-854 ` "Cosmos3OmniDiffusersPipeline": enable_cache_for_cosmos3,` / ` "Cosmos3OmniPipeline": enable_cache_for_cosmos3,`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [minor] — Confirmed: backend.py:48 now resolves `_dit_modules` entries with `attrgetter(name)(pipeline)`, so SoulXSinger's `_dit_modules = ["cfm_decoder.model.diff_estimator"]` resolves to `DiffLlama`, which declares `_cache_dit_adapter_config` with `ForwardPattern.Pattern_3`. `CacheDiTBackend.enable` now enables Cache-DiT for SoulXSinger where the old `getattr(pipeline, name, None)` check skipped the dotted name, and SoulXSingerPipeline is not in `_NO_CACHE_ACCELERATION`. Run the SoulXSinger cache-dit path (e.g. offline expansion with `cache_backend=cache_dit`) to confirm Pattern_3 on this flow-matching model, or explicitly scope the attrgetter change to the intended models. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:48 `module = attrgetter(name)(pipeline)` (PR diff replaces the old `getattr(pipeline, name, None)` skip for dotted names); vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_base.py:63 `_dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]`; vllm_omni/diffusion/models/soulx_singer/modules/llama.py:115 `_cache_dit_adapter_config = CacheDiTAdapterConfig(` with `"layers": ForwardPattern.Pattern_3,`; vllm_omni/diffusion/registry.py:332-335 `_NO_CACHE_ACCELERATION = {` ... `"NextStep11Pipeline",` `"AudioXPipeline",` (SoulXSingerPipeline absent).) + +`tests/diffusion/cache/test_cache_backends.py:?` [minor] — This diff adds `test_summary_skips_uncached_nested_dit` and `test_enable_and_refresh_nested_declared_dit` to `tests/diffusion/cache/test_cache_backends.py`, which is marked `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (line 32). The AMD Simple lane at `.buildkite/amd/test-amd-ready.yml:19` runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` and ignores only `test_teacache_extractors.py`, so these tests execute there; the Diffusion Cache lane at line 86 runs `-m 'core_model and cache and diffusion and not distributed_cuda and L4'`, which deselects this file because it has no `cache` marker. Please confirm from the failing `buildkite/vllm-omni-amd-ci` log whether the failure is caused by these new tests or is pre-existing on main before merging. (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; tests/diffusion/cache/test_cache_backends.py:121 `def test_summary_skips_uncached_nested_dit(self, mock_cache_dit):`; tests/diffusion/cache/test_cache_backends.py:139 `def test_enable_and_refresh_nested_declared_dit(self, mock_cache_dit, mock_block_adapter):`; .buildkite/amd/test-amd-ready.yml:19 `"timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/amd/test-amd-ready.yml:86 `timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [minor] — This diff switches `_dit_module_names` and `_make_pipeline_transformer_getter` to `attrgetter`, but the other `_dit_modules` consumers still use plain attribute access: `diffusion_model_runner.py:283` for torch.compile setup (whose `_compile_transformer` resolves via `getattr(self.pipeline, attr_name, None)`) and `registry.py:437` for sequence-parallel hooks (whose SP loop uses `hasattr(model, attr)`/`getattr(model, attr)`). With cache-dit plus torch.compile or SP on a nested-path model such as Bagel, the DiT would be cache-wrapped but not compiled or SP-hooked because a dotted name like `'language_model.model'` resolves to None. Update these consumers to `operator.attrgetter` for consistency, or file a follow-up so nested-path models do not silently skip compile/SP. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:78 `return cast(nn.Module, attrgetter(name)(pipeline))` — the diff's getter now resolves dotted `_dit_modules` paths. vllm_omni/diffusion/worker/diffusion_model_runner.py:283 `transformer_attrs = getattr(self.pipeline, "_dit_modules", None)` — compile setup still reads the list and then resolves each name via `getattr(self.pipeline, attr_name, None)` in `_compile_transformer`, which returns None for a dotted name. vllm_omni/diffusion/registry.py:437 `transformer_attrs = getattr(model, "_dit_modules", None)` — SP setup then gates on `hasattr(model, attr)`/`getattr(model, attr)` with the literal dotted string, which is False/None, so the nested DiT is skipped.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:65` [nit] — In `vllm_omni/diffusion/cache/cachedit/backend.py:59-66`, `cache_summary` builds `transformers` from all resolved `_dit_module_names`, skips uncached modules with `continue` inside the loop, but warns only when `transformers` itself is empty. A pipeline with resolved DiT modules that are all skipped by `BlockAdapter.is_cached` therefore emits no `cache_dit.summary` and no log. Consider a debug log when every resolved module is skipped by `is_cached`. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:59 ` transformers = [attrgetter(name)(pipeline) for name in _dit_module_names(pipeline)]`; :61 ` if not BlockAdapter.is_cached(transformer):`; :62 ` continue`; :65 ` if not transformers:`; :66 ` logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute")`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.trace.json.gz new file mode 100644 index 000000000..3f1277ff1 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/QUARANTINE_skillleak_pr5884/pr5884.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/manifest.json b/eval/dataset/arms/copilot_v13_moa_cgm/manifest.json new file mode 100644 index 000000000..c3b3c24ff --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v13_moa_cgm", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "always", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.cost.json new file mode 100644 index 000000000..e3434142c --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1656.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5509/run-20260814-215045-54de89", + "recorded_at": "2026-08-14T22:18:20", + "attempts": [ + { + "invocation_id": "inv-7c2fb30e", + "outcome": "run:run-20260814-215045-54de89", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-215045-54de89", + "llm_calls": 91, + "input_tokens": 631466, + "output_tokens": 168389, + "usd": 0.469084, + "source": "spans" + } + ], + "attempt_usd_total": 0.469084, + "llm_calls": 91, + "input_tokens": 631466, + "output_tokens": 168389, + "usd": 0.469084, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-215045-54de89", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4691, + "minutes": 27.57, + "ci_minutes": 0, + "input_tokens": 631466, + "output_tokens": 168389, + "tool_calls": 265, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8379, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 91, + "by_role": { + "moa_member": { + "usd": 0.062509, + "input_tokens": 83936, + "output_tokens": 8481, + "calls": 11 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.34831, + "input_tokens": 506798, + "output_tokens": 116994, + "calls": 76 + }, + "reducer": { + "usd": 0.058265, + "input_tokens": 40732, + "output_tokens": 42914, + "calls": 3 + } + }, + "cache_read_tokens": 3675648, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.6, + "gate": 2.89, + "review": 1647.49, + "report": 0.07 + }, + "time_to_first_result_s": 1654.04 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 42342 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.md new file mode 100644 index 000000000..af6e51a02 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.md @@ -0,0 +1,73 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 9 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] vllm_omni/diffusion/attention/backends/trtllm_attn.py:296-320 — SAGE quantize + sage_kwargs only when `self.quant.enabled`; dense path unchanged. +- [validated] vllm_omni/diffusion/attention/backends/trtllm_attn.py:194-216 — init resolves `_sage_quantize_fn` once and fails fast on missing kernel/quantize routine. +- [validated] vllm_omni/diffusion/attention/backends/trtllm_attn.py:90-95 — `_sage_kernel_available()` gates on `sage_attn_sfs` param in `trtllm_ragged_attention_deepseek` signature. +- [validated] tests/diffusion/attention/test_trtllm_attn.py:13 — `pytestmark` includes `core_model`, `diffusion`, `cuda`; tests are CI-selectable. +- [validated] tests/diffusion/attention/test_attention_config.py:34 — config tests marked `cpu`; serialization/validation covered without GPU. +- [sweep] tests/diffusion/attention/test_trtllm_attn.py:315 — arch-dependent `Missing TRTLLM-GEN kernel` skip path present for int8 on non-SM100. +- [sweep] vllm_omni/diffusion/data.py:1415-1420 — `AttnQuantSpec` rejects invalid dtypes and block sizes at construction. +- [sweep] vllm_omni/diffusion/data.py:1472-1483 — `backend_kwargs()` emits quant dict with defaults when `quant.enabled`. + +`vllm_omni/diffusion/data.py:1447` [major] — Confirmed: data.py:1447 allowlists FLASHINFER_ATTN for `quant` and docs claim it reads dtype_qk/dtype_vo/flashinfer_backend, but FlashInferAttentionImpl.__init__ takes **extra_impl_args and ignores them, so Attention (layer.py:109) passes backend_kwargs=spec.backend_kwargs() into a void — FLASHINFER+quant silently runs dense attention. AttnQuantSpec._VALID_DTYPES is a union (float16/bfloat16/int8/fp8_e4m3), so FLASHINFER_ATTN+int8 also passes spec construction without rejection. Fix: remove FLASHINFER from the allowlist (and delete the docs/test), or implement FLASHINFER quant with real rejection of unused/wrong-set fields. (evidence: vllm_omni/diffusion/data.py:1447-1450 ` if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"): + raise ValueError( + f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but " + f"backend={self.backend!r}. Remove quant or set a supported backend."`; vllm_omni/diffusion/attention/backends/flashinfer_attn.py:55-66 ` def __init__( + self, + num_heads: int, + head_size: int, + softmax_scale: float, + causal: bool = False, + num_kv_heads: int | None = None, + prefix: str = "", + **extra_impl_args, + ) -> None: + self.causal = causal + self.softmax_scale = softmax_scale`; vllm_omni/diffusion/attention/layer.py:109 ` backend_kwargs=backend_kwargs,`) + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — docs/user_guide/diffusion/attention_backends.md:232 still says 'see Known Limitations above', but this diff deletes the entire '## Known Limitations' section (LTX-2 CUDNN_ATTN torch.compile workaround / #3121 and the FA4 note), the '## Validation Guidance' section, the mixed-backend and compare-FA examples, and the '## Reproducing the Benchmark Table' section. Restore those sections or rewrite the dangling pointer; the new SAGE section (lines 170-183) is lossy and should keep the quality-comparison contract. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — grep confirms this is the only occurrence of 'Known Limitations' in the file, so the referenced section no longer exists. docs/user_guide/diffusion/attention_backends.md:182 `V is always quantized per-channel to fp8_e4m3 and K-smoothing is applied inside the routine, so` — the lossy SAGE section remains while the removed Validation Guidance contract is gone.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:196` [major] — Confirmed. AttentionSpec.__post_init__ allows quant on TRTLLM_ATTN (data.py:1449), AttnQuantSpec.enabled is true for dtype_vo-only configs (data.py:1425), and backend_kwargs() serializes it (data.py:1471). But QuantConfig.from_backend_kwargs reads only dtype_qk (trtllm_attn.py:124) and QuantConfig.enabled is dtype_qk is not None (trtllm_attn.py:130), so TRTLLM_ATTN silently runs dense attention for quant={'dtype_vo': 'fp8_e4m3'} instead of rejecting the unsupported field. Add a TRTLLM-side rejection for dtype_vo/flashinfer_backend-only quant, or gate backend_kwargs() per backend. (evidence: vllm_omni/diffusion/data.py:1425 `return self.dtype_qk is not None or self.dtype_vo is not None`; vllm_omni/diffusion/data.py:1471 `if self.quant is not None and self.quant.enabled:`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:124 `dtype_qk=q.get("dtype_qk"),`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:130 `return self.dtype_qk is not None`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:196 `if self.quant.enabled:`) + +`tests/diffusion/attention/test_trtllm_attn.py:50` [major] — This diff adds CPU-pure SAGE tests to tests/diffusion/attention/test_trtllm_attn.py, but the file is `pytestmark = [core_model, diffusion, cuda]` (line 13), so Ready CI's `Simple · Diffusion Test` (`pytest -sv tests/diffusion -m 'core_model and cpu'`, .buildkite/cuda/test-ready.yml:18) never collects them. The only CUDA diffusion step is a top-level shell glob (`pytest -sv tests/diffusion/*.py -m 'core_model and cuda'`, .buildkite/cuda/test-ready.yml:68) that does not recurse into `tests/diffusion/attention/`, and no .buildkite step references `tests/diffusion/attention` at all. On Ready CI's L4 hardware `_has_trtllm_attn()` returns False because `torch.cuda.get_device_capability()[0] < 10` (lines 16-19), and `test_sage_quant_matches_sdpa` would skip on missing TRTLLM-GEN kernels anyway. Move the CPU-pure QuantConfig tests into `tests/diffusion/attention/test_attention_config.py` (already `core_model, diffusion, cpu`) so they actually execute, and do not treat Ready CI green as SAGE kernel coverage. (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` — CPU job with `-m 'core_model and cpu'` drops this file. tests/diffusion/attention/test_trtllm_attn.py:16-19 `if not torch.cuda.is_available(): return False` / `if torch.cuda.get_device_capability()[0] < 10: return False` — L4 (sm_89) cannot run the kernel test. .buildkite/cuda/test-ready.yml:18 `timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni ...` — only cpu-marked tests collected. .buildkite/cuda/test-ready.yml:68 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"` — shell glob of top-level files only; does not recurse into tests/diffusion/attention. Grep of `.buildkite` for `tests/diffusion/attention`: no matches.) + +`vllm_omni/diffusion/data.py:?` [minor] — Docs state 'A dtype_qk from the wrong set raises', but AttentionSpec.__post_init__ (data.py:1441) only coerces quant and checks backend in {TRTLLM_ATTN, FLASHINFER_ATTN}; AttnQuantSpec._VALID_DTYPES accepts bfloat16/float16 (data.py:1410), so TRTLLM_ATTN + quant.dtype_qk='bfloat16' passes config parse and only fails later at TrtllmAttentionImpl.__init__ (trtllm_attn.py:199-203) with RuntimeError. Add per-backend dtype_qk validation in __post_init__ so JSON/CLI mistakes fail at config parse like skip_softmax does. (evidence: vllm_omni/diffusion/data.py:1410 `_VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"})`; vllm_omni/diffusion/data.py:1441 `self.quant = self._coerce(self.quant, AttnQuantSpec, "quant")` with no per-backend dtype check (only `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN")`); vllm_omni/diffusion/attention/backends/trtllm_attn.py:199-203 `if self.quant.dtype_qk not in _QK_QUANT_DTYPES: raise RuntimeError(f"TRTLLM_ATTN quant (SAGE) supports dtype_qk in {sorted(_QK_QUANT_DTYPES)}, got {self.quant.dtype_qk!r}. FLASHINFER_ATTN dtypes (float16/bfloat16) are not SAGE.")`; tests/diffusion/attention/test_trtllm_attn.py:60-62 `with pytest.raises(RuntimeError, match="supports dtype_qk"): _impl(quant={"dtype_qk": "bfloat16"})`.) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — This diff adds a second typed spec field (`quant` / `AttnQuantSpec`) and a `## TRTLLM_ATTN SAGE Quantization` section, but lines 72-73 still claim Skip-Softmax is "Today the only" typed field on the spec. Update that paragraph to name `quant` and link to the new SAGE heading. (evidence: docs/user_guide/diffusion/attention_backends.md:72-73 `A backend that needs configuration exposes it as a typed field on the spec. Today the only` / `one is `TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):`; vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None`; docs/user_guide/diffusion/attention_backends.md:170 `## TRTLLM_ATTN SAGE Quantization`) + +`vllm_omni/diffusion/data.py:1403` [minor] — `quant` is added to the shared `AttentionSpec` (vllm_omni/diffusion/data.py:1434), so a `default` spec with `quant` flows through `get_attn_backend_for_role` for both self and cross roles (vllm_omni/diffusion/attention/layer.py:77-84). Wan 2.2 cross-attn separately opts out of the KV-cache quantization path via `disable_kv_quant=True` (vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py:587; guard at vllm_omni/diffusion/attention/layer.py:222), a different quantization mechanism from TRTLLM SAGE. Please confirm default SAGE quant is not unintentionally applied to Wan cross-attn and that the quant/disable_kv_quant interaction is covered by tests. (evidence: vllm_omni/diffusion/data.py:1434 `quant: AttnQuantSpec | None = None` — added to the shared `AttentionSpec`. vllm_omni/diffusion/attention/layer.py:222 `if kv_cache_dtype is None or self._disable_kv_quant or not self._should_apply_kv_cache_quant():` — the separate KV-cache quant path honors `disable_kv_quant`. vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py:587 `disable_kv_quant=True,` — Wan cross-attn uses that separate opt-out.) + +`recipes/Wan-AI/Wan2.2-T2V.md:?` [minor] — This diff adds SAGE quantized attention to `TRTLLM_ATTN` via the new `quant` block (`vllm_omni/diffusion/data.py:1434`) and documents it at `docs/user_guide/diffusion/attention_backends.md:171` ('## TRTLLM_ATTN SAGE Quantization'), but `recipes/Wan-AI/Wan2.2-T2V.md:186` still tells users `TRTLLM_ATTN` is BF16-only (no attention-level quantization). Update that recipe bullet to mention the SAGE `quant` config or link to the SAGE section, otherwise the recipe contradicts the feature it ships next to. (evidence: recipes/Wan-AI/Wan2.2-T2V.md:186 ` - `TRTLLM_ATTN` is BF16-only (no attention-level quantization).` ; vllm_omni/diffusion/data.py:1434 ` quant: AttnQuantSpec | None = None` ; docs/user_guide/diffusion/attention_backends.md:171 `## TRTLLM_ATTN SAGE Quantization`) + +**Additional observations (beyond the comment budget):** +- `docs/user_guide/diffusion/attention_backends.md:210` [minor] docs/user_guide/diffusion/attention_backends.md:208-210 claims fp8_e4m3 SAGE kernels on SM100+SM103 and int8 on SM100-only, but the PR body evidence covers only Wan2.2-A14B I2V on B200/SM100 (Single GPU, seed 0, FlashInf +- `vllm_omni/diffusion/attention/backends/trtllm_attn.py:204` [minor] This diff gates SAGE only on FlashInfer signature/function availability, not GPU arch, so an int8 quant config on SM103 (where the docs say int8 kernels are SM100-only) passes init and fails only inside `forward_cuda` wi +- `vllm_omni/diffusion/attention/backends/trtllm_attn.py:295` [minor] vllm_omni/diffusion/attention/backends/trtllm_attn.py:294-298 gates sage_attn_sfs / num_elts_per_sage_attn_blk behind `self.quant.enabled`, but no test under tests/diffusion/attention/ monkeypatches or inspects the `trtl +- `tests/diffusion/attention/test_attention_config.py:469` [minor] tests/diffusion/attention/test_attention_config.py:445 wires per_role cross AttentionSpec with skip_softmax only; the Attention-init path (layer.py:93 spec.backend_kwargs()) is asserted at line 469 for skip_softmax but n + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.trace.json.gz new file mode 100644 index 000000000..d2b715e2f Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.cost.json new file mode 100644 index 000000000..ad049c619 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1328.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5550/run-20260814-215046-63be11", + "recorded_at": "2026-08-14T22:12:51", + "attempts": [ + { + "invocation_id": "inv-0a725b5e", + "outcome": "run:run-20260814-215046-63be11", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-215046-63be11", + "llm_calls": 63, + "input_tokens": 523251, + "output_tokens": 96793, + "usd": 0.352285, + "source": "spans" + } + ], + "attempt_usd_total": 0.352285, + "llm_calls": 63, + "input_tokens": 523251, + "output_tokens": 96793, + "usd": 0.352285, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-215046-63be11", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3523, + "minutes": 22.09, + "ci_minutes": 0, + "input_tokens": 523251, + "output_tokens": 96793, + "tool_calls": 235, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7038, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 63, + "by_role": { + "moa_member": { + "usd": 0.051141, + "input_tokens": 64238, + "output_tokens": 4033, + "calls": 11 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.279724, + "input_tokens": 437889, + "output_tokens": 78506, + "calls": 49 + }, + "reducer": { + "usd": 0.021421, + "input_tokens": 21124, + "output_tokens": 14254, + "calls": 2 + } + }, + "cache_read_tokens": 3441728, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.95, + "gate": 2.19, + "review": 1318.96, + "report": 0.06 + }, + "time_to_first_result_s": 1325.15 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 34133 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.md new file mode 100644 index 000000000..4a40d1428 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.md @@ -0,0 +1,71 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] vllm_omni/diffusion/sched/interface.py:181-197 — `validate_new_request_data_identity` checks envelope==forwarded and metadata.request_id==forwarded before RPC. +- [validated] vllm_omni/diffusion/executor/multiproc_executor.py:459-460 — identity validation runs before dispatch; mismatch raises with zero `collective_rpc` calls (test_worker_contract.py:129-141). +- [validated] vllm_omni/diffusion/executor/multiproc_executor.py:462-471 — `has_diffusion_kv_metadata` disables DLO multi-request AllGather; falls back to per-request RPC loop. +- [validated] vllm_omni/diffusion/executor/multiproc_executor.py:544-546 — metadata appended as 4th positional only when non-None; dense RPC stays `(req, od_config, kv_prefetch_job)` (test_worker_contract.py:204-221). +- [validated] vllm_omni/diffusion/worker/diffusion_model_runner.py:172-177 — dense modes reject unexpected metadata; paged accepts optional metadata (test_worker_contract.py:103-126). +- [validated] vllm_omni/diffusion/worker/diffusion_model_runner.py:636-641,750-755 — batch/step paths validate identity+metadata before forward; batch allows missing metadata in paged mode (test_worker_contract.py:144-170). +- [validated] vllm_omni/diffusion/worker/diffusion_worker.py:1036 — RPC `execute_method` dispatches positional args to `DiffusionWorker.execute_model`, so 4-tuple RPC maps to `diffusion_kv_metadata` param. +- [validated] vllm_omni/diffusion/diffusion_engine.py:191-208 — scheduler init failure calls `scheduler.close()` then `executor.shutdown()` without requiring full runtime state (test_diffusion_engine_cleanup.py:126-164). + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [major] — ARDiffusionModelRunner.execute_model (vllm_omni/experimental/ar_diffusion/runner.py:237-243) overrides DiffusionModelRunner.execute_model with the old signature `(self, req, kv_prefetch_job=None)` and calls `super().execute_model(req, kv_prefetch_job=kv_prefetch_job)`. Since DiffusionWorker.execute_model now forwards `diffusion_kv_metadata` as a kwarg when non-None (vllm_omni/diffusion/worker/diffusion_worker.py:462-465), the first metadata-carrying request to this live subclass raises TypeError before the base validation added at diffusion_model_runner.py:592. Accept and forward the kwarg (or explicitly reject scheduler-paged metadata at load). (evidence: vllm_omni/experimental/ar_diffusion/runner.py:237-243 ` def execute_model( + self, + req: OmniDiffusionRequest, + kv_prefetch_job: KVPrefetchJob | None = None, + ) -> DiffusionOutput: + if self.kv_cache is None: + return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)`; vllm_omni/diffusion/worker/diffusion_worker.py:462-465 ` kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job} + if diffusion_kv_metadata is not None: + kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata + output = self.model_runner.execute_model(req, **kwargs)`; vllm_omni/diffusion/worker/diffusion_model_runner.py:589-592 ` def execute_model( + self, + req: OmniDiffusionRequest, + kv_prefetch_job: KVPrefetchJob | None = None, + diffusion_kv_metadata: DiffusionKVMetadata | None = None, + ) -> DiffusionOutput:`) + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — This diff adds `and not has_diffusion_kv_metadata` at `vllm_omni/diffusion/executor/multiproc_executor.py:471`, making `execute_request` fall through to the per-request loop (which appends metadata as a 4-tuple RPC arg at lines 544-546) whenever any scheduled envelope carries metadata, thereby bypassing the DLO+AllGather broadcast wave. That metadata-vs-DLO gate has no coverage: `make_executor` in `tests/diffusion/diffusion_kv/test_worker_contract.py:63` pins `enable_distributed_layerwise_offload=False`, and `_make_sched_output` in `tests/diffusion/test_multiproc_engine_concurrency.py` builds `NewRequestData` without `diffusion_kv_metadata`. Add a DLO-enabled test with two metadata-carrying requests asserting `collective_rpc` is called twice with `(req, od_config, kv_prefetch_job, metadata)` 4-tuples, not once with a broadcast 3-tuple wave. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:471 ` and not has_diffusion_kv_metadata` +vllm_omni/diffusion/executor/multiproc_executor.py:544 ` args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` +vllm_omni/diffusion/executor/multiproc_executor.py:545 ` if new_req.diffusion_kv_metadata is not None:` +vllm_omni/diffusion/executor/multiproc_executor.py:546 ` args += (new_req.diffusion_kv_metadata,)` +tests/diffusion/diffusion_kv/test_worker_contract.py:63 ` enable_distributed_layerwise_offload=False,`) + +`tests/diffusion/diffusion_kv/test_metadata.py:15` [minor] — Can you add one path-level test that carries DiffusionKVMetadata through NewRequestData → MultiprocDiffusionExecutor.execute_request → DiffusionModelRunner? test_metadata.py only round-trips dataclass fields it just constructed, and the existing forwarding coverage in test_worker_contract.py stops at a mocked collective_rpc or calls the runner directly, so the full executor→runner metadata handoff is never exercised in a single test. (evidence: tests/diffusion/diffusion_kv/test_metadata.py:38 `assert metadata.sequences[0].block_ids == ([1, 2], [5, 6])` and tests/diffusion/diffusion_kv/test_metadata.py:39 `assert metadata.contexts[0].block_ids == ([7], [11])` — the test only asserts fields on the object it just constructed; no NewRequestData/executor/runner involvement. tests/diffusion/diffusion_kv/test_worker_contract.py:231-232 `NewRequestData(request_id="req-0", req=req_0, diffusion_kv_metadata=metadata_0),` with :235-238 asserting `executor.collective_rpc` call args `(req_0, executor.od_config, None, metadata_0)` — executor forwarding is covered at the unit boundary with a mocked collective_rpc, not through the real model runner, confirming no single path-level test exists.) + +`vllm_omni/diffusion/sched/base_scheduler.py:114` [minor] — Confirmed: base_scheduler.py:114 calls NewRequestData.from_state(state) without diffusion_kv_metadata, so scheduled envelopes carry diffusion_kv_metadata=None. DiffusionModelRunner.execute_model_batch (diffusion_model_runner.py:636-642) validates each new_req's identity and metadata, then builds reqs from nr.req only, dropping the metadata before _execute_request_list. Until a producer passes metadata in from_state, the PAGED_SCHEDULER metadata paths remain latent. Please confirm the linked W1 issue explicitly covers adding that scheduler producer. (evidence: vllm_omni/diffusion/sched/base_scheduler.py:114 `scheduled_new_reqs.append(NewRequestData.from_state(state))` — no diffusion_kv_metadata argument, so envelopes are metadata-free. vllm_omni/diffusion/worker/diffusion_model_runner.py:638-642 `self._validate_diffusion_kv_metadata(request_id=new_req.req.request_id, metadata=new_req.diffusion_kv_metadata,)` followed by `reqs = [nr.req for nr in scheduler_output.scheduled_new_reqs]` — metadata is validated then dropped before _execute_request_list.) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:169` [minor] — Confirm/document the fallback at diffusion_model_runner.py:169: getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY) treats an od_config missing the field as dense-legacy and rejects any attached metadata. OmniDiffusionConfig already defaults this field to DENSE_LEGACY (data.py:666), so please confirm the getattr fallback is intentional for non-dataclass/older config objects or document it explicitly. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:169 `cache_mode = getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)`; vllm_omni/diffusion/worker/diffusion_model_runner.py:172 `if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:`; vllm_omni/diffusion/worker/diffusion_model_runner.py:173 `raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")`; vllm_omni/diffusion/data.py:666 `diffusion_kv_mode: DiffusionKVCacheMode = DiffusionKVCacheMode.DENSE_LEGACY`) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:?` [minor] — This diff adds `_validate_diffusion_kv_metadata` to `execute_model_batch` (`diffusion_model_runner.py:638`) and `execute_stepwise` (`diffusion_model_runner.py:752`) before proceeding, so DENSE_LEGACY requests carrying metadata are rejected through all three entry points. The only dense-mode rejection test, `test_dense_request_rejects_metadata_before_request_forward` (`test_worker_contract.py:103`), calls `execute_model` and does not cover batch/step. Add parallel tests invoking `execute_model_batch` and `execute_stepwise` with `DiffusionKVCacheMode.DENSE_LEGACY` and non-None `diffusion_kv_metadata`, asserting the error is raised and `_execute_request_list`/`_supports_step_mode` are not reached. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:173 `raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")`; vllm_omni/diffusion/worker/diffusion_model_runner.py:638 `self._validate_diffusion_kv_metadata(`; vllm_omni/diffusion/worker/diffusion_model_runner.py:752 `self._validate_diffusion_kv_metadata(`; tests/diffusion/diffusion_kv/test_worker_contract.py:103 `def test_dense_request_rejects_metadata_before_request_forward() -> None:`; tests/diffusion/diffusion_kv/test_worker_contract.py:104 `runner = make_runner(DiffusionKVCacheMode.DENSE_LEGACY)`) + +`vllm_omni/diffusion/sched/interface.py:192` [minor] — vllm_omni/diffusion/sched/interface.py:192 adds a metadata request_id identity check and MultiprocDiffusionExecutor.execute_request calls validate_new_request_data_identity before collective_rpc, but no contract test exercises that executor path with aligned envelope/forwarded IDs and a stale metadata.request_id. Add a test driving executor.execute_request with NewRequestData(request_id='req-0', req=SimpleNamespace(request_id='req-0'), diffusion_kv_metadata=make_metadata('stale')) and assert it raises ValueError('request identity mismatch') with no collective_rpc call. (evidence: vllm_omni/diffusion/sched/interface.py:192 ` if metadata is not None and metadata.request_id != forwarded_request_id:`; vllm_omni/diffusion/executor/multiproc_executor.py:457-458 ` for new_req in new_reqs:` / ` validate_new_request_data_identity(new_req)` (runs before any collective_rpc); tests/diffusion/diffusion_kv/test_worker_contract.py:120-123 ` with pytest.raises(ValueError, match="request mismatch"):` / ` runner.execute_model(` / ` SimpleNamespace(request_id="req-0"),` / ` diffusion_kv_metadata=make_metadata("stale"),` (metadata mismatch only via runner.execute_model); tests/diffusion/diffusion_kv/test_worker_contract.py:129-133 `def test_request_rpc_rejects_envelope_request_identity_mismatch() -> None:` / ` executor, calls = make_executor()` / ` req = SimpleNamespace(request_id="actual-request-id")` / ` new_req = NewRequestData(` / ` request_id="envelope-id",` (executor path varies envelope vs forwarded IDs, not stale metadata with aligned IDs).) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:592` [nit] — `DiffusionModelRunner.execute_model` now accepts `diffusion_kv_metadata` (diffusion_model_runner.py:592), but its Args section still documents only `req`; the pre-existing `kv_prefetch_job` is also undocumented. Document both parameters here, and mirror the new `diffusion_kv_metadata` parameter in the `DiffusionWorker`/`WorkerWrapperBase` execute_model docstrings so the worker boundary contract is readable. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:591 ` kv_prefetch_job: KVPrefetchJob | None = None,`; vllm_omni/diffusion/worker/diffusion_model_runner.py:592 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,`; vllm_omni/diffusion/worker/diffusion_model_runner.py:597 ` Args:`; vllm_omni/diffusion/worker/diffusion_model_runner.py:598 ` req: A diffusion request containing a list of prompts to process.`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.trace.json.gz new file mode 100644 index 000000000..964579051 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.cost.json new file mode 100644 index 000000000..14ac237b8 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1127.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5610/run-20260814-215045-d9ee3b", + "recorded_at": "2026-08-14T22:09:31", + "attempts": [ + { + "invocation_id": "inv-d98a6f1f", + "outcome": "run:run-20260814-215045-d9ee3b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-215045-d9ee3b", + "llm_calls": 38, + "input_tokens": 290948, + "output_tokens": 50364, + "usd": 0.233112, + "source": "spans" + } + ], + "attempt_usd_total": 0.233112, + "llm_calls": 38, + "input_tokens": 290948, + "output_tokens": 50364, + "usd": 0.233112, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-215045-d9ee3b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2331, + "minutes": 18.76, + "ci_minutes": 0, + "input_tokens": 290948, + "output_tokens": 50364, + "tool_calls": 183, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5915, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 38, + "by_role": { + "planner": { + "usd": 0.000524, + "input_tokens": 105, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.220837, + "input_tokens": 278629, + "output_tokens": 42310, + "calls": 35 + }, + "reducer": { + "usd": 0.011752, + "input_tokens": 12214, + "output_tokens": 7654, + "calls": 2 + } + }, + "cache_read_tokens": 3672448, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.7, + "gate": 1.69, + "review": 1119.85, + "report": 0.07 + }, + "time_to_first_result_s": 1125.3 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 23956 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.md new file mode 100644 index 000000000..56337c7e8 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.md @@ -0,0 +1,39 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness: read `gpu_ar_model_runner.py:1595-1619` — `_should_use_async_omni_output()` guards match the compatibility table (async scheduling, prefix cache, speculative decoding, async_chunk, routed experts, model opt-in, eager postprocess). +- [validated] `qwen3_omni.py:133-164` — Thinker sets `use_async_omni_output=True`; Talker sets opt-in plus `eager_omni_postprocess_before_async_output=True` and `omni_pooler_payload_include_hidden=False`, matching stage-behavior table. +- [validated] `qwen3_tts_talker.py:351-353` — Talker opt-in flags match Qwen3-Omni Talker pattern described in doc. +- [validated] `omni_connector_model_runner_mixin.py:49-61` — `should_accumulate_full_payload_output()` returns False when `async_chunk` is enabled, supporting the doc's unreachable full-payload branch claim. +- [validated] `gpu_ar_model_runner.py:201-206,223-230` — `OmniAsyncGPUModelRunnerOutput` starts a daemon background thread and `get_output()` joins it, matching lifecycle section. +- [validated] `gpu_ar_model_runner.py:1899-1900` — connector drain happens inside `_build_omni_model_runner_output_from_snapshot`, after streaming payload build, as documented. +- [validated] `npu_ar_model_runner.py:1273-1328` — NPU fully builds `OmniModelRunnerOutput`, drains connector synchronously, then wraps `AsyncGPUModelRunnerOutput` (not `OmniAsyncGPUModelRunnerOutput`). +- [validated] Deploy YAML excerpts: `qwen3_omni_moe.yaml:15,31,43,60` and `qwen3_tts.yaml:14,52-53,80-81` match doc examples; `pipeline.py` registry keys `qwen3_omni_moe.yaml` / `qwen3_tts.yaml`. + +`docs/design/feature/omni_async_output_materialization.md:73` [minor] — docs/design/feature/omni_async_output_materialization.md:73 uses src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg". From this file's directory (docs/design/feature/) that resolves to repo-root source/architecture/, but the SVG added in this PR is at docs/source/architecture/, so the image is broken in the GitHub file preview (it only works on the built MkDocs site). Use the same absolute raw-URL pattern already used at docs/design/feature/async_chunk.md:85-86: https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/qwen3-omni-async-output-step-gap.svg. (evidence: docs/design/feature/omni_async_output_materialization.md:73 ` src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"` — from docs/design/feature/ this resolves to repo-root source/architecture/, while the SVG added in this diff is at docs/source/architecture/qwen3-omni-async-output-step-gap.svg. docs/design/feature/async_chunk.md:85-86 `` and `Async Chunk Pipeline Architecture` — the existing raw-URL pattern the comment recommends.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.trace.json.gz new file mode 100644 index 000000000..3c46b56e7 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.cost.json new file mode 100644 index 000000000..c16f948ae --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1345.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5703/run-20260814-220933-14548d", + "recorded_at": "2026-08-14T22:31:56", + "attempts": [ + { + "invocation_id": "inv-97e314b2", + "outcome": "run:run-20260814-220933-14548d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-220933-14548d", + "llm_calls": 65, + "input_tokens": 311723, + "output_tokens": 100059, + "usd": 0.259833, + "source": "spans" + } + ], + "attempt_usd_total": 0.259833, + "llm_calls": 65, + "input_tokens": 311723, + "output_tokens": 100059, + "usd": 0.259833, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-220933-14548d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2598, + "minutes": 22.38, + "ci_minutes": 0, + "input_tokens": 311723, + "output_tokens": 100059, + "tool_calls": 267, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6617, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 65, + "by_role": { + "moa_member": { + "usd": 0.045595, + "input_tokens": 61786, + "output_tokens": 5070, + "calls": 12 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.188556, + "input_tokens": 234362, + "output_tokens": 75496, + "calls": 50 + }, + "reducer": { + "usd": 0.025682, + "input_tokens": 15575, + "output_tokens": 19493, + "calls": 2 + } + }, + "cache_read_tokens": 2047488, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.26, + "gate": 2.24, + "review": 1336.98, + "report": 0.06 + }, + "time_to_first_result_s": 1342.53 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 31939 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.md new file mode 100644 index 000000000..fc412f8b9 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] vllm_omni/diffusion/models/minimax_h3/vae.py:170-176 — encode_image/encode_video now seed via torch.random.fork_rng and self.device_module.manual_seed instead of torch.cuda-only calls; CPU path still uses devices=[] +- [validated] vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:963 — sole in-repo caller of encode_image; encode_video at :1066; no signature change +- [sweep] Correctness — fork_rng + finally dtype restore preserved; no new concurrency or empty-input regressions spotted in touched hunks +- [sweep] Simplifiability — self.device_module cached at __init__ could be replaced by torch.get_device_module(parameter.device) at encode time (only one callsite pair) +- [sweep] Breaking behavior — widening `parameter.device.type == "cuda"` to `!= "cpu"` enables accelerator RNG on npu/xpu/musa/etc.; PR text limits validation to CUDA+MUSA but code has no allowlist +- [sweep] Rebase/merge — diff is small/coherent; no dropped hunks or duplicate logic seen +- [sweep] Tests — zero test files in diff; no encode_image/encode_video RNG assertions in tests/diffusion/models/minimax_h3/; PR cites missing tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py +- [sweep] Docs — recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76 labels block “validated Ref2VA configuration” but exports MODEL="${MODEL_ROOT}/FL2VA"; no Ref2VA restart note unlike MiniMax-H3-NPU.md:114-118 + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — vae.py:170 and :214 use `parameter.device.type != "cpu"`, so every non-CPU accelerator (NPU, XPU, MPS, etc.) enters `torch.random.fork_rng(devices=[parameter.device])` and `self.device_module.manual_seed(...)`. This contradicts the PR description's 'The device allowlist is limited to CUDA and MUSA. Other accelerator RNG semantics are not inferred from mocked interfaces' and changes RNG behavior for the in-tree Ascend NPU recipe without validation. Restrict to `parameter.device.type in ("cuda", "musa")` with a matching `device_type` for `fork_rng`, or add NPU/XPU fork_rng/manual_seed evidence and update the description. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` and vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []` — dispatch is gated only on 'not CPU', not on an allowlist of CUDA/MUSA; vae.py:120 `self.device_module = torch.get_device_module()`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — The diff seeds the accelerator via `self.device_module` for every non-CPU parameter device (vae.py:170-176 and :214-220) with no allowlist, contradicting the PR's 'limited to CUDA and MUSA' claim, and no test exercises that branch. The only in-tree MiniMaxH3VideoVAE.encode_video test (test_minimax_h3_contract.py:327-353) builds the VAE via `object.__new__` with a default-CPU `FakeModel.weight`, so `devices == []` and `device_module.manual_seed` never runs; the PR validation command also cites `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py`, which does not exist. Add a CPU-runnable mock test that sets fake non-CPU `parameter.device` types and asserts only CUDA/MUSA populate `fork_rng` and call `device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`, and correct the validation command to the real test files. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:175 `with self.device_module.device(device):`; vllm_omni/diffusion/models/minimax_h3/vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))`; tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:344 `video_vae = object.__new__(MiniMaxH3VideoVAE)`; PR body validation command lists `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py`, and the directory `tests/diffusion/models/minimax_h3/` contains only test_minimax_h3_contract.py, test_minimax_h3_e2e.py, test_minimax_h3_packing.py, test_minimax_h3_parallel.py, test_minimax_h3_quantization.py, test_minimax_h3_quantization_quality.py.) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — The block labeled "The validated Ref2VA configuration" (recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76) exports MODEL="${MODEL_ROOT}/FL2VA", which serves T2VA/FL2VA, not Ref2VA. Change it to export MODEL="${MODEL_ROOT}/Ref2VA" for the described TP4 Ref2VA profile (and keep a separate FL2VA block if that profile is also validated). (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72 "The validated Ref2VA configuration uses tensor parallelism across four MTT" and recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76 "export MODEL="${MODEL_ROOT}/FL2VA"" — the Ref2VA-labeled block points at the FL2VA partition; recipes/MiniMaxAI/MiniMax-H3-MUSA.md:17 "- `FL2VA` serves text-to-video+audio (`t2va`) and" and recipes/MiniMaxAI/MiniMax-H3-MUSA.md:19 "- `Ref2VA` serves image/audio or video-reference generation (`ref2va`)."; recipes/MiniMaxAI/MiniMax-H3-NPU.md:117 "export MODEL="${MODEL_ROOT}/Ref2VA"".) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — Caching a context-free backend module at init (`self.device_module = torch.get_device_module()`, vae.py:120) makes encode_image/encode_video RNG seeding depend on the init-time default accelerator, while `devices` is recomputed from the encode-time `parameter.device` (vae.py:170 and :214). If those contexts diverge, `self.device_module.device(device)` / `self.device_module.manual_seed(...)` seed the wrong backend. Prefer `torch.get_device_module(parameter.device)` inside the encode methods (or pass the constructor's `device` into `get_device_module` at init) so the module and the parameter device cannot drift apart. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 ` self.device_module = torch.get_device_module()` — captured once at init with no device argument; vae.py:175-176 ` with self.device_module.device(device):` / ` self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` — applied later to a `device` recomputed from the encode-time parameter device (vae.py:170 ` devices = [parameter.device] if parameter.device.type != "cpu" else []`).) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:1` [minor] — The PR body bullet 'adds coverage for the torchaudio, soundfile, and ffmpeg audio-loading paths' is not delivered: the diff changes only recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/README.md, and vllm_omni/diffusion/models/minimax_h3/vae.py (no test files), and grep of tests/ for `load_audio_file` returns no matches. Drop the bullet or land the audio-loader coverage. (evidence: PR #5703 body: "adds coverage for the torchaudio, soundfile, and ffmpeg audio-loading paths"; changed files: recipes/MiniMaxAI/MiniMax-H3-MUSA.md, recipes/README.md, vllm_omni/diffusion/models/minimax_h3/vae.py (none under tests/); vllm_omni/diffusion/models/minimax_h3/reference_video.py:477 `def load_audio_file(path: str) -> tuple[torch.Tensor, int]:` is not in the diff; grep `load_audio_file` in tests/ returns no matches.) + +`requirements/common.txt:1` [minor] — Confirm/document the minimum torch version required by torch.get_device_module(). requirements/common.txt (and cuda.txt/musa.txt) declare no torch floor, and vae.py:120 is the repo's first use of this API; the new MUSA recipe only says 'compatible PyTorch/torch-musa' and the NPU recipe documents PyTorch 2.10.0+cpu. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 ` self.device_module = torch.get_device_module()` — first/only repo-wide occurrence per grep; requirements/common.txt contains no `torch==`/`torch>=` pin (torch-related line is `torchsde>=0.2.6` at line 21); requirements/npu.txt:3 `torchaudio==2.10.0` pins torchaudio but not torch.) + +`gate_report:1` [minor] — gate_report line 1 — `buildkite/vllm-omni-amd-ci` is red. The diff modifies `vllm_omni/diffusion/models/minimax_h3/vae.py`, and the MiniMax H3 test modules are collected by the AMD `tests/diffusion -m 'core_model and cpu'` lane, so they are in scope; the gate report alone cannot attribute the failure to this PR. Please confirm whether the AMD CI failure is pre-existing/unrelated. (evidence: gh_ci_read(pr=5703): bucket=fail name=buildkite/vllm-omni-amd-ci state=FAILURE. .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`. tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`. vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:170` [minor] — Minor: encode_image (vae.py:170-176) and encode_video (vae.py:214-220) duplicate the same fork_rng/manual_seed block; consider a shared helper, since torch.get_device_module is only used at vae.py:120 and the repo's TID251 table prefers torch.accelerator.*. (evidence: vae.py:172 `with torch.random.fork_rng(devices=devices):` and vae.py:176 `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` are duplicated at vae.py:216 and vae.py:220; vae.py:120 `self.device_module = torch.get_device_module()` is the sole get_device_module caller.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.trace.json.gz new file mode 100644 index 000000000..508864c89 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.cost.json new file mode 100644 index 000000000..61efa6074 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 354.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5715/run-20260814-221254-1926f0", + "recorded_at": "2026-08-14T22:18:46", + "attempts": [ + { + "invocation_id": "inv-3f8eae6a", + "outcome": "run:run-20260814-221254-1926f0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-221254-1926f0", + "llm_calls": 12, + "input_tokens": 50937, + "output_tokens": 29456, + "usd": 0.056834, + "source": "spans" + } + ], + "attempt_usd_total": 0.056834, + "llm_calls": 12, + "input_tokens": 50937, + "output_tokens": 29456, + "usd": 0.056834, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-221254-1926f0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0568, + "minutes": 5.86, + "ci_minutes": 0, + "input_tokens": 50937, + "output_tokens": 29456, + "tool_calls": 37, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2291, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 12, + "by_role": { + "agent": { + "usd": 0.056834, + "input_tokens": 50937, + "output_tokens": 29456, + "calls": 12 + } + }, + "cache_read_tokens": 395520, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.44, + "gate": 1.63, + "review": 346.38, + "report": 0.09 + }, + "time_to_first_result_s": 351.51 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 12520 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.md new file mode 100644 index 000000000..27093cca7 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.md @@ -0,0 +1,41 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] README.md:19-22 news links verified against repo: recipes/MiniMaxAI/MiniMax-H3.md, recipes/OpenBMB/MiniCPM-o-4_5.md, recipes/cosmos3/Cosmos3-Nano.md all exist. +- [validated] Full-duplex runtime exists in-repo: vllm_omni/experimental/fullduplex/ and examples/online_serving/minicpmo/README.md:40 ('Experimental native full-duplex overlay'), so the feature claim is real, but the recipe linked in the news line does not document it (see review comment). +- [validated] docs/configuration/README.md:2 link updated to https://docs.vllm.ai/en/v0.26.0/configuration/index.html; grep found no remaining en/v0.16.0 link in docs. +- [validated] docs/getting_started/installation/gpu/cuda.inc.md:127 BASE_IMAGE=vllm/vllm-openai:v0.26.0 matches docker/Dockerfile.cuda:1 'ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0'; grep for 'vllm/vllm-openai:v0.22.1' across the repo returns no matches. +- [sweep] grep '0.18.0' in docs returns only the historical design doc and docs/getting_started/installation/gpu/musa.inc.md:31 'git checkout v0.18.0-dev' — the MUSA tab is unchanged by this diff but now contradicts the new parent-page compatibility note. +- [validated] NPU source-install cd handling fixed: added 'cd vllm'/'cd ..'/'cd vllm-ascend'/'cd ..' and changed 'cd /vllm-workspace/vllm-omni' to 'cd vllm-omni'; commands now chain correctly. +- [validated] docs/getting_started/installation/README.md:4 compatibility note aligns with vllm_omni/version.py warn_if_misaligned_vllm_version, which compares major.minor of vLLM vs vLLM-Omni. +- [sweep] No tests touched; all six files are documentation. Verified coverage request is satisfied by the author's claimed pre-commit + strict MkDocs build (gate_report shows no failing checks). + +`docs/getting_started/installation/README.md:?` [minor] — This new parent-page note is unqualified and sits above every platform tab, but the MUSA tab below still tells users to checkout vLLM-MUSA v0.18.0-dev — a 0.18-based fork that directly contradicts '0.26.x requires vLLM 0.26.x'. Either scope the note to upstream-vLLM platforms (or add an explicit MUSA exception/follow-up), otherwise a MUSA user following this page gets two incompatible version pins. (evidence: docs/getting_started/installation/README.md:4 'vLLM-Omni 0.26.x requires vLLM 0.26.x' vs docs/getting_started/installation/gpu/musa.inc.md:31 'git checkout v0.18.0-dev' (MUSA tab, not touched by this diff).) + +`README.md:?` [minor] — The new headline feature 'experimental full-duplex realtime runtime' links to recipes/OpenBMB/MiniCPM-o-4_5.md, but that page never mentions full-duplex or realtime, so the linked page does not support the feature claim. Point the link at examples/online_serving/minicpmo/README.md (which documents the minicpmo_4_5_duplex.yaml overlay) or add a full-duplex section to the recipe. (evidence: README.md:19 links 'MiniCPM-o 4.5' to recipes/OpenBMB/MiniCPM-o-4_5.md; grep 'duplex' and 'realtime' in that file returns no matches. examples/online_serving/minicpmo/README.md:40 '| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |') + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.trace.json.gz new file mode 100644 index 000000000..f5e569b8d Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.cost.json new file mode 100644 index 000000000..44d5fa3cc --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1651.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5840/run-20260814-221822-79f701", + "recorded_at": "2026-08-14T22:45:51", + "attempts": [ + { + "invocation_id": "inv-e174b83c", + "outcome": "run:run-20260814-221822-79f701", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-221822-79f701", + "llm_calls": 78, + "input_tokens": 664417, + "output_tokens": 140035, + "usd": 0.460061, + "source": "spans" + } + ], + "attempt_usd_total": 0.460061, + "llm_calls": 78, + "input_tokens": 664417, + "output_tokens": 140035, + "usd": 0.460061, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-221822-79f701", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4601, + "minutes": 27.49, + "ci_minutes": 0, + "input_tokens": 664417, + "output_tokens": 140035, + "tool_calls": 324, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8326, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 78, + "by_role": { + "moa_member": { + "usd": 0.061754, + "input_tokens": 98953, + "output_tokens": 5311, + "calls": 9 + }, + "agent": { + "usd": 0.344215, + "input_tokens": 517962, + "output_tokens": 97266, + "calls": 66 + }, + "reducer": { + "usd": 0.054092, + "input_tokens": 47502, + "output_tokens": 37458, + "calls": 3 + } + }, + "cache_read_tokens": 4171136, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.42, + "gate": 1.67, + "review": 1642.69, + "report": 0.06 + }, + "time_to_first_result_s": 1648.83 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 40946 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.md new file mode 100644 index 000000000..8b76efba8 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] vllm_omni/diffusion/cache/teacache/extractors.py:1252-1395 — extract_minimax_h3_context mirrors MiniMaxH3DiTModel.forward (lines 1077-1196 in minimax_h3_transformer.py); test_extractor_matches_native_forward asserts parity. +- [validated] vllm_omni/diffusion/cache/teacache/backend.py:76-101 — enable_minimax_h3_teacache rejects ref2va, allows fl2va/combined, hooks only pipeline.transformer; combined Ref2VA uses transformers_ref without hook per pipeline_minimax_h3.py:1498-1506. +- [validated] vllm_omni/diffusion/cache/teacache/config.py:96-133 — TeaCacheConfig resolves None rel_l1_thresh to 0.17 for MiniMaxH3DiTModel and 0.2 otherwise; hook.py:234-236 asserts resolved threshold. +- [sweep] tests/diffusion/cache/test_teacache_extractors.py:629-632 — img_pos_info/audio_pos_info shape[0] indexing is test-only fixture contract; sample uses 2 rows each. +- [sweep] tests/diffusion/cache/test_teacache_extractors.py:656,675 — blocks[0] used intentionally for TeaCache modulated-input probe; matches extractor blocks[0] adaln at extractors.py:1341-1342. +- [sweep] vllm_omni/diffusion/cache/teacache/extractors.py:1312-1318 — batch-size-1 and token_tags/inverse_indices length guards match forward at minimax_h3_transformer.py:1124-1130. +- [sweep] vllm_omni/diffusion/cache/teacache/extractors.py:1379-1385 — skip_mask_out_condition and update_audio_mask branches mirror forward at minimax_h3_transformer.py:1185-1195. +- [sweep] tests/diffusion/cache/test_cache_backends.py:506-507 — configured_threshold None vs explicit branches covered by parametrized minimax_h3 test. + +`vllm_omni/engine/async_omni_engine.py:?` [major] — This diff changes DiffusionCacheConfig.rel_l1_thresh from 0.2 to None and registers MiniMaxH3DiTModel's TeaCache default as 0.17, but the production stage-config path still materializes rel_l1_thresh=0.2 before that default can apply: vllm_omni/engine/async_omni_engine.py:853-856 returns {"rel_l1_thresh": 0.2} for tea_cache, and vllm_omni/engine/async_omni_engine.py:881-882 injects it whenever cache_config is omitted, so TeaCacheConfig never sees None. test_minimax_h3_only_enables_fl2va_teacache(None->0.17) constructs TeaCacheBackend(DiffusionCacheConfig()) directly and cannot catch this. Stop hardcoding 0.2 here (return {} / omit the key) so TeaCacheConfig can apply the model map, and add a factory test that _create_default_diffusion_stage_cfg({"cache_backend": "tea_cache"}) leaves rel_l1_thresh unset. (evidence: vllm_omni/engine/async_omni_engine.py:853-856 `if cache_backend == "tea_cache": return { "rel_l1_thresh": 0.2, }`; vllm_omni/engine/async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"): cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`; vllm_omni/diffusion/cache/teacache/config.py:97-100 `_MODEL_DEFAULT_REL_L1_THRESH = { "MiniMaxH3DiTModel": 0.17, }`; vllm_omni/diffusion/cache/teacache/config.py:126-128 `threshold = self.rel_l1_thresh; if threshold is None: threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)`; tests/diffusion/cache/test_cache_backends.py:504-507 `DiffusionCacheConfig() if configured_threshold is None else DiffusionCacheConfig(rel_l1_thresh=configured_threshold)`.) + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:?` [major] — MiniMaxH3QualityPolicy is backend-unaware of tea_cache: quality=="high" still builds a CacheDiTRequestSpec (quality_policy.py:149-151,165-168) and the pipeline unconditionally prepares it (pipeline_minimax_h3.py:1766). Since this diff installs the TeaCache hook on pipeline.transformer at startup and documents that TeaCache and Cache-DiT are mutually exclusive, a tea_cache server plus quality=high can activate Cache-DiT on top of the already-hooked FL2VA forward. Add a guard when _configured_backend == "tea_cache" (reject quality=high or return cache_dit=None) and add a ("tea_cache", "high", None) parametrization row to test_model_policy_owns_request_cache_target. (evidence: vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149 `if quality == "high":` then :150-151 `base_key = MINIMAX_H3_HIGH_CACHE_KEY` / `base_config = _high_quality_cache_config()` — this branch runs before any backend gate; only the omitted-quality branch checks the backend at :152 `elif self._configured_backend == "cache_dit" and quality is None:`; :165 `cache_dit=CacheDiTRequestSpec(` is returned unconditionally for quality=high. vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1766 `self._cache_dit_runtime.prepare(quality_plan.cache_dit)` applies the emitted target. The same PR diff adds vllm_omni/diffusion/cache/teacache/backend.py `apply_teacache_hook(pipeline.transformer, teacache_config)` and recipes/MiniMaxAI/MiniMax-H3.md 'TeaCache and Cache-DiT are mutually exclusive; pick one cache backend per server.' Tests cover only cache_dit/none backends (tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py:160-171), with ('none', 'high') still expecting minimax_h3.high.) + +`tests/diffusion/cache/test_cache_backends.py:?` [minor] — tests/diffusion/cache/test_cache_backends.py:490 adds the parametrized MiniMax-H3 TeaCache enable test, and line 32 marks the whole file `pytest.mark.core_model` + `pytest.mark.cpu`. The AMD CI command in .buildkite/amd/test-amd-ready.yml:19 runs `tests/diffusion` with `-m 'core_model and cpu'` and only ignores test_teacache_extractors.py, so these new tests are selected by AMD CI. buildkite/vllm-omni-amd-ci is currently FAILURE. Please confirm the red check is pre-existing on main and not caused by these new parametrized enable/reject tests before merge. (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` +tests/diffusion/cache/test_cache_backends.py:490 ` def test_minimax_h3_only_enables_fl2va_teacache(` +.buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` +gh_ci_read(5840): `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`) + +`docs/user_guide/diffusion/cache_acceleration/teacache.md:?` [minor] — This diff makes `rel_l1_thresh` optional and model-specific, but docs/user_guide/diffusion/cache_acceleration/teacache.md:122 still lists type `float` and default `0.2`. Update to `float | None` / `None` (deferred to model default; 0.17 for MiniMax-H3 FL2VA, 0.2 fallback). (evidence: vllm_omni/diffusion/data.py:422 `rel_l1_thresh: float | None = None`; vllm_omni/diffusion/cache/teacache/config.py:96 `_DEFAULT_REL_L1_THRESH = 0.2`; vllm_omni/diffusion/cache/teacache/config.py:97 `_MODEL_DEFAULT_REL_L1_THRESH = {`; vllm_omni/diffusion/cache/teacache/config.py:98 `"MiniMaxH3DiTModel": 0.17,`; vllm_omni/diffusion/cache/teacache/config.py:130 `threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)`; docs/user_guide/diffusion/cache_acceleration/teacache.md:122 `| `rel_l1_thresh` | float | `0.2` | Similarity threshold for cache reuse. Lower values prioritize quality (less caching), higher values prioritize speed (more caching). Suggested range: 0.1-0.8 |`) + +`vllm_omni/diffusion/cache/teacache/backend.py:91` [minor] — backend.py:91 applies the TeaCache hook only to pipeline.transformer (FL2VA) in the combined partition, while pipeline_minimax_h3.py:737-739 routes per-request ref2va to pipeline.transformers_ref (uncached). The new tests only cover enable-time branch reachability with mocks; there is no integration/scoping test for combined-serving Ref2VA under tea_cache. Add one or document the uncached Ref2VA path as accepted without coverage. (evidence: vllm_omni/diffusion/cache/teacache/backend.py:91 `apply_teacache_hook(pipeline.transformer, teacache_config)` — only the FL2VA transformer is hooked; vllm_omni/diffusion/cache/teacache/backend.py:95-96 `"TeaCache is enabled only for MiniMax-H3 FL2VA; " "Ref2VA requests on this combined pipeline will run without TeaCache"`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:736-739 `def _transformer_for_task(self, task: str) -> MiniMaxH3DiTModel: / if task == "ref2va" and hasattr(self, "transformers_ref"): / return self.transformers_ref / return self.transformer`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1498 `transformer = self._transformer_for_task(task)`.) + +`vllm_omni/diffusion/cache/teacache/__init__.py:20` [minor] — vllm_omni/diffusion/cache/teacache/__init__.py:20 and vllm_omni/entrypoints/cli/serve.py:586 still show `rel_l1_thresh: 0.2` examples after DiffusionCacheConfig.rel_l1_thresh became None (deferred to TeaCacheConfig: 0.17 for MiniMax-H3DiTModel, 0.2 fallback). These are explicit config values rather than hard defaults; consider a short note if the goal is to avoid implying 0.2 is the default. Minor. (evidence: vllm_omni/diffusion/cache/teacache/__init__.py:20 ` cache_config={"rel_l1_thresh": 0.2}` — vllm_omni/entrypoints/cli/serve.py:586 ` "TeaCache: '{\"rel_l1_thresh\": 0.2}'. "` — vllm_omni/diffusion/data.py:422 ` rel_l1_thresh: float | None = None`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143` [minor] — The TeaCache e2e smoke at tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143 runs num_inference_steps=2 and only calls _assert_joint_output, which checks output lengths and shapes; it has no cache-hit/call-count or quality assertion that would fail on a TeaCache regression. Could you add a cache-hit/call-count assertion or document a perf check that would actually fail on a TeaCache regression? (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143 `def test_minimax_h3_t2va_teacache_single_gpu_smoke():`; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:170 `num_inference_steps=2,`; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:186 `_assert_joint_output(outputs)`; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:22 `assert frames.shape == (107, 256, 448, 3)`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:143` [minor] — The new e2e TeaCache smoke is unreachable in CI: the module pytestmark is local_model/cuda/diffusion/parallel with no core_model marker (tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:9-14), and the only Buildkite lane sweeping tests/diffusion/models/ filters '-m core_model and cuda' (.buildkite/cuda/test-ready.yml:65), so the test is deselected; it is also skipped unless VLLM_TEST_MINIMAX_H3_FL2VA_MODEL is set. Wire it into a nightly/H100 lane (similar to the existing test_minimax_h3_quantization_quality.py entry in test-nightly.yml) or require the command in the PR checklist. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:9-14 `pytestmark = [\n pytest.mark.local_model,\n pytest.mark.cuda,\n pytest.mark.diffusion,\n pytest.mark.parallel,\n]`; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:17 `MODEL_ENV = "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL"`; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:139-141 `@pytest.mark.skipif(\n not os.environ.get(MODEL_ENV),\n reason=f"set {MODEL_ENV} to an authorized FL2VA checkpoint path",\n)`; .buildkite/cuda/test-ready.yml:65 ` - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"`) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/cache/teacache/config.py:87` [minor] The new MiniMax-H3 coefficient table (config.py:87-93) is the only calibration evidence for these five values, but no test asserts that TeaCacheConfig(transformer_type='MiniMaxH3DiTModel').coefficients resolves to them: + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.trace.json.gz new file mode 100644 index 000000000..07e562acb Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.cost.json new file mode 100644 index 000000000..56e697ae3 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 364.4, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5863/run-20260814-221847-f95376", + "recorded_at": "2026-08-14T22:24:50", + "attempts": [ + { + "invocation_id": "inv-8daacbc7", + "outcome": "run:run-20260814-221847-f95376", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-221847-f95376", + "llm_calls": 13, + "input_tokens": 131735, + "output_tokens": 31063, + "usd": 0.083997, + "source": "spans" + } + ], + "attempt_usd_total": 0.083997, + "llm_calls": 13, + "input_tokens": 131735, + "output_tokens": 31063, + "usd": 0.083997, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-221847-f95376", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.084, + "minutes": 6.04, + "ci_minutes": 0, + "input_tokens": 131735, + "output_tokens": 31063, + "tool_calls": 27, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2475, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 13, + "by_role": { + "agent": { + "usd": 0.083997, + "input_tokens": 131735, + "output_tokens": 31063, + "calls": 13 + } + }, + "cache_read_tokens": 528128, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.67, + "gate": 1.61, + "review": 357.06, + "report": 0.06 + }, + "time_to_first_result_s": 362.39 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 14766 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.md new file mode 100644 index 000000000..3c21b9c66 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.md @@ -0,0 +1,43 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [sweep] Diff is a single new markdown file; changed_symbol_consumers empty and no code symbols/defaults changed, so repo-wide caller sweep is N/A. +- [sweep] Verified all serve flags used by the recipe exist in vllm_omni/entrypoints/cli/serve.py: --num-gpus:395, --usp:461, --ring:479, --cfg-parallel-size:704, --text-encoder-tp-size:719, --vae-parallel-mode:728, --task-type:213, --enable-diffusion-pipeline-profiler:784. +- [validated] X-Stage-Durations and X-Peak-Memory-MB headers are really emitted by /v1/videos/sync (vllm_omni/entrypoints/openai/api_server.py:3510-3511), matching the recipe's profiling instructions. +- [validated] audio_reference is a multipart Form field parsed via _parse_form_json (api_server.py:3140, 3186) and the Ref2VA form shape in the recipe is structurally valid. +- [validated] TRTLLM_ATTN default requires compute_capability.major == 10 (vllm_omni/platforms/cuda/platform.py); sm120 is major 12, so the recipe's 'datacenter-Blackwell TRTLLM_ATTN default does not apply' claim is correct. +- [validated] Re-derived the per-GPU memory fit (55.4 + 22.1/U): 77.5 vs 77.49 (U1), 66.45 vs 66.44 (U2), 60.925 vs 61.07 (U4) and headroom 18.51/29.56/34.93 GiB all check out. +- [validated] aspect_ratio claims match recipes/MiniMaxAI/MiniMax-H3.md (T2VA requires a named ratio; Ref2VA defaults to 16:9), and FP8-not-compatible-with-layerwise-offload is consistent with the sibling recipe. +- [sweep] Repo-wide grep for cpunodebind/membind returns ONLY this recipe (lines 201 and 203); the serve CLI defines no such flags -> review comment (numactl fix). + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [major] — This hunk adds `--cpunodebind=0 --membind=0` (and later `--cpunodebind=1 --membind=1`) to what is presented as a `vllm serve` invocation, but `vllm serve` has no such flags — copying this command fails with 'unrecognized arguments'. These are numactl options: rewrite the two commands as `numactl --cpunodebind=0 --membind=0 vllm serve ...` (and `numactl --cpunodebind=1 --membind=1 ...`) and note that numactl must be installed. (evidence: Repo-wide grep for `cpunodebind`/`membind` matches only this file (lines 201 and 203). vllm_omni/entrypoints/cli/serve.py defines the diffusion flags this recipe uses — `--num-gpus` (line 395), `--usp` (461), `--ring` (479), `--text-encoder-tp-size` (719), `--vae-parallel-mode` (728) — but never `--cpunodebind` or `--membind`, so the parser rejects them.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:199` [minor] — This says the two-server layout doubles 'the requirements' to 400 GiB RAM, but the capacity table recommends 384 GiB for a single server, so two servers need ~768 GiB of recommended headroom — 400 GiB is only 2x the 200 GiB minimum. Clarify as '400 GiB minimum / 768 GiB recommended', otherwise a host with 400 GiB appears sufficient when it is at the bare-minimum the recipe elsewhere cautions against. (evidence: Same file, capacity table: '| Recommended system RAM | 384 GiB | 384 GiB | 384 GiB |' and '| Available system RAM | 200 GiB minimum | 200 GiB minimum | 200 GiB minimum |'.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:249` [minor] — The validated table reports an eight-GPU client E2E of 90.48 s for `duration=5.0` / 50 steps, which contradicts the PR description's headline that the 8x6000 setup 'only takes 4 minutes'. 90.48 s is about 1.5 minutes, not 4 minutes; if the 4-minute figure refers to a different workload or includes model download/setup, clarify it, otherwise correct the PR description to match the validated table. (evidence: Same file, performance table line 246: '| Client E2E | 284.76 s | 172.32 s | 90.48 s |'. The PR description states the 8x6000 setup takes about 4 minutes, conflicting with the 90.48 s eight-GPU Client E2E for `duration=5.0` / 50 steps.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.trace.json.gz new file mode 100644 index 000000000..453b0dec1 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5863.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.cost.json new file mode 100644 index 000000000..86d707ab4 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.cost.json @@ -0,0 +1,121 @@ +{ + "wall_s": 781.3, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5884/run-20260814-233136-79ca18", + "recorded_at": "2026-08-14T23:44:35", + "attempts": [ + { + "invocation_id": "inv-ac54b698", + "outcome": "run:run-20260814-233136-79ca18", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-222452-9df050", + "llm_calls": 107, + "input_tokens": 798430, + "output_tokens": 151498, + "usd": 0.574693, + "source": "spans" + }, + { + "run_dir": "run-20260814-233136-79ca18", + "llm_calls": 39, + "input_tokens": 251872, + "output_tokens": 70355, + "usd": 0.169366, + "source": "spans" + } + ], + "attempt_usd_total": 0.744059, + "llm_calls": 39, + "input_tokens": 251872, + "output_tokens": 70355, + "usd": 0.169366, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-233136-79ca18", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1694, + "minutes": 12.98, + "ci_minutes": 0, + "input_tokens": 251872, + "output_tokens": 70355, + "tool_calls": 221, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4539, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 39, + "by_role": { + "moa_member": { + "usd": 0.033949, + "input_tokens": 45514, + "output_tokens": 5277, + "calls": 10 + }, + "reducer": { + "usd": 0.01551, + "input_tokens": 9383, + "output_tokens": 11762, + "calls": 2 + }, + "agent": { + "usd": 0.119908, + "input_tokens": 196975, + "output_tokens": 53316, + "calls": 27 + } + }, + "cache_read_tokens": 604288, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.7, + "gate": 2.3, + "review": 772.79, + "report": 0.06 + }, + "time_to_first_result_s": 778.84 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5884.trace.json.gz", + "trace_bytes": 54000 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.md new file mode 100644 index 000000000..2c8db481a --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5884, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5dc24e7f6a0c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] backend.py:45 `if not isinstance(name, str)` — read hunk; non-string `_dit_modules` entries are skipped, same as pre-change behavior. +- [sweep] backend.py:51 `if module is not None` — read hunk; `attrgetter` paths whose leaf is `None` are skipped; `AttributeError` on missing intermediate segments is caught and skipped. +- [sweep] backend.py:61 `if not BlockAdapter.is_cached(transformer)` — verified cache-dit==1.3.0 `BlockAdapter.is_cached` uses `getattr(adapter, '_is_cached', False)` for `nn.Module`; Cosmos3 UND nested entry is correctly skipped in summary. +- [validated] backend.py:43-53 — `_dit_module_names` now mirrors `module_collector.py:108-110` attrgetter resolution for dotted paths. +- [validated] pipeline_bagel.py:159 — Bagel declares `_dit_modules=['language_model.model']`; generic `CacheDiTBackend.enable` now resolves and enables the nested DiT (no custom enabler). +- [validated] model_specific.py:781-804 — Cosmos3 uses `CUSTOM_DIT_ENABLERS` custom path, so generic multi-module enable loop is bypassed; only `cache_summary` needed the `is_cached` filter for dual-listed modules. +- [validated] test_cache_backends.py:32 — module-level `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` covers new tests for CI selection. +- [validated] test_cache_backends.py:139-161 — `test_enable_and_refresh_nested_declared_dit` drives real `CacheDiTBackend.enable/refresh/cache_summary` with dotted path `language_model.model`. + +`vllm_omni/diffusion/cache/cachedit/backend.py:57` [minor] — Update backend.py:57 docstring to say Cache-DiT statistics are logged only for cached DiT modules; uncached modules are skipped at line 61 via `BlockAdapter.is_cached`. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:57 `"""Log Cache-DiT statistics for every transformer on the pipeline."""`; backend.py:60-62 `for transformer in transformers:` / ` if not BlockAdapter.is_cached(transformer):` / ` continue` / ` cache_dit.summary(transformer, details=details)` — the docstring says every transformer, but the code skips uncached transformers.) + +`diffusion_model_runner.py:153` [minor] — Confirmed: `_compile_transformer` at vllm_omni/diffusion/worker/diffusion_model_runner.py:153 still resolves via `getattr(self.pipeline, attr_name, None)`, so dotted `_dit_modules` entries like Bagel `language_model.model` and SoulX `cfm_decoder.model.diff_estimator` resolve to None and are silently never compiled. This PR fixes dotted-path resolution for cache discovery/enable (cachedit/backend.py now uses `attrgetter`) but leaves the compile path unchanged. If intentionally out of scope, please track a linked follow-up issue so it isn't rediscovered later as a compile miss. (evidence: vllm_omni/diffusion/worker/diffusion_model_runner.py:153 `model = getattr(self.pipeline, attr_name, None)` — dotted `attr_name` returns None, so the module is skipped; vllm_omni/diffusion/worker/diffusion_model_runner.py:287 `self._compile_transformer(attr_name)` — passes `_dit_modules` entries directly; vllm_omni/diffusion/models/bagel/pipeline_bagel.py:159 `_dit_modules: ClassVar[list[str]] = ["language_model.model"]`; vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_base.py:63 `_dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]`.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:61` [minor] — The new `if not BlockAdapter.is_cached(transformer): continue` gate (backend.py:61) makes summaries conditional on `is_cached(transformer)`. BagelCachedAdapter only sets `block_adapter.pipe.__class__._is_cached = True` (model_specific.py:645), and `_maybe_build_block_adapter` constructs `BlockAdapter(transformer=..., ...)` without `pipe=` (backend.py:197-203), so it is not obvious that `BlockAdapter.is_cached(transformer)` is true for Bagel. Please confirm Bagel cached transformers still produce summaries or add a test exercising a Bagel-shaped cached adapter through `cache_summary`. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:61 ` if not BlockAdapter.is_cached(transformer):` +vllm_omni/diffusion/cache/cachedit/model_specific.py:645 ` block_adapter.pipe.__class__._is_cached = True` +vllm_omni/diffusion/cache/cachedit/backend.py:197-203 ` return BlockAdapter( + transformer=transformer, + blocks=[getattr(transformer, block_attribute) for block_attribute in block_attributes], + forward_pattern=list(forward_patterns), + has_separate_cfg=adapter_config.has_separate_cfg, + check_forward_pattern=adapter_config.check_forward_pattern, + )` +tests/diffusion/cache/test_cache_backends.py:126 ` transformer._is_cached = True`) + +`.buildkite/amd/test-amd-ready.yml:19` [minor] — `buildkite/vllm-omni-amd-ci` is FAILURE per `gh_ci_read(5884)`. This PR adds tests to `tests/diffusion/cache/test_cache_backends.py`, which carries `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (line 32), and `.buildkite/amd/test-amd-ready.yml:19` runs `pytest -sv tests/diffusion ... -m 'core_model and cpu'` in that lane — so the changed tests do run there. Please confirm the red AMD CI is pre-existing/unrelated to this diff, or trigger a re-run after this PR. (evidence: gh_ci_read(5884) → {"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"} +.buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` +tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:?` [minor] — `_dit_module_names` now resolves dotted paths via `attrgetter`, so `PipelineSoulXSingerSVS` (`_dit_modules=["cfm_decoder.model.diff_estimator"]`, whose `DiffLlama` declares `_cache_dit_adapter_config`) will be wrapped by cache_dit instead of raising `ValueError: ... has no declared DiT modules for Cache-DiT`. Add a SoulX Singer nested enable/refresh test (same shape as `test_enable_and_refresh_nested_declared_dit`) or an explicit allowlist guard until the nested wrap is validated. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:48 ` module = attrgetter(name)(pipeline)` (inside `_dit_module_names`, guarded by `try/except AttributeError`) — dotted `_dit_modules` paths now resolve; vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_base.py:63 ` _dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]`; vllm_omni/diffusion/models/soulx_singer/modules/llama.py:115 ` _cache_dit_adapter_config = CacheDiTAdapterConfig(` (DiffLlama declares the adapter config used by the generic `_maybe_build_block_adapter` path).) + +`tests/diffusion/cache/test_cache_backends.py:?` [minor] — The new nested test at tests/diffusion/cache/test_cache_backends.py:142 builds `CacheDiTAdapterConfig(block_forward_patterns={"blocks": ForwardPattern.Pattern_3})` with no `cached_adapter_cls`, so `_maybe_get_cached_adapter_cls` returns None and `CacheDiTBackend.enable` exercises the `cache_dit.enable_cache` fallback (asserted at line 155), not the Bagel production path. Bagel's nested transformer is `language_model.model` (`pipeline_bagel.py:159`), and `Qwen2MoTModel._cache_dit_adapter_config` sets `cached_adapter_cls=BagelCachedAdapter` (`bagel_transformer.py:894`); with that field set, `enable_cache_for_dit` calls `adapter_cls.apply` (`backend.py:143-144`). Consider adding a nested test that sets `cached_adapter_cls=BagelCachedAdapter` and asserts `BagelCachedAdapter.apply` is invoked instead of `cache_dit.enable_cache`. (evidence: tests/diffusion/cache/test_cache_backends.py:142-144 `transformer._cache_dit_adapter_config = CacheDiTAdapterConfig( + block_forward_patterns={"blocks": ForwardPattern.Pattern_3} + )` — no `cached_adapter_cls`; tests/diffusion/cache/test_cache_backends.py:155 `mock_cache_dit.enable_cache.assert_called_once()` — proves the fallback branch is exercised; vllm_omni/diffusion/cache/cachedit/config.py:30 `cached_adapter_cls: type[CachedAdapter] | None = None`; vllm_omni/diffusion/models/bagel/bagel_transformer.py:894 `cached_adapter_cls=BagelCachedAdapter,`; vllm_omni/diffusion/cache/cachedit/backend.py:143-144 `if adapter_cls is not None: + adapter_cls.apply(`) + +`tests/diffusion/cache/test_cache_backends.py:?` [minor] — The new `test_summary_skips_uncached_nested_dit` (tests/diffusion/cache/test_cache_backends.py:121-135) declares both `transformer.language_model` and `transformer` but only asserts a single `cache_dit.summary` call on `transformer`. Because the pre-fix `getattr(pipeline, name, None)` also dropped the dotted `transformer.language_model` and old `cache_summary` produced the same single summary, this test passes even if the new `attrgetter`-based dotted resolution regressed; it only proves the `is_cached` filter on whatever names were resolved. Patch `BlockAdapter.is_cached` with a side effect recording both resolved modules (nested uncached + parent cached), or otherwise assert the dotted path is resolved and then filtered, so the test fails under the old behavior. (evidence: ['tests/diffusion/cache/test_cache_backends.py:129 ` _dit_modules=["transformer.language_model", "transformer"],`', 'tests/diffusion/cache/test_cache_backends.py:135 ` mock_cache_dit.summary.assert_called_once_with(transformer, details=True)`', 'vllm_omni/diffusion/cache/cachedit/backend.py:48 ` module = attrgetter(name)(pipeline)`', 'vllm_omni/diffusion/cache/cachedit/backend.py:59 ` transformers = [attrgetter(name)(pipeline) for name in _dit_module_names(pipeline)]`', 'vllm_omni/diffusion/cache/cachedit/backend.py:61 ` if not BlockAdapter.is_cached(transformer):`', 'diff removed line in vllm_omni/diffusion/cache/cachedit/backend.py: `- return tuple(name for name in names if isinstance(name, str) and getattr(pipeline, name, None) is not None)`']) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.trace.json.gz new file mode 100644 index 000000000..9c60d8f6a Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5884.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.cost.json new file mode 100644 index 000000000..80a9aac4e --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1551.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5957/run-20260814-223159-f6b467", + "recorded_at": "2026-08-14T22:57:48", + "attempts": [ + { + "invocation_id": "inv-a3998d77", + "outcome": "run:run-20260814-223159-f6b467", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-223159-f6b467", + "llm_calls": 95, + "input_tokens": 917162, + "output_tokens": 136893, + "usd": 0.726913, + "source": "spans" + } + ], + "attempt_usd_total": 0.726913, + "llm_calls": 95, + "input_tokens": 917162, + "output_tokens": 136893, + "usd": 0.726913, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-223159-f6b467", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7269, + "minutes": 25.82, + "ci_minutes": 0, + "input_tokens": 917162, + "output_tokens": 136893, + "tool_calls": 368, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9229, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 95, + "by_role": { + "moa_member": { + "usd": 0.051847, + "input_tokens": 65205, + "output_tokens": 1284, + "calls": 7 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.621528, + "input_tokens": 748475, + "output_tokens": 112369, + "calls": 85 + }, + "reducer": { + "usd": 0.053539, + "input_tokens": 103482, + "output_tokens": 23240, + "calls": 2 + } + }, + "cache_read_tokens": 11736000, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.48, + "gate": 1.97, + "review": 1542.69, + "report": 0.07 + }, + "time_to_first_result_s": 1549.19 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5957.trace.json.gz", + "trace_bytes": 132873 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.md new file mode 100644 index 000000000..55eca3f82 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.md @@ -0,0 +1,64 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5957, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 19a6e513f28e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness: read `indextts2_talker.py` make_omni_output + `_preprocess_prefill` — v2.5 code-only defers mel codes to `request.output_token_ids`; latent v2 path unchanged; duration_factor validated 0.5–2.0. +- [validated] `vllm_omni/deploy/indextts2_5.yaml:19` sets `enable_prefix_caching: false` on both stages — default deploy avoids prefix-cache/speed salt collision. +- [sweep] Breaking behavior: `talker2s2mel_full_payload` now raises on missing fields (was warn+None); IndexTTS2 talker still emits `use_gpt_latent` every step (`indextts2_talker.py:440`). +- [validated] Native speed: `IndexTTS25Adapter.native_speed_control=True`, `_audio_encode_speed` returns 1.0 for encode path (`serving_speech.py:517-524`), streaming allows speed≠1.0 when native (`serving_speech.py:3642`). +- [validated] XPU blast radius: `qwen2_5_omni.yaml` XPU platform sets `enforce_eager: true` for stages 0–1; covered by `test_qwen2_5_omni_xpu_uses_eager_ar_stages`. +- [validated] Optional deps: `pyproject.toml` `indextts2` extra + `test_indextts2_optional_dependencies.py` gate platform-specific WeText/fugashi. +- [sweep] Indexed access `stage_configs[0]` in `end2end.py:96-140` — guarded by `if not stage_configs` and 2.5-only path; `parents[4]` correctly resolves repo root from example depth. +- [sweep] Rebase/merge: registry adds `indextts2_5` pipeline and `IndexTTS25*` arch aliases without dropping `indextts2` entries. + +`vllm_omni/model_executor/stage_input_processors/indextts2.py:?` [major] — This diff makes `talker2s2mel_full_payload` fail fast (e.g. `vllm_omni/model_executor/stage_input_processors/indextts2.py:223` missing `meta.use_gpt_latent`), but both connector senders swallow the exception before it reaches the request: `OmniConnectorModelRunnerMixin` logs and returns None (`omni_connector_model_runner_mixin.py:1927`), `send_full_payload_outputs` then `continue`s on the None payload (`~1015`), and `OmniChunkTransferAdapter._send_single_request` catches all exceptions from `custom_process_next_stage_input_func`, logs, and sends an empty `OmniPayloadStruct(finished=True)`. The new explicit diagnostic is therefore downgraded to a generic downstream Stage-1 failure/timeout. Propagate the processor `ValueError` to the request error path, or document that connector-mode failures intentionally degrade. (evidence: vllm_omni/model_executor/stage_input_processors/indextts2.py:223 ` raise ValueError(f"IndexTTS payload is missing meta.use_gpt_latent for req={rid}")`; vllm_omni/worker/omni_connector_model_runner_mixin.py:1927 ` logger.exception("custom_process_stage_input_func failed for chunk %s", request_id)`; vllm_omni/worker/omni_connector_model_runner_mixin.py:1015 ` logger.debug("[Stage-%s] send_full_payload_outputs: payload is None for %s", self._stage_id, req_id)`) + +`tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:4` [major] — The added files tests/model_executor/models/indextts2/test_configuration_and_pipeline.py, test_enhanced_codec.py, and test_recipe_defaults.py all lack a module-level pytestmark. Buildkite CUDA/AMD model_executor lanes run `pytest ... -m 'core_model and cpu'` (.buildkite/cuda/test-ready.yml:13, .buildkite/amd/test-amd-ready.yml:19) with `--strict-markers` in pyproject.toml, so these three modules are deselected and never run. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` to all three files. (evidence: .buildkite/cuda/test-ready.yml:13 ` - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/amd/test-amd-ready.yml:19 ` - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; pyproject.toml:242 ` "--strict-markers",`; tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:1-4 `# SPDX-License-Identifier: Apache-2.0` / `# SPDX-FileCopyrightText: Copyright contributors to the vLLM project` / `` / `from pathlib import Path` (entire 89-line file contains no `pytestmark`); test_enhanced_codec.py (entire 84-line file contains no `pytestmark`); test_recipe_defaults.py (entire 14-line file contains no `pytestmark`). Contrast sibling test_s2mel_v2_5.py:26 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`.) + +`docs/serving/speech_api.md:?` [major] — docs/serving/speech_api.md:149-150 still tells every HTTP client that both streaming modes require `speed` be `1.0` (or omitted), but this PR removes the schema-level restriction in audio.py and adds a model-aware gate at serving_speech.py:3642 that allows HTTP streaming speed != 1.0 when the adapter opts into native speed control (`IndexTTS25Adapter.native_speed_control = True`). Update lines 149-150 to document the model-specific exception: native-speed models (IndexTTS-2.5) accept HTTP streaming `speed` in [0.5, 2.0]; WebSocket `stream_audio` still requires `speed=1.0`. (evidence: docs/serving/speech_api.md:149-150 `Both streaming modes (`stream_format="audio"` and `"sse"`) require` / `` `response_format="pcm"` or `"wav"`, and `speed` must be `1.0` (or omitted). ``; vllm_omni/entrypoints/openai/serving_speech.py:3642 `if request.speed is not None and request.speed != 1.0 and not self._uses_native_speed_control():`; vllm_omni/entrypoints/openai/tts_adapters/indextts2.py:276 `native_speed_control: ClassVar[bool] = True`; vllm_omni/entrypoints/openai/protocol/audio.py `validate_streaming_constraints` no longer rejects `self.speed != 1.0` and only runs `if self.speed is None: self.speed = 1.0`.) + +`vllm_omni/model_executor/stage_input_processors/indextts2.py:?` [minor] — This PR makes the shared `talker2s2mel_full_payload` fail fast (indextts2.py:223) and routes both `INDEXTTS2_PIPELINE` and `INDEXTTS25_PIPELINE` through it (pipeline.py:30,66). The shared `IndexTTS2S2MelDecoder.forward` now runs `_run_cfm_groups` for both versions (indextts2_s2mel_decoder.py:587), while `vllm_omni/deploy/indextts2.yaml` still sets `s2mel_dit_cuda_graph: true` (line 50) and relies on the default `s2mel_cfm_batch_size=1` (indextts2_s2mel_decoder.py:229). Please record an IndexTTS-2 re-check (tests/model_executor/stage_input_processors/test_indextts2.py plus a 2.0 serve smoke), or note why the 2.5-only validation covers it. (evidence: vllm_omni/model_executor/stage_input_processors/indextts2.py:223 `raise ValueError(f"IndexTTS payload is missing meta.use_gpt_latent for req={rid}")`; vllm_omni/model_executor/models/indextts2/indextts2_s2mel_decoder.py:587 `mel = self._run_cfm_groups(`; vllm_omni/model_executor/models/indextts2/indextts2_s2mel_decoder.py:229 `self.s2mel_cfm_batch_size: int = int(getattr(self.config, "s2mel_cfm_batch_size", 1))`; vllm_omni/deploy/indextts2.yaml:50 `s2mel_dit_cuda_graph: true`; vllm_omni/model_executor/models/indextts2/pipeline.py:30 `custom_process_next_stage_input_func=f"{_PROC}.talker2s2mel_full_payload",`) + +`vllm_omni/deploy/qwen2_5_omni.yaml:89` [minor] — Diff flips `vllm_omni/deploy/qwen2_5_omni.yaml` XPU AR stages 0 (line 91) and 1 (line 94) from `enforce_eager: false` (previously documented as cudagraph-verified on 2x Intel Arc Pro B60) to `enforce_eager: true` for a PyTorch 2.13 XPU Dynamo duplicate-handler workaround. This user-visible perf/compile change is bundled into an IndexTTS-2.5 PR, and the only added test (`tests/config/test_config_factory.py:2279`) asserts config values, not runtime or perf. Please split this into a separate PR/commit or attach the XPU before/after evidence and toolchain-bug link. (evidence: vllm_omni/deploy/qwen2_5_omni.yaml:86-91 ` # PyTorch 2.13 XPU Dynamo raises duplicate-handler assertions while + # compiling both Qwen2.5 AR stages. Keep them eager until that toolchain + # issue is fixed; stage 2 already inherits enforce_eager: true above. + - stage_id: 0 + gpu_memory_utilization: 0.9 + enforce_eager: true`; tests/config/test_config_factory.py:2279 ` def test_qwen2_5_omni_xpu_uses_eager_ar_stages(self):` with body asserting only `deploy.stages[0].enforce_eager is True`, `deploy.stages[1].enforce_eager is True`, and `deploy.stages[2].enforce_eager is True`.) + +`vllm_omni/deploy/indextts2_5.yaml:19` [minor] — `indextts2_conditioning_cache_salt` (vllm_omni/entrypoints/openai/tts_adapters/indextts2.py:84) hashes text/voice/emotion/lang/text_normalization but not `duration_factor`/speed, while `IndexTTS25Adapter._build_params` adds `params["duration_factor"] = [1.0 / speed]`. The default deploy keeps `enable_prefix_caching: false` (indextts2_5.yaml:19 and stage 1), so the collision cannot occur by default; document this invariant or include speed/duration_factor in the salt before enabling prefix caching. (evidence: vllm_omni/deploy/indextts2_5.yaml:19 ` enable_prefix_caching: false`; vllm_omni/entrypoints/openai/tts_adapters/indextts2.py:84 `def indextts2_conditioning_cache_salt(` with key loop `for key in ("text", "voice", "voice_name", "voice_created_at", "emo_audio", "emo_vector", "emo_alpha", "emo_text", "use_emo_text", "use_random", "lang", "text_normalization"):` (no `duration_factor`/`speed`), and `IndexTTS25Adapter._build_params` ends with `params["duration_factor"] = [1.0 / speed]`; prompt assignment `prompt["cache_salt"] = indextts2_conditioning_cache_salt(request, tts_params)` (line 241).) + +`pyproject.toml:87` [minor] — IndexTTS 2.5 tokenizer_v2_5.py:251 does a lazy `import tiktoken`, but the new `indextts2` extra (pyproject.toml:87-92) and `requirements/` do not declare `tiktoken`; it currently relies on vLLM providing it transitively. Add `tiktoken` to the `indextts2` extra or document the vLLM-version guarantee so non-vLLM-bundled environments do not fail at runtime. (evidence: pyproject.toml:87-92 `indextts2 = ["fugashi>=1.2.0", "unidic-lite>=1.0.0", "wetext>=0.0.9; sys_platform != 'linux'", "WeTextProcessing; sys_platform == 'linux'", ]` — no tiktoken entry; vllm_omni/model_executor/models/indextts2/tokenizer_v2_5.py:251 ` import tiktoken`) + +`docker/Dockerfile.xpu:91` [minor] — PR #5957 body says common-path changes are 'intentionally limited to explicit model/config/pipeline registration, tokenizer-free stage initialization, a model-scoped missing-HF-config path, native-speed adapter capability, the full-payload scheduler entry, and an IndexTTS-2.5-opted request-end payload snapshot.' The diff also carries XPU/version plumbing and compat shims outside that list: docker/Dockerfile.xpu:91 `ARG VLLM_VERSION=v0.27.0` and triton-xpu 3.7.2, .buildkite/intel/pipeline-intel.yml:13 `VLLM_VERSION: "v0.27.0"`, vllm_omni/deploy/qwen2_5_omni.yaml XPU enforce_eager flips, vllm_omni/model_executor/models/common/qwen3_code_predictor.py XPU eager, vllm_omni/entrypoints/openai/serving_chat.py MistralToolCall import fallback, vllm_omni/entrypoints/openai/api_server.py renderer warmup fallback, and vllm_omni/request.py num_stale_output_tokens compat. Split these out or explicitly record them as reviewed scope expansions so risk can be attributed. (evidence: docker/Dockerfile.xpu:91 `ARG VLLM_VERSION=v0.27.0`; .buildkite/intel/pipeline-intel.yml:13 `VLLM_VERSION: "v0.27.0"`; PR #5957 body: 'Common-path changes are intentionally limited to explicit model/config/pipeline registration, tokenizer-free stage initialization, a model-scoped missing-HF-config path, native-speed adapter capability, the full-payload scheduler entry, and an IndexTTS-2.5-opted request-end payload snapshot.' — the XPU/version changes quoted above are not among the enumerated common-path changes.) + +**Additional observations (beyond the comment budget):** +- `pyproject.toml:91` [minor] Diff adds the `indextts2` extra with the Linux branch `WeTextProcessing` unpinned (pyproject.toml:91) while the non-Linux branch is `wetext>=0.0.9` (pyproject.toml:90); the Linux code at vllm_omni/model_executor/models/i +- `vllm_omni/entrypoints/openai/serving_speech.py:3642` [minor] `serving_speech.py:3642-3646` now owns stream+speed rejection for every TTS adapter with `native_speed_control` false (`_validate_speech_streaming_request`), and `protocol/audio.py` no longer raises the old streaming-spe +- `vllm_omni/engine/stage_init_utils.py:1128` [minor] Diff adds `_TokenOnlyRenderer` in `vllm_omni/engine/stage_init_utils.py` and `build_stage0_input_processor` installs it for stage 0 whenever `skip_tokenizer_init: true` is set. Stage-0 consumers are `indextts2_5.yaml:21` +- `tests/e2e/online_serving/test_indextts2_expansion.py:22` [minor] Diff adds IndexTTS-2.5 online serving plus a manual H200 HTTP/Whisper validation, but the repo's only IndexTTS speech E2E (`tests/e2e/online_serving/test_indextts2_expansion.py:22-32`) hardcodes `IndexTeam/IndexTTS-2` wi + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.trace.json.gz new file mode 100644 index 000000000..28c7423d8 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.cost.json b/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.cost.json new file mode 100644 index 000000000..7ba1b5ab3 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1785.9, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_moa_cgm/runs/pr5976/run-20260814-224554-ff00ac", + "recorded_at": "2026-08-14T23:15:37", + "attempts": [ + { + "invocation_id": "inv-0190ef44", + "outcome": "run:run-20260814-224554-ff00ac", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-224554-ff00ac", + "llm_calls": 90, + "input_tokens": 689936, + "output_tokens": 141724, + "usd": 0.531841, + "source": "spans" + } + ], + "attempt_usd_total": 0.531841, + "llm_calls": 90, + "input_tokens": 689936, + "output_tokens": 141724, + "usd": 0.531841, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-224554-ff00ac", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5318, + "minutes": 29.72, + "ci_minutes": 0, + "input_tokens": 689936, + "output_tokens": 141724, + "tool_calls": 323, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8952, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 90, + "by_role": { + "moa_member": { + "usd": 0.04374, + "input_tokens": 56766, + "output_tokens": 3237, + "calls": 8 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "agent": { + "usd": 0.42265, + "input_tokens": 537443, + "output_tokens": 102539, + "calls": 78 + }, + "reducer": { + "usd": 0.065451, + "input_tokens": 95727, + "output_tokens": 35948, + "calls": 3 + } + }, + "cache_read_tokens": 6656640, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.53, + "gate": 2.11, + "review": 1776.36, + "report": 0.06 + }, + "time_to_first_result_s": 1783.05 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 68090 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.md b/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.md new file mode 100644 index 000000000..7c73323c0 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.md @@ -0,0 +1,60 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 10 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Correctness omni_ar_scheduler.py:627-670 — replace_streaming_prompt path double-seeds num_stale_output_tokens (pre-seed + mixin._replace_streaming_session); MiniCPM-o stage-1 is LLM_AR and uses this flag. +- [sweep] Correctness omni_generation_scheduler.py:467-470 — segment-stop path lacks AR-style stale seeding, but talker is AR not generation; generation path only seeds via mixin replace (single seed). +- [validated] omni_ar_scheduler.py:336-353 + 490-491 — stale drain subtracts num_tokens_scheduled and skips append; matches new test_omni_ar_scheduler_stale_drain.py contract. +- [validated] omni_scheduler_mixin.py:105 — single seed on _replace_streaming_session when called directly (generation scheduler path). +- [sweep] Simplifiability omni_ar_scheduler.py:621-670 — compute replace_streaming_prompt before pre-seed or drop one seed site to avoid duplication. +- [validated] arg_utils.py:107-116 — unconditional ModelRegistry overwrite is intentional for v0.27 upstream arch collisions. +- [validated] qwen3_omni.py:144-148,409-417,1516-1521 — is_staged_run gates capture; bare tensor for plain vLLM runs; covered by test_qwen3_omni_forward_contract.py. +- [validated] patch.py:449-506 — inductor factorable-divisibility patch is sound (adds True only via exact polynomial division); self-extinguishes on older torch. + +`vllm_omni/core/sched/omni_ar_scheduler.py:629` [major] — Double seeding confirmed. _update_request_as_session adds num_in_flight_tokens to num_stale_output_tokens (omni_ar_scheduler.py:627-629) before the replace_streaming_prompt branch (lines 669-671) delegates to _replace_streaming_session, which adds the same num_in_flight_tokens again (omni_scheduler_mixin.py:105). On replace_streaming_prompt=True with num_in_flight_tokens > 0, the counter starts at 2x the late-frame scheduled-token sum; after the late frames each drain num_tokens_scheduled (lines 338-339), a positive residual remains and the new segment's first valid frame(s) are dropped — the audio-hole this PR targets — or a scheduled count larger than the residual trips 'assert request.num_stale_output_tokens >= 0'. Make the seed single-site for this path and add a regression test exercising replace_streaming_prompt=True with num_in_flight_tokens > 0. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:627-629 `in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0)` / `if in_flight_tokens > 0:` / `session.num_stale_output_tokens += in_flight_tokens`; vllm_omni/core/sched/omni_ar_scheduler.py:669-671 `if replace_streaming_prompt:` / `self._replace_streaming_session(session, update)` / `return`; vllm_omni/core/sched/omni_scheduler_mixin.py:105 `session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)`; vllm_omni/core/sched/omni_ar_scheduler.py:338-339 `output_is_stale = True` / `request.num_stale_output_tokens -= num_tokens_scheduled` / `assert request.num_stale_output_tokens >= 0`) + +`vllm_omni/platforms/xpu/platform.py:139` [major] — PR head CI shows buildkite/vllm-omni-amd-ci, -intel-ci, -npu-ci, and -rebase failing (checked live); main buildkite/vllm-omni and omni-release pass. This diff touches platform paths tied to intel-ci (vllm_omni/platforms/xpu/platform.py:139) and npu-ci (vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:14), so 'pre-existing' cannot be assumed from the diff alone. Please confirm these are pre-existing/non-blocking or get them green before merge. (evidence: vllm_omni/platforms/xpu/platform.py:139 `default = ["native"] if using_inductor else ["vllm_c", "native"]` — the changed XPU platform path; vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:14 `import vllm.v1.worker.mamba_utils as mamba_utils` — the changed NPU platform path. Live gh_ci_read for PR 5976 returned buckets: buildkite/vllm-omni-amd-ci FAILURE, buildkite/vllm-omni-intel-ci FAILURE, buildkite/vllm-omni-npu-ci FAILURE, buildkite/vllm-omni-rebase FAILURE, buildkite/vllm-omni SUCCESS, buildkite/omni-release SUCCESS.) + +`vllm_omni/platforms/xpu/platform.py:139` [major] — This one-line flip changes the platform-wide default IR-op priority on XPU from `xpu_kernels` to `vllm_c`. Every diffusion model on XPU inherits it: `DiffusionWorker._resolve_ir_op_priority` calls `current_omni_platform.get_default_ir_op_priority(vllm_config)` for all diffusion workers, and the registry provides a per-model override only for Cosmos3. The PR description and test plan never mention this change or any XPU validation. Either attach before/after evidence (latency and same-seed accuracy) for the full XPU diffusion set, or scope the flip (e.g. per-model `ir_op_priority_func`) to the models actually validated on XPU. (evidence: vllm_omni/platforms/xpu/platform.py:139 `default = ["native"] if using_inductor else ["vllm_c", "native"]`; vllm_omni/diffusion/worker/diffusion_worker.py:184 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`; vllm_omni/diffusion/registry.py:571 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",` and vllm_omni/diffusion/registry.py:572 `"Cosmos3OmniPipeline": "get_cosmos3_ir_op_priority_func",` (the only two entries in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`).) + +`vllm_omni/engine/arg_utils.py:116` [major] — Line 116 unconditionally overwrites the upstream global ModelRegistry for every _OMNI_MODELS entry, making any arch resolved through the global registry construct the omni class. The diff only adds the getattr(..., 'model_stage', None) or ... guard to qwen2_5_omni and qwen3_omni; the same table still registers plain-run-unsafe classes that read vllm_config.model_config.model_stage directly (step_audio2.py:57, ming_flash_omni.py:98, voxtral_tts.py:112, omnivoice.py:279), so a plain (non-staged) run resolving one of those archs will hit AttributeError: 'ModelConfig' object has no attribute 'model_stage'. Either apply the same default across every registered entry that reads model_stage, or restrict the unconditional overwrite to the upstream-registered archs that actually need it. (evidence: vllm_omni/engine/arg_utils.py:115-116 `for arch, (mod_folder, mod_relname, cls_name) in _OMNI_MODELS.items(): + ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")`; vllm_omni/model_executor/models/step_audio2/step_audio2.py:57 `raw_model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni.py:98 `self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py:112 `self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/omnivoice/omnivoice.py:279 `self.model_stage = vllm_config.model_config.model_stage`) + +`vllm_omni/core/sched/omni_ar_scheduler.py:490` [minor] — This diff seeds num_stale_output_tokens from num_in_flight_tokens at the resumable segment-stop site (omni_ar_scheduler.py:490-491) — the same contract pinned for streaming replacement in test_omni_ar_scheduler_stale_drain.py, but that file only drives _update_request_as_session and never exercises segment stop. test_resumable_segment_stop_reconciles_async_placeholders sets num_output_placeholders yet never sets num_in_flight_tokens or asserts num_stale_output_tokens, so the segment-stop seeding path is untested. Add a case: num_in_flight_tokens=3 (the stop frame's own scheduled token is settled at the top of update_from_output before the seed), run _run_resumable_segment_stop, assert num_stale_output_tokens==2, then feed two stale frames through update_from_output and confirm the next valid frame is delivered. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:490 `if request.num_in_flight_tokens > 0:`; vllm_omni/core/sched/omni_ar_scheduler.py:491 `request.num_stale_output_tokens += request.num_in_flight_tokens`; tests/core/sched/test_omni_ar_scheduler_streaming.py:119 `session.num_output_placeholders = outstanding_async_tokens` (inside test_resumable_segment_stop_reconciles_async_placeholders, which contains no num_in_flight_tokens assignment); tests/core/sched/test_omni_ar_scheduler_streaming.py:153 `session.num_in_flight_tokens = 1 # as left by schedule() for this step` (the only num_in_flight_tokens usage in the file, in a different test).) + +`vllm_omni/core/sched/omni_ar_scheduler.py:629` [minor] — PR body already lists Qwen3-Omni seed-TTS WER under Known remaining issues (build 2957: 0.3916) and confirms c2b65233 does not close it, but no tracking-issue link is present. Please add the tracking-issue link for the remaining WER regression. (evidence: gh_pr_view:5976 body: "Build 2957 (first nightly with the stale-drain fix) confirmed the drain fix does not move the seed-TTS WER metric (0.3916, in the pre-fix range) — that regression remains open under Known remaining issues." and "- **Qwen3-Omni seed-TTS WER stuck at ~0.39 (baseline 0.27, gate 0.35)** — confirmed not resolved by c2b65233 (build 2957: 0.3916, inside the pre-fix range)."; anchor vllm_omni/core/sched/omni_ar_scheduler.py:629 ` session.num_stale_output_tokens += in_flight_tokens`) + +`vllm_omni/benchmarks/patch/patch.py:1781` [minor] — probe_task is created at patch.py:1714 (asyncio.create_task(probe_loop())) before the main benchmark work, but the only stop/await is at patch.py:1781-1783 after asyncio.gather(*tasks). There is no try/finally, so if anything between task creation and that cleanup raises while probe_request_rate > 0, the probe loop is never cancelled or awaited. Wrap creation through teardown in try/finally with probe_stop.set() and await/cancel in the finally block. (evidence: vllm_omni/benchmarks/patch/patch.py:1714 `probe_task = asyncio.create_task(probe_loop())`; vllm_omni/benchmarks/patch/patch.py:1781-1783 `if probe_task is not None:` / `probe_stop.set()` / `await probe_task` — task created before the main gather and only cleaned up after it, with no intervening try/finally.) + +`tests/e2e/accuracy/test_qwen_image.py:38` [minor] — This diff adds SSIM_THRESHOLD_SDPA_FALLBACK = 0.96 (line 38) and selects it via line 262 whenever _flash_attn3_hub_available() returns False, silently relaxing the 0.97 SSIM gate on the SDPA-matched path (build 2954 landed at 0.9687). _flash_attn3_hub_available() (lines 68-84) catches any Exception from get_kernel, so transient hub/cache errors also trigger the looser gate. Skip or xfail that image when the hub kernel is absent rather than shipping a looser accuracy contract; keep 0.97 for the FA3-matched path only when both sides actually ran FA3. (evidence: tests/e2e/accuracy/test_qwen_image.py:38 `SSIM_THRESHOLD_SDPA_FALLBACK = 0.96`; :262 `ssim_threshold=SSIM_THRESHOLD if _flash_attn3_hub_available() else SSIM_THRESHOLD_SDPA_FALLBACK`; :84 ` except Exception as exc:`) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/quantization/test_quantization_quality.py:525` [minor] Confirmed. test_quantization_quality now pins enforce_eager=True for both arms (lines 525 and 537), so the nightly quality job no longer exercises the compiled FLUX-fp8 path that _patch_inductor_factorable_divisibility ( +- `vllm_omni/worker/gpu_generation_model_runner.py:847` [minor] Confirmed. `GPUGenerationModelRunner._dummy_run` accepts `randomize_inputs` (line 580) but the call at line 847 is `self.maybe_randomize_inputs(input_ids, inputs_embeds)` — the kwarg is deliberately not forwarded, unlike +- `vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py:312` [minor] This diff changes Qwen2_5OmniVideoProcessorItems.get_processor_data (qwen2_5_omni_thinker.py:296-317) to read via self.get() so vLLM 0.27's MediaWithBytes wrapper is unwrapped before the HF video processor runs. Grep fin +- `vllm_omni/benchmarks/patch/patch.py:1856` [minor] Confirmed. vllm_omni/benchmarks/patch/patch.py:1856-1868 now tolerates upstream RequestFuncOutput objects missing duplex_request_metrics/duplex_session_metrics via getattr, and tests/benchmarks/patch/test_patch.py only i + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.trace.json.gz b/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.trace.json.gz new file mode 100644 index 000000000..c67bf67ca Binary files /dev/null and b/eval/dataset/arms/copilot_v13_moa_cgm/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_test/manifest.json b/eval/dataset/arms/copilot_v13_proof_test/manifest.json new file mode 100644 index 000000000..c37f5bb8d --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v13_proof_test", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "test" + ], + "kinds": [ + "pr_review" + ], + "n_items": 5, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4762.cost.json b/eval/dataset/arms/copilot_v13_proof_test/pr4762.cost.json new file mode 100644 index 000000000..fabba3359 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4762.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1393.7, + "split": "test", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_test/runs/pr4762/run-20260813-154730-435fc7", + "recorded_at": "2026-08-13T16:10:40", + "attempts": [ + { + "invocation_id": "inv-fc3b7db5", + "outcome": "run:run-20260813-154730-435fc7", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-154730-435fc7", + "llm_calls": 172, + "input_tokens": 1566512, + "output_tokens": 245096, + "usd": 1.161101, + "source": "spans" + } + ], + "attempt_usd_total": 1.161101, + "llm_calls": 172, + "input_tokens": 1566512, + "output_tokens": 245096, + "usd": 1.161101, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-154730-435fc7", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.1611, + "minutes": 23.16, + "ci_minutes": 0, + "input_tokens": 1566512, + "output_tokens": 245096, + "tool_calls": 276, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0296, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 172, + "by_role": { + "agent": { + "usd": 1.132141, + "input_tokens": 1543707, + "output_tokens": 224398, + "calls": 170 + }, + "reducer": { + "usd": 0.02896, + "input_tokens": 22805, + "output_tokens": 20698, + "calls": 2 + } + }, + "cache_read_tokens": 17353216, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.61, + "gate": 1.68, + "review": 1384.3, + "report": 0.07 + }, + "time_to_first_result_s": 1389.65 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4762.trace.json.gz", + "trace_bytes": 44866 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4762.md b/eval/dataset/arms/copilot_v13_proof_test/pr4762.md new file mode 100644 index 000000000..42008a9ac --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4762.md @@ -0,0 +1,57 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4762, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head cdc825f03b7d) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] tests/e2e/online_serving/test_qwen3_omni.py:252 `assert not responses[0].success` — read HttpResponse/_http_response_from_requests (success = 200<=status<300) and assert_http_error (asserts status==400); the assertion is valid and the test drives the real server path. +- [sweep] config_factory.py new branches (lines 131,138,139,144,155,161,180,183,203,213,252,256,258,279,287) read in full; fallback chains mirror the pre-refactor logic except the trust_remote_code default flip (review comment 1). +- [sweep] api_server.py:514/516 hasattr/else branches — engine_client is always AsyncOmni in omni_run_server_worker, so the else is defensive-only. +- [sweep] non-test touched files mapped: endpoint_policy -> tests/config/test_endpoint_policy.py; config_factory -> tests/test_config_factory.py + tests/entrypoints/test_resolve_dreamzero_config.py; async_omni_engine/async_omni/api_server/utils/qwen3 pipeline -> tests/e2e/online_serving/test_qwen3_omni.py + updated entrypoint tests. +- [validated] tests/config/test_endpoint_policy.py pytestmark=[core_model,cpu] is selected by .buildkite/test-merge.yml:29 (`pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor`). +- [validated] shutdown_unsupported_routes unit tests cover rejection, unrestricted passthrough, idempotency, and missing-route safety; the e2e test verifies real /v1/completions rejection for qwen3_omni_moe. +- [validated] merge_pipeline_deploy only reads PipelineConfig and builds new StageConfig objects, so functools.cache on get_pipeline_config does not share mutable stage state. +- [sweep] tests/test_config_factory.py autouse clear_config_factory_caches clears the three caches after each test; test_resolve_dreamzero_config.py monkeypatches _try_infer_model_type without cache_clear, but no other test uses the same model key, so no current cross-test contamination found. + +`tests/e2e/online_serving/test_qwen3_omni.py:252` [major] — Buildkite `vllm-omni-amd-ci` and `vllm-omni-npu-ci` are red on PR 4762 head (cdc825f03b7d). The new e2e test `test_completions_rejected_for_thinker_talker` is marked `rocm: MI325` (and `cuda: H100`) at tests/e2e/online_serving/test_qwen3_omni.py:235. Please confirm whether these failures are caused by this PR or are pre-existing on main, and do not merge until the gate is green or explicitly waived. (evidence: tests/e2e/online_serving/test_qwen3_omni.py:235 `@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)` — new test is ROCm/MI325-marked. gh_ci_read(4762) returned `{"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` and `{"name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}`.) + +`vllm_omni/entrypoints/utils.py:348` [major] — The diff removes the old detection default `trust_remote_code = cli_overrides.get("trust_remote_code", True)` and `entrypoints/utils.py` now passes `cli_overrides.get("trust_remote_code", False)` into `StageConfigFactory.create_from_model`, whose new signature is `trust_remote_code: bool = False`. Because `get_hf_config` returns `None` when `get_config` raises, pipelines that are matched only via `hf_architectures` + `hf_config_predicate` (e.g. MiniCPM-o 4.5, Ming TTS) can no longer reach that branch if their custom `model_type` is not in OMNI_PIPELINES: the raw config.json fallback returns the unregistered model_type, `get_pipeline_config` returns `None`, and `load_stage_configs_from_model` falls back to legacy/default-diffusion. Keep the detection-only trust default at `True` (or read the deploy YAML's trust value for detection) and add a regression test for a predicate/arch-routed model served without `--trust-remote-code`. (evidence: diff removed: `- trust_remote_code = cli_overrides.get("trust_remote_code", True)`; `vllm_omni/config/config_factory.py:267`: `trust_remote_code: bool = False,`; `vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py:31-36`: `MiniCPM-o 4.5's HF config.json reports `model_type="minicpmo"` and `architectures=["MiniCPMO"]` ... `hf_config_predicate=lambda c: str(getattr(c, "version", "")) == "4.5",`; `vllm_omni/model_executor/models/ming_tts/pipeline.py:15,18`: `hf_architectures=("MingTTSForConditionalGeneration", "BailingMMNativeForConditionalGeneration"),` and `hf_config_predicate=lambda c: str(getattr(c, "model_type", "")) == "dense",`.) + +`vllm_omni/config/endpoint_policy.py:44` [minor] — vllm_omni/config/endpoint_policy.py:68 `restricted_endpoints = (*endpoint_restrictions, *UNSUPPORTED_ROUTES)` always applies the `/v1/chat/completions/batch` ban, but no test covers it: tests/config/test_endpoint_policy.py only targets `/v1/completions`. Add a unit test calling `shutdown_unsupported_routes(app, ())` and asserting POST `/v1/chat/completions/batch` returns 400 with `"Batched chat completions are not yet supported by vLLM Omni."` so a route mismatch cannot silently no-op. (evidence: vllm_omni/config/endpoint_policy.py:46 `EndpointRestriction( + OmniServingCapability.CHAT_COMPLETIONS_BATCH, + "Batched chat completions are not yet supported by vLLM Omni.", + ),`; vllm_omni/config/endpoint_policy.py:68 `restricted_endpoints = (*endpoint_restrictions, *UNSUPPORTED_ROUTES)`; tests/config/test_endpoint_policy.py:17 `SERVER_ROUTE = "/v1/completions"`) + +`vllm_omni/entrypoints/openai/api_server.py:514` [minor] — The call is gated behind `hasattr(engine_client, "endpoint_restrictions")` and the `else` branch only logs. Because `shutdown_unsupported_routes()` always appends the global `UNSUPPORTED_ROUTES` (the `/v1/chat/completions/batch` 400), any engine client without that attribute also skips the global batch-route shutdown. Call it unconditionally with `shutdown_unsupported_routes(app, getattr(engine_client, "endpoint_restrictions", ()))` so the batch route is always rejected. (evidence: vllm_omni/entrypoints/openai/api_server.py:514 `if hasattr(engine_client, "endpoint_restrictions"):`; :515 `shutdown_unsupported_routes(app, engine_client.endpoint_restrictions)`; :516 `else:`; :517 `logger.warning("engine client has no endpoint restrictions attribute")`. vllm_omni/config/endpoint_policy.py:44 `UNSUPPORTED_ROUTES: tuple[EndpointRestriction, ...] = (`; :46 `OmniServingCapability.CHAT_COMPLETIONS_BATCH,`; :76 `restricted_endpoints = (*endpoint_restrictions, *UNSUPPORTED_ROUTES)`.) + +`vllm_omni/config/config_factory.py:75` [minor] — `get_hf_config` is decorated with `@functools.cache` (config_factory.py:75) and, on any `get_config` exception, logs and returns the pre-initialized `hf_config = None` (config_factory.py:86-91). `functools.cache` memoizes `None` too, so one transient failure caches an empty config for that `(model, trust_remote_code)` key for the life of the process; every later `get_hf_config`/`_try_infer_model_type` call skips retry and permanently loses the hf_config-based routing path (fallbacks may still recover some models, but any model relying on the loaded HF config is stuck). Cache only successful reads, or evict/retry on failure. (evidence: vllm_omni/config/config_factory.py:75 ` @functools.cache` +vllm_omni/config/config_factory.py:76 ` def get_hf_config(cls, model: str, trust_remote_code: bool) -> PretrainedConfig | None:` +vllm_omni/config/config_factory.py:86 ` hf_config = None` +vllm_omni/config/config_factory.py:88 ` return get_config(model, trust_remote_code=trust_remote_code)` +vllm_omni/config/config_factory.py:89 ` except Exception as e:` +vllm_omni/config/config_factory.py:91 ` return hf_config`) + +`vllm_omni/config/endpoint_policy.py:42` [minor] — `UNSUPPORTED_ROUTES` in `vllm_omni/config/endpoint_policy.py` makes `/v1/chat/completions/batch` return 400 for every Omni model, and no user-facing doc or changelog mentions this route (repo-wide grep finds only the code). Please add a user-facing note or changelog entry, or confirm the silent all-models batch ban is intended. (evidence: vllm_omni/config/endpoint_policy.py:42-49 `# Routes that are not supported for any model, but are supported in vLLM.` / `# This is only temporary to avoid 500s for batched chat completions.` / `UNSUPPORTED_ROUTES: tuple[EndpointRestriction, ...] = (` / ` EndpointRestriction(` / ` OmniServingCapability.CHAT_COMPLETIONS_BATCH,` / ` "Batched chat completions are not yet supported by vLLM Omni.",` / ` ),` / `)`; vllm_omni/config/endpoint_policy.py:76 `restricted_endpoints = (*endpoint_restrictions, *UNSUPPORTED_ROUTES)`) + +`vllm_omni/config/config_factory.py:75` [minor] — vllm_omni/config/config_factory.py adds @functools.cache to get_hf_config (line 75), try_infer_model_type (line 94), and get_pipeline_config (line 189). Production has no cache invalidation: no cache_clear call exists anywhere under vllm_omni, while tests/test_config_factory.py:47-49 clears these caches only in a test fixture. Please confirm that runtime model-file changes/reloads are out of scope, or add invalidation/versioning for these caches. (evidence: vllm_omni/config/config_factory.py:75 ` @functools.cache` (immediately preceding `get_hf_config`); vllm_omni/config/config_factory.py:94 ` @functools.cache` (preceding `try_infer_model_type`); vllm_omni/config/config_factory.py:189 ` @functools.cache` (preceding `get_pipeline_config`); tests/test_config_factory.py:47 ` StageConfigFactory.get_hf_config.cache_clear()` and tests/test_config_factory.py:48-49 clear try_infer_model_type and get_pipeline_config — while grep for `cache_clear` under `vllm_omni` returns no matches, proving no production invalidation.) + +`vllm_omni/model_executor/models/qwen3_omni/pipeline.py:25` [minor] — The PR description states the completions crash is not Qwen3Omni-specific and that other models (e.g. TTS models) likely have vulnerable endpoints, yet the diff adds a COMPLETIONS `EndpointRestriction` only to the qwen3_omni pipeline (vllm_omni/model_executor/models/qwen3_omni/pipeline.py:25). Sibling thinker->talker pipelines such as qwen2_5_omni (vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py:18) define no endpoint restrictions and inherit the default empty tuple. Either add COMPLETIONS restrictions to the vulnerable siblings or track an explicit audit so the new mechanism does not ship while the acknowledged siblings remain reachable. (evidence: ['vllm_omni/model_executor/models/qwen3_omni/pipeline.py:25 ` endpoint_restrictions=(` — the only model-level endpoint_restrictions assignment in vllm_omni/', 'vllm_omni/model_executor/models/qwen2_5_omni/pipeline.py:18 `QWEN2_5_OMNI_PIPELINE = PipelineConfig(` — sibling thinker->talker pipeline with no endpoint_restrictions (defaults to ())', 'PR body: "Importantly, this issue is not Qwen3Omni specific; other models, e.g., TTS models, will probably have their own issues with things like the completion endpoint."']) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/config/config_factory.py:257` [minor] `_get_deploy_override_pipe_config` (config_factory.py:257) calls `load_deploy_config(deploy_path)` with no try/except, and its callers `get_pipeline_config` -> `get_pipeline_endpoint_restrictions`/`create_from_model` do +- `tests/config/test_endpoint_policy.py:73` [minor] `test_invalid_route_rejection` at tests/config/test_endpoint_policy.py:73 creates `app = FastAPI()` with no matching route and asserts a 400 after `shutdown_unsupported_routes`, but `shutdown_unsupported_routes` at vllm_ +- `tests/e2e/online_serving/test_qwen3_omni.py:252` [minor] The e2e test passes `err_code=400` but no `err_message`, so `send_completions_http_request` already verifies the 400 via its internal `assert_http_error` call, making the trailing `assert not responses[0].success` redund + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4762.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_test/pr4762.trace.json.gz new file mode 100644 index 000000000..452142c7c Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_test/pr4762.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4777.cost.json b/eval/dataset/arms/copilot_v13_proof_test/pr4777.cost.json new file mode 100644 index 000000000..3d8d30a7e --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4777.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 900.7, + "split": "test", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_test/runs/pr4777/run-20260813-154730-4e37a7", + "recorded_at": "2026-08-13T16:02:27", + "attempts": [ + { + "invocation_id": "inv-db32c28a", + "outcome": "run:run-20260813-154730-4e37a7", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-154730-4e37a7", + "llm_calls": 86, + "input_tokens": 683668, + "output_tokens": 109108, + "usd": 0.504718, + "source": "spans" + } + ], + "attempt_usd_total": 0.504718, + "llm_calls": 86, + "input_tokens": 683668, + "output_tokens": 109108, + "usd": 0.504718, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-154730-4e37a7", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5047, + "minutes": 14.94, + "ci_minutes": 0, + "input_tokens": 683668, + "output_tokens": 109108, + "tool_calls": 148, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6449, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 86, + "by_role": { + "planner": { + "usd": 0.000923, + "input_tokens": 1788, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.480251, + "input_tokens": 672509, + "output_tokens": 89635, + "calls": 83 + }, + "reducer": { + "usd": 0.023545, + "input_tokens": 9371, + "output_tokens": 19073, + "calls": 2 + } + }, + "cache_read_tokens": 7411456, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.21, + "gate": 2.55, + "review": 889.8, + "report": 0.07 + }, + "time_to_first_result_s": 896.61 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4777.trace.json.gz", + "trace_bytes": 23079 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4777.md b/eval/dataset/arms/copilot_v13_proof_test/pr4777.md new file mode 100644 index 000000000..e873f9f81 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4777.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4777, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 51ce34a55a74) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read vllm_omni/entrypoints/openai/image_api_utils.py in tree: line 17 `SUPPORTED_LAYERED_LAYERS_RANGE = range(2, 11)`; upper bound 10 is preserved because `stop=11` is excluded. +- [validated] image_api_utils.py:115-125 `validate_layered_layers` passes None through unchanged; `2 in range(2,11)` is accepted, `1` and `11` raise; error string uses `range.start` / `range.stop-1`, so the message is automatically 'between 2 and 10 inclusive' — no off-by-one or stale literal. +- [validated] vllm_omni/entrypoints/openai/protocol/images.py:60-63 field description updated to 'Supported range: 2-10.', matching the new constant. +- [sweep] All `validate_layered_layers` consumers share the helper and have no separate hardcoded 3: protocol/images.py:93, api_server.py:2070, serving_chat.py:3287. +- [sweep] Pipeline vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py:667 reads `req.sampling_params.layers`; prepare_latents/_unpack_latents use `layers+1` and forward() drops the first frame via `latents = latents[:, :, 1:]`, so layers=2 is semantically valid (2 output images) with no `layers >= 3` assertion. +- [sweep] Repo-wide grep for 'between 3 and 10' and '3-10' found no other Qwen-Image-Layered range references; the only '3-10' hits are unrelated TTS reference-audio docs. +- [sweep] Diff-touched tests verified in tree: test_image_server.py:1247/1666/1680 now use layers=1 lower-bound and 'between 2 and 10 inclusive'; test_qwen_image_layered_expansion.py:10/116/120 restored `layers=2`. +- [sweep] changed_symbol_consumers checked: the two dfx reliability tests are NOT in the diff and still assert the old '3 to 10' contract — see review comments. + +`tests/dfx/reliability/invalid_param_test/test_invalid_image_generation.py:?` [major] — The diff changes SUPPORTED_LAYERED_LAYERS_RANGE to range(2, 11) in vllm_omni/entrypoints/openai/image_api_utils.py:17, so layers=2 now passes validation. But tests/dfx/reliability/invalid_param_test/test_invalid_image_generation.py:90 still submits {"layers": 2} and expects 400 with ('layers', 'value_error', '3', '10'), so the case will now fail. Update the below-min param to {"layers": 1} and the expected error pieces to ('layers', 'value_error', '2', '10'). (evidence: vllm_omni/entrypoints/openai/image_api_utils.py:17 `SUPPORTED_LAYERED_LAYERS_RANGE = range(2, 11)` (2 is inside the valid range, so layers=2 no longer raises) versus tests/dfx/reliability/invalid_param_test/test_invalid_image_generation.py:90 `pytest.param({"layers": 2}, ("layers", "value_error", "3", "10"), id="layers_below_min"),` (still expects a 400 with the old bounds 3/10).) + +`tests/dfx/reliability/invalid_param_test/test_invalid_image_editing.py:?` [major] — The diff widens SUPPORTED_LAYERED_LAYERS_RANGE to range(2, 11), so /v1/images/edits now accepts layers=2 (Form value "2" coerced to int 2 passes validate_layered_layers). tests/dfx/reliability/invalid_param_test/test_invalid_image_editing.py:65 still sends layers="2" and line 66 expects 400 with ("Invalid layers", "3", "1"); the server will now return 200 and the test fails. Change the below-min input to "1" (or 1) and update the expected substrings to ("Invalid layers", "2", "10"). (evidence: tests/dfx/reliability/invalid_param_test/test_invalid_image_editing.py:65 `{"prompt": "edit", "model": _IMAGE_EDITS_SERVER_MODEL, "layers": "2"},` ; tests/dfx/reliability/invalid_param_test/test_invalid_image_editing.py:66 `("Invalid layers", "3", "1"),` ; vllm_omni/entrypoints/openai/image_api_utils.py:17 `SUPPORTED_LAYERED_LAYERS_RANGE = range(2, 11)` ; vllm_omni/entrypoints/openai/image_api_utils.py:119 ` if layers not in SUPPORTED_LAYERED_LAYERS_RANGE:` ; vllm_omni/entrypoints/openai/api_server.py:2070 ` layers = validate_layered_layers(layers)`) + +`vllm_omni/entrypoints/openai/image_api_utils.py:17` [minor] — `SUPPORTED_LAYERED_LAYERS_RANGE = range(2, 11)` (image_api_utils.py:17) feeds all three generic validation call sites — protocol/images.py:93, serving_chat.py:3287, and api_server.py:2070 — and each path copies the value onto `gen_params.layers` (serving_chat.py:3334, api_server.py:1853, api_server.py:2124). The only live read of `sampling_params.layers` is pipeline_qwen_image_layered.py:667 (`layers = req.sampling_params.layers`), so other diffusion models served through these endpoints will now accept `layers=2` without consuming it. Is accepting-and-ignoring intended, or should the bound be scoped to Qwen-Image-Layered? (evidence: vllm_omni/entrypoints/openai/image_api_utils.py:17 `SUPPORTED_LAYERED_LAYERS_RANGE = range(2, 11)`; vllm_omni/entrypoints/openai/protocol/images.py:93 `return validate_layered_layers(v)`; vllm_omni/entrypoints/openai/serving_chat.py:3287 `layers = validate_layered_layers(layers)`; vllm_omni/entrypoints/openai/api_server.py:2070 `layers = validate_layered_layers(layers)`; vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py:667 `layers = req.sampling_params.layers` — repo-wide grep for `sampling_params.layers` returns no other consumer.) + +`tests/entrypoints/openai_api/test_image_server.py:1247` [minor] — The diff changes the lower-bound rejection case at tests/entrypoints/openai_api/test_image_server.py:1247 from `layers=2` to `layers=1`, so the newly valid `layers=2` is never positively asserted in this file. Add `ImageGenerationRequest(prompt="test", layers=2)` (and ideally an HTTP 200 path) to lock in the range(2, 11) boundary. (evidence: tests/entrypoints/openai_api/test_image_server.py:1247 ` ImageGenerationRequest(prompt="test", layers=1)` — inside `with pytest.raises(ValueError):`, with no positive `layers=2` assertion in the file; vllm_omni/entrypoints/openai/image_api_utils.py:17 `SUPPORTED_LAYERED_LAYERS_RANGE = range(2, 11)` — makes 2 the newly valid lower bound.) + +`tests/e2e/online_serving/test_qwen_image_layered_expansion.py:?` [minor] — Verified against the PR-time tree: LAYERS_GUARD_CASES now passes expected_layers=2 with id="layers_guard_001_layers2" (lines 119-120), and test_layered_output_image_count asserts len(image_items) == expected_layers at line 200. PR body confirms the H100 L2 regression was not run. Note PR 4777 is already MERGED, so the ask should be post-merge/CI execution of pytest -m "diffusion and full_model" tests/e2e/online_serving/test_qwen_image_layered_expansion.py::test_layered_output_image_count -k layers_guard_001_layers2 rather than 'before merge'. (evidence: tests/e2e/online_serving/test_qwen_image_layered_expansion.py:119 `2,`; tests/e2e/online_serving/test_qwen_image_layered_expansion.py:120 `id="layers_guard_001_layers2",`; tests/e2e/online_serving/test_qwen_image_layered_expansion.py:200 `assert len(image_items) == expected_layers, (`) + +`vllm_omni/entrypoints/openai/image_api_utils.py:17` [nit] — CI shows only `buildkite/vllm-omni-amd-ci` FAILING (all other reported lanes SUCCESS). Since this diff changes shared OpenAI image entrypoint validation (`SUPPORTED_LAYERED_LAYERS_RANGE = range(2, 11)` at image_api_utils.py:17), please confirm in the PR thread that the AMD failure is pre-existing on `main` and not introduced by the range change; if introduced, fix before merge. (evidence: vllm_omni/entrypoints/openai/image_api_utils.py:17 `SUPPORTED_LAYERED_LAYERS_RANGE = range(2, 11)` — the exact changed shared-entrypoint validation line the comment anchors on. gh_ci_read(4777) returns `buildkite/vllm-omni-amd-ci` with state FAILURE as the only failing check, while build (3.11), build (3.12), pre-commit, DCO, buildkite/vllm-omni, buildkite/vllm-omni-intel-ci, buildkite/vllm-omni-npu-ci, and docs/readthedocs.org:vllm-omni are all SUCCESS.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4777.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_test/pr4777.trace.json.gz new file mode 100644 index 000000000..71719e485 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_test/pr4777.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4834.cost.json b/eval/dataset/arms/copilot_v13_proof_test/pr4834.cost.json new file mode 100644 index 000000000..3279c0127 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4834.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1924.4, + "split": "test", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_test/runs/pr4834/run-20260813-154730-99b324", + "recorded_at": "2026-08-13T16:19:30", + "attempts": [ + { + "invocation_id": "inv-7a4522ed", + "outcome": "run:run-20260813-154730-99b324", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-154730-99b324", + "llm_calls": 139, + "input_tokens": 1042484, + "output_tokens": 264226, + "usd": 0.838421, + "source": "spans" + } + ], + "attempt_usd_total": 0.838421, + "llm_calls": 139, + "input_tokens": 1042484, + "output_tokens": 264226, + "usd": 0.838421, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-154730-99b324", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8384, + "minutes": 32.01, + "ci_minutes": 0, + "input_tokens": 1042484, + "output_tokens": 264226, + "tool_calls": 290, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0526, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 139, + "by_role": { + "agent": { + "usd": 0.809709, + "input_tokens": 1022906, + "output_tokens": 242961, + "calls": 137 + }, + "reducer": { + "usd": 0.028712, + "input_tokens": 19578, + "output_tokens": 21265, + "calls": 2 + } + }, + "cache_read_tokens": 9863040, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.1, + "gate": 2.12, + "review": 1914.01, + "report": 0.07 + }, + "time_to_first_result_s": 1920.27 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4834.trace.json.gz", + "trace_bytes": 38961 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4834.md b/eval/dataset/arms/copilot_v13_proof_test/pr4834.md new file mode 100644 index 000000000..f5c72df68 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4834.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4834, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5793cd6f4bba) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [validated] async_omni.py:311 `if _sleeping_tags:` — guard is reachable on every generate() and rejects while any tag is asleep; no off-by-one/None issue in normal construction. +- [validated] async_omni.py:934 `if not hasattr(self, '_sleeping_tags')` and 986 `if hasattr(...)` — redundant; __init__ always sets _sleeping_tags (line 144) and _level2_sleeping. +- [validated] async_omni.py:937 `if level == 2:` — sets _level2_sleeping=True after sleep RPC; flag is sticky because wake_up raises before it can be cleared. +- [sweep] async_omni.py:944 `if getattr(self, '_level2_sleeping', False):` — read in situ; blanket raise blocks diffusion level-2 wake (see review comment 1). +- [validated] async_omni.py:952/954/956 tag filtering — None -> all current tags, provided tags filtered to sleeping ones, empty -> [] idempotent no-op; sequential duplicate wake works as the new test asserts. +- [validated] async_omni.py:990 `if not getattr(self, '_sleeping_tags', None)` — unreachable reset branch: level-2 flag True makes wake_up raise before the discard loop, so this can never execute. +- [validated] vllm_omni/diffusion/data.py CuMemTag added; only consumer is async_omni.py:936 — no other callers need updating. +- [sweep] vllm_omni/diffusion/data.py:1489 hunk — read OmniACK/OmniSleepTask/OmniWakeTask definitions; OmniACK.status is a required str ('SUCCESS'/'ERROR') used by the finding about failed wake. + +`vllm_omni/entrypoints/async_omni.py:944` [blocker] — wake_up() raises NotImplementedError as soon as _level2_sleeping is set (async_omni.py:944), and sleep() sets it for every sleep(level=2) call (async_omni.py:938). DiffusionWorker.sleep(level=2) does save named buffers to CPU (diffusion_worker.py:539) and wake_up restores them (diffusion_worker.py:589), so the blanket guard breaks supported diffusion sleep/wake and makes existing sleep(level=2)->wake_up() tests fail (test_omni_sleep_mode.py:212/221, 269/272, 357/376, 488/492, 544/547). Scope the guard to stages whose worker truly discards weights without a restore path, or update the tests/CI if a global level-2 contract is intentional. (evidence: vllm_omni/entrypoints/async_omni.py:938 `if level == 2:` / `self._level2_sleeping = True`; vllm_omni/entrypoints/async_omni.py:944 `if getattr(self, "_level2_sleeping", False):` / `raise NotImplementedError(`; vllm_omni/diffusion/worker/diffusion_worker.py:539 `self._sleep_saved_buffers = {name: buffer.cpu().clone() for name, buffer in model.named_buffers()}`; vllm_omni/diffusion/worker/diffusion_worker.py:589 `buffer.data.copy_(self._sleep_saved_buffers[name].data)`; tests/entrypoints/test_omni_sleep_mode.py:212 `acks = await diffusion_engine.sleep(stage_ids=[0], level=2)` and :221 `await diffusion_engine.wake_up(stage_ids=[0])`; tests/entrypoints/test_omni_sleep_mode.py:544 `acks = await engine.sleep(stage_ids=[0, 1], level=2)` and :547 `await engine.wake_up(stage_ids=[0, 1])`) + +`vllm_omni/entrypoints/async_omni.py:985` [major] — wake_up() collects worker ACKs with `final_acks.append(ack)` (async_omni.py:981) and then unconditionally discards every requested tag from `_sleeping_tags` (async_omni.py:985-987) without checking `ack.status`. Workers return `OmniACK(status="ERROR")` on failure (worker/base.py:317, diffusion_worker.py:695), so a failed wake clears the guard and lets generate() (async_omni.py:310-317) admit requests against a stage that is still asleep. Only discard tags whose ACK status is "SUCCESS", or raise before mutating state when any ERROR ACK is received. (evidence: vllm_omni/entrypoints/async_omni.py:981 `final_acks.append(ack)`; vllm_omni/entrypoints/async_omni.py:985-987 `for t in requested_tags:` / `if hasattr(self, "_sleeping_tags"):` / `self._sleeping_tags.discard(t)`; vllm_omni/entrypoints/async_omni.py:310-317 `_sleeping_tags = getattr(self, "_sleeping_tags", None)` / `if _sleeping_tags:` / `raise RuntimeError(...)`; vllm_omni/worker/base.py:317 `return OmniACK(task_id=tid, status="ERROR", error_msg=str(e))`; vllm_omni/diffusion/worker/diffusion_worker.py:695 `return OmniACK(task_id=task.task_id, status="ERROR", error_msg=str(e))`) + +`vllm_omni/entrypoints/async_omni.py:936` [major] — sleep() marks a single global `_sleeping_tags` set (async_omni.py:936) without recording which `stage_ids` it slept, and wake_up() computes `requested_tags = list(_current_tags)` (async_omni.py:953) then discards those tags after RPC-ing only the requested stages (async_omni.py:985/987). So sleep(stage_ids=[0,1], level=1) then wake_up(stage_ids=[0]) empties `_sleeping_tags` while stage 1 remains offloaded, and generate()'s guard (async_omni.py:310-311) passes. Track tags per stage (dict[stage_id, set[str]]) or reject partial sleep/wake explicitly; the current tests only wake the same subset they slept, so this asymmetric case is not covered. (evidence: vllm_omni/entrypoints/async_omni.py:936 `self._sleeping_tags.update([CuMemTag.WEIGHTS.value, CuMemTag.KV_CACHE.value])` (global set updated regardless of `stage_ids`); :953 `requested_tags = list(_current_tags)` (full wake defaults to all tracked tags); :985 `for t in requested_tags:`; :987 `self._sleeping_tags.discard(t)` (discard after waking only requested stages); :310-311 `_sleeping_tags = getattr(self, "_sleeping_tags", None)` / `if _sleeping_tags:` (generate guard checks only the global set).) + +`vllm_omni/entrypoints/async_omni.py:936` [major] — sleep() unconditionally records both CuMemTag.WEIGHTS and CuMemTag.KV_CACHE in the engine-global _sleeping_tags (async_omni.py:936), but diffusion stages only allocate a 'weights' CuMem pool (diffusion_model_runner.py:191). On a diffusion-only engine, sleep(level=1) leaves a phantom kv_cache tag: generate() remains rejected after wake_up(tags=["weights"]) because _sleeping_tags is still non-empty, and wake_up(tags=None) forwards ["weights","kv_cache"] to DiffusionWorker.wake_up -> allocator.wake_up(tags) for a pool that was never created. Derive the tag set from each selected stage backend, or add kv_cache only for stages that actually allocate it. (evidence: vllm_omni/entrypoints/async_omni.py:936 `self._sleeping_tags.update([CuMemTag.WEIGHTS.value, CuMemTag.KV_CACHE.value])`; vllm_omni/diffusion/worker/diffusion_model_runner.py:191 `return memory_pool_context_fn(tag="weights")`; vllm_omni/diffusion/worker/diffusion_worker.py:572-573 `allocator.wake_up(tags)` after `def wake_up(self, tags: list[str] | None = None) -> bool` at line 570; vllm_omni/entrypoints/async_omni.py:310-317 `_sleeping_tags = getattr(self, "_sleeping_tags", None)` / `if _sleeping_tags: raise RuntimeError(...)`.) + +`tests/entrypoints/test_omni_sleep_mode.py:?` [major] — The three new regression tests in TestOmniSleepMode (test_omni_sleep_mode.py:412 ff) are marked only @pytest.mark.asyncio + @hardware_test(...). The module pytestmark (lines 21-24) is only advanced_model + usefixtures, hardware_test adds only platform/resource/distributed marks (never omni), and .buildkite/test-amd.yaml:118 selects -m "advanced_model and omni and MI325". The new tests are therefore silently deselected on the MI325 lane and the claimed regression coverage never executes. Add @pytest.mark.omni to each new test (the module-level e2e tests already do at lines 457/501/554) or adjust the lane marker expression. (evidence: tests/entrypoints/test_omni_sleep_mode.py:21-24 `pytestmark = [ + pytest.mark.advanced_model, + pytest.mark.usefixtures("clean_gpu_memory_between_tests"), +]`; tests/entrypoints/test_omni_sleep_mode.py:410-412 ` @pytest.mark.asyncio + @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) + async def test_level2_sleep_wake_raises(self, llm_engine: AsyncOmni):`; .buildkite/test-amd.yaml:118 ` - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"`) + +`vllm_omni/entrypoints/openai/api_server.py:3388` [major] — PR makes AsyncOmni.wake_up raise NotImplementedError after sleep(level=2), but the /v1/omni/wakeup endpoint (api_server.py:3388) neither catches it nor maps it to a client error, so a level-2 sleep (the OmniSleepRequest default) followed by wake returns an unhandled 500 and leaves app.state.sleeping_stages stale. The mock tests (e.g. test_sleep_then_wakeup_roundtrip) don't exercise the real raise and still assert the old 200-and-cleared contract; update the endpoint and tests to the new level-2 contract, or explicitly reject level-2 sleep via HTTP. (evidence: vllm_omni/entrypoints/async_omni.py:944 ` if getattr(self, "_level2_sleeping", False):` +vllm_omni/entrypoints/async_omni.py:945 ` raise NotImplementedError(` +vllm_omni/entrypoints/openai/api_server.py:3388 ` acks = await engine_client.wake_up(stage_ids=request.stage_ids)` +vllm_omni/entrypoints/openai/api_server.py:3389-3391 ` for sid in request.stage_ids:` / ` if sid in sleeping_set:` / ` sleeping_set.remove(sid)` +tests/entrypoints/openai_api/test_omni_sleep_wakeup.py:194 ` assert app.state.sleeping_stages == {0, 1}` +tests/entrypoints/openai_api/test_omni_sleep_wakeup.py:198 ` assert app.state.sleeping_stages == set()`) + +`vllm_omni/entrypoints/async_omni.py:990` [minor] — Confirmed. `wake_up()`'s level-2 clear at async_omni.py:990-991 is dead code: the guard at async_omni.py:944 raises `NotImplementedError` before line 991 whenever `_level2_sleeping` is True, and when the flag is False the assignment is a no-op. Remove lines 990-991 and the future-support comment, or move a real full-wake clear ahead of the level-2 guard. (evidence: async_omni.py:938 ` self._level2_sleeping = True`; async_omni.py:944 ` if getattr(self, "_level2_sleeping", False):`; async_omni.py:945 ` raise NotImplementedError(`; async_omni.py:990 ` if not getattr(self, "_sleeping_tags", None):`; async_omni.py:991 ` self._level2_sleeping = False`. The only assignments to `_level2_sleeping` are line 938 (set True inside sleep(level=2)) and line 991 (set False), and line 991 is only reachable after the line 944 guard has already returned/raised whenever the flag is True.) + +`tests/diffusion/test_diffusion_streaming_output.py:?` [minor] — AsyncOmni.__init__ now replaces `_is_sleeping` with `_sleeping_tags`/`_level2_sleeping` (async_omni.py:144-145), but tests/diffusion/test_diffusion_streaming_output.py:431 still sets `omni._is_sleeping = False` on a manually-constructed AsyncOmni; generate() and is_sleeping() now read `_sleeping_tags` (async_omni.py:310-311, 1001), so that assignment is dead and the test's awake-state setup is silently ineffective. Replace it with `omni._sleeping_tags = set()` and `omni._level2_sleeping = False`, or delete it. (evidence: tests/diffusion/test_diffusion_streaming_output.py:431 ` omni._is_sleeping = False`; vllm_omni/entrypoints/async_omni.py:144 ` self._sleeping_tags: set[str] = set()`; vllm_omni/entrypoints/async_omni.py:310 ` _sleeping_tags = getattr(self, "_sleeping_tags", None)`; vllm_omni/entrypoints/async_omni.py:311 ` if _sleeping_tags:`; vllm_omni/entrypoints/async_omni.py:1001 ` return bool(getattr(self, "_sleeping_tags", None))`) + +**Additional observations (beyond the comment budget):** +- `tests/entrypoints/test_omni_sleep_mode.py:412` [minor] The new tests declare res={"cuda":"H100","rocm":"MI325"} but the PR Test Plan says the standalone probe ran on 4 x A100 80GB, which would not select these tests under the H100 marker. Please align the hardware markers wi +- `docs/features/sleep_mode.md:60` [minor] wake_up() now raises NotImplementedError after `sleep(level=2)` (vllm_omni/entrypoints/async_omni.py:944-950, gated on `_level2_sleeping` set at line 938), but docs/features/sleep_mode.md still shows `acks = await engine +- `vllm_omni/entrypoints/async_omni.py:953` [minor] wake_up() snapshots `_sleeping_tags` at async_omni.py:952-953 and only discards tags at async_omni.py:985-987, after `await self.collective_rpc(...)` at async_omni.py:974; there is no lock or in-flight marking. Since the + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4834.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_test/pr4834.trace.json.gz new file mode 100644 index 000000000..be24b43d2 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_test/pr4834.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4849.cost.json b/eval/dataset/arms/copilot_v13_proof_test/pr4849.cost.json new file mode 100644 index 000000000..f167dbd0d --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4849.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 824.7, + "split": "test", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_test/runs/pr4849/run-20260813-154730-b0930c", + "recorded_at": "2026-08-13T16:01:11", + "attempts": [ + { + "invocation_id": "inv-6496f756", + "outcome": "run:run-20260813-154730-b0930c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-154730-b0930c", + "llm_calls": 94, + "input_tokens": 634811, + "output_tokens": 145744, + "usd": 0.507182, + "source": "spans" + } + ], + "attempt_usd_total": 0.507182, + "llm_calls": 94, + "input_tokens": 634811, + "output_tokens": 145744, + "usd": 0.507182, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-154730-b0930c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5072, + "minutes": 13.68, + "ci_minutes": 0, + "input_tokens": 634811, + "output_tokens": 145744, + "tool_calls": 161, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6192, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 94, + "by_role": { + "agent": { + "usd": 0.484433, + "input_tokens": 619102, + "output_tokens": 128950, + "calls": 92 + }, + "reducer": { + "usd": 0.022749, + "input_tokens": 15709, + "output_tokens": 16794, + "calls": 2 + } + }, + "cache_read_tokens": 6498688, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.03, + "gate": 1.96, + "review": 814.51, + "report": 0.07 + }, + "time_to_first_result_s": 820.54 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4849.trace.json.gz", + "trace_bytes": 30108 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4849.md b/eval/dataset/arms/copilot_v13_proof_test/pr4849.md new file mode 100644 index 000000000..fbb554c68 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4849.md @@ -0,0 +1,41 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4849, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f7bf64f2b275) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] Grep 'ar2diffusion' repo-wide: only production caller is the diffusion stage via pipeline config (vllm_omni/model_executor/models/hunyuan_image3/pipeline.py:45); all other hits are the two updated tests or unrelated GLM code. No stale list-expecting consumer remains. +- [sweep] Read vllm_omni/engine/orchestrator.py:1204-1290 _forward_to_next_stage: diffusion branch builds diffusion_source_outputs = [output, *companion_outputs] (line 1211), calls _fn(diffusion_source_outputs, req_state.prompt, requires_multimodal_data), then handles None/empty-list and leaves a single dict intact; new return type is consumed correctly. +- [sweep] Read vllm_omni/model_executor/stage_input_processors/glm_image.py:215-410: GLM ar2diffusion already returns dict|None with source_outputs[0] and identical prompt normalization; Hunyuan now matches that sibling contract. +- [validated] hunyuan_image3.py:114-115 'if not source_outputs: return None' makes line 118 'ar_output = source_outputs[0]' safe against IndexError on empty input. +- [validated] hunyuan_image3.py:118-119 'output = ar_output.outputs[0]' is unchanged from the old loop body; orchestrator parent outputs always carry at least one completion in this path. +- [validated] hunyuan_image3.py:125 'original_prompt = prompt[0] if prompt else {}' is guarded by the ternary; branches at 124/126/128 are equivalent to the old normalization for the single parent case. +- [validated] Branches at hunyuan_image3.py:155, 197, 199, 206, 208, 210, 214, 216, 221 are all reachable and carry over the old loop-body semantics verbatim; no dead/redundant branch introduced. +- [sweep] Read tests/model_executor/stage_input_processors/test_hunyuan_image3_bridge.py fully: new tests cover empty source_outputs -> None, companion presence -> parent-first, single-dict return, ratio override and custom system prompt forwarding; file has pytestmark core_model+cpu. + +`tests/model_executor/stage_input_processors/test_hunyuan_image3_bridge.py:48` [minor] — The diff rewrites `ar2diffusion` to return `dict[str, Any] | None` and adds `test_ar2diffusion_returns_one_request_payload_for_request_level_batching` at tests/model_executor/stage_input_processors/test_hunyuan_image3_bridge.py:48; the file declares `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (line 19). .buildkite/test-amd-merge.yml:32 runs `tests/model_executor -m 'core_model and cpu'`, so the AMD lane selects this test. CI shows `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` FAILURE while `buildkite/vllm-omni` and `buildkite/vllm-omni-intel-ci` pass. Confirm whether the two failing lanes are red on main or broke on this PR, and attach the failing logs before treating the PR as clean. (evidence: tests/model_executor/stage_input_processors/test_hunyuan_image3_bridge.py:48 `def test_ar2diffusion_returns_one_request_payload_for_request_level_batching():`; tests/model_executor/stage_input_processors/test_hunyuan_image3_bridge.py:19 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; .buildkite/test-amd-merge.yml:32 `- "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`) + +`tests/dfx/perf/tests/test_hunyuan_image3_it2i.json:?` [minor] — The diff changes the shared HunyuanImage3 bridge `ar2diffusion` to return a single `dict[str, Any] | None` (vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:99). PR #4849's Validation section lists only `test_image_to_image_alignment_online`; issue #4832 also lists `tests/dfx/perf/scripts/run_diffusion_benchmark.py::test_diffusion_performance_benchmark[0-test_hunyuan_image3_ti2i_inline]`, and tests/dfx/perf/tests/test_hunyuan_image3_it2i.json:3 names that node. Please run that benchmark node (or explicitly track it separately) before claiming `Fixes #4832`. (evidence: vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:99 `) -> dict[str, Any] | None:` +tests/dfx/perf/tests/test_hunyuan_image3_it2i.json:3 ` "test_name": "test_hunyuan_image3_ti2i_inline",` +PR #4849 body: `- \`test_image_to_image_alignment_online\` passed without the list-prompt error.` (Validation lists no benchmark node; only compileall/git diff --check accompany it) +Issue #4832 body: `**Test nodes:** ... \`tests/dfx/perf/scripts/run_diffusion_benchmark.py::test_diffusion_performance_benchmark[0-test_hunyuan_image3_ti2i_inline]\`` and `The diffusion performance benchmark also completed \`0/96\` requests ... then failed the completion assertion.`) + +`vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:?` [minor] — The new `if isinstance(prompt, list): original_prompt = prompt[0] if prompt else {}` path (vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:123-124) silently discards prompt entries 1..N for a multi-entry list, whereas the old loop mapped each prompt entry to its output. Reject `len(prompt) > 1` at the bridge boundary or remove `list` from the signature so the stated contract ('genuine multi-request lists remain rejected') is enforced instead of truncating. (evidence: vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:123-124 `if isinstance(prompt, list):\n original_prompt = prompt[0] if prompt else {}`) + +`tests/engine/test_orchestrator_stage_input_bridge.py:?` [minor] — The diff changes `ar2diffusion` to return `dict | None` (vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:99), and the orchestrator's diffusion branch still contains list-specific handling at vllm_omni/engine/orchestrator.py:1264 after building `diffusion_source_outputs = [output, *companion_outputs]` at line 1211. tests/engine/test_orchestrator_stage_input_bridge.py:30 only drives `stage_type = "llm"` and never exercises this diffusion path, so the new dict-return contract lacks an orchestrator-level integration test. Add a `stage_type="diffusion"` fake whose `custom_process_input_func` returns the dict for `[parent, *companions]`, and assert `submit_initial` receives the dict unchanged without list iteration. (evidence: vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:99 `) -> dict[str, Any] | None:`; vllm_omni/engine/orchestrator.py:1211 `diffusion_source_outputs = [output, *companion_outputs]`; vllm_omni/engine/orchestrator.py:1264 `if isinstance(diffusion_prompt, list):`; tests/engine/test_orchestrator_stage_input_bridge.py:30 `self.stage_type = "llm"`) + +`vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:97` [nit] — The rewritten `ar2diffusion` still declares `requires_multimodal_data: bool = False` (vllm_omni/model_executor/stage_input_processors/hunyuan_image3.py:97) and its docstring says it controls whether multimodal data is forwarded (line 108), but the body reads and forwards `multi_modal_data` unconditionally when present (lines 205-211) and never reads the parameter. Guard the forwarding on the flag, or remove the parameter and update the docstring. (evidence: hunyuan_image3.py:97 ` requires_multimodal_data: bool = False,`; hunyuan_image3.py:108 ` requires_multimodal_data: Whether to forward multimodal data.`; hunyuan_image3.py:205 ` mm_data = original_prompt.get("multi_modal_data")`; hunyuan_image3.py:211 ` diffusion_input["multi_modal_data"] = {"image": prompt_images}` — the parameter is never read anywhere in the function body.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4849.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_test/pr4849.trace.json.gz new file mode 100644 index 000000000..775e93497 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_test/pr4849.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4954.cost.json b/eval/dataset/arms/copilot_v13_proof_test/pr4954.cost.json new file mode 100644 index 000000000..6ae2f46a5 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4954.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1517.0, + "split": "test", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_test/runs/pr4954/run-20260813-154730-d2dafe", + "recorded_at": "2026-08-13T16:12:43", + "attempts": [ + { + "invocation_id": "inv-ff963c10", + "outcome": "run:run-20260813-154730-d2dafe", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-154730-d2dafe", + "llm_calls": 153, + "input_tokens": 1070904, + "output_tokens": 262912, + "usd": 0.863861, + "source": "spans" + } + ], + "attempt_usd_total": 0.863861, + "llm_calls": 153, + "input_tokens": 1070904, + "output_tokens": 262912, + "usd": 0.863861, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-154730-d2dafe", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8639, + "minutes": 25.22, + "ci_minutes": 0, + "input_tokens": 1070904, + "output_tokens": 262912, + "tool_calls": 264, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.965, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 153, + "by_role": { + "agent": { + "usd": 0.818728, + "input_tokens": 1049315, + "output_tokens": 227237, + "calls": 150 + }, + "reducer": { + "usd": 0.045134, + "input_tokens": 21589, + "output_tokens": 35675, + "calls": 3 + } + }, + "cache_read_tokens": 10574592, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.62, + "gate": 2.16, + "review": 1507.05, + "report": 0.07 + }, + "time_to_first_result_s": 1512.88 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4954.trace.json.gz", + "trace_bytes": 32463 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4954.md b/eval/dataset/arms/copilot_v13_proof_test/pr4954.md new file mode 100644 index 000000000..8021cf118 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_test/pr4954.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4954, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0b2ef4aea1c2) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 11 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read all diff hunks in situ: tests/helpers/assertions.py:19-64 and 628-664, tests/helpers/test_assertions.py (whole), tests/model_executor/models/voxtral_tts/test_tts_preprocess.py (whole), vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py:209-221. +- [sweep] Indexed accesses test_tts_preprocess.py:60 and :77 (`embed_multimodal_calls[0]`) are each preceded by `assert len(model.model.embed_multimodal_calls) == 1` (lines 59 and 76), so the index is safe. +- [sweep] New branches assertions.py:51/56 guard empty preprocessed text/word lists (reachable and correct); assertions.py:658 is the containment gate; voxtral_tts.py:214 legacy fallback is reachable when `codes` is absent or lacks `audio`. +- [sweep] Confirmed `codes` in `model_intermediate_buffer` is always a plain dict (gpu_model_runner.py `_update_intermediate_buffer` builds `existing.setdefault(k, {})`), so `isinstance(codes, Mapping)` holds for the feedback path. +- [validated] Central fix: producer writes nested codes at voxtral_tts_audio_generation.py:1062-1063 (`audio_list = list(torch.split(audio_codes.unsqueeze(1), 1, dim=0))`; `mm_tokens = {"codes": {"audio": audio_list}}`), and the new voxtral_tts.py:212-213 reads the same nested key. +- [validated] Legacy path preserved: voxtral_tts.py:217 still falls back to `info_dict.pop("audio", None)`; test_tts_preprocess_keeps_legacy_top_level_audio_feedback exercises it. +- [validated] Deploy config voxtral_tts.yaml sets `enable_prefix_caching: false` for both stages, so the postprocess/preprocess feedback loop is not affected by the flattened prefix-cache multimodal payload path. +- [sweep] Shared-helper blast radius: `assert_audio_speech_response` is called from tests/helpers/runtime.py:1888/1933/1970 and tests/e2e/offline_inference/test_glm_tts_expansion.py:131, so the new fallback applies to all speech models; only Voxtral e2e was validated in the PR. + +`tests/helpers/test_assertions.py:9` [major] — tests/helpers/test_assertions.py has no pytestmark, so both new tests are deselected by the `-m 'core_model and cpu'` expression in the 'Simple · Other Test' lane (.buildkite/test-merge.yml:29). Repo-wide grep found no pytest_collection_modifyitems/add_marker hook that auto-marks unmarked tests, so nothing compensates for the omission. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (as tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:10 does) so the new assertion tests actually run in CI. (evidence: ['.buildkite/test-merge.yml:29 "timeout 20m pytest -sv -m \'core_model and cpu\' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"', 'tests/helpers/test_assertions.py:9 `def test_short_transcript_repeat_passes_containment_fallback():` — the new 24-line file has only `import pytest` plus two unmarked tests; there is no `pytestmark` line.', 'tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:10 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`']) + +`tests/helpers/test_assertions.py:9` [major] — tests/helpers/test_assertions.py has no `pytestmark`, and the only merge-pipeline lane that could collect it, 'Simple · Other Test' (.buildkite/test-merge.yml:29), selects with `-m 'core_model and cpu'`, so both new tests are deselected and CI never runs them. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (both registered in pyproject.toml) or wire the file into a path-based lane; the PR body's `4 passed` only proves a local run. (evidence: tests/helpers/test_assertions.py:1-25 (whole file) contains only `# SPDX...`, `import pytest`, `from tests.helpers.assertions import _assert_transcript_matches`, `def test_short_transcript_repeat_passes_containment_fallback():` and `def test_short_transcript_unrelated_text_still_fails():` — no `pytestmark` assignment anywhere. .buildkite/test-merge.yml:29 `- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`. .buildkite/test-merge.yml:13 `- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (the other simple lane only targets those two dirs). pyproject.toml:236 `"core_model: L1&L2 tests (run in each PR)",` and the markers block also registers `"cpu: Tests that run on CPU",`.) + +`tests/helpers/test_assertions.py:9` [major] — `tests/helpers/test_assertions.py` (tests at lines 9 and 16) is not collected by the merge gate: `.buildkite/test-merge.yml:29` selects `pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor`, and this file has no `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. Add that marker (as `tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:11` does) so the containment-fallback branch is actually exercised in CI. (evidence: ['.buildkite/test-merge.yml:29 ` - "timeout 20m pytest -sv -m \'core_model and cpu\' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`', 'tests/helpers/test_assertions.py:4 `import pytest` (file read shows no `pytestmark` line anywhere in the file)', 'tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:11 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`']) + +`tests/helpers/assertions.py:?` [major] — tests/helpers/assertions.py:658 — after a cosine miss, `_short_transcript_contains_expected` can accept a negated transcript. With transcript='not how are you' and expected='how are you', `preprocess_text` leaves both texts intact, the short-text (12<=15) and word-count (4<=3+2) guards pass, and 'how are you' in 'not how are you' is True, so the fallback passes a transcript that contradicts the expected text. The substring check has no word-boundary or negation handling. (evidence: tests/helpers/assertions.py:59 ` short_text = min(len(transcript_clean), len(expected_clean)) <= 15`; tests/helpers/assertions.py:60 ` small_word_delta = len(transcript_words) <= len(expected_words) + 2`; tests/helpers/assertions.py:61 ` return short_text and small_word_delta and expected_clean in transcript_clean`; tests/helpers/assertions.py:658-660 ` if _short_transcript_contains_expected(transcript, expected):` / ` print("short speech containment check passed")` / ` return`) + +`tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:51` [minor] — Confirmed. The test feeds a 2D `[1, 4]` tensor as `codes.audio` (test_tts_preprocess.py:51), while the real producer emits 3D `[1, 1, C]` frames (`compute_mm_logits` → `torch.split(audio_codes.unsqueeze(1), 1, dim=0)`), and the real `embed_multimodal` asserts `audio_tokens.dim() == 3` (voxtral_tts_audio_generation.py:1023). `FakeAudioGeneration.embed_multimodal` only records kwargs, so the test never exercises the shape contract and would pass with a shape the real model rejects. Use the production `[1, 1, C]` shape or drive `_parse_and_validate_audio_arrays`. Minor. (evidence: tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:51 ` audio_tokens = torch.tensor([[1, 2, 3, 4]])`; tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:32-34 ` def embed_multimodal(self, **kwargs):` / ` self.embed_multimodal_calls.append(kwargs)` / ` return [torch.full((1, 4), 7.0)]`; vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py:1023 ` assert audio_tokens.dim() == 3, f"{audio_tokens.ndim=} {audio_tokens.shape=}"`; vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py:1062 ` audio_list = list(torch.split(audio_codes.unsqueeze(1), 1, dim=0))`) + +`tests/helpers/assertions.py:658` [minor] — This containment fallback is a repo-wide relaxation of the speech transcript gate: _assert_transcript_matches is reached from assert_audio_speech_response (tests/helpers/assertions.py:738) for every TTS model that uses the shared speech helper, not just Voxtral. The PR verification only runs Voxtral's test_speech_english_basic plus the new direct unit tests, so no other model's short-clip gate has been re-validated under the new pass condition. Either scope the fallback to the flaky case or run one short-utterance speech test for another model to show the loosened gate does not mask a regression. (evidence: tests/helpers/assertions.py:658 ` if _short_transcript_contains_expected(transcript, expected):` followed by `print("short speech containment check passed")` and `return` — the fallback lives inside the shared `_assert_transcript_matches`. tests/helpers/assertions.py:738 ` _assert_transcript_matches(` with `threshold=0.9` and `escalation_model=request_config.get("transcript_escalation_model")` inside `assert_audio_speech_response`. tests/helpers/runtime.py:1888 ` assert_audio_speech_response(omni_resp, request_config, run_level=self.run_level)`. tests/e2e/offline_inference/test_glm_tts_expansion.py:131 ` assert_audio_speech_response(` passing `run_level="full_model"` with `{"input": synth_text, "response_format": "wav"}`.) + +`tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:10` [minor] — CI shows `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` FAILURE while `buildkite/vllm-omni` is SUCCESS. This PR adds `tests/model_executor/models/voxtral_tts/test_tts_preprocess.py` (marked `core_model`, `cpu`) and changes shared helper `tests/helpers/assertions.py`, so the AMD/NPU lanes may plausibly run this code. Please confirm the red lanes are pre-existing on main or link the failing logs, since the logs are not visible here. (evidence: tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:10 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; gh_ci_read(4954) returned `{"name": "buildkite/vllm-omni-amd-ci", "state": "FAILURE"}`, `{"name": "buildkite/vllm-omni-npu-ci", "state": "FAILURE"}`, and `{"name": "buildkite/vllm-omni", "state": "SUCCESS"}`.) + +`tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:51` [minor] — This test passes `codes.audio` as a 2-D tensor (`(1, 4)`), but the real producer emits a list of 3-D frames (`(1, 1, C)`) via `compute_mm_logits`, and the real consumer `_parse_and_validate_audio_arrays` asserts `audio_tokens.dim() == 3` for tensor input. FakeAudioGeneration bypasses that assertion, so the test stays green on an invalid shape. Pin the real shape (e.g. `(1, 1, 4)`) or have the fake mirror the dim contract. (evidence: tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:51 `audio_tokens = torch.tensor([[1, 2, 3, 4]])`; vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py:1062 `audio_list = list(torch.split(audio_codes.unsqueeze(1), 1, dim=0))`; vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py:1063 `mm_tokens = {"codes": {"audio": audio_list}}`; vllm_omni/model_executor/models/voxtral_tts/voxtral_tts_audio_generation.py:1023 `assert audio_tokens.dim() == 3, f"{audio_tokens.ndim=} {audio_tokens.shape=}"`; tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:32-34 `def embed_multimodal(self, **kwargs):` / `self.embed_multimodal_calls.append(kwargs)` / `return [torch.full((1, 4), 7.0)]`) + +**Additional observations (beyond the comment budget):** +- `tests/model_executor/models/voxtral_tts/test_tts_preprocess.py:10` [minor] CI for this PR: `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are FAILURE while `buildkite/vllm-omni` is SUCCESS. The new file is CPU/core_model-marked (tests/model_executor/models/voxtral_tts/test_tts_pr +- `tests/helpers/assertions.py:59` [minor] Confirmed: `_short_transcript_contains_expected` (tests/helpers/assertions.py:59-61) is not language-aware. `preprocess_text` strips whitespace between CJK characters (tests/helpers/media.py:556), so a CJK transcript `.s +- `vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py:213` [minor] This hunk switches tts_preprocess to consume nested `codes.audio` (voxtral_tts.py:213). PR #4954's e2e verification runs only `test_speech_english_basic`, while issue #4949 reports five failing expansion tests (`test_spe + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_test/pr4954.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_test/pr4954.trace.json.gz new file mode 100644 index 000000000..ed1c38277 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_test/pr4954.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_val/manifest.json b/eval/dataset/arms/copilot_v13_proof_val/manifest.json new file mode 100644 index 000000000..c4e7ae70d --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v13_proof_val", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 5, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4810.cost.json b/eval/dataset/arms/copilot_v13_proof_val/pr4810.cost.json new file mode 100644 index 000000000..148be6d9a --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4810.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1540.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_val/runs/pr4810/run-20260813-150851-b382ad", + "recorded_at": "2026-08-13T15:34:28", + "attempts": [ + { + "invocation_id": "inv-5a3df287", + "outcome": "run:run-20260813-150851-b382ad", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-150851-b382ad", + "llm_calls": 148, + "input_tokens": 1325930, + "output_tokens": 266573, + "usd": 1.02102, + "source": "spans" + } + ], + "attempt_usd_total": 1.02102, + "llm_calls": 148, + "input_tokens": 1325930, + "output_tokens": 266573, + "usd": 1.02102, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-150851-b382ad", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.021, + "minutes": 25.61, + "ci_minutes": 0, + "input_tokens": 1325930, + "output_tokens": 266573, + "tool_calls": 255, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0256, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 148, + "by_role": { + "agent": { + "usd": 0.996112, + "input_tokens": 1310850, + "output_tokens": 247662, + "calls": 146 + }, + "reducer": { + "usd": 0.024908, + "input_tokens": 15080, + "output_tokens": 18911, + "calls": 2 + } + }, + "cache_read_tokens": 13695872, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.9, + "gate": 1.64, + "review": 1530.68, + "report": 0.07 + }, + "time_to_first_result_s": 1536.26 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 32497 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4810.md b/eval/dataset/arms/copilot_v13_proof_val/pr4810.md new file mode 100644 index 000000000..bf69eebce --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4810.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] grep 'get_cache_scale' in vllm_omni -> hits: diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 (stale removed-API call), mimo_audio_llm.py:1157 and mammoth_moda2.py:487 (new get_cache_scale_mapper calls only). +- [sweep] grep 'get_cache_scale_mapper' in vllm_omni -> only the two new direct-loader call sites (mammoth_moda2.py:487, mimo_audio_llm.py:1157); no other direct loader applies the mapper. +- [validated] hunyuan_image3.py:222-226 removed get_cache_scale branch; outer HunyuanImage3ForConditionalGeneration.load_weights at 2178 uses AutoWeightsLoader, so HunyuanModel receives mapped names. +- [validated] mammoth_moda2.py:483-489 applies get_cache_scale_mapper before the loop; test_direct_custom_loaders_apply_cache_scale_mapper asserts mapper.called and mapped load. +- [validated] mimo_audio_llm.py:1153-1160 applies mapper before the loop; outer MiMoAudioForConditionalGeneration.load_weights (mimo_audio.py:701-714) routes model-prefixed weights to this loader. +- [validated] qwen2_old.py:335-340 removed the getattr(get_cache_scale) branch; parent Qwen2ForCausalLM.load_weights at 443-448 uses AutoWeightsLoader. +- [sweep] read all four changed hunks in situ plus the new test file; test has pytestmark=[core_model,cpu] so it is marker-selected. +- [sweep] registry.py:175 maps MiMoAudioLLMModel -> mimo_audio_llm.MiMoAudioLLMForConditionalGeneration, confirming the changed MiMo loader is a registered direct-load path. + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:?` [major] — The PR's purpose is to remove every `get_cache_scale` call, but `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` still calls `self.quant_config.get_cache_scale(name)` inside `HunyuanImage3Model.load_weights`. `pipeline_hunyuan_image3.py:386` instantiates this model with `quant_config=quant_config` and the pipeline's `load_weights` delegates to it via `AutoWeightsLoader`, so a quantized HunyuanImage3 diffusion checkpoint still hits the removed API and crashes. The new `_STALE_API_FILES` omits this path, so the added regression test passes without covering it. Fix the diffusion loader (use `get_cache_scale_mapper()` or the outer loader) and include it in `_STALE_API_FILES`. (evidence: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 ` if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`; vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py:386 ` self.model = HunyuanImage3Model(self.hf_config, quant_config=quant_config)`; tests/model_executor/models/test_kv_cache_scale_mapper.py:15-20 `_STALE_API_FILES = [` ` "vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py",` ` "vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py",` ` "vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py",` ` "vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py",` `]` — the diffusion transformer path is absent.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:?` [minor] — tests/model_executor/models/test_kv_cache_scale_mapper.py:81 `assert ".get_cache_scale(" not in source` only matches method-call syntax. The removed qwen2_old.py branch was `get_cache_scale = getattr(self.quant_config, "get_cache_scale", None)` followed by `if get_cache_scale is not None and (scale_name := get_cache_scale(name)):`, which contains no `.get_cache_scale(` substring, so the static scan passes for qwen2_old.py both before and after this PR. The Qwen2 behavioral case uses `_QuantConfigWithoutOldCacheScale`, which defines only `get_cache_scale_mapper`; pre-PR, `getattr(self.quant_config, "get_cache_scale", None)` would be `None` and the mapped weight would load via the fall-through path, so this test would not catch a regression. Strengthen the scan (e.g. also reject `getattr(self.quant_config, "get_cache_scale"`) or give the Qwen2 fake quant_config a `get_cache_scale` so the removed branch is actually exercised. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:81 `assert ".get_cache_scale(" not in source`; tests/model_executor/models/test_kv_cache_scale_mapper.py:39-43 `_QuantConfigWithoutOldCacheScale` defines only `get_cache_scale_mapper`; diff-removed qwen2_old.py lines: `get_cache_scale = getattr(self.quant_config, "get_cache_scale", None)` and `if get_cache_scale is not None and (scale_name := get_cache_scale(name)):` — no `.get_cache_scale(` substring, so the static scan and the old-code fall-through both pass.) + +`vllm_omni/model_executor/models/mimo_audio_llm.py:1157` [minor] — This load_weights path now relies on `quant_config.get_cache_scale_mapper()` (external vLLM API) at vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1157, but requirements/*.txt and setup.py declare no vLLM version floor (vLLM is user-provided). Should the PR document or enforce a minimum vLLM version that provides `get_cache_scale_mapper`? (evidence: vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1157 `cache_scale_mapper = self.quant_config.get_cache_scale_mapper()` — the changed load_weights calls the external API. setup.py:244 `install_requires = get_install_requires()` and setup.py:249 `install_requires=install_requires,` — dependencies come only from `requirements/{device}.txt`; `requirements/common.txt` contains no `vllm` requirement, so no floor for the API is declared.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:144` [minor] — The direct-loader regression only feeds a scalar `torch.ones(())` (line 144), and `_RecordingCacheScaleMapper.apply` passes the tensor through unchanged, so it never exercises the dim>0 shape that the removed `loaded_weight.dim() == 0` slicing used to normalize. Add a non-scalar scale input for mammoth_moda2/mimo_audio and a `model.`-prefixed variant for MiMo (the mapper now runs before the `name.startswith("model.")` rewrite), or verify the mapped `.attn.k_scale` weight_loader accepts the full tensor. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:144 ` loaded = load_weights(fake_model, [(_SOURCE_SCALE_NAME, torch.ones(()))])`; tests/model_executor/models/test_kv_cache_scale_mapper.py:36 ` yield name.replace(".k_proj.output_scale", ".attn.k_scale"), weight`; vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:489 ` weights = cache_scale_mapper.apply) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:?` [minor] — The test `test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale` binds `load_weights = getattr(module, class_name).load_weights` (line 110) and calls it directly at line 112 with the already-mapped `_MAPPED_SCALE_NAME`; no `AutoWeightsLoader` is ever constructed in this file (grep for `AutoWeightsLoader`: no matches). It therefore proves only that the delegated sub-loaders accept a pre-mapped name, not that the outer `AutoWeightsLoader` applies `get_cache_scale_mapper` — the premise making removal of the Hunyuan/Qwen branches safe. Drive the top-level `AutoWeightsLoader` path with a source `...k_proj.output_scale` name, or rename the test to state what it actually asserts. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:23 `_MAPPED_SCALE_NAME = "layers.0.self_attn.attn.k_scale"`; tests/model_executor/models/test_kv_cache_scale_mapper.py:110 `load_weights = getattr(module, class_name).load_weights`; tests/model_executor/models/test_kv_cache_scale_mapper.py:112 `loaded = load_weights(fake_model, [(_MAPPED_SCALE_NAME, torch.ones(()))])`; grep `AutoWeightsLoader` in tests/model_executor/models/test_kv_cache_scale_mapper.py returns no matches.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:12` [nit] — The new test file declares `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (line 12), and `.buildkite/test-amd-merge.yml:32` runs `timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml`, which collects this new file in the AMD merge lane while `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are FAILURE and `buildkite/vllm-omni` is SUCCESS. Before merging, confirm from logs or main history that these lanes are pre-existing red; if this new test file is the trigger, adjust test collection or lane selection rather than leaving unexplained red gates. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; .buildkite/test-amd-merge.yml:32 `"timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; gh_ci_read PR 4810: `buildkite/vllm-omni-amd-ci` state FAILURE, `buildkite/vllm-omni-npu-ci` state FAILURE, `buildkite/vllm-omni` state SUCCESS.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:15` [nit] — Keep this CPU regression test as the guard against reintroducing the removed `get_cache_scale` API; the PR body says it will be removed after validation. Once the diffusion transformer path is fixed, add `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py` to `_STALE_API_FILES` (it still calls `get_cache_scale` at line 2238) rather than deleting the coverage. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:15 `_STALE_API_FILES = [`; tests/model_executor/models/test_kv_cache_scale_mapper.py:81 ` assert ".get_cache_scale(" not in source`; vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 ` if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`) + +`vllm_omni/model_executor/models/mammoth_moda2.py:487` [nit] — Could you confirm whether an outer MammothModa2ForConditionalGeneration / AutoWeightsLoader already applies `get_cache_scale_mapper` before reaching MammothModa2Qwen2ForCausalLM.load_weights? If it does, please clarify that the new per-loader mapper call here is not applying the mapping twice. This could not be verified from repo sources; if it remains unverified, please note it as a follow-up. (evidence: [unverified] MammothModa2ARForConditionalGeneration defines no load_weights and inherits from vLLM Qwen2_5_VLForConditionalGeneration, while top-level MammothModa2ForConditionalGeneration.load_weights delegates to `self.ar.load_weights`; whether an outer AutoWeightsLoader already applies the mapper before reaching MammothModa2Qwen2ForCausalLM.load_weights could not be verified from repo sources, so no double-mapper comment was filed.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_val/pr4810.trace.json.gz new file mode 100644 index 000000000..7a3fc97b6 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_val/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4816.cost.json b/eval/dataset/arms/copilot_v13_proof_val/pr4816.cost.json new file mode 100644 index 000000000..226ef170e --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4816.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1979.7, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_val/runs/pr4816/run-20260813-150851-c82d91", + "recorded_at": "2026-08-13T15:41:47", + "attempts": [ + { + "invocation_id": "inv-78e49ca6", + "outcome": "run:run-20260813-150851-c82d91", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-150851-c82d91", + "llm_calls": 109, + "input_tokens": 1012000, + "output_tokens": 180967, + "usd": 0.763727, + "source": "spans" + } + ], + "attempt_usd_total": 0.763727, + "llm_calls": 109, + "input_tokens": 1012000, + "output_tokens": 180967, + "usd": 0.763727, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-150851-c82d91", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7637, + "minutes": 32.94, + "ci_minutes": 0, + "input_tokens": 1012000, + "output_tokens": 180967, + "tool_calls": 183, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0352, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 109, + "by_role": { + "planner": { + "usd": 0.000515, + "input_tokens": 87, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.730978, + "input_tokens": 994731, + "output_tokens": 155537, + "calls": 105 + }, + "reducer": { + "usd": 0.032234, + "input_tokens": 17182, + "output_tokens": 25030, + "calls": 3 + } + }, + "cache_read_tokens": 10793472, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.28, + "gate": 1.65, + "review": 1971.25, + "report": 0.07 + }, + "time_to_first_result_s": 1976.22 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 24035 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4816.md b/eval/dataset/arms/copilot_v13_proof_val/pr4816.md new file mode 100644 index 000000000..da56acd85 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4816.md @@ -0,0 +1,36 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] api_server.py:754,1017,1157,1248,1286,1329 read in situ — pure-diffusion branch sets `serving_tokenization = None`, LLM branch sets `ServingTokenization(...)`, and all four getattr sites use the new name consistently. +- [sweep] grep `openai_serving_tokenization` repo-wide -> no matches; no stale reader/consumer remains. +- [sweep] grep `serving_tokenization` repo-wide -> only api_server.py (6 sites) and tests/entrypoints/openai_api/test_serving_speech.py (3 sites); all updated. +- [sweep] grep `base(raw_request)` -> single caller `_create_speech_error_json_response` at api_server.py:376; grep `instrumentator` -> only the import at api_server.py:67. +- [sweep] test_serving_speech.py:2306-2636 read in situ — `_patch_api_server_base` (2339-2352) patches `api_server_module.base`, so list_voices/upload_voice/delete_voice tests bypass the real base()/state path; only create_speech/batch/audio_generate tests drive the new getattr. +- [validated] test_serving_speech.py:2533,2546,2615 updated tests set `serving_tokenization` and would fail if the create_* getattr rename were reverted. +- [sweep] grep `init_app_state` repo-wide -> only `omni_init_app_state` is used; no upstream `init_app_state` call to reconcile. +- [sweep] pyproject.toml + requirements/common.txt read -> vllm is not pinned in requirements; docs/CI reference vllm==0.24.0 (cuda.inc.md:23), VLLM_VERSION v0.23.0 (.buildkite/pipeline-intel.yaml:13), and vllm==0.22.0 (apps/lvsa-omni-showcase/README.md:35). + +`tests/entrypoints/openai_api/test_serving_speech.py:?` [major] — The three updated tests (2533/2546/2615) only hit the direct `getattr(raw_request.app.state, "serving_tokenization", None)` branches of create_speech/create_speech_batch/create_audio_generate and return `base_server.create_error_response(...)`; they never reach `_create_speech_error_json_response` (call sites begin at api_server.py:1374). The real path — `_create_speech_error_json_response` -> `base(raw_request).create_error_response(...)` (api_server.py:376) — reads app.state via upstream base(), but test_serving_speech.py:2352 mocks `api_server_module.base` for the voice routes, so the rename at api_server.py:754/1017 has no consumer test that would fail if it regressed. Add a regression test that sets `app.state.serving_tokenization` and calls list_voices/upload_voice WITHOUT patching `api_server_module.base`. (evidence: vllm_omni/entrypoints/openai/api_server.py:376 ` err = base(raw_request).create_error_response(` — real upstream base() is the path the fix must feed. vllm_omni/entrypoints/openai/api_server.py:1017 ` state.serving_tokenization = ServingTokenization(` — the renamed assignment with no un-mocked consumer test. tests/entrypoints/openai_api/test_serving_speech.py:2352 ` mocker.patch.object(api_server_module, "base", return_value=fake_base)` — voice-route tests bypass the real base(). tests/entrypoints/openai_api/test_serving_speech.py:2533 ` raw_request.app.state.serving_tokenization = fake_base` — updated test sets state directly and never reaches `_create_speech_error_json_response`.) + +`vllm_omni/entrypoints/openai/api_server.py:?` [minor] — This hunk renames the pure-diffusion slot to `serving_tokenization` but keeps its value `None`. Upstream `base()` (imported at api_server.py:67) now returns `request.app.state.serving_tokenization`, and `_create_speech_error_json_response` calls `base(raw_request).create_error_response(...)` with no None guard, so on a pure-diffusion server every speech error route (`upload_voice`/`delete_voice` ValueError, or `list_voices` with no handler) still raises `AttributeError: 'NoneType' object has no attribute 'create_error_response'` and returns 500 — the exact failure mode this PR is meant to eliminate, only shifted from a missing `State` attribute to `NoneType`. Guard the `base()` result in `_create_speech_error_json_response` (fall back to a tokenization-independent error builder when None), or give diffusion mode a real error-response handler; then add a pure-diffusion test that POSTs an undecodable audio_sample and asserts a 400. (evidence: vllm_omni/entrypoints/openai/api_server.py:754 'state.serving_tokenization = None'; :376 'err = base(raw_request).create_error_response('; :67 'from vllm.entrypoints.serve.instrumentator.basic import base'; :1483 'return _create_speech_error_json_response(raw_request, str(e))' inside upload_voice's `except ValueError`. PR description quotes upstream tokenization() as 'return request.app.state.serving_tokenization'. vllm_omni/entrypoints/openai/serving_speech.py `for_diffusion` (line 416) creates a speech handler that calls `_init_speaker_storage` and its `upload_voice` (line 1214) raises ValueError on undecodable audio, so the None path is reachable in pure diffusion mode.) + +`vllm_omni/entrypoints/openai/api_server.py:1017` [minor] — The rename is internally complete (no `openai_serving_tokenization` references remain; all in-repo readers and the serving_speech tests are updated), but buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, and buildkite/vllm-omni-npu-ci are FAILURE while buildkite/vllm-omni is SUCCESS. Before merging, confirm those three reds are pre-existing on main and not caused by this shared-entrypoint rename. (evidence: vllm_omni/entrypoints/openai/api_server.py:1017 ` state.serving_tokenization = ServingTokenization(` — writer renamed; vllm_omni/entrypoints/openai/api_server.py:1157 ` base_server = getattr(raw_request.app.state, "serving_tokenization", None)` — reader updated; tests/entrypoints/openai_api/test_serving_speech.py:2533 ` raw_request.app.state.serving_tokenization = fake_base` — test updated; repo-wide grep for `openai_serving_tokenization` returns no matches.) + +`vllm_omni/entrypoints/openai/api_server.py:?` [minor] — This rename only works when the installed vLLM's `vllm/entrypoints/serve/instrumentator/basic.py` reads `request.app.state.serving_tokenization` (upstream PR #46022). vLLM is not pinned in requirements, while docs/CI reference several versions (vllm==0.24.0, VLLM_VERSION v0.23.0, vllm==0.22.0). On any of those that predate #46022 the old `base()` reader (`request.app.state.openai_serving_tokenization`) now misses and `_create_speech_error_json_response` regresses to the same AttributeError. Verify the minimum supported vLLM revision and pin/document it. (evidence: requirements/common.txt contains no vllm dependency (grep over requirements/ only hits a comment); docs/getting_started/installation/gpu/cuda.inc.md:23 `uv pip install vllm==0.24.0 --torch-backend=auto`; .buildkite/pipeline-intel.yaml:13 `VLLM_VERSION: "v0.23.0"`; apps/lvsa-omni-showcase/README.md:35 `pip install "vllm==0.22.0"`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_val/pr4816.trace.json.gz new file mode 100644 index 000000000..a39373c8a Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_val/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4825.cost.json b/eval/dataset/arms/copilot_v13_proof_val/pr4825.cost.json new file mode 100644 index 000000000..2fd3c0def --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4825.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1052.5, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_val/runs/pr4825/run-20260813-150849-f93cce", + "recorded_at": "2026-08-13T15:26:20", + "attempts": [ + { + "invocation_id": "inv-858057cc", + "outcome": "run:run-20260813-150849-f93cce", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-150849-f93cce", + "llm_calls": 121, + "input_tokens": 707195, + "output_tokens": 195578, + "usd": 0.600237, + "source": "spans" + } + ], + "attempt_usd_total": 0.600237, + "llm_calls": 121, + "input_tokens": 707195, + "output_tokens": 195578, + "usd": 0.600237, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-150849-f93cce", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6002, + "minutes": 17.51, + "ci_minutes": 0, + "input_tokens": 707195, + "output_tokens": 195578, + "tool_calls": 231, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7334, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 121, + "by_role": { + "agent": { + "usd": 0.57778, + "input_tokens": 695266, + "output_tokens": 178122, + "calls": 119 + }, + "reducer": { + "usd": 0.022457, + "input_tokens": 11929, + "output_tokens": 17456, + "calls": 2 + } + }, + "cache_read_tokens": 7191040, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.61, + "gate": 1.61, + "review": 1045.14, + "report": 0.07 + }, + "time_to_first_result_s": 1050.41 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 30778 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4825.md b/eval/dataset/arms/copilot_v13_proof_val/pr4825.md new file mode 100644 index 000000000..48776047f --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4825.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4825, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac3a2ce5a9b9) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read the production hunk in situ (vllm_omni/diffusion/lora/manager.py:368-383); the only production change is adding `"unet"` to `default_components`, no other logic is modified. +- [sweep] Read the added test in situ (tests/diffusion/lora/test_lora_manager.py:580-619); the `if isinstance(layer, _FakeLinearBase)` branch at line 587 is reachable for the `proj` leaf and intentionally not for plain nn.Module containers. +- [validated] The new test is a real regression guard: without `"unet"` in `default_components`, the loop would never scan `pipeline.unet`, leaving `replace_calls`/`_lora_modules` empty and failing the asserts at lines 616-617. +- [validated] Blast radius in-tree: `self.unet` at top level exists only in `pipeline_sdxl.py:110`; the `ovis_image`/`flux2_klein` `unet` hits are docstring text only, and `soulx_singer/rmvpe.py:236` is a nested submodule, not a top-level pipeline attribute. +- [validated] The `_lora_components` opt-in path (sensenova_u1 `pipeline_sensenova_u1.py:543`) is unaffected; a duplicate component in defaults+opt-ins would be skipped by the `full_module_name in self._lora_modules` guard. +- [validated] SDXL packed QKV is wired: `SDXLUNet2DConditionModel.load_weights` sets `stacked_params_mapping` (sdxl_unet.py:823-829) during model load, before `DiffusionLoRAManager` init (diffusion_worker.py `load_model` then `init_lora_manager`), so `_compute_packed_modules_mapping` derives `to_qkv -> to_q/to_k/to_v`. +- [sweep] All `changed_symbol_consumers` references (`_fake_from_layer_diffusion`, `test_lora_manager_discovers_unet_component`, `_matches_target`) resolve inside the touched files; no stale in-repo caller of the changed behavior found. +- [sweep] CI attribution: `gh_ci_read` shows `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` FAILURE; build(3.11/3.12), pre-commit, DCO, buildkite/vllm-omni, intel-ci and docs pass. + +`vllm_omni/diffusion/lora/manager.py:380` [major] — Confirmed at manager.py:380. Adding `"unet"` to default_components routes SDXL `unet.*` names through `_match_target_modules`, but utils.py:25 suffix-matches `.*\.to_out$`; a PEFT `target_modules` list entry `to_out` (without `.0`) therefore matches the `nn.ModuleList` container `...attn1.to_out` while the actual `RowParallelLinear` at `...attn1.to_out.0` fails the match. `from_layer_diffusion` returns the container unchanged, so `to_out` is silently skipped while `to_q/to_k/to_v` are wrapped — a partially-applied adapter with no error. Normalize `to_out.0`/`to_out` before matching, or explicitly reject/document the `to_out.0` spelling, and add a regression test using real SDXL `unet.*.attn1.to_out.0` names. (evidence: ['vllm_omni/diffusion/lora/manager.py:380 ` "unet",` — newly added to default_components, enabling SDXL unet.* matching.', 'vllm_omni/diffusion/lora/utils.py:25 ` re.match(rf".*\\.{target_module}$", module_name) or target_module == module_name` — suffix matcher that matches `...to_out` but not `...to_out.0`.', 'vllm_omni/diffusion/models/sdxl/sdxl_unet.py:151-153 ` self.to_out = nn.ModuleList(` / ` [` / ` RowParallelLinear(inner_dim, dim, bias=True),` — the real linear layer lives at `.to_out.0`, while the container occupies `.to_out`.']) + +`vllm_omni/diffusion/lora/manager.py:380` [minor] — Concrete change: the diff adds "unet" to the hardcoded default_components tuple in vllm_omni/diffusion/lora/manager.py (lines 375-381). This tuple is now another in-repo enumeration of denoiser components alongside vllm_omni/diffusion/registry.py:427 (transformer_attrs = ["transformer", "transformer_2", "dit", "unet"], SP-scoped) and vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py:46 (_dit_modules = ["unet"], per-pipeline). The bug being fixed was exactly a component missing from this parallel hardcoded list, so a separate manual enumeration invites the same drift. Where the mapping allows, derive scanned components from a shared source; otherwise pin the lists together with a comment/test (non-blocking). (evidence: vllm_omni/diffusion/lora/manager.py:375-381 `default_components = (` / `"transformer",` / `"transformer_2",` / `"dit",` / `"bagel",` / `"unet",` / `)`. vllm_omni/diffusion/registry.py:427 `transformer_attrs = ["transformer", "transformer_2", "dit", "unet"]`. vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py:46 `_dit_modules: list[str] = ["unet"]`.) + +`vllm_omni/diffusion/lora/manager.py:380` [minor] — Concrete change confirmed: `vllm_omni/diffusion/lora/manager.py:380` adds `"unet"` to `default_components`. CI read shows `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` as FAILURE while `buildkite/vllm-omni` is SUCCESS. The AMD lane runs `pytest -s -v tests/diffusion/lora/test_images_generations_lora.py` (.buildkite/test-amd.yaml:23), the NPU-ready lane runs `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'` (.buildkite/test-npu-ready.yaml:21), and the merge lane runs the touched CPU unit test via `timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' ...` (.buildkite/test-amd-merge.yml:21). These failures may be pre-existing and should be attributed against main before treating them as blockers. (evidence: vllm_omni/diffusion/lora/manager.py:380 ` "unet",`; .buildkite/test-amd.yaml:23 ` - pytest -s -v tests/diffusion/lora/test_images_generations_lora.py`; .buildkite/test-npu-ready.yaml:21 ` - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; .buildkite/test-amd-merge.yml:21 ` - "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; gh_ci_read(4825): `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`, `{"bucket":"fail","name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}`, `{"bucket":"pass","name":"buildkite/vllm-omni","state":"SUCCESS"}`.) + +`tests/diffusion/lora/test_lora_manager.py:580` [minor] — New test `test_lora_manager_discovers_unet_component` (`tests/diffusion/lora/test_lora_manager.py:580`) uses `peft_helper = type("_PH", (), {"r": 1})()` (`:613`) with no `target_modules`, so `_matches_target` takes the `target_modules_list is None -> return True` branch and replaces every `_FakeLinearBase` under `unet`; it exercises only the component-scan plumbing, not the `target_modules`-filtered path (`_match_target_modules`) that real SDXL adapters (e.g. `to_q`/`to_k`/`to_v`/`to_out.0`) rely on. The only LoRA e2e (`tests/diffusion/lora/test_images_generations_lora.py:30,57`) targets Z-Image `transformer.layers.0.attention.to_qkv` and never touches `unet`. Minor: consider adding an SDXL/unet test that sets `target_modules` (including a `to_out` variant) or an SDXL LoRA e2e to prove the claimed fix. (evidence: ['tests/diffusion/lora/test_lora_manager.py:613 `peft_helper = type("_PH", (), {"r": 1})()` — helper has no `target_modules`, so the manager\'s `target_modules_list` remains `None`.', 'tests/diffusion/lora/test_lora_manager.py:580 `def test_lora_manager_discovers_unet_component(monkeypatch):`', 'vllm_omni/diffusion/lora/manager.py:380 ` "unet",`', 'vllm_omni/diffusion/lora/manager.py (inside `_replace_layers_with_lora`): `if target_modules_list is None:\n return True\n return _match_target_modules(module_name, target_modules_list)` — proves all modules match when no `target_modules` is present.', 'tests/diffusion/lora/test_images_generations_lora.py:30 `MODEL = "Tongyi-MAI/Z-Image-Turbo"`', 'tests/diffusion/lora/test_images_generations_lora.py:57 ` module_name = "transformer.layers.0.attention.to_qkv"`']) + +`vllm_omni/diffusion/lora/manager.py:376` [minor] — manager.py:376-382: `unet` is now in the global `default_components` tuple, and `_replace_layers_with_lora` iterates it for every pipeline via `hasattr(self.pipeline, component_name)` with no opt-out for out-of-tree pipelines. Please confirm custom pipelines with a top-level `unet` are expected to be scanned, or document how they can opt out when `unet` is not a denoiser. (evidence: vllm_omni/diffusion/lora/manager.py:374 `default_components = (` ... vllm_omni/diffusion/lora/manager.py:380 ` "unet",` vllm_omni/diffusion/lora/manager.py:382 ` extra_components = tuple(getattr(self.pipeline, "_lora_components", ()) or ())` vllm_omni/diffusion/lora/manager.py:383 ` for component_name in (*default_components, *extra_components):` vllm_omni/diffusion/lora/manager.py:384 ` if not hasattr(self.pipeline, component_name):`) + +`tests/diffusion/lora/test_lora_manager.py:580` [nit] — The proposed/actual SDXL LoRA e2e uses `cosmic-tf/sdxl_finetuned_8styles`, but this adapter's config could not be fetched. In this repo SDXL FFN modules are `ff.geglu.proj`/`ff.out_proj`, while diffusers adapters commonly target `ff.net.0.proj`/`ff.net.2`; if the linked adapter uses those names, only attention layers may actually match, weakening the e2e. Please confirm the adapter's `target_modules` or select an adapter that matches SDXL FFN naming so the added SDXL path is meaningfully covered. (evidence: Could not fetch the linked adapter `cosmic-tf/sdxl_finetuned_8styles` config; SDXL FFN naming here is `ff.geglu.proj`/`ff.out_proj` while diffusers adapters commonly target `ff.net.0.proj`/`ff.net.2`, so only attention layers may actually match. Recorded as residual risk, not a review comment.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_val/pr4825.trace.json.gz new file mode 100644 index 000000000..46ce4f22f Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_val/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4837.cost.json b/eval/dataset/arms/copilot_v13_proof_val/pr4837.cost.json new file mode 100644 index 000000000..27d4cafa8 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4837.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1179.2, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_val/runs/pr4837/run-20260813-150850-f17b75", + "recorded_at": "2026-08-13T15:28:27", + "attempts": [ + { + "invocation_id": "inv-c06c2a1e", + "outcome": "run:run-20260813-150850-f17b75", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-150850-f17b75", + "llm_calls": 151, + "input_tokens": 1081831, + "output_tokens": 206241, + "usd": 0.812744, + "source": "spans" + } + ], + "attempt_usd_total": 0.812744, + "llm_calls": 151, + "input_tokens": 1081831, + "output_tokens": 206241, + "usd": 0.812744, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-150850-f17b75", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8127, + "minutes": 19.61, + "ci_minutes": 0, + "input_tokens": 1081831, + "output_tokens": 206241, + "tool_calls": 238, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8516, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 151, + "by_role": { + "agent": { + "usd": 0.792538, + "input_tokens": 1072116, + "output_tokens": 190291, + "calls": 149 + }, + "reducer": { + "usd": 0.020206, + "input_tokens": 9715, + "output_tokens": 15950, + "calls": 2 + } + }, + "cache_read_tokens": 10880896, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.91, + "gate": 2.36, + "review": 1170.31, + "report": 0.07 + }, + "time_to_first_result_s": 1176.63 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 28443 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4837.md b/eval/dataset/arms/copilot_v13_proof_val/pr4837.md new file mode 100644 index 000000000..418df85bf --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4837.md @@ -0,0 +1,45 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4837, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] Read orchestrator.py:1178-1310 in situ; confirmed the new branch `if len(diffusion_prompt) == 1:` (line 1290) unwraps single-element lists for BOTH `submit_update` and `submit_initial` paths. +- [sweep] Read StagePool.submit_initial (stage_pool.py:932) and submit_update (stage_pool.py:1001); both raise ValueError on `isinstance(request, list)`, so the unwrap is required on the initial path and now happens there. +- [sweep] Read thinker2imagegen (ming_flash_omni.py:346); it returns exactly `[{"prompt": "", "extra": extra}]`, so `len(diffusion_prompt) == 1` covers the reported Ming-flash-omni case. +- [sweep] Grepped all `custom_process_input_func` assignments repo-wide; the len>1 list case for diffusion stages remains rejected by StagePool, which is the intended 'no diffusion list batching' contract. +- [sweep] Read deploy/ming_flash_omni_image.yaml; async_chunk=false and a single diffusion stage, so the initial-submit path is the one the issue exercised. +- [validated] pipeline_z_image.py:720 returns `DiffusionOutput(output=image, stage_durations=stage_durations)`, so the new `return outputs` preserves `stage_durations` plus `error/finished/peak_memory_mb/chunk_index/total_chunks` that the old `DiffusionOutput(output=raw)` dropped. +- [sweep] Read tests/engine/test_orchestrator_stage_input_bridge.py and tests/engine/test_orchestrator_kv_sender_info.py; neither drives a non-empty single-element list through the initial-submit diffusion path (kv_sender_info covers empty-list error and plain dict prompt only). +- [sweep] gh_ci_read for PR 4837: pre-commit/build/DCO/buildkite/vllm-omni/intel/docs pass; buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE — recorded as a minor pre-existing-vs-introduced question, not a blocker. + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449` [blocker] — MingImagePipeline.forward calls super().forward(z_req, prompt=None, height=..., width=..., num_inference_steps=..., guidance_scale=..., prompt_embeds=..., negative_prompt_embeds=..., generator=..., output_type="pt", return_dict=True) at pipeline_ming_imagegen.py:449-462, but the inherited ZImagePipeline.forward signature is def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput: (pipeline_z_image.py:413). That call raises TypeError: ZImagePipeline.forward() got an unexpected keyword argument 'prompt' on the first image-gen request. Additionally, _ZPipelineSamplingParams (pipeline_ming_imagegen.py:63-76) lacks latents, cfg_normalize, extra_args, and output_type, which ZImagePipeline.forward reads at pipeline_z_image.py:464-471, so fixing the signature alone still raises AttributeError. Pass only z_req and add those fields; then rerun the serve+curl smoke. (evidence: vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413 `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:`; vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449 `outputs: DiffusionOutput = super().forward(` with keyword args `prompt=None, height=height, width=width, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, generator=generator, output_type="pt", return_dict=True`; vllm_omni/diffusion/models/z_image/pipeline_z_image.py:464 `latents = req.sampling_params.latents`, :466 `cfg_normalization = req.sampling_params.cfg_normalize`, :467 `cfg_truncation = req.sampling_params.extra_args.get("cfg_truncation", 1.0)`, :471 `output_type = req.sampling_params.output_type or "pil"`.) + +`vllm_omni/engine/orchestrator.py:?` [minor] — The removal of `already_submitted and` is the intended fix: a single-element diffusion bridge list is now unwrapped on the initial submit path too, matching the update path and avoiding stage_pool's list-prompt ValueError. The change is correct but untested. Add a regression test in tests/engine/test_orchestrator_kv_sender_info.py whose custom_process_input_func returns [{"prompt": ""}] and assert diffusion_stage.calls[0]["prompt"] is the unwrapped dict, not a list. (evidence: vllm_omni/engine/orchestrator.py:1290 `if len(diffusion_prompt) == 1:` / vllm_omni/engine/orchestrator.py:1291 `diffusion_prompt = diffusion_prompt[0]` — runs before vllm_omni/engine/orchestrator.py:1295 `if already_submitted:`, so it applies to both initial submit and update paths. tests/engine/test_orchestrator_kv_sender_info.py:196 `diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: []` — the only non-None custom_process_input_func test case in tests/engine; all other files set it to None.) + +`vllm_omni/engine/orchestrator.py:1290` [minor] — Confirmed: with `already_submitted` False, the unwrapped single element is now passed to `submit_initial` (orchestrator.py:1290-1291, 1296-1302), so the len==1 unwrap runs for every diffusion bridge with a custom_process_input_func, not just the Ming path. Previously single-element lists reached initial submit as lists; multi-element lists still pass through unchanged. Please confirm the broader blast radius is intended. (evidence: ['vllm_omni/engine/orchestrator.py:1290 `if len(diffusion_prompt) == 1:`', 'vllm_omni/engine/orchestrator.py:1291 `diffusion_prompt = diffusion_prompt[0]`', 'vllm_omni/engine/orchestrator.py:1296 `await next_pool.submit_update(req_id, req_state, diffusion_prompt)`', 'vllm_omni/engine/orchestrator.py:1301 `diffusion_prompt,` (argument to `submit_initial` when `already_submitted` is False)']) + +`vllm_omni/engine/orchestrator.py:1290` [minor] — Confirmed: PR 4837 CI shows buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci as FAILURE while buildkite/vllm-omni and buildkite/vllm-omni-intel-ci pass. This is a triage question only (pre-existing vs introduced), correctly rated minor and non-blocking. (evidence: orchestrator.py:1290 `if len(diffusion_prompt) == 1:` (the diff's changed line; line 1291 is `diffusion_prompt = diffusion_prompt[0]`). gh_ci_read PR 4837 returned: pre-commit SUCCESS, build (3.12) SUCCESS, build (3.11) SUCCESS, DCO SUCCESS, buildkite/vllm-omni-amd-ci FAILURE, buildkite/vllm-omni-npu-ci FAILURE, buildkite/vllm-omni SUCCESS, buildkite/vllm-omni-intel-ci SUCCESS, docs/readthedocs.org:vllm-omni SUCCESS.) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:411` [minor] — pipeline_ming_imagegen.py:411-413 — the comment 'matches ZImagePipeline's contract when prompt_embeds are pre-computed' is stale: ZImagePipeline.forward (pipeline_z_image.py:413) now only accepts `req` and has no `prompt_embeds` parameter on the forward path. Update or remove the comment. (evidence: vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:411-413 + # Format prompt_embeds / negative_prompt_embeds as list[Tensor] + # (one entry per request) — matches ZImagePipeline's contract when + # prompt_embeds are pre-computed. +vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413 + def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:?` [nit] — The diff changes `return DiffusionOutput(output=raw)` to `return outputs` (pipeline_ming_imagegen.py:475), so the returned object now carries `stage_durations` from `ZImagePipeline.forward` instead of discarding them. `MingImagePipeline.__init__` enables the diffusion pipeline profiler, and request_batch.py:47 / output_formatter.py:200 read `diffusion_output.stage_durations`; if propagating those profiling timings is intended, add a one-line comment at `return outputs` to make that explicit — no code change required. (evidence: vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:475 `return outputs`; vllm_omni/diffusion/models/z_image/pipeline_z_image.py:719 `stage_durations = self.stage_durations if hasattr(self, "stage_durations") else None`; vllm_omni/diffusion/models/z_image/pipeline_z_image.py:720 `return DiffusionOutput(output=image, stage_durations=stage_durations)`; vllm_omni/diffusion/worker/request_batch.py:47 `stage_durations=result.stage_durations,`; vllm_omni/diffusion/output_formatter.py:200 `stage_durations=diffusion_output.stage_durations,`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_val/pr4837.trace.json.gz new file mode 100644 index 000000000..6d33f1fd4 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_val/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4893.cost.json b/eval/dataset/arms/copilot_v13_proof_val/pr4893.cost.json new file mode 100644 index 000000000..c2128a545 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4893.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1117.9, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_proof_val/runs/pr4893/run-20260813-150850-29bbab", + "recorded_at": "2026-08-13T15:27:26", + "attempts": [ + { + "invocation_id": "inv-7f701d67", + "outcome": "run:run-20260813-150850-29bbab", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-150850-29bbab", + "llm_calls": 133, + "input_tokens": 1001920, + "output_tokens": 208039, + "usd": 0.703189, + "source": "spans" + } + ], + "attempt_usd_total": 0.703189, + "llm_calls": 133, + "input_tokens": 1001920, + "output_tokens": 208039, + "usd": 0.703189, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-150850-29bbab", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7032, + "minutes": 18.58, + "ci_minutes": 0, + "input_tokens": 1001920, + "output_tokens": 208039, + "tool_calls": 238, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7928, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 133, + "by_role": { + "agent": { + "usd": 0.677661, + "input_tokens": 985311, + "output_tokens": 188943, + "calls": 131 + }, + "reducer": { + "usd": 0.025528, + "input_tokens": 16609, + "output_tokens": 19096, + "calls": 2 + } + }, + "cache_read_tokens": 7549184, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.12, + "gate": 1.97, + "review": 1108.73, + "report": 0.07 + }, + "time_to_first_result_s": 1114.87 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 35345 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4893.md b/eval/dataset/arms/copilot_v13_proof_val/pr4893.md new file mode 100644 index 000000000..48dffb179 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_proof_val/pr4893.md @@ -0,0 +1,56 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4893, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] Indexed access hunyuan_fused_moe.py:91 hidden_states.shape[0] — read in situ: hidden_states comes from kwargs.get('hidden_states') or args[0], both guarded for None; shape[0] is the MoE batch dim (contract: input rank>=2), consistent with the existing _set_forward_context_num_tokens call. +- [sweep] Branch tests/diffusion/distributed/test_expert_parallel_layout.py:24 (if reduce_scatter is not None) — mirrors vLLM GroupCoordinator capability; reachable only for vllm_* fakes (fake_init_vllm_model_parallel_group passes reduce_scatter), diffusion fakes leave it unset; no dead arm. +- [sweep] Branch parallel_state.py:868/875 (TP vllm-vs-diffusion) — both arms reachable: MoE mapping -> vLLM coordinator, otherwise diffusion tensor group; covered by test_moe_ep_maps... and test_cfg_parallel_keeps_diffusion_dp_without_ep. +- [sweep] Branch parallel_state.py:882 (DP remap) — condition dropped cfg_parallel_size>1 so it now runs for all MoE mappings (incl. cfgp1), matching the tp2_sp2 fix intent; verified test asserts vllm _DP distinct from diffusion _DP with world_size 4 for cfgp2. +- [sweep] Branch hunyuan_fused_moe.py:31/34/38/47 — all reachable: context unavailable -> return; existing dp_metadata -> return; dp_group None or world_size<=1 -> return; any(count is None) -> RuntimeError; error path verified by code read. +- [validated] parallel_state.py:816 sets vllm_parallel_state._DP = diffusion _DP; only the use_moe_parallel_mapping block at :882-885 replaces it with a vLLM coordinator; destroy_model_parallel :919-921 destroys it when distinct — lifecycle is symmetric on the happy path. +- [validated] destroy_model_parallel handles the new vLLM _PCP/_TP/_DP/_EP (parallel_state.py:919-945); _PCP is no longer aliased to _SP so the pre-existing 'is not _SP' guard now destroys it correctly. +- [validated] grep 'vllm_parallel_state._' across vllm_omni finds no consumer of _PCP/_TP/_DP/_EP outside parallel_state.py, so no in-repo caller is left assuming the old diffusion-coordinator API on those slots. + +`tests/diffusion/distributed/test_expert_parallel_layout.py:73` [major] — tests/diffusion/distributed/test_expert_parallel_layout.py:92 monkeypatches `parallel_state.init_vllm_model_parallel_group` with `fake_init_vllm_model_parallel_group`, so the real wrapper (vllm_omni/diffusion/distributed/parallel_state.py:515-527) never runs in CI. The fake injects `device_communicator=object()` and `reduce_scatter=...` (lines 73-74), which are exactly what the new assertions check; deleting `use_device_communicator=True` from the real wrapper would keep every changed test green. Make the fake record/assert the forwarded kwargs, or add a focused test that calls the real wrapper with a mocked `vllm_parallel_state.init_model_parallel_group` and asserts `use_device_communicator=True`. (evidence: tests/diffusion/distributed/test_expert_parallel_layout.py:92 `monkeypatch.setattr(parallel_state, "init_vllm_model_parallel_group", fake_init_vllm_model_parallel_group)` — replaces the real wrapper, so initialize_model_parallel() calls the fake. tests/diffusion/distributed/test_expert_parallel_layout.py:73-74 `device_communicator=object(),` / `reduce_scatter=lambda tensor, **kwargs: tensor,` — the fake injects exactly the two attributes the new assertions check. vllm_omni/diffusion/distributed/parallel_state.py:526 `use_device_communicator=True,` — the real behavior that no test exercises.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:91` [minor] — The diff adds `_set_forward_context_dp_metadata(hidden_states.shape[0])` unconditionally inside `_num_tokens_pre_hook` (hunyuan_fused_moe.py:91). The helper only early-returns when `_vllm_ps._DP` is None or `world_size <= 1` (hunyuan_fused_moe.py:38), so on multi-DP runs where MoE expert-parallel mapping is off `_vllm_ps._DP` is still the diffusion DP coordinator (parallel_state.py:816; only the `use_moe_parallel_mapping` block at :882-885 swaps in a vLLM coordinator) and every forward performs the new `all_gather_object` and sets vLLM `dp_metadata` from a diffusion group. Gate the helper/call to the MoE-mapping path (e.g. require `device_communicator` on `_vllm_ps._DP`) so non-EP multi-DP runs do not take the new collective and set unvalidated vLLM dp_metadata. (evidence: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:91 ` _set_forward_context_dp_metadata(hidden_states.shape[0])` — unconditional in the pre-hook. vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:37 ` dp_group = getattr(_vllm_ps, "_DP", None)`; :38 ` if dp_group is None or getattr(dp_group, "world_size", 1) <= 1:`; :39 ` return`; :42 ` torch.distributed.all_gather_object(`; :45 ` group=dp_group.cpu_group,`; :50 ` forward_context.dp_metadata = _vllm_fc.DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64))` — the helper runs the collective whenever _DP.world_size > 1, regardless of MoE-mapping. vllm_omni/diffusion/distributed/parallel_state.py:816 ` vllm_parallel_state._DP = _DP` — non-mapping path keeps the diffusion coordinator. vllm_omni/diffusion/distributed/parallel_state.py:882 ` if use_moe_parallel_mapping:`; :885 ` vllm_parallel_state._DP = init_vllm_model_parallel_group(` — only this block swaps in the vLLM coordinator.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:91` [minor] — `_num_tokens_pre_hook` (hunyuan_fused_moe.py:85-91) is the only caller of `_set_forward_context_dp_metadata`, but no test invokes it. The new `TestSetForwardContextDPMetadata` tests only the helper directly, so dropping the line 91 call or passing `shape[1]` would pass the suite. Add a test that mocks the FusedMoE factory and calls the registered hook with 1024-row `hidden_states`, asserting both `num_tokens` and `dp_metadata` on the forward context. (evidence: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:91 ` _set_forward_context_dp_metadata(hidden_states.shape[0])` inside `_num_tokens_pre_hook`, registered at line 93 ` moe_runner.register_forward_pre_hook(_num_tokens_pre_hook, with_kwargs=True)`; tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:74 ` hunyuan_moe._set_forward_context_dp_metadata(1024)` calls the helper directly; grep for `_num_tokens_pre_hook` under `tests/` returns no matches.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:47` [minor] — New TestSetForwardContextDPMetadata covers only the success and keep-existing branches. The `None`-entry failure path at hunyuan_fused_moe.py:47-48 (`if any(count is None ...)` / `raise RuntimeError(...)`) is untested. Add a test where mocked `all_gather_object` leaves a `None` entry and assert `RuntimeError('Failed to gather MoE DP token counts: ...')` fires, so a later silent-default change is caught. (evidence: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:47 ` if any(count is None for count in gathered_num_tokens):`; vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:48 ` raise RuntimeError(f"Failed to gather MoE DP token counts: {gathered_num_tokens}")`; tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:56 ` def test_sets_dp_metadata_from_vllm_dp_group(self, mocker):` and :79 ` def test_keeps_existing_dp_metadata(self, mocker):` are the only tests calling `_set_forward_context_dp_metadata`, and no test references the RuntimeError string.) + +`vllm_omni/diffusion/distributed/parallel_state.py:515` [minor] — The new `init_vllm_model_parallel_group` (parallel_state.py:515) calls `vllm_parallel_state.init_model_parallel_group(..., use_device_communicator=True)` and hunyuan_fused_moe.py:50 constructs `_vllm_fc.DPMetadata`, but pyproject.toml/requirements do not declare a vllm lower bound; only docker/Dockerfile.ci pins `VLLM_BASE_TAG=v0.24.0`. The monkeypatched distributed test exercises the fake wrapper, not the real vLLM 0.24 GroupCoordinator path, so CI does not prove the installed vLLM provides these APIs. Consider adding an explicit version constraint/pin or a compatibility test that exercises the real `init_model_parallel_group(use_device_communicator=True)`. (evidence: vllm_omni/diffusion/distributed/parallel_state.py:515 `def init_vllm_model_parallel_group(` ... vllm_omni/diffusion/distributed/parallel_state.py:526 `use_device_communicator=True,` — wrapper passes the new vLLM kwarg. vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:50 `forward_context.dp_metadata = _vllm_fc.DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64))` — new upstream API used with no version guard. pyproject.toml:10 `dynamic = ["version", "dependencies"]` and pyproject.toml:17 `# Dependencies are now managed dynamically via setup.py...`; requirements/common.txt has only a comment mentioning vllm, no pin. docker/Dockerfile.ci:5 `ARG VLLM_BASE_TAG=v0.24.0` — only CI image pin. tests/diffusion/distributed/test_expert_parallel_layout.py:92 `monkeypatch.setattr(parallel_state, "init_vllm_model_parallel_group", fake_init_vllm_model_parallel_group)` — the real wrapper is bypassed in the test.) + +`vllm_omni/diffusion/distributed/parallel_state.py:882` [minor] — parallel_state.py:882 — the DP remap now runs for every MoE mapping (the `cfg_parallel_size > 1` gate is gone), including cfg_parallel_size=1, but test_expert_parallel_layout.py only exercises cfg_parallel_size=2 (vllm `_DP.world_size == 4`, local_group [0,8,16,24]). Please confirm the broadened cfgp1 behavior is intended and add a cfg_parallel_size=1 MoE-mapping case. (evidence: parallel_state.py:882 ` if use_moe_parallel_mapping:`; parallel_state.py:885-886 ` vllm_parallel_state._DP = init_vllm_model_parallel_group(` / ` group_ranks=rank_generator.get_ranks("cfg-dp"),` — no `cfg_parallel_size > 1` gate remains. PR diff hunk: `- if use_moe_parallel_mapping and cfg_parallel_size > 1:` -> `+ if use_moe_parallel_mapping:`. tests/diffusion/distributed/test_expert_parallel_layout.py:106 ` cfg_parallel_size=2,`; :116 ` assert parallel_state.vllm_parallel_state._DP.world_size == 4` — the only MoE-mapping case asserts the cfgp2 configuration, and a grep for `cfg_parallel_size=1` in that file returns no matches.) + +`vllm_omni/diffusion/distributed/parallel_state.py:885` [minor] — The diff switches `_PCP`/`_TP`/`_DP`/`_EP` to real vLLM `GroupCoordinator`s (`use_device_communicator=True`) whenever `enable_expert_parallel` is selected (parallel_state.py:853-907), and `initialize_model_parallel` runs for every diffusion worker (diffusion_worker.py:306). This is not CUDA-only: the XPU stage 1 config sets `enable_expert_parallel: true` (hunyuan_image_3_moe.yaml:131), and the AMD/NPU CI lanes are currently failing. Confirm `init_vllm_model_parallel_group(use_device_communicator=True)` is validated on XPU/ROCm/NPU, or gate the coordinator switch to CUDA, before merge. (evidence: vllm_omni/diffusion/distributed/parallel_state.py:526 ` use_device_communicator=True,` +vllm_omni/diffusion/distributed/parallel_state.py:885 ` vllm_parallel_state._DP = init_vllm_model_parallel_group(` +vllm_omni/diffusion/worker/diffusion_worker.py:306 ` initialize_model_parallel(` +vllm_omni/deploy/hunyuan_image_3_moe.yaml:131 ` enable_expert_parallel: true`) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:50` [nit] — At vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:50, `forward_context.dp_metadata = _vllm_fc.DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64))` is set whenever metadata is absent. Add a brief comment (or assert on `len(gathered_num_tokens) == dp_group.world_size`) noting this is the diffusion-scoped forward context and that `all_gather_object` results are ordered by DP rank. (evidence: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:34 ` if getattr(forward_context, "dp_metadata", None) is not None:` ; vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:42-50 ` gathered_num_tokens: list[int | None] = [None] * dp_group.world_size + torch.distributed.all_gather_object( + gathered_num_tokens, + int(num_tokens), + group=dp_group.cpu_group, + ) + if any(count is None for count in gathered_num_tokens): + raise RuntimeError(f"Failed to gather MoE DP token counts: {gathered_num_tokens}") + + forward_context.dp_metadata = _vllm_fc.DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64))`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v13_proof_val/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v13_proof_val/pr4893.trace.json.gz new file mode 100644 index 000000000..41f919a47 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_proof_val/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_wave2/manifest.json b/eval/dataset/arms/copilot_v13_wave2/manifest.json new file mode 100644 index 000000000..6ef41347e --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v13_wave2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5509.cost.json b/eval/dataset/arms/copilot_v13_wave2/pr5509.cost.json new file mode 100644 index 000000000..f017a6694 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5509.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1504.9, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_wave2/runs/pr5509/run-20260813-174808-e8e814", + "recorded_at": "2026-08-13T18:13:09", + "attempts": [ + { + "invocation_id": "inv-d411dd38", + "outcome": "run:run-20260813-174808-e8e814", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-174808-e8e814", + "llm_calls": 161, + "input_tokens": 995826, + "output_tokens": 264366, + "usd": 0.847056, + "source": "spans" + } + ], + "attempt_usd_total": 0.847056, + "llm_calls": 161, + "input_tokens": 995826, + "output_tokens": 264366, + "usd": 0.847056, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-174808-e8e814", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8471, + "minutes": 25.02, + "ci_minutes": 0, + "input_tokens": 995826, + "output_tokens": 264366, + "tool_calls": 303, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9559, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 161, + "by_role": { + "agent": { + "usd": 0.817402, + "input_tokens": 972880, + "output_tokens": 243072, + "calls": 159 + }, + "reducer": { + "usd": 0.029653, + "input_tokens": 22946, + "output_tokens": 21294, + "calls": 2 + } + }, + "cache_read_tokens": 10643712, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.41, + "gate": 1.85, + "review": 1495.94, + "report": 0.07 + }, + "time_to_first_result_s": 1501.27 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5509.trace.json.gz", + "trace_bytes": 44396 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5509.md b/eval/dataset/arms/copilot_v13_wave2/pr5509.md new file mode 100644 index 000000000..a2ccb51da --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5509.md @@ -0,0 +1,70 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5509, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 58131b8dc1cb) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read all diff hunks in situ: vllm_omni/diffusion/attention/backends/trtllm_attn.py (full file), vllm_omni/diffusion/data.py (AttnQuantSpec/AttentionSpec/AttentionConfig region), tests/diffusion/attention/test_trtllm_attn.py and test_attention_config.py, docs/user_guide/diffusion/attention_backends.md (full file). +- [validated] trtllm_attn.py:198-218 init order is correct: dtype_qk membership check -> _sage_kernel_available() -> _sage_quantize_fn() None check; self._sage_quantize_fn is resolved once at init so forward_cuda never hits the lru_cache getter under torch.compile. +- [validated] data.py:1415/1418 AttnQuantSpec rejects dtype_qk/dtype_vo outside {float16,bfloat16,int8,fp8_e4m3} and block sizes outside {1,4,16}; test_attention_config.py:82-91 exercises int4 and k_block_size=8 rejection. +- [sweep] trtllm_attn.py:90-97 _sage_kernel_available uses inspect.signature(trtllm_ragged_attention_deepseek) to detect sage_attn_sfs; cannot execute flashinfer in this environment, so this relies on the PR's B200 evidence that the production path works (assumption recorded). +- [sweep] trtllm_attn.py:198 vs 296: quant.enabled branch in forward is only reachable after init validation, and self._sage_quantize_fn is non-None there; no None-deref path found. +- [sweep] data.py:1447 admits quant for FLASHINFER_ATTN, but grep for dtype_vo/flashinfer_backend found no consumer anywhere in code, and flashinfer_attn.py:55-63 __init__ has no backend_kwargs param (swallowed by **extra_impl_args) -> silent ignore (comment 1). +- [sweep] layer.py:320-327 ring path guards skip_softmax but not quant; ring_runner.run_attention bypasses self.attention so quant is dropped under ring SP -> silent ignore (comment 3). +- [sweep] docs diff deletes '## Known Limitations' (LTX-2 CUDNN crash workaround + FA4 status), '### Mixed backends across roles', '### Compare against FlashAttention', '## Validation Guidance', '## Reproducing the Benchmark Table'; grep confirms none of these sections exist elsewhere in the repo, and docs:232 still says 'see Known Limitations above' (comment 2). + +`vllm_omni/diffusion/data.py:?` [major] — This diff makes `quant` valid for `FLASHINFER_ATTN` and serializes it in `backend_kwargs()` (data.py:1447-1451, 1479-1482), and the new doc text claims `FLASHINFER_ATTN` reads `dtype_qk`/`dtype_vo`/`flashinfer_backend`. But `FlashInferAttentionImpl.__init__` (flashinfer_attn.py:55-66) has no `backend_kwargs` handling, accepts `**extra_impl_args`, and only sets `self.causal`/`self.softmax_scale`, so the `quant` dict passed by layer.py:109 is silently dropped and the user gets dense attention. Implement the FLASHINFER_ATTN quant path, or remove FLASHINFER_ATTN from the allowed set in `AttentionSpec.__post_init__` and delete/soften the doc claim. (evidence: vllm_omni/diffusion/data.py:1447-1449 `if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"):` / `raise ValueError(` / `f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but "`. vllm_omni/diffusion/attention/layer.py:109 `backend_kwargs=backend_kwargs,`. vllm_omni/diffusion/attention/backends/flashinfer_attn.py:54-66 `class FlashInferAttentionImpl(AttentionImpl):` / `def __init__(` / `self,` / `num_heads: int,` / `head_size: int,` / `softmax_scale: float,` / `causal: bool = False,` / `num_kv_heads: int | None = None,` / `prefix: str = "",` / `**extra_impl_args,` / `) -> None:` / `self.causal = causal` / `self.softmax_scale = softmax_scale`) + +`docs/user_guide/diffusion/attention_backends.md:?` [major] — docs/user_guide/diffusion/attention_backends.md:232 still reads '- **Workaround for known issues**: see Known Limitations above.', but this PR deleted the `## Known Limitations` heading it points to. `## Known Limitations` (including the LTX-2.0 `CUDNN_ATTN` torch.compile workaround for #3121 and the FA4 status), `## Validation Guidance`, `## Reproducing the Benchmark Table`, `### Mixed backends across roles`, and `### Compare against FlashAttention` are all gone and were not moved within the file. Restore those sections or move them deliberately with a working link, and at minimum delete/repair the dangling bullet; if the removals are intentional, say so in the PR body rather than silently dropping them in a kernel PR. (evidence: docs/user_guide/diffusion/attention_backends.md:232 `- **Workaround for known issues**: see Known Limitations above.` — the surviving bullet whose target heading was deleted. grep for `Known Limitations` in this file returns only line 232 (so `## Known Limitations` is gone); grep for `Validation Guidance`, `Reproducing the Benchmark Table`, `FA4 not yet integrated`, and `Mixed backends across roles` each return no matches.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:?` [major] — Diff applies quant only in `TrtllmAttentionImpl.forward_cuda` (trtllm_attn.py:296-299) after `self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)` (trtllm_attn.py:194). `_run_ring_attention` (layer.py:319-329) guards only `skip_softmax` and then delegates to `RingParallelAttention`, which bypasses the backend impl, so with ring_degree>1 a TRTLLM_ATTN quant config passes init validation and is silently ignored. Extend the guard to raise `NotImplementedError` when `getattr(self.attention, "quant", None).enabled` is true (or route quant through the ring path), and add a test mirroring `test_ring_sp_with_skip_softmax_rejected`. (evidence: vllm_omni/diffusion/attention/layer.py:319-329 `def _run_ring_attention(self, query, key, value, attn_metadata): + skip = getattr(self.attention, "skip", None) + if skip is not None and getattr(skip, "configured", False): + raise NotImplementedError( + "Skip-Softmax (TRTLLM_ATTN) is not supported with ring sequence parallelism: " + "the ring path bypasses the backend, so the skip config would be silently ignored. " + "Use Ulysses SP instead, or remove the skip_softmax config." + ) + # Delegate to RingParallelAttention strategy if available + if self.ring_runner is not None: + return self.ring_runner.run_attention( + query, key, value, attn_metadata, softmax_scale=self.softmax_scale, causal=self.causal + )` — the guard checks only `skip` and has no `quant` check before delegating to the ring path. +vllm_omni/diffusion/attention/backends/trtllm_attn.py:194 `self.quant = QuantConfig.from_backend_kwargs(backend_kwargs)`; and trtllm_attn.py forward_cuda: `if self.quant.enabled: q, k, v, sage_attn_sfs, sage_block_sizes = self.quant.quantize(q, k, v, self._sage_quantize_fn)` — quant is consumed only in `forward_cuda`, which `_run_ring_attention` does not call.) + +`tests/diffusion/attention/test_trtllm_attn.py:50` [major] — The new QuantConfig tests are in a cuda-marked file CI never selects: test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` means `tests/diffusion -m 'core_model and cpu'` (test-ready.yml:18, test-merge.yml:18) deselects them, and `tests/diffusion/*.py -m 'core_model and cuda'` (test-ready.yml:68) is a shell glob that only matches top-level files, not tests/diffusion/attention/. Move test_quant_config_from_backend_kwargs, test_quant_rejects_non_sage_dtype, test_quant_quantize_requires_flashinfer_routine, and test_quant_quantize_calls_routine_and_shapes_sfs into the cpu-marked test_attention_config.py, and ensure test_sage_quant_matches_sdpa runs explicitly on SM100 hardware so the new QuantConfig/SAGE logic gets CI coverage. (evidence: tests/diffusion/attention/test_trtllm_attn.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` — cuda-marked module. tests/diffusion/attention/test_trtllm_attn.py:48 `def test_quant_config_from_backend_kwargs():` — first of four CPU-runnable QuantConfig tests plus `test_sage_quant_matches_sdpa`, all in this cuda-marked file. .buildkite/cuda/test-ready.yml:18 `- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — the only diffusion cpu lane, deselects cuda-marked tests. .buildkite/cuda/test-ready.yml:68 `- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"` — shell glob matches only top-level files, never tests/diffusion/attention/. .buildkite/cuda/test-merge.yml:18 `- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — merge pipeline repeats the cpu-only diffusion command with no attention/ cuda lane.) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — This diff adds `quant: AttnQuantSpec | None = None` to `AttentionSpec` (vllm_omni/diffusion/data.py:1434) and documents the new SAGE quant section, but the Configuration prose still says `skip_softmax` is 'the only' typed spec field (docs/user_guide/diffusion/attention_backends.md:72-73) and the design doc repeats it (docs/design/module/dit_module.md:543). Update both to mention `quant` and point to the SAGE section so the docs match the new API. (evidence: vllm_omni/diffusion/data.py:1434 ` quant: AttnQuantSpec | None = None`; docs/user_guide/diffusion/attention_backends.md:72 `A backend that needs configuration exposes it as a typed field on the spec. Today the only` and :73 `one is `TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)):`; docs/design/module/dit_module.md:543 `A backend that needs configuration exposes it as a typed field on `AttentionSpec`; `AttentionSpec.backend_kwargs()` serializes it into the `backend_kwargs` the impl consumes. Today the only such field is `skip_softmax` (TRTLLM_ATTN), so e.g. `target_sparsity` can be set without changing model code.`) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:?` [minor] — SAGE init gates only on `_sage_kernel_available()` (trtllm_attn.py:204), which checks solely whether `sage_attn_sfs` exists in `trtllm_ragged_attention_deepseek`'s signature (trtllm_attn.py:93). The docs state int8 QK kernels are SM100-only (attention_backends.md:209-210), but there is no compute-capability check for `dtype_qk='int8'` at init, so on SM103 an int8 config passes init and later fails in `forward_cuda`. The new test turns that `Missing TRTLLM-GEN kernel` failure into `pytest.skip` (test_trtllm_attn.py:314-316), keeping CI green. Add an SM-arch check for int8 at init and make the test distinguish 'unsupported arch' from 'kernel missing' rather than skipping. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:93 `return "sage_attn_sfs" in inspect.signature(trtllm_ragged_attention_deepseek).parameters`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:204 `if not _sage_kernel_available():`; docs/user_guide/diffusion/attention_backends.md:209-210 `availability is arch-dependent: `fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103** (B300); `int8` QK kernels are compiled for **SM100 only**.`; tests/diffusion/attention/test_trtllm_attn.py:314-316 `except RuntimeError as e: # pragma: no cover - arch-dependent (int8 is SM100-only)` / `if "Missing TRTLLM-GEN kernel" in str(e):` / `pytest.skip(f"no trtllm-gen SAGE kernel for dtype_qk={dtype_qk} on this GPU arch")`) + +`tests/diffusion/attention/test_trtllm_attn.py:306` [minor] — The PR body's B200 LPIPS/timing tables are manual, and the only automated SAGE parity check is test_sage_quant_matches_sdpa (tests/diffusion/attention/test_trtllm_attn.py:306), gated by requires_sage to Blackwell SM100+ with FlashInfer >= 0.6.16rc1. .buildkite pools are only l4/h100 and benchmarks/diffusion/bench_attention_backends.py does not exercise the SAGE quant path, so maintainers cannot validate the claimed speedups through CI or a committed benchmark. Consider adding a reproducible SAGE benchmark or lane. (evidence: tests/diffusion/attention/test_trtllm_attn.py:298-301 `requires_sage = pytest.mark.skipif( + not (_has_trtllm_attn() and tg._sage_quantize_fn() is not None), + reason="requires Blackwell SM100+ GPU with flashinfer >= 0.6.16rc1 (trtllm_sage_attention_quantize)", +)`; tests/diffusion/attention/test_trtllm_attn.py:306 `def test_sage_quant_matches_sdpa(dtype_qk):`; .buildkite/common/ci_mirror_hardwares.yml:7 `# l4_1, l4_4, h100_1 … h100_4` — CI pools only L4/H100, no SM100 lane, and grep for sm100|b200|blackwell under .buildkite returns no matches.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:214` [minor] — Confirmed: docs pin SAGE on FlashInfer >= 0.6.16rc1, but pyproject.toml, setup.py, requirements/, and docker/ declare no flashinfer minimum. The SAGE path is enforced only at runtime — `_sage_kernel_available()` probes for the `sage_attn_sfs` kwarg (vllm_omni/diffusion/attention/backends/trtllm_attn.py ~l.90) and `__init__` raises RuntimeErrors when the kernel path or `trtllm_sage_attention_quantize` is missing (version message at l.214). Non-breaking for dense/Skip-Softmax; consider declaring the minimum if the SAGE path should be CI-covered. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:214 `"trtllm_sage_attention_quantize (added in flashinfer >= 0.6.16rc1). Upgrade "` — runtime-only version guard; grep for `flashinfer` in pyproject.toml, setup.py, requirements/, and docker/ returned no matches, confirming the >=0.6.16rc1 minimum documented in the diff is not declared as an in-repo dependency bound.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/attention/backends/trtllm_attn.py:204` [minor] The `_sage_kernel_available()==False` gating branch at trtllm_attn.py:204-210 has no direct test; only the monkeypatch-to-True path in test_quant_quantize_requires_flashinfer_routine is covered. Add a test that monkeypat +- `vllm_omni/diffusion/attention/backends/trtllm_attn.py:131` [minor] Two quant-enabled predicates diverge: AttnQuantSpec.enabled is true when dtype_vo alone is set (data.py:1425), so backend_kwargs() emits a quant block; but QuantConfig.enabled only checks dtype_qk (trtllm_attn.py:131), s + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5509.trace.json.gz b/eval/dataset/arms/copilot_v13_wave2/pr5509.trace.json.gz new file mode 100644 index 000000000..4387942ce Binary files /dev/null and b/eval/dataset/arms/copilot_v13_wave2/pr5509.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5550.cost.json b/eval/dataset/arms/copilot_v13_wave2/pr5550.cost.json new file mode 100644 index 000000000..2b7758977 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5550.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 945.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_wave2/runs/pr5550/run-20260813-174808-f0eaa6", + "recorded_at": "2026-08-13T18:03:50", + "attempts": [ + { + "invocation_id": "inv-241d6a63", + "outcome": "run:run-20260813-174808-f0eaa6", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-174808-f0eaa6", + "llm_calls": 121, + "input_tokens": 958983, + "output_tokens": 202232, + "usd": 0.824243, + "source": "spans" + } + ], + "attempt_usd_total": 0.824243, + "llm_calls": 121, + "input_tokens": 958983, + "output_tokens": 202232, + "usd": 0.824243, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-174808-f0eaa6", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8242, + "minutes": 15.69, + "ci_minutes": 0, + "input_tokens": 958983, + "output_tokens": 202232, + "tool_calls": 234, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7779, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 121, + "by_role": { + "agent": { + "usd": 0.799189, + "input_tokens": 936003, + "output_tokens": 185127, + "calls": 119 + }, + "reducer": { + "usd": 0.025055, + "input_tokens": 22980, + "output_tokens": 17105, + "calls": 2 + } + }, + "cache_read_tokens": 12698624, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 37.57, + "gate": 1.56, + "review": 902.44, + "report": 0.06 + }, + "time_to_first_result_s": 941.62 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5550.trace.json.gz", + "trace_bytes": 38779 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5550.md b/eval/dataset/arms/copilot_v13_wave2/pr5550.md new file mode 100644 index 000000000..5946140de --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5550.md @@ -0,0 +1,66 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5550, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 20ca4b93de6e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] tests/diffusion/diffusion_kv/test_metadata.py:37,38,68,109 — each indexed `[0]` access is preceded by construction of a non-empty sequence/context tuple in the same test; no unsafe access (file read in situ). +- [sweep] tests/diffusion/diffusion_kv/test_worker_contract.py:221 `calls[0][1][0]` — guarded by the preceding `assert result.request_ids == ["req-0"]` and the full `calls == [...]` assertion; safe. +- [sweep] vllm_omni/diffusion/diffusion_engine.py:199 `if scheduler_to_close is not None` — currently `self.scheduler` is assigned before `initialize()` in both `_init_scheduler` branches, so the None arm is unreachable today; the getattr default is defensive and harmless. +- [sweep] vllm_omni/diffusion/executor/multiproc_executor.py:545 metadata-attach branch — both arms exercised: dense legacy positional-shape test (None arm) and `test_request_rpc_sends_attached_metadata_for_only_that_request` (not-None arm). +- [sweep] vllm_omni/diffusion/sched/interface.py:184 and :192 — envelope-vs-forwarded arm is tested; the metadata-vs-forwarded arm (envelope == forwarded, metadata differs) has no direct test. +- [sweep] vllm_omni/diffusion/worker/diffusion_model_runner.py:172 and :175 — dense-reject and stale-id arms are tested; the positive PAGED_SCHEDULER accept path is untested (see review_comments). +- [sweep] vllm_omni/diffusion/worker/diffusion_worker.py:463 and :1372 — both arms covered by the wrapper-forwarding test and the dense-worker no-metadata test. +- [validated] vllm_omni/diffusion/sched/base_scheduler.py:114 is the only production `NewRequestData.from_state` caller and passes no metadata, so the keyword-only change is non-breaking. + +`vllm_omni/experimental/ar_diffusion/runner.py:?` [major] — ARDiffusionModelRunner.execute_model (vllm_omni/experimental/ar_diffusion/runner.py:237-255) is an in-repo subclass override that was not updated for the new base execute_model kwarg. It still declares only (self, req, kv_prefetch_job=None) and both super().execute_model(req, kv_prefetch_job=kv_prefetch_job) calls (runner.py:243, runner.py:255) drop diffusion_kv_metadata. Since DiffusionWorker.execute_model now forwards a non-None diffusion_kv_metadata as **kwargs and ARDiffusionEngine routes to this runner, a metadata-bearing request raises TypeError before base _validate_diffusion_kv_metadata. Add the parameter and forward it at both super calls, or explicitly reject metadata, plus a contract test. (evidence: vllm_omni/experimental/ar_diffusion/runner.py:237-243: " def execute_model(\n self,\n req: OmniDiffusionRequest,\n kv_prefetch_job: KVPrefetchJob | None = None,\n ) -> DiffusionOutput:\n if self.kv_cache is None:\n return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)"; runner.py:255: " output = super().execute_model(req, kv_prefetch_job=kv_prefetch_job)"; vllm_omni/diffusion/worker/diffusion_worker.py:462-465: " kwargs: dict[str, Any] = {\"kv_prefetch_job\": kv_prefetch_job}\n if diffusion_kv_metadata is not None:\n kwargs[\"diffusion_kv_metadata\"] = diffusion_kv_metadata\n output = self.model_runner.execute_model(req, **kwargs)"; vllm_omni/diffusion/worker/diffusion_model_runner.py:592: " diffusion_kv_metadata: DiffusionKVMetadata | None = None,"; vllm_omni/experimental/ar_diffusion/engine.py:40: " default_diffusion_model_runner_cls = AR_DIFFUSION_MODEL_RUNNER_CLS") + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — multiproc_executor.py:471 — `and not has_diffusion_kv_metadata` disables the DLO+AllGather DP broadcast for the whole wave when any NewRequestData carries diffusion_kv_metadata. The wave then falls into the per-request loop (multiproc_executor.py:541-551) with `unique_reply_rank=0, exec_all_ranks=True`, so every DP rank executes the same request and only rank 0's reply is used: the wave is serialized and the dp_size concurrency set by DiffusionEngine._init_runtime_state is lost, including for metadata-free requests in the same wave. (Note: the production scheduler does not yet emit metadata, so this is currently a forward-looking/contract gap rather than a live regression.) Either implement metadata-aware DP broadcast or explicitly guard/document the serial fallback, and add a test with enable_distributed_layerwise_offload=True, dlo_use_allgather=True and two metadata-bearing NewRequestData asserting two serial 4-tuple RPCs instead of one broadcast call. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)`; :471 ` and not has_diffusion_kv_metadata`; :545 ` if new_req.diffusion_kv_metadata is not None:`; :546 ` args += (new_req.diffusion_kv_metadata,)`; :550 ` unique_reply_rank=0,`; :551 ` exec_all_ranks=True,`) + +`vllm_omni/diffusion/diffusion_engine.py:191` [minor] — vllm_omni/diffusion/diffusion_engine.py:191-208 bundles engine init-failure cleanup (scheduler.close()/executor.shutdown() before re-raise) that the PR body neither lists under Main changes nor in the Test plan, so it sits outside the stated W0 DiffusionKVMetadata plumbing scope. It is small and covered by tests/diffusion/test_diffusion_engine_cleanup.py::test_scheduler_initialization_failure_closes_scheduler_and_executor — please confirm it is intended to ship here rather than in a follow-up. (evidence: vllm_omni/diffusion/diffusion_engine.py:191-208 ` try: + self._init_scheduler(od_config, scheduler) + self._init_runtime_state() + self._init_execute_fn() + self._log_execution_mode(od_config) + except Exception: + # close() cannot be used because runtime synchronization state may not exist yet. + scheduler_to_close = getattr(self, "scheduler", None) + if scheduler_to_close is not None: + try: + scheduler_to_close.close() + except Exception: + logger.exception("Failed to close Scheduler after DiffusionEngine initialization failed") + try: + self.executor.shutdown() + except Exception: + logger.exception("Failed to shut down Executor after DiffusionEngine initialization failed") + raise`) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:172` [minor] — Confirmed. `_validate_diffusion_kv_metadata` (diffusion_model_runner.py:163-175) and `validate_new_request_data_identity` (sched/interface.py:181-194) are only covered by rejection tests. Add: (1) PAGED_SCHEDULER + matching-metadata positive tests for `execute_model`, `execute_model_batch`, and `execute_stepwise` asserting `_execute_request_list`/forward is reached; (2) a `validate_new_request_data_identity` case where `new_req.request_id == new_req.req.request_id` but `metadata.request_id` differs. (evidence: tests/diffusion/diffusion_kv/test_worker_contract.py:88-101 `def test_paged_request_without_metadata_reaches_request_forward() -> None: ... assert runner.execute_model(req) is expected_output` — the only PAGED_SCHEDULER success-path test passes no metadata. tests/diffusion/diffusion_kv/test_worker_contract.py:116-126 `def test_paged_request_rejects_mismatched_metadata_identity() -> None: ... with pytest.raises(ValueError, match="request mismatch"): runner.execute_model(SimpleNamespace(request_id="req-0"), diffusion_kv_metadata=make_metadata("stale"))` — the only runner test with metadata is a rejection. vllm_omni/diffusion/sched/interface.py:191-195 `metadata = new_req.diffusion_kv_metadata + if metadata is not None and metadata.request_id != forwarded_request_id: + raise ValueError(...)` — this branch has no test with envelope==forwarded request and mismatched metadata; existing mismatch tests (test_worker_contract.py:129-142, 158-171, 188-202) all fail on the envelope-vs-request check first.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:?` [minor] — The diff adds validate_new_request_data_identity(new_req) and the comment 'cannot be forwarded or converted into a worker output' only in MultiprocDiffusionExecutor.execute_request (multiproc_executor.py:460). execute_batch (multiproc_executor.py:616-617) forwards the whole scheduler_output via collective_rpc("execute_model_batch", args=(scheduler_output, self.od_config), ...) and execute_step (multiproc_executor.py:652-653) forwards via collective_rpc("execute_stepwise", args=(scheduler_output,), ...) without this pre-dispatch check. In those paths an identity mismatch is only caught worker-side by DiffusionModelRunner.execute_model_batch (diffusion_model_runner.py:637) / execute_stepwise (diffusion_model_runner.py:751). Either add the same pre-dispatch validation loop before these two collective_rpc calls or narrow the executor comment to the request-mode path only. (evidence: multiproc_executor.py:460 ` validate_new_request_data_identity(new_req)` — present only in execute_request. multiproc_executor.py:616-617 ` "execute_model_batch",\n args=(scheduler_output, self.od_config),` — execute_batch forwards without the loop. multiproc_executor.py:652-653 ` "execute_stepwise",\n args=(scheduler_output,),` — execute_step forwards without the loop. diffusion_model_runner.py:637 ` validate_new_request_data_identity(new_req)` — worker-side check in execute_model_batch. diffusion_model_runner.py:751 ` validate_new_request_data_identity(new_req)` — worker-side check in execute_stepwise.) + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:592` [nit] — The diff adds `diffusion_kv_metadata` to `DiffusionModelRunner.execute_model` (diffusion_model_runner.py:592), `DiffusionWorker.execute_model` (diffusion_worker.py:426), and `WorkerWrapperBase.execute_model` (diffusion_worker.py:1358), but the public signatures do not document it. In diffusion_model_runner.py the execute_model Args section lists only `req` (kv_prefetch_job is also omitted); in DiffusionWorker.execute_model there is no Args section. Add an Args entry on each signature stating `diffusion_kv_metadata` is the trusted scheduler allocation snapshot accepted only in `paged_scheduler` mode, and fix the runner's missing `kv_prefetch_job` entry while there. (evidence: diffusion_model_runner.py:592 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,`; diffusion_model_runner.py:597-598 ` Args: + req: A diffusion request containing a list of prompts to process.`; diffusion_worker.py:426 ` diffusion_kv_metadata: DiffusionKVMetadata | None = None,`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5550.trace.json.gz b/eval/dataset/arms/copilot_v13_wave2/pr5550.trace.json.gz new file mode 100644 index 000000000..d333087a6 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_wave2/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5610.cost.json b/eval/dataset/arms/copilot_v13_wave2/pr5610.cost.json new file mode 100644 index 000000000..0b1d6e9b1 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5610.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 751.6, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_wave2/runs/pr5610/run-20260813-174808-eb3f93", + "recorded_at": "2026-08-13T18:00:36", + "attempts": [ + { + "invocation_id": "inv-ef021155", + "outcome": "run:run-20260813-174808-eb3f93", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-174808-eb3f93", + "llm_calls": 68, + "input_tokens": 471830, + "output_tokens": 99665, + "usd": 0.404797, + "source": "spans" + } + ], + "attempt_usd_total": 0.404797, + "llm_calls": 68, + "input_tokens": 471830, + "output_tokens": 99665, + "usd": 0.404797, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-174808-eb3f93", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4048, + "minutes": 12.47, + "ci_minutes": 0, + "input_tokens": 471830, + "output_tokens": 99665, + "tool_calls": 115, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5511, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 68, + "by_role": { + "planner": { + "usd": 0.001021, + "input_tokens": 2153, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.389349, + "input_tokens": 456506, + "output_tokens": 89414, + "calls": 65 + }, + "reducer": { + "usd": 0.014427, + "input_tokens": 13171, + "output_tokens": 9851, + "calls": 2 + } + }, + "cache_read_tokens": 6213760, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 34.71, + "gate": 1.92, + "review": 711.35, + "report": 0.06 + }, + "time_to_first_result_s": 748.02 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5610.trace.json.gz", + "trace_bytes": 27821 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5610.md b/eval/dataset/arms/copilot_v13_wave2/pr5610.md new file mode 100644 index 000000000..4fcb36fac --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5610.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5610, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 50126edf2f50) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] docs/.nav.yml:118-119 read in situ — new bare-path entry `- design/feature/omni_async_output_materialization.md` matches sibling entries and the target file exists; the duplicate `design/feature/vae_parallel.md` entries are pre-existing and not introduced by this diff. +- [sweep] docs/design/index.md:10 read in situ — `[Async Omni Output Materialization](feature/omni_async_output_materialization.md)` targets an existing file. +- [sweep] docs/design/feature/omni_async_output_materialization.md read end-to-end — TOC anchors match headings; fenced bash/yaml/text blocks parse; admonition syntax (`!!! note`/`!!! warning`) is standard MkDocs. +- [sweep] docs/source/architecture/qwen3-omni-async-output-step-gap.svg inspected in diff — `aria-labelledby="title desc"` matches the two ids; no external entities; 2.8ms→41µs ≈ 68× math consistent. +- [validated] Compatibility table == `GPUARModelRunner._should_use_async_omni_output` (vllm_omni/worker/gpu_ar_model_runner.py:1595-1622): use_async_scheduling, omni_prefix_cache, speculative_config, async_chunk, enable_return_routed_experts, model use_async_omni_output, and has_postprocess→eager_omni_postprocess_before_async_output. +- [validated] Qwen3-Omni opt-ins: qwen3_omni.py:134 (thinker `use_async_omni_output=True`), :162 (talker sets `use_async_omni_output`, `eager_omni_postprocess_before_async_output=True`, `omni_pooler_payload_include_hidden=False`). +- [validated] Qwen3-TTS opt-ins: qwen3_tts_talker.py:351-353 sets `use_async_omni_output`, eager postprocess, and `omni_pooler_payload_include_hidden=False`. +- [validated] Deploy examples: qwen3_omni_moe.yaml and qwen3_tts.yaml both `async_chunk: true`; AR `async_scheduling` default is True via stage_config.py:941-944 (`engine_args.get('async_scheduling', True)` → OmniARAsyncScheduler). + +`docs/design/feature/omni_async_output_materialization.md:119` [minor] — Performance table (lines 108-112) and 'hashes matched' (lines 122-125) are attributed to the Qwen3-Omni optimization blog and PR #4476; this docs-only PR carries no independently re-derived benchmark or hash evidence. Please confirm whether citing the blog/PR is the intended scope, or if independent verification is required before merge. (evidence: docs/design/feature/omni_async_output_materialization.md:110 `| CUDA Graph + async chunk | 9.3 req/s | 655 ms | 0.63 |`; docs/design/feature/omni_async_output_materialization.md:119 `[Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization)`; docs/design/feature/omni_async_output_materialization.md:123 `The text and audio output hashes matched, confirming that overlap does not`; docs/design/feature/omni_async_output_materialization.md:125 `[PR #4476](https://github.com/vllm-project/vllm-omni/pull/4476) for the`) + +`docs/design/feature/omni_async_output_materialization.md:83` [minor] — The new `docs/design/feature/omni_async_output_materialization.md` uses `https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization` at lines 83, 119, and 383, while all other repo blog citations use `https://blog.vllm.ai/...` (docs/.nav.yml:140, docs/README.md:66, docs/features/sleep_mode.md:5). Please switch these three links to the repo's `blog.vllm.ai` host/format (or confirm the `vllm.ai/blog/...` URL resolves) so the performance table and adapted figure retain a valid cited source. (evidence: docs/design/feature/omni_async_output_materialization.md:83 ` `; docs/design/feature/omni_async_output_materialization.md:119 `[Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization)`; docs/design/feature/omni_async_output_materialization.md:383 `- [Qwen3-Omni optimization blog](https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization):`; docs/.nav.yml:140 ` - Blog: https://blog.vllm.ai`; docs/README.md:66 `- [vllm-omni announcement blogpost](https://blog.vllm.ai/2025/11/30/vllm-omni.html)`; docs/features/sleep_mode.md:5 `This feature is inherited from [vLLM’s Sleep Mode](https://blog.vllm.ai/2025/10/26/sleep-mode.html) ...`) + +`docs/design/feature/omni_async_output_materialization.md:73` [minor] — PR test result records `mkdocs build --strict` as blocked before MkDocs startup by the existing qwen-asr==0.0.6 / transformers>=5.5.3 conflict, while the custom image-reference check passed. This page introduces the first local relative image under docs/design/feature/ (line 73: src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"), which does resolve to the added docs/source/architecture/qwen3-omni-async-output-step-gap.svg. Confirm via `mkdocs build --strict` once the dependency conflict is resolved (or in a venv excluding qwen-asr), since the custom checker does not model MkDocs page-relative URL resolution. (evidence: ['docs/design/feature/omni_async_output_materialization.md:73 `src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg"`', 'docs/design/feature/async_chunk.md:86 `Async Chunk Pipeline Architecture`', 'PR #5610 body Test Result: `mkdocs build --strict`: blocked before MkDocs startup by the existing dependency-resolution conflict where `qwen-asr==0.0.6` pins `transformers==4.57.6` while the project requires `transformers>=5.5.3`.']) + +`docs/design/feature/omni_async_output_materialization.md:64` [minor] — Confirmed. In `vllm_omni/worker/gpu_ar_model_runner.py`, `_clone_cuda_tensor_payload` adds tensors to the async-source list only when `value.device.type == "cuda"` (line 73), and `_snapshot_tensor_payload_to_cpu_async` uses `torch.cuda.current_stream()` / `torch.cuda.Event()` (lines 135-136). `vllm_omni/platforms/xpu/utils.py` aliases only `torch.cuda.Stream`, `default_stream`, `current_stream`, and `stream` to XPU — no `Event` alias. Non-CUDA tensors therefore skip the async-source branch and the function returns before creating the event, so XPU/MUSA do not get the dedicated-stream/pinned-D2H overlap. Add a clause to the doc's platform note to make this explicit. (evidence: vllm_omni/worker/gpu_ar_model_runner.py:73 `if value.device.type == "cuda":`; vllm_omni/worker/gpu_ar_model_runner.py:135 `source_stream = torch.cuda.current_stream()`; vllm_omni/worker/gpu_ar_model_runner.py:136 `ready_event = torch.cuda.Event()`; vllm_omni/platforms/xpu/utils.py:10-13 `torch.cuda.Stream = torch.xpu.Stream`, `torch.cuda.default_stream = torch.xpu.current_stream`, `torch.cuda.current_stream = torch.xpu.current_stream`, `torch.cuda.stream = torch.xpu.stream` (no `torch.cuda.Event` alias).) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5610.trace.json.gz b/eval/dataset/arms/copilot_v13_wave2/pr5610.trace.json.gz new file mode 100644 index 000000000..234877715 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_wave2/pr5610.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5703.cost.json b/eval/dataset/arms/copilot_v13_wave2/pr5703.cost.json new file mode 100644 index 000000000..a54314b2e --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5703.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1158.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_wave2/runs/pr5703/run-20260813-174808-d91016", + "recorded_at": "2026-08-13T18:07:23", + "attempts": [ + { + "invocation_id": "inv-040fdbb5", + "outcome": "run:run-20260813-174808-d91016", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-174808-d91016", + "llm_calls": 144, + "input_tokens": 769300, + "output_tokens": 219726, + "usd": 0.6564, + "source": "spans" + } + ], + "attempt_usd_total": 0.6564, + "llm_calls": 144, + "input_tokens": 769300, + "output_tokens": 219726, + "usd": 0.6564, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-174808-d91016", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6564, + "minutes": 19.25, + "ci_minutes": 0, + "input_tokens": 769300, + "output_tokens": 219726, + "tool_calls": 267, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7874, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 144, + "by_role": { + "agent": { + "usd": 0.611457, + "input_tokens": 742183, + "output_tokens": 185581, + "calls": 141 + }, + "reducer": { + "usd": 0.044943, + "input_tokens": 27117, + "output_tokens": 34145, + "calls": 3 + } + }, + "cache_read_tokens": 7666304, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 35.17, + "gate": 1.91, + "review": 1117.7, + "report": 0.08 + }, + "time_to_first_result_s": 1154.83 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5703.trace.json.gz", + "trace_bytes": 37901 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5703.md b/eval/dataset/arms/copilot_v13_wave2/pr5703.md new file mode 100644 index 000000000..1d8607260 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5703.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5703, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1c6ee28f627c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] diff hunks: read vllm_omni/diffusion/models/minimax_h3/vae.py in situ (lines 97-220); both encode_image (:170) and encode_video (:214) hunks and the :120 constructor hunk match the diff. +- [validated] vae.py:170/214 CUDA/ROCm behavior is unchanged: old predicate `parameter.device.type == "cuda"` and new `!= "cpu"` both include cuda devices, and ROCm reports device.type == 'cuda'. +- [sweep] changed_symbol_consumers: `encode_image`/`encode_video` matches in dreamzero/wan2_2/lance are unrelated same-name methods; the only minimax_h3 consumers are pipeline_minimax_h3.py:963 (`_encode_visual_condition` -> encode_image) and :1066 (`_encode_video_conditions` -> encode_video), both read and hold for cuda/musa. +- [sweep] blast radius of `!= "cpu"`: Ascend NPU (recipes/MiniMaxAI/MiniMax-H3-NPU.md, Atlas 800I A3) and Intel XPU now enter the new RNG branch; NPU is documented/validated, XPU is unvalidated — this is the source of comment 1. +- [validated] vae.py encode_image restores `model.parallel_tiling` in its finally block (vae.py:178-181) — no regression there. +- [validated] recipes/MiniMaxAI/MiniMax-H3-MUSA.md soundfile-fallback claim matches reference_video.py:468-476 `_soundfile_to_waveform` (`sf.read(..., dtype="float32", always_2d=True)` then `.t().contiguous()`, preserving (C,T), FP32, native rate). +- [sweep] recipes/README.md line 45 hunk: the updated MiniMax-H3 table row and new MUSA row are internally consistent with the new recipe. +- [sweep] tests: diff touches no test files (sweep_targets TEST FILES TOUCHED: NONE); repo-wide grep for `test_minimax_h3_vae` returns no matches, so the PR validation command references a non-existent file. + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — In `encode_image` (vae.py:170) and `encode_video` (vae.py:214), `devices = [parameter.device] if parameter.device.type != "cpu" else []` enables the `fork_rng` / `device_module.manual_seed` path for every non-CPU accelerator (e.g. NPU/XPU), contradicting the PR's stated 'allowlist is limited to CUDA and MUSA'. NPU is a documented MiniMax H3 serving path (recipes/MiniMaxAI/MiniMax-H3-NPU.md), so scope the predicate to `parameter.device.type in ("cuda", "musa")` or add NPU/XPU RNG validation evidence and update the description. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vllm_omni/diffusion/models/minimax_h3/vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; PR 5703 body: 'The device allowlist is limited to CUDA and MUSA. Other accelerator RNG semantics are not inferred from mocked interfaces.'; recipes/MiniMaxAI/MiniMax-H3-NPU.md:18 'Runs on a CPU-only (aarch64) PyTorch build with torch_npu; no CUDA runtime is present.'; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:481 `self.device = get_local_device()`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:545 `self.video_vae = MiniMaxH3VideoVAE(`.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [major] — `devices` now includes any non-CPU device (e.g. musa:0), but `torch.random.fork_rng(devices=devices)` at vae.py:172 and vae.py:216 still uses fork_rng's default `device_type="cuda"`. On MUSA that forks/restores torch.cuda RNG state (or raises when CUDA is unavailable) instead of the torch.musa generator seeded by `self.device_module.manual_seed(...)` inside the loop, so the MUSA RNG state this PR promises to restore is not actually forked. Pass `device_type=parameter.device.type` (or derive it from `self.device_module`) in both `encode_image` and `encode_video`. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()` +vllm_omni/diffusion/models/minimax_h3/vae.py:170 `devices = [parameter.device] if parameter.device.type != "cpu" else []` +vllm_omni/diffusion/models/minimax_h3/vae.py:172 `with torch.random.fork_rng(devices=devices):` +vllm_omni/diffusion/models/minimax_h3/vae.py:175 `with self.device_module.device(device):` +vllm_omni/diffusion/models/minimax_h3/vae.py:176 `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` +vllm_omni/diffusion/models/minimax_h3/vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []` +vllm_omni/diffusion/models/minimax_h3/vae.py:216 `with torch.random.fork_rng(devices=devices):`) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76` [major] — The validated Ref2VA configuration command exports the FL2VA partition, not Ref2VA. The recipe states FL2VA serves only t2va/fl2va (lines 17-18) and instructs `Set MODEL to ... Ref2VA for Ref2VA` (line 69), and the pipeline rejects a ref2va task against a non-ref2va partition. A user following the headline Ref2VA profile serves the wrong partition and every Ref2VA request fails; change line 76 to `export MODEL="${MODEL_ROOT}/Ref2VA"` (and note the Ref2VA partition download). (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:76 `export MODEL="${MODEL_ROOT}/FL2VA"` +recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72 `The validated Ref2VA configuration uses tensor parallelism across four MTT` +recipes/MiniMaxAI/MiniMax-H3-MUSA.md:69 `Set MODEL to FL2VA for T2VA and FL2VA, or to Ref2VA for Ref2VA.` +vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:597 `f"checkpoint partition {self.partition!r} supports {sorted(self.supported_tasks)}, got task={task!r}"`) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — The failing check is buildkite/vllm-omni-amd-ci (SUCCESS: vllm-omni, -npu-ci, -intel-ci). On ROCm, torch reports device.type `cuda`, so the change from `== "cuda"` to `!= "cpu"` at vae.py:170/214 is behaviorally identical for the AMD lane and is unlikely to be the regression; confirm the red against main rather than attributing it to this diff. (evidence: ['vllm_omni/diffusion/models/minimax_h3/vae.py:170 ` devices = [parameter.device] if parameter.device.type != "cpu" else []`', 'vllm_omni/diffusion/models/minimax_h3/vae.py:214 ` devices = [parameter.device] if parameter.device.type != "cpu" else []`', 'pr_diff old line: `- devices = [parameter.device] if parameter.device.type == "cuda" else []` — for any device whose type is `cuda` (ROCm included), old and new expressions both evaluate to `[parameter.device]`, so the branch is behaviorally identical.', 'gh_ci_read(5703): buildkite/vllm-omni-amd-ci state FAILURE; buildkite/vllm-omni, buildkite/vllm-omni-npu-ci, buildkite/vllm-omni-intel-ci state SUCCESS.']) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — The new device-aware RNG branch in encode_image (vae.py:175-176) and encode_video (vae.py:219-220) is only entered when `devices` is non-empty, and `devices = [parameter.device] if parameter.device.type != "cpu" else []` (vae.py:170/214) makes it empty for the CPU FakeModel used by the only repo encode_video test (test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))`; called at line 353). So no repo test reaches the changed branch. Also the PR validation command references `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py`, which is absent from the tree. Add a test that drives this branch through a non-CPU device_module (asserting manual_seed is invoked inside device_module.device and RNG state is restored) and fix the validation command. (evidence: vae.py:214 `devices = [parameter.device] if parameter.device.type != "cpu" else []`; vae.py:219-220 `with self.device_module.device(device):` / `self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)`; test_minimax_h3_contract.py:335 `self.weight = torch.nn.Parameter(torch.ones(1))`; test_minimax_h3_contract.py:353 `rows, shape = video_vae.encode_video("frames")` — CPU parameter makes devices empty, so the branch is skipped; list_dir tests/diffusion/models/minimax_h3 shows no test_minimax_h3_vae.py.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:?` [minor] — vae.py:120 binds `self.device_module = torch.get_device_module()` with no device even though `__init__` receives the VAE's `device`. On hosts where the process-default accelerator differs from the VAE device, `self.device_module.device(device)`/`manual_seed` at lines 175-176 and 219-220 will target the wrong module; pass `torch.get_device_module(device)` (or resolve per encode from `parameter.device.type`). Separately, lines 170 and 214 gate on `parameter.device.type != "cpu"`, which seeds every non-CPU accelerator (NPU/XPU/...) and does not implement the PR body's claimed CUDA/MUSA allowlist. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 ` self.device_module = torch.get_device_module()` — called with no device argument while the same __init__ takes `device: torch.device` and moves the VAE to it; vllm_omni/diffusion/models/minimax_h3/vae.py:170 and vllm_omni/diffusion/models/minimax_h3/vae.py:214 ` devices = [parameter.device] if parameter.device.type != "cpu" else []` — any non-CPU accelerator is seeded with no CUDA/MUSA allowlist.) + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` [minor] — At vllm_omni/diffusion/models/minimax_h3/vae.py:120 the new `torch.get_device_module()` call (and `torch.random.fork_rng(devices=...)` at lines 173 and 217) relies on torch APIs whose minimum version is not pinned in pyproject.toml or requirements/*.txt. Please confirm the oldest supported torch (including the externally routed torch-musa stack) provides these APIs, or add an explicit lower-bound/guard. (evidence: vllm_omni/diffusion/models/minimax_h3/vae.py:120 `self.device_module = torch.get_device_module()`; vllm_omni/diffusion/models/minimax_h3/vae.py:173 `with torch.random.fork_rng(devices=devices):`; vllm_omni/diffusion/models/minimax_h3/vae.py:217 `with torch.random.fork_rng(devices=devices):`. Dependency scan: pyproject.toml contains only `"torchmetrics==1.9.0", # for accuracy similarity metrics` (line 59) and `"torchdiffeq==0.2.5", # for covo_audio Test` (line 69) as torch-adjacent pins, and requirements/*.txt contain no `torch>=` lower bound (common.txt has torchsde, musa.txt has torchada>=0.1.52, npu.txt has torchaudio==2.10.0).) + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:64` [minor] — PR description promises 'adds coverage for the torchaudio, soundfile, and ffmpeg audio-loading paths', but the diff only adds the MUSA recipe, README entry, and device-aware RNG handling in vae.py — no test file and no audio-loading code. Please point to the test that guards the soundfile/ffmpeg fallback described at recipes/MiniMaxAI/MiniMax-H3-MUSA.md:64, or remove the coverage claim; the manual MUSA smoke does not guard the regression. (evidence: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:64 `When torchaudio cannot load TorchCodec, vLLM-Omni automatically falls back to`; PR 5703 body: `adds coverage for the torchaudio, soundfile, and ffmpeg audio-loading paths`; PR 5703 files: `recipes/MiniMaxAI/MiniMax-H3-MUSA.md` (147 additions), `recipes/README.md` (2 additions/1 deletion), `vllm_omni/diffusion/models/minimax_h3/vae.py` (7 additions/6 deletions) — no test or audio-loading file in the diff.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5703.trace.json.gz b/eval/dataset/arms/copilot_v13_wave2/pr5703.trace.json.gz new file mode 100644 index 000000000..3bb65b26b Binary files /dev/null and b/eval/dataset/arms/copilot_v13_wave2/pr5703.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5715.cost.json b/eval/dataset/arms/copilot_v13_wave2/pr5715.cost.json new file mode 100644 index 000000000..f47de856c --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5715.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 388.0, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_wave2/runs/pr5715/run-20260813-174808-4fdb44", + "recorded_at": "2026-08-13T17:54:32", + "attempts": [ + { + "invocation_id": "inv-1de51fe9", + "outcome": "run:run-20260813-174808-4fdb44", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-174808-4fdb44", + "llm_calls": 12, + "input_tokens": 121265, + "output_tokens": 26111, + "usd": 0.075861, + "source": "spans" + } + ], + "attempt_usd_total": 0.075861, + "llm_calls": 12, + "input_tokens": 121265, + "output_tokens": 26111, + "usd": 0.075861, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-174808-4fdb44", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0759, + "minutes": 6.41, + "ci_minutes": 0, + "input_tokens": 121265, + "output_tokens": 26111, + "tool_calls": 40, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2536, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 12, + "by_role": { + "agent": { + "usd": 0.075861, + "input_tokens": 121265, + "output_tokens": 26111, + "calls": 12 + } + }, + "cache_read_tokens": 533248, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 35.39, + "gate": 2.08, + "review": 346.8, + "report": 0.06 + }, + "time_to_first_result_s": 384.32 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5715.trace.json.gz", + "trace_bytes": 12109 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5715.md b/eval/dataset/arms/copilot_v13_wave2/pr5715.md new file mode 100644 index 000000000..91271682c --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5715.md @@ -0,0 +1,32 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5715, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 765caa2a121a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] README.md:19-21 read in situ — 0.26.0 entry, retained 0.24.0 entry, and consolidated 0.14-0.22 cadence entry all present; no dropped/duplicated hunks. +- [validated] README.md:19 links recipes/MiniMaxAI/MiniMax-H3.md and recipes/OpenBMB/MiniCPM-o-4_5.md — both files exist (list_dir recipes/MiniMaxAI and recipes/OpenBMB). +- [validated] Full-duplex runtime is real repo functionality: vllm_omni/experimental/fullduplex/ exists and examples/online_serving/minicpmo/README.md:58-59 documents 'experimental native duplex backend' with vllm_omni/deploy/minicpmo_4_5_duplex.yaml. +- [sweep] README.md:55 feature-list addition 'Full-duplex realtime serving with streaming audio input and output (experimental)' verified in situ; experimental qualifier present. +- [sweep] docs/README.md:52-61 read in situ — model list and feature list mirror README.md changes; no bold-formatting regression (original was already unbolded). +- [sweep] docs/configuration/README.md:5 — link now points to https://docs.vllm.ai/en/v0.26.0/configuration/index.html; grep across docs shows no other docs.vllm.ai/en/v0.* configuration links remain. +- [sweep] docs/getting_started/installation/README.md:4 — compatibility admonition added; repo-wide 0.26 pins in cuda.inc.md/rocm.inc.md/quickstart.md are consistent with it. +- [sweep] docs/getting_started/installation/gpu/cuda.inc.md:5,127 — BASE_IMAGE vllm/vllm-openai:v0.26.0 matches docker/Dockerfile.cuda:1 ARG default; grep confirms v0.22.1 no longer appears anywhere in the repo. + +`README.md:?` [minor] — This diff announces the full-duplex realtime runtime and anchors it to recipes/OpenBMB/MiniCPM-o-4_5.md, but that recipe never mentions the duplex runtime, so a reader following the link cannot find the announced feature. Repo evidence: grep for 'duplex' and 'realtime' in recipes/OpenBMB/MiniCPM-o-4_5.md returns no matches, while examples/online_serving/minicpmo/README.md:58-59 actually documents it — 'To start the experimental native duplex backend, use vllm_omni/deploy/minicpmo_4_5_duplex.yaml.' Point the anchor at examples/online_serving/minicpmo/README.md (or vllm_omni/experimental/fullduplex/DESIGN.md) instead of, or in addition to, the recipe. (evidence: recipes/OpenBMB/MiniCPM-o-4_5.md: grep 'duplex'/'realtime' -> no matches; examples/online_serving/minicpmo/README.md:58-59: 'To start the experimental native duplex backend, use `vllm_omni/deploy/minicpmo_4_5_duplex.yaml`.') + +`docs/getting_started/installation/README.md:?` [minor] — This diff adds a blanket compatibility statement at the top of the installation page, but it is not scoped, and the MUSA page reachable from the same index still instructs a much older dependency line. Repo evidence: docs/getting_started/installation/gpu/musa.inc.md:31 says 'git checkout v0.18.0-dev' for vLLM-MUSA. Either qualify the note (e.g. upstream vLLM on CUDA/ROCm/XPU, not platform forks) or add a one-line MUSA caveat so MUSA users aren't told to expect vLLM 0.26.x. (evidence: docs/getting_started/installation/gpu/musa.inc.md:31: 'git checkout v0.18.0-dev' (vLLM-MUSA install instructions), unchanged by this PR.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5715.trace.json.gz b/eval/dataset/arms/copilot_v13_wave2/pr5715.trace.json.gz new file mode 100644 index 000000000..c1d211f51 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_wave2/pr5715.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5840.cost.json b/eval/dataset/arms/copilot_v13_wave2/pr5840.cost.json new file mode 100644 index 000000000..577ea7db1 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5840.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1228.2, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_wave2/runs/pr5840/run-20260813-175435-e248a1", + "recorded_at": "2026-08-13T18:15:00", + "attempts": [ + { + "invocation_id": "inv-fc5191af", + "outcome": "run:run-20260813-175435-e248a1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-175435-e248a1", + "llm_calls": 164, + "input_tokens": 1319182, + "output_tokens": 267191, + "usd": 1.09085, + "source": "spans" + } + ], + "attempt_usd_total": 1.09085, + "llm_calls": 164, + "input_tokens": 1319182, + "output_tokens": 267191, + "usd": 1.09085, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-175435-e248a1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0909, + "minutes": 20.42, + "ci_minutes": 0, + "input_tokens": 1319182, + "output_tokens": 267191, + "tool_calls": 261, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9583, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 164, + "by_role": { + "agent": { + "usd": 1.056502, + "input_tokens": 1291749, + "output_tokens": 242730, + "calls": 162 + }, + "reducer": { + "usd": 0.034349, + "input_tokens": 27433, + "output_tokens": 24461, + "calls": 2 + } + }, + "cache_read_tokens": 16324480, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.7, + "gate": 1.69, + "review": 1219.86, + "report": 0.06 + }, + "time_to_first_result_s": 1225.3 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5840.trace.json.gz", + "trace_bytes": 47296 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5840.md b/eval/dataset/arms/copilot_v13_wave2/pr5840.md new file mode 100644 index 000000000..cfef72b3c --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5840.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5840, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 75cd90708932) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] backend.py:76-99 custom enabler read; ref2va raises, combined warns and hooks only pipeline.transformer; pipeline_minimax_h3.py:736-738 confirms Ref2VA routes to transformers_ref, which stays unhooked in combined mode as documented. +- [validated] config.py:97-133 read; TeaCacheConfig.__post_init__ resolves None -> 0.17 for MiniMaxH3DiTModel and 0.2 otherwise, so hook.py:234-236's `assert rel_l1_thresh is not None` cannot fire. +- [validated] extractors.py:1252-1404 read side-by-side with minimax_h3_transformer.py:1077-1190 forward; kwargs contract, _embed, block loop, final_layer, masks and video_layout forwarding match native. +- [validated] tests/diffusion/cache/test_teacache_extractors.py:625-641 native-parity test and 680-699 guaranteed-cache-hit test actually drive the real MiniMaxH3DiTModel with fake linears and assert block_calls == 1. +- [sweep] indexed accesses at extractors.py:1312-1342,1381 and test lines 629-698 are guarded by explicit shape checks / fixtures with known 2-row img/audio counts; no unchecked [0] indexing on production code. +- [sweep] branches at backend.py:79,83,93 reachable (ref2va/fl2va/combined/None); the None/unexpected-partition raise path has no unit test but is defensive and low-risk. +- [sweep] changed_symbol_consumers: all in-repo rel_l1_thresh readers grepped — async_omni_engine.py:855 is the stale hardcoded-0.2 consumer (comment 1); hunyuan_image3_transformer.py:3156 reads the already-resolved field; cli/serve.py:586 help text still shows a 0.2 example (cosmetic, not broken). +- [sweep] non-test touched files read in situ: backend.py, config.py, extractors.py, hook.py, data.py, docs/user_guide/diffusion_features.md (table hunk), recipes/MiniMaxAI/MiniMax-H3.md TeaCache section and Known limitations. + +`vllm_omni/engine/async_omni_engine.py:?` [major] — vllm_omni/engine/async_omni_engine.py:854-856 still returns {'rel_l1_thresh': 0.2} for 'tea_cache', and _normalize_cache_config (async_omni_engine.py:881-882) injects that dict whenever cache_config is omitted. That explicit 0.2 flows to DiffusionCacheConfig/TeaCacheConfig as an explicit value, so MiniMax-H3's model-specific default of 0.17 (teacache/config.py:97-98,129-133) is never exercised by Omni(cache_backend='tea_cache') or vllm serve --cache-backend tea_cache without --cache-config, contradicting recipes/MiniMaxAI/MiniMax-H3.md:762. Leave the absent-config path empty/None so the model default applies, or document 0.2 as the serving default, and add a regression test. (evidence: vllm_omni/engine/async_omni_engine.py:855 `"rel_l1_thresh": 0.2,` (inside `_get_default_cache_config` tea_cache branch); vllm_omni/engine/async_omni_engine.py:881-882 `if cache_config is None and cache_backend not in (None, "", "none"):` / `cache_config = AsyncOmniEngine._get_default_cache_config(cache_backend)`; vllm_omni/diffusion/cache/teacache/config.py:97-98 `_MODEL_DEFAULT_REL_L1_THRESH = {` / ` "MiniMaxH3DiTModel": 0.17,`; vllm_omni/diffusion/cache/teacache/config.py:129-130 `if threshold is None:` / ` threshold = _MODEL_DEFAULT_REL_L1_THRESH.get(self.transformer_type, _DEFAULT_REL_L1_THRESH)`; recipes/MiniMaxAI/MiniMax-H3.md:762 `The model-specific default and examples use \`rel_l1_thresh=0.17\``.) + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:?` [major] — The diff adds MiniMaxH3Pipeline to the TeaCache custom enablers in vllm_omni/diffusion/cache/teacache/backend.py and documents 'TeaCache and Cache-DiT are mutually exclusive; pick one cache backend per server' (recipes/MiniMaxAI/MiniMax-H3.md:760), but the request-scoped Cache-DiT path still installs Cache-DiT when a request sends quality='high' on a TeaCache server: MiniMaxH3QualityPolicy.resolve returns a CacheDiTRequestSpec for quality=='high' without checking _configured_backend, and RequestScopedCacheDiTRuntime.prepare calls CacheDiTBackend.enable(self._pipeline), so the FL2VA transformer can end up with both the TeaCache hook and Cache-DiT hooks. Reject quality='high' when _configured_backend == 'tea_cache' (or explicitly disable TeaCache first), and add a regression test that a tea_cache server with quality='high' does not install Cache-DiT. (evidence: quality_policy.py:149 `if quality == "high":`; quality_policy.py:152 `elif self._configured_backend == "cache_dit" and quality is None:` (the high branch has no backend guard); vllm_omni/diffusion/cache/cachedit/runtime.py:72-73 `if self._backend is None:\n backend = CacheDiTBackend(spec.cache_config)\n backend.enable(self._pipeline)`; pipeline_minimax_h3.py:1578 `quality = sampling.quality`, :1761 `quality_plan = self._quality_policy.resolve(`, :1766 `self._cache_dit_runtime.prepare(quality_plan.cache_dit)`; recipes/MiniMaxAI/MiniMax-H3.md:760 `TeaCache and Cache-DiT are mutually exclusive; pick one cache backend per server.`) + +`.buildkite/amd/test-amd-ready.yml:19` [minor] — .buildkite/amd/test-amd-ready.yml:19 runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py`, and tests/diffusion/cache/test_cache_backends.py carries `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, so this PR's modified file IS exercised by the failing buildkite/vllm-omni-amd-ci lane. Please confirm whether that AMD failure is pre-existing or introduced by this diff. (evidence: ['.buildkite/amd/test-amd-ready.yml:19 ` - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m \'core_model and cpu\' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`', 'tests/diffusion/cache/test_cache_backends.py:28 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`', 'gh_ci_read(5840): `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`']) + +`cli/serve.py:586` [minor] — vllm_omni/entrypoints/cli/serve.py:586 — the --cache-config help example still reads TeaCache: '{"rel_l1_thresh": 0.2}'. Since this PR introduces model-specific defaults (MiniMax-H3 0.17, generic fallback 0.2), consider updating the example/help to mention the model-specific default and 0.2 fallback; cosmetic only, the 0.2 example is still a valid fallback value. (evidence: vllm_omni/entrypoints/cli/serve.py:586 `"TeaCache: '{\"rel_l1_thresh\": 0.2}'. "`) + +`docs/user_guide/diffusion/cache_acceleration/teacache.md:?` [minor] — docs/user_guide/diffusion/cache_acceleration/teacache.md:122 still lists `rel_l1_thresh` Default as `0.2`. The diff changes TeaCacheConfig to `rel_l1_thresh: float | None = None` and resolves None to a model-specific default (`MiniMaxH3DiTModel` -> 0.17, otherwise 0.2) in vllm_omni/diffusion/cache/teacache/config.py:96-99 and 122. Update the table to describe the model-specific default/fallback behavior so the guide matches the new config resolution. (evidence: docs/user_guide/diffusion/cache_acceleration/teacache.md:122 `| `rel_l1_thresh` | float | `0.2` | Similarity threshold for cache reuse. Lower values prioritize quality (less caching), higher values prioritize speed (more caching). Suggested range: 0.1-0.8 |`; vllm_omni/diffusion/cache/teacache/config.py:122 `rel_l1_thresh: float | None = None`; vllm_omni/diffusion/cache/teacache/config.py:96 `_DEFAULT_REL_L1_THRESH = 0.2`; vllm_omni/diffusion/cache/teacache/config.py:97 `_MODEL_DEFAULT_REL_L1_THRESH = {`; vllm_omni/diffusion/cache/teacache/config.py:98 ` "MiniMaxH3DiTModel": 0.17,`) + +`tests/diffusion/cache/test_teacache_extractors.py:499` [minor] — The diff adds _MiniMaxH3FakeAttention.forward returning q (tests/diffusion/cache/test_teacache_extractors.py:499), so test_extractor_matches_native_forward compares native and extractor paths through an attention stub that discards metadata; a regression in the extractor's cu_seqlens_q/max_seqlen_q forwarding would not change the output and the test would still pass. Extend test_run_transformer_blocks_forwards_packed_layout to also capture and assert kwargs['cu_seqlens'] == sample_inputs['packed_seq_params']['cu_seqlens_q'] and kwargs['max_seqlen'] == sample_inputs['packed_seq_params']['max_seqlen_q'] from the block call. (evidence: tests/diffusion/cache/test_teacache_extractors.py:499 `return q`; :661 `received.append((kwargs["packed_total"], kwargs["video_layout"]))`; :668 `assert received == [(sample_inputs["x"].shape[1], video_layout)]`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:136` [minor] — test_minimax_h3_t2va_teacache_single_gpu_smoke (lines 145-186) passes cache_backend='tea_cache' and num_inference_steps=2 but asserts only via _assert_joint_output(outputs) at line 186, which checks output shape/audio metadata (lines 19-26). TeaCache always computes the first step, so with no hook-registration or cache-hit/block-call assertion the test would still pass if tea_cache were silently ignored (selector returns None, hook not registered, or zero hits). Add a cheap engagement assertion or rename it as a plumbing smoke that does not claim TeaCache acceleration. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:186 ` _assert_joint_output(outputs)` — the sole assertion in test_minimax_h3_t2va_teacache_single_gpu_smoke; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:19-26 `def _assert_joint_output(outputs): ... assert frames.shape == (107, 256, 448, 3) ... assert multimodal["fps"] == 24` — none of these asserts TeaCache selection, registration, or cache hits.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:22` [minor] — The diff changes the shared _assert_joint_output expectation from 39 to 107 frames and adds duration=4.0 / aspect_ratio=16:9 to test_minimax_h3_t2va_fp8_single_gpu_smoke (tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:22,125), making that smoke's workload roughly 2.7x the previous frame count, while the PR description only reports running the new TeaCache smoke. Please re-run and report the modified FP8 smoke: python -m pytest -q tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py::test_minimax_h3_t2va_fp8_single_gpu_smoke. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:22 ` assert frames.shape == (107, 256, 448, 3)`; tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:125 ` "duration": 4.0,`; PR #5840 body Testing section: `The current-main one-GPU E2E smoke also completed successfully:` followed by `python -m pytest -q tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py::test_minimax_h3_t2va_teacache_single_gpu_smoke` — no re-run of test_minimax_h3_t2va_fp8_single_gpu_smoke is reported.) + +**Additional observations (beyond the comment budget):** +- `.buildkite/amd/test-amd-ready.yml:19` [minor] The diff adds CPU-marked TestMiniMaxH3Extractor tests in tests/diffusion/cache/test_teacache_extractors.py (class at lines 569-570), but both AMD CPU lanes in .buildkite/amd/test-amd-ready.yml (lines 19 and 30) run `--ig + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5840.trace.json.gz b/eval/dataset/arms/copilot_v13_wave2/pr5840.trace.json.gz new file mode 100644 index 000000000..916758414 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_wave2/pr5840.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5863.cost.json b/eval/dataset/arms/copilot_v13_wave2/pr5863.cost.json new file mode 100644 index 000000000..7bc2316c7 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5863.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1607.5, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_wave2/runs/pr5863/run-20260813-180038-2961ca", + "recorded_at": "2026-08-13T18:27:23", + "attempts": [ + { + "invocation_id": "inv-000b184b", + "outcome": "run:run-20260813-180038-2961ca", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-180038-2961ca", + "llm_calls": 124, + "input_tokens": 904929, + "output_tokens": 165985, + "usd": 0.691976, + "source": "spans" + } + ], + "attempt_usd_total": 0.691976, + "llm_calls": 124, + "input_tokens": 904929, + "output_tokens": 165985, + "usd": 0.691976, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-180038-2961ca", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.692, + "minutes": 26.75, + "ci_minutes": 0, + "input_tokens": 904929, + "output_tokens": 165985, + "tool_calls": 235, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9227, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 124, + "by_role": { + "agent": { + "usd": 0.672371, + "input_tokens": 894437, + "output_tokens": 150769, + "calls": 122 + }, + "reducer": { + "usd": 0.019605, + "input_tokens": 10492, + "output_tokens": 15216, + "calls": 2 + } + }, + "cache_read_tokens": 9817088, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.7, + "gate": 1.88, + "review": 1599.2, + "report": 0.07 + }, + "time_to_first_result_s": 1604.84 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 29820 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5863.md b/eval/dataset/arms/copilot_v13_wave2/pr5863.md new file mode 100644 index 000000000..0cdfe2746 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5863.md @@ -0,0 +1,50 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 4 finding(s) below | + +**Validated:** +- [validated] Attention-backend default: vllm_omni/platforms/cuda/platform.py:206-225 gates TRTLLM_ATTN default on compute_capability.major == 10 and routes sm_120 Blackwell to CUDNN_ATTN; recipe claim 'TRTLLM_ATTN default does not apply here' is correct. +- [validated] t2va aspect_ratio requirement: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:347-348 raises 't2va requires an explicit aspect_ratio' when value is None, independent of width/height; recipe line 306 is correct. +- [validated] Rank grouping: initialize_model_parallel uses order 'tp-sp-pp-cfg-dp' and set_seq_parallel_pg(use_ulysses_low=True); TP ranks are consecutive and Ulysses ranks strided by TP. Recipe's TP2/USP4 groups (0,2,4,6)/(1,3,5,7) with TP pairs (0,1),(2,3),(4,5),(6,7) and both dual-socket device-order tables verified (vllm_omni/diffusion/distributed/parallel_state.py). +- [validated] CLI flags: --usp, --ring, --num-gpus, --tensor-parallel-size, --text-encoder-tp-size, --vae-patch-parallel-size, --vae-parallel-mode, --vae-use-tiling, --diffusion-attention-backend, --task-type all exist in vllm_omni/entrypoints/cli/serve.py. +- [validated] CFG-distilled constraint: MiniMaxH3Pipeline.__init__ raises ValueError when cfg_parallel_size != 1 (pipeline_minimax_h3.py); recipe's '--cfg-parallel-size must remain 1' is correct. +- [validated] X-Stage-Durations and X-Peak-Memory-MB response headers exist in vllm_omni/entrypoints/openai/api_server.py:3510-3511. +- [validated] Internal arithmetic recomputes: 278.55/168.73=1.65x and 168.73/87.90=1.92x denoise; 5.396/2.791=1.93x and 2.791/1.798=1.55x VAE; stage sums are ~0.75s below E2E in all three rows; headroom 18.5/29.6/34.9 GiB; 107.58/90.48=18.9% first-request slowdown; memory fit 55.4+22.1/usp predicts 60.9 GiB vs 61.07 measured. +- [sweep] Ref2VA request path checked against _parse_video_form (api_server.py) and serving_video.py: input_reference image + audio_reference JSON flows work, but /root/audio_ref.json's required JSON shape is undocumented (see review comment). + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [minor] — Lines 201 and 203 present `--cpunodebind`/`--membind` as if they were `vllm serve` flags, but they are numactl(8) options and are not registered by the serve parser. Reword to `numactl --cpunodebind=0 --membind=0 vllm serve ... --port 8091` for FL2VA and `numactl --cpunodebind=1 --membind=1 vllm serve ... --port 8092` for Ref2VA, or state explicitly that these flags belong to numactl. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201 `Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and`; recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:203 ``--cpunodebind=1 --membind=1`, and `--port 8092` for the Ref2VA server.`; repo-wide grep for `cpunodebind`/`membind` matches only these two lines, and `vllm_omni/entrypoints/cli/serve.py` constructs the serve parser without any `cpunodebind`/`membind` argument (parser flags such as `--num-gpus`, `--usp`, `--vae-patch-parallel-size` are present; no numactl option is).) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340` [minor] — recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340 sends `-F "audio_reference= unsqueeze(0) -> [1,B,T] with num_quantizers=1 is correct (N=1,batch=B,time=T); output [B,2T,hidden]. No batch-as-time bug. +- [sweep] full-payload accumulator retention: read omni_connector_model_runner_mixin.accumulate_full_payload_output — flat meta.* conditioning emitted only on first decode step persists because absent keys are not pruned; defer path snapshots CUDA tensors. + +`tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:16` [major] — tests/model_executor/models/indextts2/{test_configuration_and_pipeline.py,test_enhanced_codec.py,test_recipe_defaults.py} all lack `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (and `import pytest`), so `.buildkite/cuda/test-merge.yml:13` (`pytest -sv tests/model_executor -m 'core_model and cpu'`) deselects every test in them; the pinned pipeline registration, use_gpt_latent=false, s2mel_cfm_batch_size=4 and EnhancedCodec behavior never run in CI. Add the marker to all three files. (evidence: tests/model_executor/models/indextts2/test_configuration_and_pipeline.py:17 `def test_indextts25_defaults_are_distinct_from_v2():` — file imports only `from pathlib import Path` and `import yaml`, with no `pytestmark`; tests/model_executor/models/indextts2/test_enhanced_codec.py:1 `import torch` with no `pytestmark`; tests/model_executor/models/indextts2/test_recipe_defaults.py:1 `from pathlib import Path` with no `pytestmark`; .buildkite/cuda/test-merge.yml:13 `timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' ...`; tests/model_executor/models/indextts2/test_s2mel_v2_5.py:26 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +`vllm_omni/deploy/qwen2_5_omni.yaml:89` [minor] — vllm_omni/deploy/qwen2_5_omni.yaml:90,93 flips the Qwen2.5-Omni `xpu` override for AR stages 0/1 from `enforce_eager: false` to `true`, disabling CUDA graphs for an existing model on XPU. This accelerator-behavior change is bundled inside the IndexTTS 2.5 PR, covered only by config-load assertions (tests/config/test_config_factory.py::test_qwen2_5_omni_xpu_uses_eager_ar_stages) with no XPU runtime/accuracy evidence, and is absent from the PR body's 'Common-path changes' scope list. Split it out or add the XPU perf/accuracy rationale plus the lane that actually runs Qwen2.5-Omni on XPU. (evidence: vllm_omni/deploy/qwen2_5_omni.yaml:90 ` enforce_eager: true`; vllm_omni/deploy/qwen2_5_omni.yaml:93 ` enforce_eager: true`; tests/config/test_config_factory.py:2285 ` assert deploy.stages[0].enforce_eager is True`) + +`.buildkite/amd/test-amd-ready.yml:?` [minor] — Confirmed: .buildkite/amd/test-amd-ready.yml:19 and :30 are the same `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml` command; the AMD lane therefore duplicates a model_executor bucket and does not run tests/worker or tests/entrypoints, so the PR's request.py/gpu_ar_model_runner/mm_outputs changes are uncovered there. buildkite/vllm-omni-amd-ci is FAILURE while other checks are SUCCESS. (evidence: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` and .buildkite/amd/test-amd-ready.yml:30 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — identical commands, and the file contains no tests/worker or tests/entrypoints step.) + +`stage_init_utils.py:1140` [minor] — The `_TokenOnlyRenderer` branch at `stage_init_utils.py:1140` is hit by every skip_tokenizer_init stage-0 pipeline (indextts2.yaml, indextts2_low_latency.yaml, personaplex.yaml, indextts2_5.yaml), but both existing unit tests monkeypatch `InputProcessor` (and the skip-specific test also monkeypatches `_build_token_only_renderer`), so the real `InputProcessor(vllm_config=..., renderer=_TokenOnlyRenderer(stage_vllm_config, tokenizer=None))` construction is never exercised. Please add a regression test that instantiates the real path (or otherwise verify the two-positional-arg `BaseRenderer` call) for the pre-existing skip_tokenizer_init pipelines. (evidence: vllm_omni/engine/stage_init_utils.py:1140 `if bool(getattr(stage_vllm_config.model_config, "skip_tokenizer_init", False)):` — gate that routes all skip_tokenizer_init stage-0 configs to the new renderer; vllm_omni/engine/stage_init_utils.py:1133 `return _TokenOnlyRenderer(stage_vllm_config, tokenizer=None)` — the unverified two-positional-arg construction; tests/engine/test_async_omni_engine_stage_init.py:851 `monkeypatch.setattr(init_mod, "InputProcessor", DummyInputProcessor)` and tests/engine/test_async_omni_engine_stage_init.py:854 `"_build_token_only_renderer",` / `lambda _config: token_only_renderer,` — both real objects are mocked in the only tests that cover the function.) + +`.buildkite/intel/pipeline-intel.yml:13` [minor] — This diff upgrades the Intel XPU CI path to vLLM v0.27.0 (`.buildkite/intel/pipeline-intel.yml:13` `VLLM_VERSION: "v0.27.0"`, `docker/Dockerfile.xpu:91` `ARG VLLM_VERSION=v0.27.0`) and sets Qwen2.5-Omni XPU AR stages to eager (`vllm_omni/deploy/qwen2_5_omni.yaml:90` and `:93` `enforce_eager: true`), but the PR body Test Plan still says **vLLM Version:** 0.26.0 and the Common-path changes list omits the XPU/0.27 alignment. The 0.26 E2E evidence therefore does not cover the 0.27 XPU CI path Buildkite now runs. Update the body to state the rebase target and the XPU eager/perf tradeoff, or split the Intel XPU/0.27 alignment out of this new-model PR. (evidence: .buildkite/intel/pipeline-intel.yml:13 ` VLLM_VERSION: "v0.27.0"`; docker/Dockerfile.xpu:91 `ARG VLLM_VERSION=v0.27.0`; vllm_omni/deploy/qwen2_5_omni.yaml:90 ` enforce_eager: true` and :93 ` enforce_eager: true`; PR body (gh_pr_view #5957) `**vLLM Version:** 0.26.0`.) + +`pyproject.toml:91` [minor] — pyproject.toml:91 declares `WeTextProcessing; sys_platform == 'linux'` with no version lower bound, while text_processing_v2_5.py:117-123 constructs `tn.*.normalizer.Normalizer` with kwargs (`remove_interjections`, `remove_erhua`, `overwrite_cache`) that may not exist in every historical release. Pin a minimum known-good version and update test_indextts2_optional_dependencies.py:85 (currently asserting an empty specifier), or verify and document that the oldest published release supports all used kwargs. (evidence: pyproject.toml:91 ` "WeTextProcessing; sys_platform == 'linux'",`; vllm_omni/model_executor/models/indextts2/text_processing_v2_5.py:117-121 ` return Normalizer(` / ` remove_interjections=False,` / ` remove_erhua=False,` / ` overwrite_cache=False,` and :123 ` return Normalizer(overwrite_cache=False)`; tests/test_indextts2_optional_dependencies.py:85 ` ("wetextprocessing", "", 'sys_platform == "linux"'),`) + +`tests/model_executor/models/indextts2/test_enhanced_codec.py:22` [minor] — The only real-codec `EnhancedCodec.decode` test uses B=1 (`codec.decode(torch.tensor([[0, 1, 2]]))`), while `IndexTTS2S2MelDecoder._build_semantic_embedding` feeds `(B, max_code_len)` batch tensors into `codec.decode(mel_codes)` and Stage 1 ships with `max_num_seqs: 4`. The batched semantic path is covered only by `_FakeEnhancedCodec`, which does not exercise the real `[B,T] -> unsqueeze(0) -> [N,B,T]` decode contract. Add a B=2 real-codec decode case asserting `(2, 2*T, hidden_size)` to guard the leading-dimension interpretation. (evidence: vllm_omni/model_executor/models/indextts2/codec/models.py:95-98 `def decode(self, codes: torch.Tensor) -> torch.Tensor: + """Decode `[B,T]` or `[N,B,T]` codes to `[B,T*scale,hidden]`.""" + if codes.ndim == 2: + codes = codes.unsqueeze(0)`; vllm_omni/model_executor/models/indextts2/indextts2_s2mel_decoder.py:917 `semantic = codec.decode(mel_codes)` (consumer, called with `(B, max_code_len)` from `_build_batched_inputs`); vllm_omni/deploy/indextts2_5.yaml:41 `max_num_seqs: 4`; tests/model_executor/models/indextts2/test_enhanced_codec.py:22 `decoded = codec.decode(torch.tensor([[0, 1, 2]]))`; tests/model_executor/models/indextts2/test_s2mel_v2_5.py:29 `class _FakeEnhancedCodec:`) + +`vllm_omni/model_executor/models/indextts2/preprocess_utils.py:282` [minor] — preprocess_utils.py:282 now raises RuntimeError('IndexTTS requires torchaudio Kaldi fbank support') from exc, replacing the prior logger.warning + torch.zeros fallback. indextts2_talker.py:634 still calls fbank = compute_fbank(wav_16k, device) from the versioned talker that serves both indextts2 and indextts2_5, so IndexTTS-2 also hard-fails when torchaudio Kaldi fbank is unavailable. The fail-fast direction is defensible; no test or doc note covers this changed shared path. (evidence: vllm_omni/model_executor/models/indextts2/preprocess_utils.py:282 ` raise RuntimeError("IndexTTS requires torchaudio Kaldi fbank support") from exc`; vllm_omni/model_executor/models/indextts2/indextts2_talker.py:634 ` fbank = compute_fbank(wav_16k, device)`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5957.trace.json.gz b/eval/dataset/arms/copilot_v13_wave2/pr5957.trace.json.gz new file mode 100644 index 000000000..7dc262ee7 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_wave2/pr5957.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5976.cost.json b/eval/dataset/arms/copilot_v13_wave2/pr5976.cost.json new file mode 100644 index 000000000..b3e5b7e97 --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5976.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1317.8, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v13_wave2/runs/pr5976/run-20260813-181311-4736a9", + "recorded_at": "2026-08-13T18:35:07", + "attempts": [ + { + "invocation_id": "inv-c441b962", + "outcome": "run:run-20260813-181311-4736a9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-181311-4736a9", + "llm_calls": 131, + "input_tokens": 1550782, + "output_tokens": 265325, + "usd": 1.096469, + "source": "spans" + } + ], + "attempt_usd_total": 1.096469, + "llm_calls": 131, + "input_tokens": 1550782, + "output_tokens": 265325, + "usd": 1.096469, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-181311-4736a9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0965, + "minutes": 21.92, + "ci_minutes": 0, + "input_tokens": 1550782, + "output_tokens": 265325, + "tool_calls": 225, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9876, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 131, + "by_role": { + "agent": { + "usd": 1.060321, + "input_tokens": 1504882, + "output_tokens": 243761, + "calls": 129 + }, + "reducer": { + "usd": 0.036148, + "input_tokens": 45900, + "output_tokens": 21564, + "calls": 2 + } + }, + "cache_read_tokens": 14292608, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.37, + "gate": 1.78, + "review": 1309.09, + "report": 0.06 + }, + "time_to_first_result_s": 1315.29 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5976.trace.json.gz", + "trace_bytes": 64250 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5976.md b/eval/dataset/arms/copilot_v13_wave2/pr5976.md new file mode 100644 index 000000000..ad22d712f --- /dev/null +++ b/eval/dataset/arms/copilot_v13_wave2/pr5976.md @@ -0,0 +1,48 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5976, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 25b80bb995bd) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] Double-seed defect confirmed: omni_ar_scheduler.py:628-629 adds num_in_flight_tokens to num_stale_output_tokens unconditionally; omni_ar_scheduler.py:669-670 then calls _replace_streaming_session on replace_streaming_prompt=True, and omni_scheduler_mixin.py:105 adds the same value again. omni_generation_scheduler.py `_update_request_as_session` only calls the mixin (line 604), so it is single-seeded — the bug is AR-specific. +- [validated] New stale-drain tests do not exercise the replacement branch: tests/core/sched/test_omni_ar_scheduler_stale_drain.py `_replace_streaming_session` builds a StreamingUpdate with no additional_information/model_intermediate_buffer, so replace_streaming_prompt is False and only the top seed runs (assert num_stale_output_tokens == 1 at line 145); tests/core/sched/test_omni_ar_scheduler_streaming.py drives the real replacement path but never asserts num_stale_output_tokens. +- [sweep] Indexed accesses: tests/diffusion/test_compile.py:107,110 `model.blocks[0]` is safe (the test's `_Model.__init__` appends exactly one block); qwen2_5_omni_thinker.py:308 `videos[0]` is a comment quoting transformers internals, and the fix switches to `self.get(index)` to unwrap MediaWithBytes. +- [sweep] New branches all reachable/checked against the diff: hasattr guards in stale-drain test (0.26 dev compat), `if patched:` in test_inductor (wrapper-install guard), bagel `if not all(...)` (probe aggregation), qwen_image `_FLASH_ATTN3_HUB_AVAILABLE` lazy cache and threshold branch, qwen3 `return_hidden_states` branch, `probe_request_rate > 0`/`probe_task is not None`, scheduler `num_stale_output_tokens > 0`/`output_is_stale`, parallel_state `"use_all2all" in signature`, and `not replica_id or replica_id <= 0`. +- [validated] vllm_omni/patch.py `_provably_factorable_multiple` is sound: it can only ADD True, requires sympy.cancel(num/den) to yield an integer-coefficient polynomial (identity num == den*q for every symbol assignment); FloorDiv/ModularIndexing atoms and >20 symbols are rejected; install probe self-extinguishes on torch <=2.11 and the wrapper install is idempotent. +- [validated] vllm_omni/engine/stage_engine_startup.py `_scoped_replica_compile_cache_env` restores VLLM_CACHE_ROOT in finally; lock order cache->device is never inverted (cache lock only acquired inside scoped_spawn_device_env); replica 0 keeps shared cache and per-(stage_id, replica_id) dirs are unique. +- [validated] qwen_image SDPA fallback is genuinely matched: DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA maps to set_attention_backend("native") in pipeline_diffusers_adapter.py (`case "TORCH_SDPA": attention_backend_attempts.append("native")`) and the reference `_set_reference_attention_backend` also uses "native"; 0.97->0.96 loosening is gated to the fallback path only. +- [validated] New test files all carry pytestmark matching CI lanes: core/sched, engine, model_executor and diffusion files use `pytest.mark.core_model`/`cpu` and are selected by the `-m 'core_model and cpu'` commands in .buildkite/cuda/test-merge.yml and test-ready.yml. + +`vllm_omni/core/sched/omni_ar_scheduler.py:628` [blocker] — `_update_request_as_session` seeds `num_stale_output_tokens += in_flight_tokens` at omni_ar_scheduler.py:628-629 before the `replace_streaming_prompt` branch, and that branch then calls `_replace_streaming_session` (omni_ar_scheduler.py:669-670), which adds `num_in_flight_tokens` to the same counter again (omni_scheduler_mixin.py:105). On the `replace_streaming_prompt=True` handoff this seeds the counter to 2xN; after the N pre-replacement frames drain it, the leftover N makes `update_from_output` drop the first valid new-segment frame via `if output_is_stale: continue` (omni_ar_scheduler.py:348-352). Gate the top seed to the non-replacement branch (or drop the mixin's duplicate seed) and add a regression with `num_in_flight_tokens=1` on the replacement path asserting `num_stale_output_tokens == 1`. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:628-629 `if in_flight_tokens > 0:` / ` session.num_stale_output_tokens += in_flight_tokens` (top seed); vllm_omni/core/sched/omni_ar_scheduler.py:669-670 ` if replace_streaming_prompt:` / ` self._replace_streaming_session(session, update)`; vllm_omni/core/sched/omni_scheduler_mixin.py:105 ` session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0)` (second seed, same counter); vllm_omni/core/sched/omni_ar_scheduler.py:348-352 ` if output_is_stale:` / ` continue` drops the frame while the counter remains positive.) + +`vllm_omni/engine/arg_utils.py:116` [major] — The unconditional registration in `register_omni_models_to_vllm` now selects omni constructors for every `_OMNI_MODELS` arch in non-staged runs, but only `qwen2_5_omni.py` and `qwen3_omni.py` received the `getattr(vllm_config.model_config, 'model_stage', None) or 'thinker'` fallback. The remaining constructors still read the attribute directly (verified: `step_audio2.py:57`, `ming_flash_omni.py:98`, `voxtral_tts.py:112`, `omnivoice.py:279`, `mimo_audio.py:554`, `ming_tts.py:64`, `minicpmo_4_5_omni.py:94`, `covo_audio.py:218`, `cosyvoice3.py:462`, `mammoth_moda2.py:890`), so a plain non-staged `vllm serve` of those archs can raise `AttributeError: 'ModelConfig' object has no attribute 'model_stage'`. Add the same fallback (and a forward-contract test analogous to `tests/model_executor/models/qwen3_omni/test_qwen3_omni_forward_contract.py`) to every unconditionally registered constructor, or gate the overwrite to archs proven to construct non-staged. (evidence: vllm_omni/engine/arg_utils.py:117 ` ModelRegistry.register_model(arch, f"vllm_omni.model_executor.models.{mod_folder}.{mod_relname}:{cls_name}")` (inside the unconditional `for arch, (mod_folder, mod_relname, cls_name) in _OMNI_MODELS.items():` loop); vllm_omni/model_executor/models/voxtral_tts/voxtral_tts.py:112 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/ming_tts/ming_tts.py:64 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/step_audio2/step_audio2.py:57 ` raw_model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/ming_flash_omni/ming_flash_omni.py:98 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/omnivoice/omnivoice.py:279 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/mimo_audio/mimo_audio.py:554 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py:94 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/covo_audio/covo_audio.py:218 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py:462 ` self.model_stage = vllm_config.model_config.model_stage`; vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:890 ` self.model_stage = vllm_config.model_config.model_stage`) + +`vllm_omni/platforms/npu/worker/npu_model_runner.py:?` [major] — The diff adds `randomize_inputs: bool = False` to `OmniGPUModelRunner._dummy_run` (vllm_omni/worker/gpu_model_runner.py:881) and `GPUGenerationModelRunner._dummy_run` (vllm_omni/worker/gpu_generation_model_runner.py:580), but the NPU overrides `OmniNPUModelRunner._dummy_run` (vllm_omni/platforms/npu/worker/npu_model_runner.py:112) and `NPUGenerationModelRunner._dummy_run` (vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:641) still end with `profile_cpp: bool = False` and declare no `randomize_inputs`. Since those overrides precede the updated base in their MROs (npu_model_runner.py:37 and npu_generation_model_runner.py:44), a keyword call `_dummy_run(..., randomize_inputs=True)` on NPU raises TypeError. Add `randomize_inputs: bool = False` to both NPU overrides; the generation override can ignore it, matching `GPUGenerationModelRunner`. (evidence: vllm_omni/worker/gpu_model_runner.py:881 ` randomize_inputs: bool = False,` (added to `OmniGPUModelRunner._dummy_run`). vllm_omni/platforms/npu/worker/npu_model_runner.py:112 ` def _dummy_run(` whose signature ends ` profile_seq_lens: int | None = None,` / ` profile_cpp: bool = False,` with no `randomize_inputs`; class MRO at vllm_omni/platforms/npu/worker/npu_model_runner.py:37 `class OmniNPUModelRunner(OmniGPUModelRunner, NPUModelRunner):`. vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:641 ` def _dummy_run(` whose signature likewise ends ` profile_seq_lens: int | None = None,` / ` profile_cpp: bool = False,` with no `randomize_inputs`; class MRO at vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:44 `class NPUGenerationModelRunner(OmniNPUModelRunner, OmniConnectorModelRunnerMixin):`.) + +`benchmarks/patch.py:1712` [minor] — The benchmark probe/duplex changes in vllm_omni/benchmarks/patch/patch.py (probe_request_rate background probe, signature line 1545 and gate lines 1688-1714; duplex-metric tolerance around line 1849) have no unit test under tests/ and exercise the manual benchmark path. Is that intentional for this PR, or should a lightweight test/gate be added? (evidence: vllm_omni/benchmarks/patch/patch.py:1545 `probe_request_rate: float = 0.0,`; vllm_omni/benchmarks/patch/patch.py:1712 `if probe_request_rate > 0:`; vllm_omni/benchmarks/patch/patch.py:1849 `duplex_request_metrics = [` (diff hunk @@ -1819,11 +1849,19 @@). Repo-wide grep for `probe_request_rate` matched only vllm_omni/benchmarks/patch/patch.py; grep of tests/ for `probe_request_rate|duplex_request_metrics|MixRequestFuncOutput` returned no matches.) + +`vllm_omni/platforms/xpu/platform.py:139` [minor] — Confirmed: the diff flips the XPU platform-wide diffusion IR-op default from xpu_kernels to vllm_c in get_default_ir_op_priority (vllm_omni/platforms/xpu/platform.py:139), and _resolve_ir_op_priority consumes it platform-wide at vllm_omni/diffusion/worker/diffusion_worker.py:184. Repo grep shows no remaining xpu_kernels reference, and there is no XPU counterpart to tests/platforms/test_cuda_ir_op_priority.py. Please state/cite the upstream XPUPlatform default if this is mechanical 0.27 alignment, or add an XPU ir_op_priority test/validation if it is a deliberate behavioral change. (evidence: vllm_omni/platforms/xpu/platform.py:139 `default = ["native"] if using_inductor else ["vllm_c", "native"]`; vllm_omni/diffusion/worker/diffusion_worker.py:184 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`) + +`vllm_omni/core/sched/omni_ar_scheduler.py:348` [nit] — The drain invariant `assert request.num_stale_output_tokens >= 0` assumes each arriving frame's num_tokens_scheduled matches the seeded in-flight sum. The sweep notes spec asymmetry and prefill chunks are covered, but preemption-driven re-scheduling of token counts between seed and drain is not covered by any test. Should we add a test or guard for that preemption case? (evidence: [sweep] Residual (not a posted comment, no verified failure): the drain `assert request.num_stale_output_tokens >= 0` assumes each arriving frame's num_tokens_scheduled matches the seeded in-flight sum; spec asymmetry and prefill chunks are covered, but preemption-driven re-scheduling of token counts between seed and drain is not covered by any test.) + +`vllm_omni/core/sched/omni_ar_scheduler.py:479` [nit] — The diff keeps `request.async_tokens_to_discard = outstanding_async_tokens` at `vllm_omni/core/sched/omni_ar_scheduler.py:479` and a second write `session.async_tokens_to_discard = 1` at line 634 in `_update_request_as_session` (the streaming-update discard path, not the `_replace_streaming_session` helper), with a comment calling it an 'omni-only signal'. After the vLLM 0.27 rebase, repo-wide grep shows no vllm_omni production reader — only these writes, production comments, and test assertions (`tests/core/sched/test_omni_ar_scheduler_streaming.py:124,221,243`). Remove the dead writes and the 'omni-only signal' comment (and the assertions that pin the writes), or name the production consumer that still reads the attribute. (evidence: vllm_omni/core/sched/omni_ar_scheduler.py:479 `request.async_tokens_to_discard = outstanding_async_tokens`; vllm_omni/core/sched/omni_ar_scheduler.py:634 `session.async_tokens_to_discard = 1`; tests/core/sched/test_omni_ar_scheduler_streaming.py:124 `assert session.async_tokens_to_discard == outstanding_async_tokens`; tests/core/sched/test_omni_ar_scheduler_streaming.py:221 `assert session.async_tokens_to_discard == 1` — the only non-comment references in production are writes; the only reads are test assertions.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v13_wave2/pr5976.trace.json.gz b/eval/dataset/arms/copilot_v13_wave2/pr5976.trace.json.gz new file mode 100644 index 000000000..61ac02a86 Binary files /dev/null and b/eval/dataset/arms/copilot_v13_wave2/pr5976.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/manifest.json b/eval/dataset/arms/copilot_v14_holdout3_r1/manifest.json new file mode 100644 index 000000000..638948725 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v14_holdout3_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout3" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.cost.json new file mode 100644 index 000000000..ba61791da --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1957.7, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr5678/run-20260815-034501-afbe24", + "recorded_at": "2026-08-15T04:17:36", + "attempts": [ + { + "invocation_id": "inv-df81b8c4", + "outcome": "run:run-20260815-034501-afbe24", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-034501-afbe24", + "llm_calls": 192, + "input_tokens": 2007974, + "output_tokens": 325445, + "usd": 1.527987, + "source": "spans" + } + ], + "attempt_usd_total": 1.527987, + "llm_calls": 192, + "input_tokens": 2007974, + "output_tokens": 325445, + "usd": 1.527987, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-034501-afbe24", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.528, + "minutes": 32.58, + "ci_minutes": 0, + "input_tokens": 2007974, + "output_tokens": 325445, + "tool_calls": 324, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2854, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 192, + "by_role": { + "agent": { + "usd": 1.503213, + "input_tokens": 1978086, + "output_tokens": 310294, + "calls": 190 + }, + "reducer": { + "usd": 0.024774, + "input_tokens": 29888, + "output_tokens": 15151, + "calls": 2 + } + }, + "cache_read_tokens": 23253504, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.62, + "gate": 1.64, + "review": 1949.28, + "report": 0.08 + }, + "time_to_first_result_s": 1954.59 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5678.trace.json.gz", + "trace_bytes": 50865 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.md new file mode 100644 index 000000000..567c07b46 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5678, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head eeaa56144818) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] `_resolve_model_tokenizer_paths`/Audex/tokenizer precedence was preserved by extracting it into `_finalize_engine_args_dict` and both builders call it — no dropped legacy logic found. +- [claim-verified] PR claim 'current runtime, headless, and diffusion startup consumers remain on the stable compatibility entry point': grep found only stage_runtime.py:376 and entrypoints/cli/serve.py:941 call `build_engine_args_dict`, and stage_init_utils.py:983-995 delegates to `build_legacy_engine_args_dict`. +- [claim-verified] 'Every currently registered pipeline' backend parity is asserted over all direct `PipelineConfig` registry entries by test_typed_engine_args_match_current_registry_backend_semantics; `qwen3_omni_moe` (the only callable) is skipped — review comment 2. +- [validated] tests/engine/test_stage_engine_args.py (new file) contains the all-registry parity test `test_typed_engine_args_match_current_registry_backend_semantics` that builds legacy+typed args per pipeline stage and compares `_effective_backend_values` — matches the PR body claim. +- [validated] `build_legacy_engine_args_dict` now detaches via `_to_dict(stage_config.engine_args)` then pops a None `tensor_parallel_size`, so the legacy config object is no longer mutated; covered by `test_legacy_engine_args_drop_none_tp_without_mutating_stage_config`. +- [validated] `_stage_engine_values` inserts `topology.omni_kv_config` between deploy extras and CLI overrides, preserving the documented precedence (topology over deploy, CLI over topology). +- [validated] `_stage_engine_overrides` filters `_LEGACY_STAGE_METADATA_EXTRA_FIELDS` (final_output/final_output_type/is_comprehension) out of engine_extras, matching the PR comment that these are topology-owned. +- [validated] CI selection: .buildkite/cuda/test-merge.yml:23 runs `pytest tests/entrypoints tests/engine -m 'core_model and cpu'`; the new file's `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (test_stage_engine_args.py:36) means it is collected in the passing CUDA lane. +- [validated] Stable API preserved: `build_engine_args_dict` delegates to `build_legacy_engine_args_dict` (stage_init_utils.py:983-999); in-repo callers stage_runtime.py:376, serve.py:941, stage_init_utils.py:1042/1365, test_arg_utils.py:325, test_async_omni_engine_stage_init.py still call the stable symbol. +- [validated] Legacy non-mutation: test_legacy_engine_args_drop_none_tp_without_mutating_stage_config asserts `stage_config.engine_args == {'tensor_parallel_size': None}` after the call; the pop is applied to the detached `_to_dict` copy (stage_init_utils.py:963-967). +- [validated] Subdir/CLI tokenizer precedence: test_typed_engine_args_preserve_deploy_subdirectory_precedence and test_typed_llm_engine_args_preserve_legacy_adapter_behavior assert typed==legacy through the shared `_finalize_engine_args_dict`. +- [validated] Diffusion attention normalization and generation cache defaults: test_typed_diffusion_engine_args_use_structured_diffusion_config and test_typed_engine_args_preserve_explicit_backend_default_overrides exercise `parse_attention_config` and generation `setdefault` defaults. +- [validated] Ownership validation: tests/config/test_omni_config.py:215-258 drives `_validate_stage_engine_override_ownership` for unowned CLI fields, nested parallel_config, and diffusion-only `kv_cache_dtype` routing. +- [validated] Diffusion batch default deferral: test_typed_ming_image_engine_args_defer_diffusion_batch_default loads ming_flash_omni_image.yaml and asserts `max_num_seqs` absent plus `OmniDiffusionConfig(...).max_num_seqs == 1`. + +`vllm_omni/engine/stage_init_utils.py:~370` [major] — `_apply_rocm_attention_backend` (added by this diff) sets `engine_args["attention_backend"] = "ROCM_AITER_FA"` at line 370 when `rocm_aiter_ops.is_enabled()` is True, but line 376 then unconditionally overwrites it with `"TRITON_ATTN"`, so the aiter-enabled ROCm default is dead on both the legacy and typed paths. Return after setting ROCM_AITER_FA in the enabled branch (or gate the TRITON_ATTN assignment with `else`), and parametrize `test_typed_engine_args_own_rocm_attention_default` over `is_enabled()` True/False, asserting `ROCM_AITER_FA` for the True case. (evidence: vllm_omni/engine/stage_init_utils.py:369-370 (added by this diff in `_apply_rocm_attention_backend`): `if rocm_aiter_ops.is_enabled():` / ` engine_args["attention_backend"] = "ROCM_AITER_FA"` — then vllm_omni/engine/stage_init_utils.py:376 unconditionally executes `engine_args["attention_backend"] = "TRITON_ATTN"`, so the enabled assignment is always overwritten. The only new ROCm test, tests/engine/test_stage_engine_args.py:432-443, monkeypatches `rocm_aiter_ops.is_enabled=lambda: False` and asserts `typed_args["attention_backend"] == "TRITON_ATTN"`, leaving the True branch unexercised.) + +`vllm_omni/config/omni_config.py:~542` [major] — `_DiffusionConfigProjection.scheduler_port` defaults to `None` and `enrich_config()` forwards every projection field into `OmniDiffusionConfig(**kwargs)`, whose `scheduler_port` default is `5555`. On an unset projection this passes `scheduler_port=None`, bypassing the 5555 default (a validated dataclass would reject the None rather than silently override). Default this field to `5555` or skip `None` values when building the `enrich_config` kwargs. (evidence: vllm_omni/config/omni_config.py:542 `scheduler_port: int | None = None` (added by this PR) — projection default is None. vllm_omni/diffusion/data.py:764 `scheduler_port: int = 5555` (unchanged by this diff, present in the PR-time tree) — real default. vllm_omni/config/omni_config.py `_DiffusionConfigProjection.enrich_config()` builds `kwargs = {name: _copy_value(getattr(self, name)) for name in _DIFFUSION_CONFIG_FIELDS if name in omni_diffusion_fields}` and then `omni_diffusion_config = OmniDiffusionConfig(**kwargs)`, while `_DIFFUSION_CONFIG_FIELDS = frozenset(f.name for f in fields(_DiffusionConfigProjection))` includes `scheduler_port`; therefore an unset projection forwards `scheduler_port=None`, so the `5555` default never applies.) + +`vllm_omni/config/omni_config.py:615` [minor] — Extract the shared diffusion dtype-normalization and validation logic (string dtype_map, cache_config -> DiffusionCacheConfig, quantization propagation/resolution, attention-config building, kv-skip selector parsing, max_cpu_loras check, and diffusers kwargs validation) into a common helper used by both `OmniDiffusionConfig.__post_init__` and `_DiffusionConfigProjection.__post_init__`. Do not call `OmniDiffusionConfig.__post_init__` directly: `_DiffusionConfigProjection` intentionally skips its startup side effects (port probing, HF metadata loading). The field-coverage test only guards field membership, so the duplicated behavior can drift silently. (evidence: vllm_omni/config/omni_config.py:615 `def __post_init__(self) -> None:` (in `_DiffusionConfigProjection`) contains `dtype_map = {"auto": torch.bfloat16, "bfloat16": torch.bfloat16, "bf16": torch.bfloat16, "float16": torch.float16, "fp16": torch.float16, "half": torch.float16, "float32": torch.float32, "fp32": torch.float32, "float": torch.float32,}` plus `DiffusionCacheConfig.from_dict`, `build_quant_config`, `build_attention_config`, `parse_kv_cache_skip_selector`, `max_cpu_loras >= 1`, and diffusers kwargs validation. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/data.py:597 `class OmniDiffusionConfig:` — its `__post_init__` contains the identical `dtype_map` and the same validation sequence.) + +`tests/engine/test_stage_engine_args.py:486` [minor] — test_typed_engine_args_match_current_registry_backend_semantics sweeps audex_tts/audex_tta/audex_thinker_only/audex_s2s with model='/tmp' and only asserts legacy-vs-typed backend-field parity; the file has no 'audex' reference and never exercises the Audex snapshot/tokenizer resolution path (the os.path.isdir branch at stage_init_utils.py:74 short-circuits on '/tmp'). Add a dedicated assertion for those branches. (evidence: tests/engine/test_stage_engine_args.py:472 `@pytest.mark.parametrize("model_type", sorted(OMNI_PIPELINES))`; tests/engine/test_stage_engine_args.py:486 `model="/tmp",`; tests/engine/test_stage_engine_args.py:508 `assert typed_effective_args == legacy_effective_args, (` — the sweep only compares backend args and grep shows no 'audex' reference anywhere in the file. Unchanged by this diff, present in the PR-time tree: vllm_omni/engine/stage_init_utils.py:74 `if os.path.isdir(model):` — a directory model like '/tmp' takes this short-circuit, so no download/snapshot path runs.) + +`vllm_omni/config/omni_config.py:~876` [minor] — Confirmed: `_global_stage_cli_fields()` (vllm_omni/config/omni_config.py:895-910) admits all `fields(OmniEngineArgs)`, and since `OmniEngineArgs` inherits upstream `EngineArgs` (vllm_omni/engine/arg_utils.py:117), inherited-but-unowned valid fields flow through `_stage_cli_overrides` into `_validate_stage_engine_override_ownership`, which raises at line 876/890-894 because `owner_fields` is the finite local structured union (lines 861-865), not the full EngineArgs surface. The new tests only cover `enable_lora`, `parallel_config.cfg_parallel_size`, and `kv_cache_dtype`. Enumerate the intended allowlist/denylist and add coverage for a representative inherited EngineArgs field before the cutover. (evidence: vllm_omni/config/omni_config.py:876 `unowned_fields = set(overrides) - owner_fields`; vllm_omni/config/omni_config.py:901 `candidates = (frozenset(f.name for f in fields(OmniEngineArgs)) | frozenset(_STAGE_DEPLOY_ENGINE_FIELDS) | frozenset(_PIPELINE_DEPLOY_CLI_FIELDS))`; vllm_omni/engine/arg_utils.py:117 `class OmniEngineArgs(EngineArgs):` — unchanged by this diff, present in the PR-time tree, showing the inherited upstream field surface exceeds the local structured owner sets at vllm_omni/config/omni_config.py:861-865.) + +`vllm_omni/config/omni_config.py:~885` [minor] — `_validate_stage_engine_override_ownership` checks nested `parallel_config` fields only `if isinstance(parallel_config, Mapping)`. A programmatic caller supplying `engine_extras={'parallel_config': }` passes validation unchecked, and `_build_parallel_config` then runs `_mapping_or_empty(engine.get('parallel_config'))`, yielding `{}` and silently dropping the caller's parallelism settings. Reject non-Mapping `parallel_config` in the validator, or normalize dataclass instances to a mapping before building. (evidence: vllm_omni/config/omni_config.py:885 `if isinstance(parallel_config, Mapping):` — new ownership validator only descends into nested fields for Mapping values, so non-Mapping values bypass validation. vllm_omni/config/omni_config.py:913-914 `def _mapping_or_empty(value: Any) -> Mapping[str, Any]:` / `return value if isinstance(value, Mapping) else {}` — unchanged by this diff, present in the PR-time tree. vllm_omni/config/omni_config.py:1441 `parallel_config = _mapping_or_empty(engine.get("parallel_config"))` — unchanged by this diff; a non-Mapping value becomes `{}`, so downstream kwargs contribute nothing and the caller's parallelism settings are discarded.) + +`tests/engine/test_stage_engine_args.py:476` [minor] — The registry-wide parity test `test_typed_engine_args_match_current_registry_backend_semantics` calls `resolve_pipeline_config(model_type)` without an HF config and skips whenever it returns `None` (tests/engine/test_stage_engine_args.py:475-477). For the only callable registry entry, `qwen3_omni_moe`, the `@pipeline_cfg_resolver` wrapper returns `None` when `hf_config is None`, so the test skips it entirely and exercises NEITHER resolver branch (full 3-stage vs thinker-only, selected by `enable_audio_output`). Add a sibling parametrization that passes a Qwen3OmniMoeConfig with `enable_audio_output=True` and `False` (or resolves the two PipelineConfig constants directly) so both variants are parity-checked against legacy backend args. (evidence: tests/engine/test_stage_engine_args.py:475-477 `pipeline = resolve_pipeline_config(model_type)` / `if pipeline is None:` / ` pytest.skip(f"Pipeline {model_type!r} requires an HF config to resolve")` — new file in this diff; the test resolves each registry key with no HF config and skips on None. vllm_omni/config/stage_config.py:38-39 (unchanged by this diff, present in the PR-time tree) `if hf_config is None or not isinstance(hf_config, config_type):` / ` return None` — the @pipeline_cfg_resolver wrapper makes resolve_qwen3_omni_pipeline return None when no config is passed. vllm_omni/model_executor/models/qwen3_omni/pipeline.py:99-114 (unchanged by this diff) `if not hf_config.enable_audio_output:` / ` return QWEN3_OMNI_THINKER_ONLY_PIPELINE` / `return QWEN3_OMNI_PIPELINE` — two variants exist but are unreachable when hf_config is None. vllm_omni/config/pipeline_registry.py:118 (unchanged by this diff) `"qwen3_omni_moe": resolve_qwen3_omni_pipeline` — the only callable registry value; all other entries are PipelineConfig instances.) + +`.buildkite/amd/test-amd-merge.yml:~141` [minor] — The diff adds `_apply_rocm_attention_backend` and a monkeypatched ROCm unit test (`tests/engine/test_stage_engine_args.py::test_typed_engine_args_own_rocm_attention_default`), but the AMD 'Engine Test' step at `.buildkite/amd/test-amd-merge.yml:141` runs only `tests/engine/test_async_omni_engine_abort.py`, and no Buildkite lane references `test_stage_engine_args.py`. The changed ROCm defaulting path (now applied at `vllm_omni/engine/stage_init_utils.py:389`) is therefore never exercised by `buildkite/vllm-omni-amd-ci`. Add `tests/engine/test_stage_engine_args.py` (or at least the ROCm test) to the AMD lane so the changed path is covered on AMD hardware. (evidence: Unchanged by this diff, present in the PR-time tree: `.buildkite/amd/test-amd-merge.yml:141` ` - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py` — this is the sole engine-test command in the AMD 'Engine Test' step. Added by this diff: `tests/engine/test_stage_engine_args.py:426` `def test_typed_engine_args_own_rocm_attention_default(monkeypatch, tmp_path):`. Added by this diff: `vllm_omni/engine/stage_init_utils.py:389` ` _apply_rocm_attention_backend(engine_args, stage_type)` (inside `extract_legacy_stage_metadata`). A `.buildkite`-scoped grep for `test_stage_engine_args` returns no matches, proving no Buildkite lane selects the new test file.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.trace.json.gz new file mode 100644 index 000000000..caed603d5 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5678.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.cost.json new file mode 100644 index 000000000..4f264f35d --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2092.6, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr5691/run-20260815-034502-380d73", + "recorded_at": "2026-08-15T04:19:50", + "attempts": [ + { + "invocation_id": "inv-7825e99c", + "outcome": "run:run-20260815-034502-380d73", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-034502-380d73", + "llm_calls": 177, + "input_tokens": 1940984, + "output_tokens": 352283, + "usd": 1.554828, + "source": "spans" + } + ], + "attempt_usd_total": 1.554828, + "llm_calls": 177, + "input_tokens": 1940984, + "output_tokens": 352283, + "usd": 1.554828, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-034502-380d73", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.5548, + "minutes": 34.81, + "ci_minutes": 0, + "input_tokens": 1940984, + "output_tokens": 352283, + "tool_calls": 315, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3241, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 177, + "by_role": { + "agent": { + "usd": 1.461506, + "input_tokens": 1746188, + "output_tokens": 315321, + "calls": 174 + }, + "reducer": { + "usd": 0.093322, + "input_tokens": 194796, + "output_tokens": 36962, + "calls": 3 + } + }, + "cache_read_tokens": 23824128, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.19, + "gate": 1.81, + "review": 2081.48, + "report": 0.06 + }, + "time_to_first_result_s": 2088.53 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5691.trace.json.gz", + "trace_bytes": 119133 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.md new file mode 100644 index 000000000..2cd3a8f05 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5691, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0eee3b18645a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] prior concern — text_encoder_tp_size dropped from stage→worker conversion: resolved; _PARALLEL_CONFIG_ENGINE_FIELDS is derived from _ParallelConfigEngineOverrides.__annotations__ which includes `text_encoder_tp_size`, and OmniDiffusionConfig.__post_init__ converts the parallel_config Mapping via DiffusionParallelConfig.from_dict. +- [resolved] packed-metadata blast radius: `AttentionMetadata.extra` cu_seqlens keys are produced only by vllm_omni/diffusion/models/minimax_h3/{minimax_h3_transformer,denoise_loop}.py; other models (magi_human, lingbot, mammoth_moda2) pass cu_seqlens_q as explicit kwargs to their own backend calls, so the shared FlashAttentionImpl packed branch is MiniMax-H3-only. +- [resolved] cache_dit dependency concern: requirements/common.txt:19 `cache-dit==1.3.0` is a core dependency, so `from cache_dit import ForwardPattern` in minimax_h3_transformer.py and test_minimax_h3_parallel.py resolves on CPU CI. +- [claim-verified] test_shifted_sigma_schedule_matches_reference_values: shifted=12*b/(1+11*b) for b in linspace(1,0,5) gives [1.0, 0.97297297, 0.92307692, 0.8, 0.0], matching the test's expected [1.0, 0.9729729891, 0.9230769277, 0.8000000119, 0.0] (no trailing 0 appended since shifted[-1]==0). +- [claim-verified] e2e test arithmetic: num_frames=29 -> align to 39 (39%17==5), asserted frames.shape==(39,256,448,3); video_latent_t(39)=12, consistent. +- [claim-verified] pyproject `fa4` pin `flash-attn-4[cu13]==4.0.0b18` (pyproject.toml:102) matches the doc statement 'Version 4.0.0b18 is required' in attention_backends.md. +- [claim-verified] diff_stat confirms 45 files changed / 7356 insertions matching the provided evidence; no files in the diff are missing from the archived pr_diff. +- [claim-refuted] PR body has no checkable claims ('PLEASE FILL IN THE PR DESCRIPTION HERE'; empty Test Plan/Result). +- [claim-refuted] single-video Ref2VA via singular `input_reference` is NOT handled: _parse_video_form only persists paths for plural `input_references` (api_server.py:3117); the singular path decodes to VideoFrames(list[Image.Image]) (video_api_utils.py:31) and the pipeline's prepare_reference_videos raises TypeError for non-path values — MINOR finding (review comment 2). +- [claim-refuted] PR body test plan/results are empty template text; the only end-to-end test (test_minimax_h3_e2e.py) is skip-gated on `VLLM_TEST_MINIMAX_H3_FL2VA_MODEL` and 8 GPUs, so no semantic parity evidence for H3 is shown in the PR. +- [claim-refuted] PR body Test Plan / Test Result are the unfilled template (`PLEASE FILL IN THE PR DESCRIPTION HERE`), so the PR carries no evidence that any of the listed tests were run. +- [validated] config wiring text_encoder_tp_size: OrchestratorArgs→_create_default_diffusion_stage_cfg→DiffusionParallelConfig (async_omni_engine.py:968,989); deploy path via _ParallelConfigEngineOverrides→_build_parallel_config→OmniDiffusionConfig.from_kwargs→DiffusionParallelConfig.from_dict (data.py:936-937) all carry the field. +- [validated] DiT Ulysses validation vs recipe: `--num-gpus 4 --usp 4` with TP=1 gives local_heads=56, 56%4==0, so the four-GPU recipe config passes MiniMaxH3DiTModel.__init__ (minimax_h3_transformer.py:851-858). +- [validated] sigma schedule test values: recomputed 12*0.75/(1+11*0.75)=0.97297, 12*0.5/(1+11*0.5)=0.92308, 12*0.25/(1+11*0.25)=0.8 — match test_minimax_h3_contract.py expected list. + +`vllm_omni/diffusion/attention/parallel/ring.py:~193` [major] — The FA4 preference added here is unreachable in the exact install the [fa4] extra targets. ring.py:193 forces backend_pref="sdpa" whenever HAS_FA3/HAS_FLASH_ATTN/HAS_AITER are all False, and ring.py:207-221 returns ring_pytorch_attn_func before ring.py:227's `if HAS_FA4 ...` is evaluated. With `flash-attn-4[cu13]==4.0.0b18`, HAS_FA4=True (via flash_attn.cute) while the other three flags are False, so Ring attention silently falls back to SDPA instead of the new CuTe path. Fix: change the guard to `elif not HAS_FA3 and not HAS_FLASH_ATTN and not HAS_AITER and not HAS_FA4:` and add a test with HAS_FA4=True while HAS_FA3=HAS_FLASH_ATTN=HAS_AITER=False. (evidence: UNCHANGED BY THIS DIFF, present in the PR-time tree — vllm_omni/diffusion/attention/parallel/ring.py:193 ` elif not HAS_FA3 and not HAS_FLASH_ATTN and not HAS_AITER:` then ring.py:207 ` if backend_pref in {"sdpa", "torch", "torch_sdpa"}:` and ring.py:210 ` return ring_pytorch_attn_func(` return before the DIFF-ADDED FA4 branch at ring.py:227 ` if HAS_FA4 and query.is_cuda and torch.cuda.get_device_capability(query.device)[0] >= 10:`. ring_globals.py:11-12 ` import flash_attn # noqa: F401` / ` from flash_attn.flash_attn_interface import _flash_attn_forward # noqa: F401` set HAS_FLASH_ATTN only via FA2; ring_globals.py:31-35 ` from flash_attn.cute import flash_attn_func as fa4_attn_func # noqa: F401` / ` HAS_FA4 = True` set HAS_FA4 independently; ring_globals.py:41-43 ` from flash_attn_interface import _flash_attn_forward as fa3_fwd_func # noqa: F401` / ` from flash_attn_interface import flash_attn_func as fa3_attn_func # noqa: F401` / ` HAS_FA3 = True` require FA3 packages. The diff's [fa4] extra is pyproject.toml:102-103 `fa4 = [` / ` "flash-attn-4[cu13]==4.0.0b18",` — installing only the cute-based package, so HAS_FA4=True with HAS_FA3=HAS_FLASH_ATTN=HAS_AITER=False is the real [fa4] state and the FA4 branch is dead.) + +`vllm_omni/diffusion/attention/backends/utils/fa.py:65` [major] — fa.py:65 makes `_is_blackwell()` processes rebind the module-global `flash_attn_func`/`flash_attn_varlen_func` to `flash_attn.cute`, a process-wide backend swap not scoped to MiniMax H3. Unchanged consumers import these exact globals and rely on the FA2/FA3 positional contract: qwen3_tts `whisper_encoder.py:25,143` (`flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen, dropout_p=0.0)` then `x.reshape(...)`), and ming_flash_omni `audio_encoder.py:18,65` (`flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen)` then `.contiguous().view(...)`). The PR's own FA4 path (`ring_kernels.py` `flash_attn4_func_forward`) uses `softmax_scale`/`softcap`/`return_lse` and unpacks `(out, softmax_lse)`, showing the CuTe API diverges from that convention. Confirm `flash_attn.cute.flash_attn_varlen_func` accepts `dropout_p` and the positional cu_seqlens signature and returns a single tensor, or gate the FA4 preference behind opt-in/per-model capability so unvalidated models keep the FA3/FA2 chain. (evidence: In diff: vllm_omni/diffusion/attention/backends/utils/fa.py:65 ` from flash_attn.cute import flash_attn_func, flash_attn_varlen_func # noqa: F401` — inside `if _is_blackwell():` (fa.py:63), so it rebinds the module globals for every Blackwell process where the import succeeds. Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/qwen3_tts/tokenizer_25hz/vq/whisper_encoder.py:25 `from vllm_omni.diffusion.attention.backends.utils.fa import HAS_FLASH_ATTN, flash_attn_varlen_func`; :143 `x = flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen, dropout_p=0.0)`; :144 `x = x.reshape(n_ctx, n_state)`. vllm_omni/model_executor/models/ming_flash_omni/audio_encoder.py:18 `from vllm_omni.diffusion.attention.backends.utils.fa import HAS_FLASH_ATTN, flash_attn_varlen_func`; :65 `attn_output = flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen)`; :68 `attn_output = attn_output.contiguous().view(n_ctx, n_state)`. Corroborating CuTe keyword/return shape in diff: vllm_omni/diffusion/attention/backends/ring/ring_kernels.py `flash_attn4_func_forward` → `out, softmax_lse = fa4_attn_func(q, k, v, softmax_scale=softmax_scale, causal=causal, window_size=(None if left == -1 else left, None if right == -1 else right), softcap=softcap or 0.0, return_lse=True)`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:182` [major] — No test in this PR exercises `text_encoder_tp_size > 1`: the only encoder construction test is the parameter-free stub (tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:175-182 `load_model=False, encoder_group=None` ... `assert encoder.tp_size == 1`), the only grep hit for the parameter in tests/ is the field-name set entry tests/config/test_omni_config.py:443, and the e2e test uses the default (tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:35 `DiffusionParallelConfig(ulysses_degree=8)`). Add a CPU test of the multi-rank TP path (weight loaders + replicated forward) or a `--text-encoder-tp-size` config-propagation test. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:175-182 ` encoder = MiniMaxH3Qwen3VLEncoder(\n "/nonexistent/text_encoder",\n device=torch.device("cpu"),\n load_model=False,\n encoder_group=None,\n )\n assert not encoder.is_loaded\n assert encoder.tp_size == 1` — the only encoder construction test is the parameter-free stub. tests/config/test_omni_config.py:443 ` "text_encoder_tp_size",` — the only other tests/ hit for the parameter is a field-name set entry. tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:35 ` parallel_config=DiffusionParallelConfig(ulysses_degree=8),` — uses the default text_encoder_tp_size=1. grep in tests/ for `text-encoder-tp-size` returns no matches. All quoted lines are within this PR's diff.) + +`vllm_omni/diffusion/attention/backends/utils/fa.py:~65` [minor] — This diff makes FA4-on-Blackwell selection depend on a single import that must bind BOTH `flash_attn_func` and `flash_attn_varlen_func`; if `flash_attn.cute` in the pinned `flash-attn-4==4.0.0b18` wheel does not export a varlen entry point, the combined import raises, the `except Exception` block at fa.py:68-72 only logs at debug, `flash_attn_func` is never bound, and the advertised FA4 path silently falls back to FA3/FA2 (fa.py:76). Other FA4 consumers import only the dense API — ring_globals.py:31 `from flash_attn.cute import flash_attn_func as fa4_attn_func` and bench_attention_backends.py:175 `from flash_attn.cute import flash_attn_func`. Split the imports (bind `flash_attn_func` first, try `flash_attn_varlen_func` separately) so a missing varlen export cannot disable dense FA4. (evidence: fa.py:65 `from flash_attn.cute import flash_attn_func, flash_attn_varlen_func # noqa: F401` (inside `if _is_blackwell(): try:`); fa.py:72 `logger.debug("CuTe FlashAttention-4 is unavailable: %s", exc)`; fa.py:76 `if flash_attn_func is None:` begins the FA3/FA2 fallback, so a varlen ImportError leaves the dense symbol None. Contrast consumers: ring_globals.py:31 `from flash_attn.cute import flash_attn_func as fa4_attn_func # noqa: F401` and bench_attention_backends.py:175 `from flash_attn.cute import flash_attn_func` (unchanged benchmark file in the PR-time tree).) + +`.buildkite/cuda/test-merge.yml:92` [minor] — DCO check is ACTION_REQUIRED on this head (commits lack Signed-off-by). The repo requires DCO sign-off per docs/contributing/README.md:92. Add the Signed-off-by trailer to all commits in this PR. Note: DCO is a GitHub check, not a step in .buildkite/cuda/test-merge.yml. (evidence: The draft's anchor is wrong — `.buildkite/cuda/test-merge.yml:18` reads `- label: "Simple · Diffusion Test"`, and that file contains no DCO gate. The real DCO gate is a GitHub check: live CI readout for PR 5691 returns `{"bucket":"fail","name":"DCO","state":"ACTION_REQUIRED"}`. The repo rule, unchanged by this diff and present in the PR-time tree: `docs/contributing/README.md:92` — `Commits must include a `Signed-off-by:` header which certifies agreement with the terms of the DCO.`) + +`recipes/MiniMaxAI/MiniMax-H3.md:305` [minor] — recipes/MiniMaxAI/MiniMax-H3.md:305-306 assert measured 86.964s and 784.394s results and line 56 claims an 'accuracy-qualified reference path', but the PR contains no result JSON, server log, or benchmark artifact in its 45 changed files. Provide the artifact behind these numbers, or mark them explicitly as unverified author assertions rather than measured results. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:305 `| FL2VA, 209 frames, 1248x768 | no offload, U4, VPP4 tile, regional compile | 86.964 s mean HTTP client latency |` and recipes/MiniMaxAI/MiniMax-H3.md:306 `| Two-video Ref2VA, 362 frames, 1344x768 | no offload, U4, VPP4 tile, regional compile | 784.394 s accounted model-stage mean |`; recipes/MiniMaxAI/MiniMax-H3.md:56 `This matches the accuracy-qualified reference path and prevents the Qwen3-VL`. The PR's diff_stat lists 45 changed files (docs, tests, and vllm_omni source only) — no benchmark result JSON, server log, or artifact file is present, so the measured-performance claims are unsubstantiated in-tree.) + +`vllm_omni/diffusion/models/minimax_h3/denoise_loop.py:~101` [minor] — This diff sets `max_seqlen_q` to the length of the FIRST packed document only (`int(cu[1])`), but `cu_seqlens` has three entries — `[0, used, seq_len]` — i.e. two documents where the second is the alignment-padding doc of length `seq_len - used`. FlashAttention's `max_seqlen_q` must be the maximum over all documents; whenever `seq_len - used > used` (possible for any packed layout with `used < seq_len/2`, e.g. small text-only or short-clip shapes), the value understates the true max and the varlen kernel reads past the intended bound. Compute `max_seqlen_q = max(cu[1:] - cu[:-1])` (and use the same max in `MiniMaxH3Attention._run_packed_attention`, which derives `used = int(cu_seqlens[1].item())` for the padding mask) instead of assuming the first document is longest. (evidence: packed_sequence.py returns `"cu_seqlens": cu` where `cu = torch.tensor([0, used, seq_len], dtype=torch.int32)` (packed_sequence.py, both minimax_h3_packed_sequence and minimax_h3_packed_sequence_ref2va_blocks), and denoise_loop.py:101 records `"max_seqlen_q": int(cu[1])` into `packed_seq_params`; `_forward_varlen_packed` passes it straight to `flash_attn_varlen_func(max_seqlen_q=...)` (flash_attn.py:181-186).) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~759` [minor] — In the distributed-VAE path this diff broadcasts the rank-0 `prepared_videos` list (whose `prepared_path` entries point into rank 0's `tempfile.TemporaryDirectory`) to every DiT rank, and then ALL ranks call `load_video_frames(item["prepared_path"])` on those paths. That only works on a single node with a shared filesystem (`/tmp`); on any multi-node deployment the non-zero ranks read a path that does not exist on their host. The recipe documents only single-node 4x B300, but nothing in the code guards or documents this single-node assumption — add an explicit single-node guard or copy/stream the prepared bytes instead of broadcasting local paths. (evidence: pipeline_minimax_h3.py:664-665 `if rank != 0:` / `return None` — `_prepare_reference_videos` only materializes files on rank 0. pipeline_minimax_h3.py:740 `distributed_encode = self.video_vae.is_distributed_enabled()`. pipeline_minimax_h3.py:744-751 `prepared_videos_list = [prepared_videos]` / `dist.broadcast_object_list(prepared_videos_list, src=0, group=group, device=self.device)` / `prepared_videos = prepared_videos_list[0]` — the rank-0 path list is broadcast. pipeline_minimax_h3.py:755 `if rank == 0 or distributed_encode:` and pipeline_minimax_h3.py:759 `self.video_vae.encode_video(load_video_frames(item["prepared_path"])) for item in prepared_videos` — every rank in the distributed path opens the rank-0-local paths. pipeline_minimax_h3.py:1077 (approx.) `with tempfile.TemporaryDirectory(prefix="minimax_h3_ref2va_") as workdir:` — the paths live only in rank 0's temporary directory. All lines are in the PR-time tree (this file is added by the diff).) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/models/minimax_h3/reference_video.py:157` [minor] The new pipeline only accepts file paths for video references, but the singular `input_reference` form field never produces paths — it decodes uploads to PIL frames (`VideoFrames`). A one-video Ref2VA request sent throug +- `vllm_omni/entrypoints/openai/api_server.py:2952` [minor] The persisted multi-video temp files can be orphaned: `_run_video_generation_job` runs `job = await VIDEO_STORE.get(video_id)` and `if job is None: return` (api_server.py:2873-2876) before entering `try`, and `delete_vid +- `vllm_omni/entrypoints/openai/api_server.py:2982` [minor] This new multipart field is accepted by the shared `/v1/videos` and `/v1/videos/sync` dependency for every diffusion model and changes the payload contract: `_run_and_extract` passes `multi_modal_data["video"] = input_vi +- `vllm_omni/diffusion/models/minimax_h3/denoise_loop.py:128` [minor] This diff adds the CFG-distilled denoise loop and `MiniMaxH3DenoiseBranch.forward_kwargs` timestep/index_copy/index_select layout, but no CPU test drives it: add a CPU smoke test that builds a `MiniMaxH3DenoiseBranch` fr + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.trace.json.gz new file mode 100644 index 000000000..95dc3213c Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5691.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.cost.json new file mode 100644 index 000000000..0ee4e1d8c --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1514.4, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr5713/run-20260815-034500-f425cc", + "recorded_at": "2026-08-15T04:10:12", + "attempts": [ + { + "invocation_id": "inv-124860a9", + "outcome": "run:run-20260815-034500-f425cc", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-034500-f425cc", + "llm_calls": 94, + "input_tokens": 792498, + "output_tokens": 178457, + "usd": 0.595771, + "source": "spans" + } + ], + "attempt_usd_total": 0.595771, + "llm_calls": 94, + "input_tokens": 792498, + "output_tokens": 178457, + "usd": 0.595771, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-034500-f425cc", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5958, + "minutes": 25.2, + "ci_minutes": 0, + "input_tokens": 792498, + "output_tokens": 178457, + "tool_calls": 156, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8604, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 94, + "by_role": { + "planner": { + "usd": 0.001066, + "input_tokens": 2319, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.542872, + "input_tokens": 758434, + "output_tokens": 138788, + "calls": 90 + }, + "reducer": { + "usd": 0.051833, + "input_tokens": 31745, + "output_tokens": 39269, + "calls": 3 + } + }, + "cache_read_tokens": 6870144, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.67, + "gate": 1.77, + "review": 1506.37, + "report": 0.07 + }, + "time_to_first_result_s": 1511.86 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5713.trace.json.gz", + "trace_bytes": 34413 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.md new file mode 100644 index 000000000..34c9c4cee --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5713, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6a464218a760) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] shared-server lock contention: tests/helpers/fixtures/runtime.py defines only module `omni_server` (line 36) and function `omni_server_function` (line 25); the new file never requests `omni_server_function`, so no second server contends for `omni_fixture_lock`. +- [claim-verified] No production `vllm_omni/` changes: `diff_stat` shows only 3 test files; all 224 insertions are in tests/dfx and tests/entrypoints. +- [claim-verified] 'shared module server... level=2 discards weights': `_QWEN_IMAGE` uses `omni_server` module fixture (tests/helpers/fixtures/runtime.py:36) and test order is enforced only by source ordering; invalid-JSON params run before the new test in the same file. +- [claim-verified] 'No vllm_omni production error-mapping change': `vllm_omni/entrypoints/openai/api_server.py` registers handlers only for EngineGenerateError/EngineDeadError (lines 303-304) and omni_sleep/omni_wakeup have no NotImplementedError handler; upstream `build_openai_app` supplies the 501 path. +- [claim-verified] timing arithmetic: 2100.9s = 35.0min and 901.24s = 15.0min; the description's '49min' before figure is off by 14min (2940s), and the headline 2100.9->901.24 is a 57% reduction, not '49min->35min'. +- [claim-verified] 'No production vllm_omni/ code changes' holds: diff_stat lists exactly the 3 test files above. +- [claim-verified] dfx comment about iter_omni_server/omni_fixture_lock is accurate: iter_omni_server wraps its whole yield in `with omni_fixture_lock:` (tests/helpers/runtime.py) and omni_server is module-scoped (tests/helpers/fixtures/runtime.py). +- [claim-refuted] PR body timing pair is internally inconsistent: 2100.9s = 35.0min and 901.24s = 15.0min (calc), so '49min -> 35min' does not match the quoted seconds; 35min even equals the 'before' seconds, suggesting swapped or different-scope columns. +- [validated] test_wakeup_route_propagates_not_implemented_error (tests/entrypoints/openai_api/test_omni_sleep_wakeup.py:178): mock AsyncMock(side_effect=NotImplementedError) + TestClient(raise_server_exceptions=True) exercises the real wakeup route and the sleeping_set-not-cleared assertion; pytestmark core_model+cpu runs on CPU with no vLLM engine import (uses bare router), so it does exercise the route behavior it names. +- [validated] test_wakeup_after_level2_sleep_fails (tests/dfx/reliability/invalid_param_test/test_invalid_server_control.py:109): asserts live sleep level=2 returns 200 SUCCESS then wakeup returns 501 with both message substrings; assert_http_error requires all substrings in serialized body (tests/helpers/assertions.py:884-895). +- [validated] sleep_resps[0] at test_invalid_server_control.py:125 is safe: send_omni_sleep_http_request always returns [resp] (tests/helpers/runtime.py send_omni_sleep_http_request body returns [resp]). +- [validated] invalid-JSON cases unaffected by adding --enable-sleep-mode: /v1/omni/sleep and /v1/omni/wakeup are registered unconditionally (vllm_omni/entrypoints/openai/api_server.py:3485,3497) and FastAPI validates the request body before the handler; the flag only changes whether engine.sleep is reached on VALID bodies. +- [validated] test_wakeup_route_propagates_not_implemented_error exercises real route order: omni_wakeup removes sleeping_stages only after `await engine_client.wake_up(stage_ids=request.stage_ids)` succeeds (api_server.py), so a raise leaves sleeping_stages=={0}. +- [validated] test_wakeup_after_level2_sleep_fails 501/message matches production: AsyncOmni.wake_up raises NotImplementedError("wake_up() after sleep(level=2) is not yet implemented: ...") (vllm_omni/entrypoints/async_omni.py), containing both 'sleep(level=2)' and 'not yet implemented'. + +`tests/entrypoints/test_omni_sleep_mode.py:111` [major] — The class-scoped fixtures rely on `_ensure_awake` between cases, but this helper swallows every wake failure and only logs a warning (tests/entrypoints/test_omni_sleep_mode.py:111-116). A failed wake silently leaves the shared engine asleep/partially asleep (`_sleeping_tags` still set) for the next test, so the next case fails for an unrelated reason or its sleep-state assertions run from a corrupted baseline. Re-raise on failure only when it won't mask a primary test failure, or add an explicit awake-state assert before each dependent test rather than a best-effort warning. (evidence: tests/entrypoints/test_omni_sleep_mode.py:111-116 `async def _ensure_awake(engine: AsyncOmni, stage_ids: list[int]) -> None:` / ` """Best-effort full wake so the next shared-engine test starts active."""` / ` try:` / ` await engine.wake_up(stage_ids=stage_ids)` / ` except Exception as e:` / ` logger.warning("ensure_awake failed (stage_ids=%s): %s", stage_ids, e)`. This helper is called in `finally` blocks after each shared-engine case (lines 232, 247, 261, 283, 306, 385). Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/async_omni.py:514-521 `_sleeping_tags = getattr(self, "_sleeping_tags", None)` / `if _sleeping_tags:` / ` raise RuntimeError(` / ` f"Generation rejected: Engine is partially or fully asleep. "` / ` f"Currently sleeping tags: {list(_sleeping_tags)}. "` / ` f"Please perform a full wake_up before generating."` / ` )`.) + +`tests/entrypoints/test_omni_sleep_mode.py:113` [minor] — `_ensure_awake` is the only isolation between class-scoped engine tests and it swallows `Exception`, so a failed wake in a test's `finally` passes the owning test and leaves stage 0 asleep for the next case (which calls `sleep(stage_ids=[0], level=1)` with no awake-state precondition). Re-raise the wake failure in `_ensure_awake` (or assert `_sleeping_tags` is empty after wake) so the suite fails at the test that actually left the engine asleep. (evidence: Changed by this diff — tests/entrypoints/test_omni_sleep_mode.py:111-116: `async def _ensure_awake(engine: AsyncOmni, stage_ids: list[int]) -> None:` / ` """Best-effort full wake so the next shared-engine test starts active."""` / ` try:` / ` await engine.wake_up(stage_ids=stage_ids)` / ` except Exception as e:` / ` logger.warning("ensure_awake failed (stage_ids=%s): %s", stage_ids, e)`. Changed by this diff — tests/entrypoints/test_omni_sleep_mode.py:242 `await llm_engine.wake_up(stage_ids=[0], tags=["weights"])`; :247 `await _ensure_awake(llm_engine, [0])`; :256 `await llm_engine.sleep(stage_ids=[0], level=1)` (next test, no awake-state precondition). Unchanged by this diff, present in the PR-time tree — vllm_omni/entrypoints/async_omni.py:1194 `self._sleeping_tags.update([CuMemTag.WEIGHTS.value, CuMemTag.KV_CACHE.value])`; vllm_omni/entrypoints/async_omni.py:1245 `self._sleeping_tags.discard(t)` — wake with tags=["weights"] discards only WEIGHTS, leaving KV_CACHE in `_sleeping_tags`, so a swallowed full-wake failure leaves the shared engine dirty.) + +`PR description:1` [minor] — PR body Merge CI section pairs 2100.9s -> 901.24s with 49min -> 35min, but 2100.9s = 35.0min and 901.24s = 15.0min. Make the minutes match the quoted seconds (35min -> 15min) or correct the seconds to match 49min -> 35min (2940s -> 2100.9s). (evidence: PR #5713 description: "Merge CI (**2100.9s -> 901.24s**, **49min -> 35min**)" — 2100.9/60 = 35.015 min and 901.24/60 = 15.021 min, so the quoted seconds correspond to 35min -> 15min, not 49min -> 35min.) + +`tests/entrypoints/test_omni_sleep_mode.py:~207` [minor] — TestOmniLlmSleepMode.test_llm_sleep_ack uses the shared class-scoped llm_engine fixture, which builds two process stages on devices '0' and '1', but the test is still gated num_cards=1. tests/helpers/mark.py cuda_marks returns early for num_cards==1 and never emits a skipif, so on a single visible GPU this test is not skipped and the fixture will attempt to place stage 1 on device '1', failing or misbehaving. This two-device mismatch predates this diff (the fixture was already 2-stage at base), so it is not newly introduced by the function→class scope change; severity is major only if the merge lane actually runs this test on one H100. (evidence: tests/entrypoints/test_omni_sleep_mode.py:207 `@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=1)` — the test now consumes the class-scoped fixture defined at tests/entrypoints/test_omni_sleep_mode.py:151-155 (`@pytest_asyncio.fixture(scope="class", loop_scope="class")` / `stages, connectors = _build_llm_stages()`), and _build_llm_stages at tests/entrypoints/test_omni_sleep_mode.py:133 hardcodes stage 0 on `"devices": "0"` and at :140 hardcodes stage 1 on `"devices": "1"`. Unchanged by this diff, tests/helpers/mark.py:19-24 shows `if num_cards == 1: return marks` before any skipif, so num_cards=1 does not skip on a single-GPU host — the fixture init rather than the marker will fail when only one device is visible.) + +`tests/entrypoints/test_omni_sleep_mode.py:~227` [minor] — get_device_global_memory_used_gib returns 0.0 on any measurement exception (tests/entrypoints/test_omni_sleep_mode.py:80-82), and both reclaim audits are OR-gates — line 227 `assert freed_gib > 5.0 or drop_gib > 3.0` and line 300 `assert freed_gb > 14.0 or drop_gib > 8.0`. When the global-drop branch silently collapses to 0.0, the ACK freed_bytes branch alone decides the test, so a broken worker-process memory measurement degrades the audit to a signal check without indication. Consider asserting drop_gib > 0 (or guarding the measurement) so the global branch cannot silently pass a broken measurement. (evidence: tests/entrypoints/test_omni_sleep_mode.py:82 ` return 0.0` (inside `get_device_global_memory_used_gib` except handler at lines 80-82) — unchanged by this diff, present in the PR-time tree; tests/entrypoints/test_omni_sleep_mode.py:227 ` assert freed_gib > 5.0 or drop_gib > 3.0, (`; tests/entrypoints/test_omni_sleep_mode.py:300 ` assert freed_gb > 14.0 or drop_gib > 8.0, (`.) + +`tests/entrypoints/test_omni_sleep_mode.py:500` [minor] — Confirmed: `test_pure_diffusion_sleep_wake` at tests/entrypoints/test_omni_sleep_mode.py:500-501 is marked `core_model` + `hardware_test(res={"cuda": "L4", ...})` on top of the module-level `advanced_model` mark, so the listed merge command `pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"` does not collect it (it expands to the L4 mark, not H100). Either list an L4 lane command for this test or explicitly state where the claimed L4 coverage runs. (evidence: tests/entrypoints/test_omni_sleep_mode.py:24-26 `pytestmark = [ + pytest.mark.advanced_model, +]` (module-level, unchanged by this diff, present in the PR-time tree); tests/entrypoints/test_omni_sleep_mode.py:500 `@pytest.mark.core_model`; tests/entrypoints/test_omni_sleep_mode.py:501 `@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)`; tests/helpers/mark.py:14 ` test_resource = pytest.mark.L4` (cuda_marks maps res="L4" to L4, not H100). PR body merge command: `pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"` — requires the H100 mark, which this test does not carry.) + +`tests/entrypoints/test_omni_sleep_mode.py:506` [minor] — tests/entrypoints/test_omni_sleep_mode.py:506 docstring says "Diffusion TP=2 sleep/wake is covered by TestOmniDiffusionSleepMode", but that suite runs an explicit-stage BAGEL engine (`model_class_name: "BagelPipeline"`, model=ByteDance-Seed/BAGEL-7B-MoT) with `parallel_config.tensor_parallel_size: 2`. `test_pure_diffusion_sleep_wake` now hardcodes `tensor_parallel_size: 1` for MODEL_DIFF (riverclouds/qwen_image_random), so the default-config pure-diffusion TP=2 path from the removed `test_pure_diffusion_scenario` (parametrized tp_size [1,2]) is no longer exercised; the removed `test_diffusion_model_sleep_tp` also dropped BAGEL default-config TP=1. Either keep a default-config TP=2 pure-diffusion case or correct the docstring to state TP=2 coverage remains only for the explicit-stage BagelPipeline path. (evidence: PR-time tree, tests/entrypoints/test_omni_sleep_mode.py:506 ` Diffusion TP=2 sleep/wake is covered by TestOmniDiffusionSleepMode`; :515 ` "tensor_parallel_size": 1,` (inside `test_pure_diffusion_sleep_wake`, which uses `"model": MODEL_DIFF`); :185 ` "model_class_name": "BagelPipeline",` (inside the `diffusion_engine` fixture, whose engine is built with `AsyncOmni(model=MODEL, stages=stages, ...)`). PR-time grep finds no `test_pure_diffusion_scenario` and no `test_diffusion_model_sleep_tp` definition (only a docstring mention at line 318). Merge-base file (unchanged-by-diff proof the removed tests existed): `@pytest.mark.parametrize("tp_size", [1, 2])` on `test_pure_diffusion_scenario` with `"model": MODEL_DIFF` and `"tensor_parallel_size": tp_size`; `@pytest.mark.parametrize("tp_size", [1])` on `test_diffusion_model_sleep_tp` with `"model": MODEL`.) + +`.buildkite/amd/test-amd-merge.yml:~21` [minor] — The failing `buildkite/vllm-omni-amd-ci` check does not run any of this PR's touched tests: diff_stat shows only `tests/dfx/reliability/invalid_param_test/test_invalid_server_control.py`, `tests/entrypoints/openai_api/test_omni_sleep_wakeup.py`, and `tests/entrypoints/test_omni_sleep_mode.py` changed, and no pytest command in `.buildkite/amd/*.yml` targets `tests/entrypoints` or `tests/dfx` (grep for both returns no matches). The only path-less command, `.buildkite/amd/test-amd-ready.yml:86` (`-m "core_model and cache and diffusion and not distributed_cuda and L4"`), also excludes these files by marker. Please confirm the failure is pre-existing on main (or from a step outside these pytest commands) rather than introduced by this PR. (evidence: Unchanged by this diff, present in the PR-time tree: +.buildkite/amd/test-amd-merge.yml:21 `- "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` +.buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` +.buildkite/amd/test-amd-ready.yml:86 `- timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"` (path-less; marker excludes the touched files) +Grep for `entrypoints` and `tests/dfx` across `.buildkite/amd` returns no matches; diff_stat shows only the three test files changed.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.trace.json.gz new file mode 100644 index 000000000..38b23f5ec Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5713.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.cost.json new file mode 100644 index 000000000..35f41e1a2 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1199.5, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr5843/run-20260815-034500-f3d634", + "recorded_at": "2026-08-15T04:04:57", + "attempts": [ + { + "invocation_id": "inv-c71907f6", + "outcome": "run:run-20260815-034500-f3d634", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-034500-f3d634", + "llm_calls": 119, + "input_tokens": 1275564, + "output_tokens": 213781, + "usd": 0.931458, + "source": "spans" + } + ], + "attempt_usd_total": 0.931458, + "llm_calls": 119, + "input_tokens": 1275564, + "output_tokens": 213781, + "usd": 0.931458, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-034500-f3d634", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9315, + "minutes": 19.95, + "ci_minutes": 0, + "input_tokens": 1275564, + "output_tokens": 213781, + "tool_calls": 233, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8985, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 119, + "by_role": { + "agent": { + "usd": 0.904942, + "input_tokens": 1258728, + "output_tokens": 193842, + "calls": 117 + }, + "reducer": { + "usd": 0.026517, + "input_tokens": 16836, + "output_tokens": 19939, + "calls": 2 + } + }, + "cache_read_tokens": 13033216, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.58, + "gate": 1.63, + "review": 1190.7, + "report": 0.06 + }, + "time_to_first_result_s": 1196.95 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5843.trace.json.gz", + "trace_bytes": 34473 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.md new file mode 100644 index 000000000..940ca36ac --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.md @@ -0,0 +1,60 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5843, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head d6ba7d145815) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Structured-config path concern: _DiffusionConfigProjection.request_batch_max_wait_ms (vllm_omni/config/omni_config.py:532) is a plain float with no Field constraint, but production builds OmniDiffusionConfig from it (omni_config.py:755-756 and stage_init_utils.build_diffusion_config), which re-runs __post_init__ and enforces the finite non-negative check. +- [resolved] prior concern: removing the ming_flash_omni_tts LEGACY_TTS_DETECTORS entry could break stage-key detection — resolved because the adapter registry now owns that stage key; grep shows no deploy/config depends on the legacy detector entry (only test_tts_detection.py and check_tts_adapter.py reference it). +- [claim-verified] PR body claim 'Added unit tests in test_diffusion_scheduler' is true (diff adds 89 lines there); 'All passed locally' is not verifiable and gh_ci_read shows buildkite/vllm-omni-amd-ci FAILURE. +- [claim-verified] PR description claim 'Added unit tests in test_diffusion_scheduler to test the abstracted admission waiting policy' — diff adds tests at tests/diffusion/test_diffusion_scheduler.py:247-322, but one of them is mislabeled (see review comment). +- [claim-verified] PR body claim 'Added unit tests in test_diffusion_scheduler to test the abstracted admission waiting policy' — confirmed at tests/diffusion/test_diffusion_scheduler.py:249-323. +- [validated] vllm_omni/diffusion/data.py:951 now uses `if not math.isfinite(self.request_batch_max_wait_ms) or self.request_batch_max_wait_ms < 0:` and `import math` is present at top of file (line 5); negative/nan/inf all raise ValueError. +- [validated] vllm_omni/entrypoints/cli/serve.py:53 `_nonneg_finite_float` plus `import math` (line 9) reject nan/inf/-1 at parse time; docs hook provides `_nonneg_finite_float: float` no-op so docs generation still executes subparser_init. +- [validated] vllm_omni/diffusion/sched/base_scheduler.py:162-169 default `get_admission_wait_decision` returns `_AdmissionWaitDecision(should_wait=False)` and `should_end_admission_wait` returns True, so StepScheduler/custom schedulers remain compatible with the new engine call. +- [validated] TTS detection parity: vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py:19-20 declares `stage_keys = frozenset({"ming_tts"})` and `name = "ming_flash_omni_tts"`, matching the removed LegacyDetector's stage_keys/name; MingTTSAdapter remains architecture-identified with detect_priority=200. +- [validated] serve.py:53 `_nonneg_finite_float` rejects nan/inf/-1; `import math` present at serve.py:10; test_serve_cli_rejects_invalid_request_batch_max_wait_ms (tests/entrypoints/test_async_omni_diffusion_config.py:392) drives nan/inf/-inf/-1 through the real parser and expects SystemExit. +- [validated] interface.py:35-41 `_AdmissionWaitDecision` frozen dataclass defaults — consumed consistently by engine, RequestScheduler, StepScheduler/BaseScheduler, and test stubs. +- [validated] tts_adapters/__init__.py LEGACY_TTS_DETECTORS emptied — MingFlashOmniTTSAdapter (stage_keys={'ming_tts'}, default priority 100) and MingTTSAdapter (arch-only, priority 200) keep detect_tts_model_type('ming_tts','MingTTSForConditionalGeneration')=='ming_flash_omni_tts'; existing test tests/entrypoints/openai_api/test_tts_detection.py:249 covers it. +- [validated] tools/pre_commit/check_tts_adapter.py budgets 29/1 -> 27/0 — pre-commit lane is SUCCESS in gh_ci_read; the TTS cleanup is outside the stated refactor scope but required to keep the ratchet green. +- [validated] old method `_wait_for_request_batch_admission_locked` has no remaining in-repo references (grep returned no matches). + +`tests/diffusion/test_diffusion_scheduler.py:~284` [minor] — `get_admission_wait_decision` short-circuits at `if max_wait_ms <= 0.0 or self.max_num_running_reqs <= 1:` (request_scheduler.py:49) before reaching `if self.num_running_requests() > 0:` (request_scheduler.py:51). `test_admission_wait_disabled_with_zero_max_wait` initializes with only `request_batch_max_wait_ms=0.0` (so max_num_running_reqs defaults to 1), and `test_admission_wait_disabled_while_wave_is_running` initializes `max_num_seqs=1` (line 284) — so both exit on the first disjunct and neither would fail if `max_wait_ms <= 0.0` or `num_running_requests() > 0` were deleted. Tighten: set `max_num_seqs=2` for the zero-wait test, and `max_num_seqs=2` with one request scheduled first for the running-wave test, then assert `should_wait is False`. (evidence: vllm_omni/diffusion/sched/request_scheduler.py:49 `if max_wait_ms <= 0.0 or self.max_num_running_reqs <= 1:` +vllm_omni/diffusion/sched/request_scheduler.py:51 `if self.num_running_requests() > 0:` +tests/diffusion/test_diffusion_scheduler.py:284 `max_num_seqs=1,` +tests/diffusion/test_diffusion_scheduler.py:285 `request_batch_max_wait_ms=1000.0,` +All four lines are part of this PR's diff; with max_num_running_reqs=1 the test at line 284 exits at line 49 and never reaches line 51, so the running-wave arm has no failing-on-deletion coverage.) + +`tests/diffusion/test_diffusion_scheduler.py:278` [minor] — `RequestScheduler.get_admission_wait_decision` computes `stable_window_s = min(0.3, max_wait_s / 2.0) if dp_concurrent else min(0.05, max_wait_s / 5.0)` (vllm_omni/diffusion/sched/request_scheduler.py:55), but `test_admission_wait_decision_encodes_coalescing_policy` parameterizes only `request_batch_max_wait_ms=1000.0`, so both expectations (0.3 and 0.05) exercise only the `min()` cap. Add cases with small waits — e.g. `dp_concurrent=True` at 100 ms (expect 0.05) and `dp_concurrent=False` at 200 ms (expect 0.04) — so the proportional arms `max_wait_s / 2.0` and `max_wait_s / 5.0` are actually executed and asserted. (evidence: tests/diffusion/test_diffusion_scheduler.py:264-266 `@pytest.mark.parametrize( + ("dp_concurrent", "expected_stable_window_s"), + [(False, 0.05), (True, 0.3)], + )`; tests/diffusion/test_diffusion_scheduler.py:267 `request_batch_max_wait_ms=1000.0,`; tests/diffusion/test_diffusion_scheduler.py:278 `assert decision.stable_window_s == expected_stable_window_s`; vllm_omni/diffusion/sched/request_scheduler.py:55 `stable_window_s = min(0.3, max_wait_s / 2.0) if dp_concurrent else min(0.05, max_wait_s / 5.0)`. With `max_wait_ms=1000.0`, `max_wait_s=1.0`, so both arms return their `min()` caps: `min(0.3, 0.5)=0.3` and `min(0.05, 0.2)=0.05`; the proportional branches `max_wait_s / 2.0` and `max_wait_s / 5.0` are never executed by the test.) + +`tests/entrypoints/test_async_omni_diffusion_config.py:393` [minor] — Strengthen `test_serve_cli_rejects_invalid_request_batch_max_wait_ms` to assert the parser error message and exit code for each invalid value (nan/inf/-inf/-1); it currently only asserts `SystemExit`, so it does not verify the new `_nonneg_finite_float` validation failure mode. (evidence: tests/entrypoints/test_async_omni_diffusion_config.py:393 `def test_serve_cli_rejects_invalid_request_batch_max_wait_ms(bad_wait: str):` — tests/entrypoints/test_async_omni_diffusion_config.py:401-407: ` with pytest.raises(SystemExit):` / ` parser.parse_args(` / ` [` / ` "serve",` / ` "Qwen/Qwen-Image",` / ` "--omni",` / ` "--request-batch-max-wait-ms",` / ` bad_wait,` — the test asserts only that SystemExit is raised; no assertion exists on the error message or exit code.) + +`vllm_omni/diffusion/sched/base_scheduler.py:170` [minor] — The diff moves admission coalescing from engine-side flags (`supports_request_batch or dp_concurrent`) into the scheduler. `_busy_loop` now calls `_wait_for_admission_if_needed_locked()` unconditionally, but `BaseScheduler.get_admission_wait_decision` defaults to `_AdmissionWaitDecision(should_wait=False)` and `BaseScheduler.should_end_admission_wait` defaults to `True`, so a custom scheduler injected via `DiffusionEngine(scheduler=...)` that relied on engine-side coalescing silently loses admission wait unless it overrides both new methods. Update the base scheduler/SchedulerInterface docs and docs/design/module/archive/dit_module.md to list the new required overrides, or restore a compat path. (evidence: vllm_omni/diffusion/diffusion_engine.py:437 `self._wait_for_admission_if_needed_locked()` — called unconditionally; the old `if self.supports_request_batch or self.dp_concurrent:` gate was removed by this diff. vllm_omni/diffusion/sched/base_scheduler.py:170 `return _AdmissionWaitDecision(should_wait=False)` — base default admission decision. vllm_omni/diffusion/sched/base_scheduler.py:177 `return True` — base default should_end_admission_wait ends any wait immediately. docs/design/module/archive/dit_module.md:85 (unchanged by this diff, present in PR-time tree) `- **Scheduler Selection**: Uses \`StepScheduler\` for step-batch execution and \`RequestScheduler\` for request-batch execution. Tests and custom integrations may inject a \`SchedulerInterface\` instance explicitly.`) + +`.buildkite/amd/test-amd-ready.yml:~19` [minor] — `.buildkite/amd/test-amd-ready.yml:19` runs `tests/diffusion` with `-m 'core_model and cpu'`, and `buildkite/vllm-omni-amd-ci` is FAILING while the other checks pass. The diff does replace the engine-side admission gate with `_wait_for_admission_if_needed_locked()` (vllm_omni/diffusion/diffusion_engine.py:437) and adds admission tests. Note: `test_request_batch_admission_exits_early_when_waiting_queue_stable` is modified by this diff, but `test_async_add_req_and_stream_response` (tests/diffusion/test_diffusion_engine.py:761) is in the lane's blast radius and NOT modified by the diff. Please link the AMD job log and name the failing test to establish pre-existing vs introduced. (evidence: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; vllm_omni/diffusion/diffusion_engine.py:437 `self._wait_for_admission_if_needed_locked()` (changed by this diff); tests/diffusion/test_diffusion_engine.py:562 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]`; tests/diffusion/test_diffusion_scheduler.py:29 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`. CI check state (not a code line): gh_ci_read(5843) -> `{"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.trace.json.gz new file mode 100644 index 000000000..f50bcf78e Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5843.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.cost.json new file mode 100644 index 000000000..70307057e --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1858.3, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr5853/run-20260815-034501-559bab", + "recorded_at": "2026-08-15T04:15:56", + "attempts": [ + { + "invocation_id": "inv-aef7e5c7", + "outcome": "run:run-20260815-034501-559bab", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-034501-559bab", + "llm_calls": 175, + "input_tokens": 2122466, + "output_tokens": 284106, + "usd": 1.462506, + "source": "spans" + } + ], + "attempt_usd_total": 1.462506, + "llm_calls": 175, + "input_tokens": 2122466, + "output_tokens": 284106, + "usd": 1.462506, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-034501-559bab", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.4625, + "minutes": 30.91, + "ci_minutes": 0, + "input_tokens": 2122466, + "output_tokens": 284106, + "tool_calls": 295, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2427, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 175, + "by_role": { + "agent": { + "usd": 1.439549, + "input_tokens": 2094116, + "output_tokens": 270229, + "calls": 173 + }, + "reducer": { + "usd": 0.022957, + "input_tokens": 28350, + "output_tokens": 13877, + "calls": 2 + } + }, + "cache_read_tokens": 21367552, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.53, + "gate": 1.9, + "review": 1849.29, + "report": 0.06 + }, + "time_to_first_result_s": 1854.76 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5853.trace.json.gz", + "trace_bytes": 47825 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.md new file mode 100644 index 000000000..6bf5bf423 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.md @@ -0,0 +1,60 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5853, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head a24ff8f2ad80) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] diff_stat returns exactly the 28 files listed in the PR context (merge-base a874b8e09bb5); no hidden additions/removals beyond the evidence diff. +- [claim-verified] gh_ci_read shows buildkite/vllm-omni-amd-ci FAILURE with pre-commit, build 3.11/3.12, DCO, vllm-omni, intel-ci, npu-ci, and readthedocs all SUCCESS; attribution to this diff cannot be established from the read-only review (prior-round major comment stands). +- [claim-refuted] PR body says 'Without startup Cache-DiT capability ... high returns an error', but vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149-151 resolves `quality == "high"` unconditionally (`base_key = MINIMAX_H3_HIGH_CACHE_KEY; base_config = _high_quality_cache_config()`), and test_minimax_h3_quality_policy.py `("none","high") -> minimax_h3.high` confirms it installs without startup backend. The updated docs table (cache_dit.md) is the correct one. +- [claim-refuted] PR body claims 'Without startup Cache-DiT capability ... high returns an error', but vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149-151 (`if quality == "high": base_key = ... base_config = _high_quality_cache_config()`) has no backend gate and tests parametrize ('none','high') as installing the profile; prior-round comment already covers this. +- [claim-refuted] recipes/MiniMaxAI/MiniMax-H3.md:592 says '4× NVIDIA H200' while the PR description attributes the identical medians (85.49s/63.36s) to '4x NVIDIA L20X'; one provenance string is wrong → review comment. +- [validated] vllm_omni/inputs/data.py:213-215 adds `quality: str | None = None` with `DIFFUSION_QUALITY_LEVELS = ("lossless", "high")`; __post_init__ at :347-348 raises ValueError for unsupported values. +- [validated] New test files carry `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`, so they are collected and selected under the core/cpu marker lanes. +- [validated] vllm_omni/diffusion/sched/interface.py:60,118 adds quality to both batching keys; base_scheduler.py:281 and request_scheduler.py:37 build keys via field-name getattr so quality is automatically propagated; None vs 'lossless' stays distinct. +- [validated] tests/diffusion/test_diffusion_scheduler.py:380-442 and 987-1018 exercise RequestScheduler and StepScheduler quality isolation through the real scheduling path. +- [validated] tests/diffusion/cache/test_cache_dit_request_runtime.py:65,94 event-sequence assertions agree with runtime implementation (adopt no-op on same key, disable on key change, refresh on repeated same key). +- [validated] tests/diffusion/cache/test_cache_dit.py:77-91 updated public-API assertion includes CacheDiTRequestSpec and RequestScopedCacheDiTRuntime; __init__.py export list matches. +- [validated] tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:13-83 verifies _quality_policy.resolve is called with the request fields and _cache_dit_runtime.prepare(cache_spec) runs before diffuse. +- [validated] vllm_omni/entrypoints/openai/serving_video.py:203-204 and serving_video_output_stream.py:538-539 forward quality only when explicitly provided; explicit null writes None (no semantic change). +- [validated] vllm_omni/entrypoints/openai/api_server.py:2814 `quality=req.quality or "default"` maps omitted→'default' and keeps high/lossless for the async job response; VideoResponse quality default is 'default'. + +`vllm_omni/diffusion/cache/cachedit/backend.py:1` [major] — Resolve the failing `buildkite/vllm-omni-amd-ci` check before merge. The PR adds test modules that import `vllm_omni.diffusion.cache.cachedit.runtime` -> backend -> `import cache_dit`, plus scheduler/policy tests, so a collection-time import on the AMD lane cannot be ruled out. (evidence: [gate] buildkite/vllm-omni-amd-ci is the single failing check. The diff adds test modules importing vllm_omni.diffusion.cache.cachedit.runtime -> backend -> `import cache_dit`, plus scheduler/policy tests, so a collection-time import on the AMD lane cannot be ruled out; attribution to this PR vs pre-existing main was not verifiable read-only.) + +`vllm_omni/inputs/data.py:~215` [minor] — `OmniDiffusionSamplingParams` gains `quality` in vllm_omni/inputs/data.py:215 and serving_video.py:204 / serving_video_output_stream.py:539 forward it for every model, but the only model consumer is MiniMax H3 (pipeline_minimax_h3.py:1575 -> MiniMaxH3QualityPolicy.resolve). Consequently `quality="high"`/`"lossless"` is accepted and effectively ignored on non-H3 pipelines. Since this is a user-visible API contract, either reject `quality` at the serving boundary for models that do not implement it, or document the per-model no-op. (evidence: vllm_omni/inputs/data.py:215 `quality: str | None = None` — new field. vllm_omni/entrypoints/openai/serving_video.py:204 `gen_params.quality = request.quality` — forwarded for every model. vllm_omni/entrypoints/openai/serving_video_output_stream.py:539 `gen_params.quality = request.quality` — forwarded for every model. vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1575 `quality = sampling.quality` — the only `sampling.quality` consumer in the tree (grep for `sampling.quality` matches only this line).) + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:~149` [minor] — Confirmed. `MiniMaxH3QualityPolicy.resolve` (vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149-151) sets the high-quality Cache-DiT profile for `quality == "high"` without any `_configured_backend` gate, so `high` installs Cache-DiT even when the server was started without it. The docs table (docs/user_guide/diffusion/cache_acceleration/cache_dit.md:173) matches this ('no Cache-DiT | high | Install H3's conservative Cache-DiT profile'), but the PR description still says 'Without startup Cache-DiT capability ... `high` returns an error'. Update the PR description to match the shipped behavior. (evidence: vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149-151 `if quality == "high":` / `base_key = MINIMAX_H3_HIGH_CACHE_KEY` / `base_config = _high_quality_cache_config()` with no `_configured_backend` check (the only backend gate is the `elif self._configured_backend == "cache_dit" and quality is None` branch for omitted quality). docs/user_guide/diffusion/cache_acceleration/cache_dit.md:173 `| no Cache-DiT | `high` | Install H3's conservative Cache-DiT profile |`. PR body returned by gh_pr_view: `Without startup Cache-DiT capability, omitted and `lossless` are no-ops, `high` returns an error.` — the PR description contradicts the code and its own docs table.) + +`vllm_omni/diffusion/cache/cachedit/backend.py:~243` [minor] — `CacheDiTBackend.enable`'s custom-enabler branch (backend.py:243-244) records `self._refresh_funcs = [custom_enabler(pipeline, self.config)]` but sets `self._cache_targets = [_default_get_pipeline_transformer(pipeline)]` — only `pipeline.transformer`. The new `disable()` (backend.py:270-282) calls `cache_dit.disable_cache(target)` only for `self._cache_targets`. A custom enabler can install hooks on a different/wider target than the default transformer: `enable_cache_for_wan22` (model_specific.py:71, unchanged by this diff) in dual-transformer mode enables a `BlockAdapter(transformer=[pipeline.transformer, pipeline.transformer_2], ...)` and refreshes both transformers, so a future `disable()` would call `disable_cache` on the raw `pipeline.transformer` and leave the installed BlockAdapter hooks in place. Today only MiniMax H3 calls `disable()` and it uses the generic non-custom path, so this is latent; make the shared lifecycle symmetric by having custom enablers return/register every exact target they enabled. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:243 ` self._refresh_funcs = [custom_enabler(pipeline, self.config)]` +vllm_omni/diffusion/cache/cachedit/backend.py:244 ` self._cache_targets = [_default_get_pipeline_transformer(pipeline)]` +vllm_omni/diffusion/cache/cachedit/backend.py:280 ` cache_dit.disable_cache(target)` +unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/cache/cachedit/model_specific.py:71 `def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> RefreshCacheContextFunc:` — its dual-transformer branch calls `cache_dit.enable_cache(BlockAdapter(transformer=[pipeline.transformer, pipeline.transformer_2], ...))` and later builds `refresh_trans_two = _build_cache_context_refresh(cache_config, lambda pipeline: pipeline.transformer_2)`.) + +`recipes/MiniMaxAI/MiniMax-H3.md:592` [minor] — recipes/MiniMaxAI/MiniMax-H3.md:592 states the request-scoped quality table was measured on 4× NVIDIA H200, while PR #5853's description attributes the identical median numbers (85.49 s, 63.36 s, 25.9%, SSIM 0.9709, PSNR 34.98 dB) to 4× NVIDIA L20X. Reconcile the hardware name so the documented benchmark provenance is internally consistent. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:592 `The following result was measured on 4× NVIDIA H200 with SP4, text-encoder` (followed by table `| lossless | 85.49 s | ... | 0.9709 | 34.98 dB |`, `| high | 63.36 s | 1.35× | 0.9709 | 34.98 dB |`). PR #5853 body (not in the diff, but authoritative PR metadata): `The online server was exercised on 4x NVIDIA L20X with SP4, text-encoder TP4, 1344x768, 124 frames, 24 FPS, and 50 steps.` followed by `| Median | 85.49 s | 63.36 s | 25.9% | 0.9709 | 34.98 dB |` — same median figures, contradictory hardware attribution.) + +`tests/diffusion/test_diffusion_model_runner.py:165` [minor] — This hunk adds `test_request_scoped_cache_dit_lifecycle_is_pipeline_opt_in` (tests/diffusion/test_diffusion_model_runner.py:165) with no per-test markers while the file-level `pytestmark` is only `pytest.mark.diffusion`. The Simple · Diffusion Test lane runs `pytest -sv tests/diffusion -m 'core_model and cpu'` (.buildkite/cuda/test-merge.yml:18), so this new test is collected but deselected and the claimed routing coverage never runs. Add `@pytest.mark.core_model` and `@pytest.mark.cpu`, matching the sibling tests in this file including the newly added `test_execute_model_cache_summary_follows_pipeline_owned_cache_dit_state`. (evidence: tests/diffusion/test_diffusion_model_runner.py:16 `pytestmark = [pytest.mark.diffusion]` — file-level marker only, no core_model/cpu. tests/diffusion/test_diffusion_model_runner.py:165 `def test_request_scoped_cache_dit_lifecycle_is_pipeline_opt_in():` — new test with no decorators (confirmed by reading the PR-time file region; both lines are within the diff's added hunk). .buildkite/cuda/test-merge.yml:18 `"timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — the CI lane for this directory selects only core_model-and-cpu tests, so the unmarked new test is deselected. The changed-file marker convention is proven by the diff's next added test, `test_execute_model_cache_summary_follows_pipeline_owned_cache_dit_state`, which carries `@pytest.mark.core_model` (tests/diffusion/test_diffusion_model_runner.py).) + +`vllm_omni/diffusion/cache/cachedit/backend.py:243` [nit] — Validate `force_refresh` `once` semantics when `num_outputs_per_prompt > 1`: the current behavior is one refresh per request with multiple denoise loops, and this was not checked against cache-dit's once-policy clearing. (evidence: Unverified suspicion (not posted as a comment): force_refresh `once` semantics with num_outputs_per_prompt>1 (one refresh per request, multiple denoise loops) were not checked against cache-dit's once-policy clearing.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.trace.json.gz new file mode 100644 index 000000000..2bfa4c687 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5853.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.cost.json new file mode 100644 index 000000000..2cfbf8449 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 932.9, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr5857/run-20260815-040500-749e83", + "recorded_at": "2026-08-15T04:20:30", + "attempts": [ + { + "invocation_id": "inv-58652d6b", + "outcome": "run:run-20260815-040500-749e83", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-040500-749e83", + "llm_calls": 72, + "input_tokens": 546231, + "output_tokens": 116382, + "usd": 0.446975, + "source": "spans" + } + ], + "attempt_usd_total": 0.446975, + "llm_calls": 72, + "input_tokens": 546231, + "output_tokens": 116382, + "usd": 0.446975, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-040500-749e83", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.447, + "minutes": 15.51, + "ci_minutes": 0, + "input_tokens": 546231, + "output_tokens": 116382, + "tool_calls": 127, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6324, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 72, + "by_role": { + "agent": { + "usd": 0.429096, + "input_tokens": 538461, + "output_tokens": 102070, + "calls": 70 + }, + "reducer": { + "usd": 0.017879, + "input_tokens": 7770, + "output_tokens": 14312, + "calls": 2 + } + }, + "cache_read_tokens": 6350848, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.62, + "gate": 1.61, + "review": 925.04, + "report": 0.06 + }, + "time_to_first_result_s": 930.32 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5857.trace.json.gz", + "trace_bytes": 20889 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.md new file mode 100644 index 000000000..167ed3505 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5857, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4aa748dc54f4) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [claim-verified] 'uses serving options available on main': all flags above are present in serve.py at PR head. +- [claim-verified] 69,219 MiB = 67.60 GiB; 4-GPU headroom 73,415-69,219=4,196 MiB ~= 4.1 GiB; 8-GPU headroom 73,415-46,930=26,485 MiB ~= 25.9 GiB. +- [claim-refuted] 2-GPU headroom: 73,415/1024 - 37,250/1024 = 35.3 GiB (calc), but the recipe says 'about 34.7 GiB' — internal inconsistency (see review comment). +- [validated] All CLI flags in the three serving blocks exist at head in vllm_omni/entrypoints/cli/serve.py: --omni, --num-gpus, --usp/--ulysses-degree, --ring/--ring-degree, --text-encoder-tp-size (serve.py:718), --vae-patch-parallel-size (serve.py:710), --vae-parallel-mode (choices include 'tile'), --vae-use-tiling, --diffusion-attention-backend, --enable-distributed-layerwise-offload, --dlo-no-use-allgather (store_false for dlo_use_allgather), --dlo-resident-layers (serve.py:663). +- [validated] CUDNN_ATTN is a real backend enum: vllm_omni/diffusion/attention/backends/registry.py:63 `CUDNN_ATTN = "vllm_omni.diffusion.attention.backends.cudnn_attn.CuDNNAttentionBackend"`; docs/user_guide/diffusion/attention_backends.md:19 documents it as the Blackwell default. +- [validated] 50-steps->49-updates claim: pipeline_minimax_h3.py uses `progress_bar(total=len(video_sigmas) - 1)` and time_request.py `_time_shift_sigmas(num_steps=50)` returns 50 sigmas (linspace(1.0,0.0,50) ends at 0.0 so the extra-0 append is skipped) -> 49 denoise updates. +- [validated] Per-step arithmetic (calc): 504.69/49=10299.8->10,300; 541.90/49=11059.2->11,059; 278.73/49=5688.4->5,688; 299.85/49=6119.4->6,119; 159.40/49=3253.1->3,253; 167.25/49=3413.3->3,413 — all match the doc table. +- [validated] 124-frame claim: pipeline_minimax_h3.py `_resolve_shape` computes requested_frames=round(5.0*24)=120 then `minimax_h3_align_frame_count` snaps to 17n+5 = 124 (time_request.py `_align_frame_count`). +- [validated] T2VA curl example is a working payload: serving_video.py maps top-level flow_shift->extra_args['flow_shift'] and aspect_ratio->extra_args['aspect_ratio'], sets num_inference_steps/seed/fps, and merges extra_params into extra_args; pipeline_minimax_h3.py reads extra.get('task'), extra.get('duration'), extra.get('audio_flow_shift') (pipeline lines 642, 1615-1617). +- [sweep] diff_stat: merge-base d219d93bbb4d -> only recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md changed (198 insertions), docs-only, no tests touched. +- [sweep] recipes/README.md 'Available Recipes' table lists MiniMax-H3.md (line 48) and MUSA but not the new RTX-PRO-5000 recipe; 5090/NPU siblings are also unlisted. +- [sweep] Sibling contrast vs MiniMax-H3.md and MiniMax-H3-5090.md: 200 GiB min / 384 GiB rec RAM and 135 GiB partition storage match siblings; the two-GPU route intentionally differs (TP1 x Ulysses2 vs siblings' TP2) and states its own memory table; no contradictory flags. +- [sweep] Mode matrix: recipe uses /v1/videos/sync (api_server.py:3446); async /v1/videos and sync share _parse_video_form, so the documented fields behave identically on both. +- [sweep] The recipe's flag/pipeline paths are already covered by existing tests: tests/entrypoints/test_async_omni_diffusion_config.py:274-332 (MiniMax-H3 + dlo/attention flags) and tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py (DLO offload plan, sigma schedule, shape contract) — no test changes needed for a docs-only file. + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:109` [minor] — The PR-body 50-step matrix lists an 8-GPU TP2 x Ulysses4 latency winner (T2VA 147.77 s / first-frame I2VA 155.70 s, 70.38 GiB peak), but the Eight GPUs section of the recipe ships only TP4 x Ulysses2. Add the TP2 x Ulysses4 route to the Eight GPUs section or state there why the slower TP4 x Ulysses2 was chosen as the production recommendation. (evidence: PR #5857 body (PR description, not part of the diff): '| 8 | T2VA | TP2 x Ulysses4 | resident | **147.77** | 0.04 | n/a | **144.06** | 2.56 | **2,940** | 70.38 | Passed; performance winner; 1.3 GiB headroom |' and 'On eight GPUs, TP2 x Ulysses4 is the latency winner, while TP4 x Ulysses2 is the production recommendation because it retains substantially more memory headroom.' — recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:109 '### Eight GPUs' and :111 'The recommended eight-GPU route uses TP4 x Ulysses2, text-encoder TP8, VAE patch parallelism 8, and host-memory interleaving across both NUMA nodes.' A repo grep for 'Ulysses4' in that file returns no matches, proving the recipe omits the winner.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:~177` [minor] — Change 'about 34.7 GiB' to 'about 35.3 GiB': 73,415 MiB is 71.69 GiB and 71.69 - 36.38 = 35.31 GiB. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:169 `| 2 | T2VA | TP1 x Ulysses2, DLO resident 20 | 515.57 | 1.19 | 504.69 | 8.79 | 10,300 | 36.38 |`; recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:177 `nvidia-smi`. The two-GPU route leaves about 34.7 GiB below the reported`; recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:178 `73,415 MiB device capacity, the four-GPU route leaves about 4.1 GiB, and the`. 73,415/1024 = 71.69 GiB; 71.69 - 36.38 = 35.31 GiB, not 34.7.) + +`recipes/README.md:~48` [minor] — Confirmed as drafted: the PR adds recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md without touching recipes/README.md, so the new recipe is missing from the 'Available Recipes' index. Add a row at recipes/README.md:48-49, e.g. 'MiniMax-H3 RTX PRO 5000 | T2VA/FL2VA/Ref2VA | 2/4/8x RTX PRO 5000 72 GiB'; the sibling MiniMax-H3-5090.md and MiniMax-H3-NPU.md variants are also absent and can be indexed in the same pass. (evidence: recipes/README.md:48 `| [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |` and recipes/README.md:49 `| [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |` are the only MiniMaxAI H3 rows in the 'Available Recipes' table (no RTX PRO 5000 row follows before `| [`krea/Krea-2.md`]...`). diff_stat reports this PR changes exactly `recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md | 198 +++++++++++++++++++++++++++` and does not change recipes/README.md (README line 48 is therefore unchanged by this diff but present in the PR-time tree); directory listing of recipes/MiniMaxAI shows MiniMax-H3-5090.md, MiniMax-H3-NPU.md, and MiniMax-H3-RTX-PRO-5000.md exist but are not indexed.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:80` [minor] — The four-GPU section documents only TP2 x Ulysses2, whose 67.58 GiB peak leaves about 4.1 GiB of headroom (recipe lines 171 and 178), while the PR description's 50-step matrix already measured TP4 x Ulysses1 at 50.25 GiB as 'Passed; safer-memory alternative'. Add the TP4 x Ulysses1 command to the Four GPUs section, or state why it was omitted, so readers approaching the 4.1 GiB wall see the measured safer-memory alternative. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:80 `The validated baseline uses TP2 x Ulysses2, text-encoder TP4, VAE patch` — this file is the PR's only changed file (ADDED); the Four GPUs block through line 106 contains no TP4 x Ulysses1 route. recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:171 `| 4 | T2VA | TP2 x Ulysses2 | 284.01 | 0.03 | 278.73 | 4.36 | 5,688 | 67.58 |`. recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:178 `73,415 MiB device capacity, the four-GPU route leaves about 4.1 GiB, and the`. Present in the PR body (outside the diff): `| 4 | T2VA | TP4 x Ulysses1 | resident | 329.56 | 0.03 | n/a | 324.34 | 4.35 | 6,619 | 50.25 | Passed; safer-memory alternative |`.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:~149` [nit] — The validation block pins PyTorch, CUDA, and driver but not the vLLM-Omni commit, so the posted latency and memory numbers cannot be reproduced against a specific tree. Add the vLLM-Omni commit/version to the validation block, as the sibling MiniMax-H3 recipe does with `ae6577ea` (`0.26.1.dev14+gae6577ea`). (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:149 `eight RTX PRO 5000 GPUs. The run used PyTorch 2.11.0+cu130, CUDA 13.0, driver` and :150 `580.95.05, 1344x768 output, 124 frames, and two warmups.` — no commit pinned anywhere in this new file (grep for `commit` returns no matches). Sibling recipe, unchanged by this diff and present in the PR-time tree: recipes/MiniMaxAI/MiniMax-H3.md:164 `At vLLM-Omni commit \`ae6577ea\`, one full 50-step T2VA request completed on`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.trace.json.gz new file mode 100644 index 000000000..29352b697 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr5857.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.cost.json new file mode 100644 index 000000000..406f7d4f0 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 722.4, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr6045/run-20260815-041015-14fdb7", + "recorded_at": "2026-08-15T04:22:15", + "attempts": [ + { + "invocation_id": "inv-c855ab77", + "outcome": "run:run-20260815-041015-14fdb7", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-041015-14fdb7", + "llm_calls": 70, + "input_tokens": 460251, + "output_tokens": 83500, + "usd": 0.365531, + "source": "spans" + } + ], + "attempt_usd_total": 0.365531, + "llm_calls": 70, + "input_tokens": 460251, + "output_tokens": 83500, + "usd": 0.365531, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-041015-14fdb7", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3655, + "minutes": 12.0, + "ci_minutes": 0, + "input_tokens": 460251, + "output_tokens": 83500, + "tool_calls": 152, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.524, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 70, + "by_role": { + "agent": { + "usd": 0.354039, + "input_tokens": 448987, + "output_tokens": 75852, + "calls": 68 + }, + "reducer": { + "usd": 0.011492, + "input_tokens": 11264, + "output_tokens": 7648, + "calls": 2 + } + }, + "cache_read_tokens": 5533824, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.33, + "gate": 1.74, + "review": 714.69, + "report": 0.07 + }, + "time_to_first_result_s": 719.81 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6045.trace.json.gz", + "trace_bytes": 26636 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.md new file mode 100644 index 000000000..16241dbe8 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6045, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head b112c7fdeed2) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Benchmark arithmetic checked with calc: 168/27=6.22 (doc says 6.2x) and 283/56=5.05 (doc says 5.1x); the speedup columns are consistent with the sequential/multi-thread times. +- [claim-verified] Pipeline Parallelism is removed from the primary User Guide nav (no parallelism/pipeline_parallel.md entry under Features>Diffusion Acceleration) and remains by direct link, as features/README.md states. +- [claim-verified] Custom Diffusion Pipeline and ComfyUI are grouped under a new Integrations nav section; Multi-Thread Weight Loading anchor is preserved in diffusion_features.md. +- [validated] Startup/loading defaults claim: vllm_omni/entrypoints/cli/serve.py:635-645 defines --disable-multithread-weight-load (action=store_false, dest=enable_multithread_weight_load, default=True) and --num-weight-load-threads (default=4); vllm_omni/config/omni_config.py:343-344 `enable_multithread_weight_load: bool = True` / `num_weight_load_threads: int = Field(default=4, ge=1)`. +- [validated] Loader actually uses the knobs: vllm_omni/diffusion/model_loader/diffusers_loader.py:242-252 gates on use_safetensors + enable_multithread_weight_load + safetensors_load_strategy != torchao and passes max_workers=num_threads to multi_thread_safetensors_weights_iterator. +- [validated] HunyuanImage3 step-execution matrix cell: docs/user_guide/diffusion/execution_modes.md:97-101 documents TORCH_SDPA requirement for --max-num-seqs>1; vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py:1846-1853 raises unless backend==TORCH_SDPA for grouped batching; tests/diffusion/models/hunyuan_image3/test_hunyuan_image3_step_execution.py exists. +- [validated] Helios single-request step-execution cell: docs/user_guide/diffusion/execution_modes.md:103-104 'Helios supports single-request step execution only: use --step-execution --max-num-seqs 1'; vllm_omni/diffusion/models/helios/pipeline_helios.py:661 denoise_step and :780 step_scheduler are implemented. +- [validated] HSDP compatibility changes: docs/user_guide/diffusion/parallelism/hsdp.md:26 'Combined Mode: HSDP overlays on top of other parallelism (Ulysses-SP, CFG-Parallel)'; config validation (vllm_omni/config/omni_config.py OmniStageDiffusionParallelConfig.__post_init__) allows HSDP with SP/CFG and rejects TP/DP; tests/diffusion/distributed/test_hsdp.py:358,390 exercise cfg_parallel_size=2 with HSDP. +- [validated] Distributed-layerwise anchor: docs/user_guide/diffusion/cpu_offload.md:182 '## Distributed Layerwise Offloading' (anchor distributed-layerwise-offloading); the compatibility matrix it points at exists in docs/design/feature/distributed_layerwise_offload.md:91 '## Parallelism compatibility'. +- [sweep] diff_stat: exactly 5 files changed (docs/.nav.yml, docs/design/index.md, docs/features/README.md, docs/user_guide/diffusion/startup_and_loading.md, docs/user_guide/diffusion_features.md), 211 insertions / 118 deletions, 0 test files. +- [sweep] Nav/link existence: list_dir confirmed every nav target exists (docs/features/{README,sleep_mode,session_state_manager,custom_pipeline,comfyui}.md, docs/user_guide/diffusion/{execution_modes,attention_backends,regional_compilation,frame_interpolation,startup_and_loading,cpu_offload,lora,feature_compatibility}.md, parallelism/*, cache_acceleration/*, docs/user_guide/quantization/*, docs/design/feature/*, docs/configuration/stage_configs.md). +- [sweep] Taxonomy defect found: docs/design/index.md:53 '#### Attention optimization' is followed (after the Skip-Softmax bullet at :59) by Cache-DiT/TeaCache/Diffusion Continuous Batching/Distributed Layerwise Offload bullets at :61-64 with no intervening heading, so they render under 'Attention optimization' even though docs/.nav.yml:127-132 makes them siblings of Attention Optimization. +- [sweep] GGUF inconsistency: docs/.nav.yml quantization list drops GGUF (now ModelOpt -> AutoRound, lines 56-57), but docs/user_guide/diffusion_features.md:95 and docs/user_guide/quantization/overview.md:49 still advertise GGUF as a supported method; docs/user_guide/quantization/gguf.md:8-10 says GGUF 'has migrated to OOT vllm-gguf-plugin'. +- [sweep] features/README.md:52 cache-acceleration row links only cache_dit.md while the same cell names TeaCache and the nav exposes teacache.md (docs/.nav.yml:65). + +`docs/design/index.md:53` [minor] — The diff renames `#### Attention Backends` to `#### Attention optimization` and inserts the Skip-Softmax paragraph/bullet, but leaves the immediately following `Cache-DiT`, `TeaCache`, `Diffusion Continuous Batching`, and `Distributed Layerwise Offload` bullets under the same heading (docs/design/index.md:53-64). In docs/.nav.yml those four are siblings of `Attention Optimization` under `Diffusion Acceleration`, not children. Add subheadings (e.g. `#### Cache acceleration`, `#### Runtime`, `#### Offloading`) or move the attention paragraph and Skip-Softmax bullet below those four bullets so the page taxonomy matches the nav this PR is aligning. (evidence: docs/design/index.md:53 `#### Attention optimization`; docs/design/index.md:59 `- [Skip-Softmax](feature/skip_softmax.md)`; docs/design/index.md:61 `- [Cache-DiT](feature/cache_dit.md)`; docs/design/index.md:62 `- [TeaCache](feature/teacache.md)`; docs/design/index.md:63 `- [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)`; docs/design/index.md:64 `- [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md)` — no heading between line 59 and 64. Contrast docs/.nav.yml:127 `- Attention Optimization:` followed by :128 `- Skip-Softmax: design/feature/skip_softmax.md`, then :129 `- Cache-DiT: design/feature/cache_dit.md`, :130 `- TeaCache: design/feature/teacache.md`, :131 `- Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md`, :132 `- Distributed Layerwise Offload: design/feature/distributed_layerwise_offload.md` at the same indentation as Attention Optimization.) + +`docs/.nav.yml:~13` [nit] — The diff removes the GGUF entry from the Features/Quantization nav (docs/.nav.yml:56 ` - ModelOpt: user_guide/quantization/modelopt.md`, :57 ` - AutoRound: user_guide/quantization/autoround.md`; GGUF no longer appears in the file), but the user-facing tables still advertise GGUF (docs/user_guide/diffusion_features.md:95 and docs/user_guide/quantization/overview.md:49), and docs/user_guide/quantization/gguf.md:13 notes GGUF has migrated to the OOT vllm-gguf-plugin. Mirror that de-prioritization in the tables (add the plugin caveat or annotate/drop the GGUF rows), or restore the nav entry. (evidence: ['docs/.nav.yml:56 ` - ModelOpt: user_guide/quantization/modelopt.md` and docs/.nav.yml:57 ` - AutoRound: user_guide/quantization/autoround.md`; a repo grep for `GGUF` in docs/.nav.yml returns no matches, so GGUF is absent from the nav (entry removed by this diff).', 'docs/user_guide/diffusion_features.md:95 `| **[GGUF](quantization/gguf.md)** | `quantization="gguf"` | Native GGUF transformer-only weights (Q4, Q8, etc.) | Memory reduction on consumer GPUs |` — unchanged by this diff, still advertises GGUF.', 'docs/user_guide/quantization/overview.md:49 `| GGUF | [GGUF](gguf.md) | Pre-quantized transformer weights | Qwen-Image | Validated where a model-specific GGUF adapter exists |` — still advertises GGUF.', 'docs/user_guide/quantization/gguf.md:13 `GGUF support has migrated to OOT [vllm-gguf-plugin](https://github.com/vllm-project/vllm-gguf-plugin). Make sure you have GGUF plugin installed before serving a GGUF model.` — supports the migration rationale (comment cited lines 8-10; actual note line is 13).']) + +`docs/features/README.md:52` [nit] — At docs/features/README.md:52, the row for 'Reuse denoising computation' links its User guide cell only to [Cache Acceleration](../user_guide/diffusion/cache_acceleration/cache_dit.md), while its Related design contract cell names both Cache-DiT and TeaCache, and docs/.nav.yml:65-66 exposes both guides. Add a TeaCache link to the User guide cell or split the row so users are not routed only to Cache-DiT. (evidence: docs/features/README.md:52 `| Reuse denoising computation | [Cache Acceleration](../user_guide/diffusion/cache_acceleration/cache_dit.md) | [Cache-DiT](../design/feature/cache_dit.md), [TeaCache](../design/feature/teacache.md) |`; docs/.nav.yml:65 ` - TeaCache: user_guide/diffusion/cache_acceleration/teacache.md`; docs/.nav.yml:66 ` - Cache-DiT: user_guide/diffusion/cache_acceleration/cache_dit.md`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.trace.json.gz new file mode 100644 index 000000000..baa803fca Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6045.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.cost.json new file mode 100644 index 000000000..777b4ff50 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1262.3, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr6049/run-20260815-041558-52293d", + "recorded_at": "2026-08-15T04:36:59", + "attempts": [ + { + "invocation_id": "inv-9ff9b455", + "outcome": "run:run-20260815-041558-52293d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-041558-52293d", + "llm_calls": 160, + "input_tokens": 1094563, + "output_tokens": 221883, + "usd": 0.857673, + "source": "spans" + } + ], + "attempt_usd_total": 0.857673, + "llm_calls": 160, + "input_tokens": 1094563, + "output_tokens": 221883, + "usd": 0.857673, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-041558-52293d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8577, + "minutes": 21.0, + "ci_minutes": 0, + "input_tokens": 1094563, + "output_tokens": 221883, + "tool_calls": 289, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8925, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 160, + "by_role": { + "agent": { + "usd": 0.82699, + "input_tokens": 1070949, + "output_tokens": 199820, + "calls": 158 + }, + "reducer": { + "usd": 0.030683, + "input_tokens": 23614, + "output_tokens": 22063, + "calls": 2 + } + }, + "cache_read_tokens": 11780352, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.48, + "gate": 1.91, + "review": 1254.56, + "report": 0.06 + }, + "time_to_first_result_s": 1259.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6049.trace.json.gz", + "trace_bytes": 45264 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.md new file mode 100644 index 000000000..1391977a6 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6049, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0db924bc5444) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Prior concern 'no in-repo caller left stale after public signature change': repo-wide grep found only updated examples/tests/registry call sites; no stale consumer. +- [resolved] Prior concern 'LingBot offline I2V loses num_frames after removing the builder': resolved by request_utils.py:395-398 falling back to sampling.num_frames. +- [resolved] prior concern that LingBot would now receive an unvalidated I2V envelope: vllm_omni/diffusion/models/lingbot_video/request_utils.py:158-177 `resolve_lingbot_mode` rejects `multi_modal_data` with video/audio, enforces `modalities` and rejects >1 image — the pipeline owns validation as the PR claims. +- [resolved] prior concern that removing the VACE I2V builder's `num_frames`/height/width keys breaks the pipeline: vllm_omni/model_extras/vace.py build_image_to_video_prompt only returns `video`/`mask` under multi_modal_data and vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py:536-539 reads `additional_info.get("mask")` plus frame conditioning, then receives num_frames via sampling params — no envelope key dependence found. +- [resolved] Concern 'removed LingBot invalid-media validation leaves no guard' is resolved at tests/diffusion/models/lingbot_video/test_request_utils.py:58-76 `test_resolve_lingbot_mode_rejects_invalid_media_combinations`, which asserts the new envelope validation messages. +- [resolved] Concern 'LingBot I2V no longer receives prompt num_frames' is resolved at vllm_omni/diffusion/models/lingbot_video/request_utils.py:395-398, which derives explicit frames from `sampling.num_frames` (`if explicit_num_frames is None and sampling.num_frames not in (None, 1): explicit_num_frames = sampling.num_frames`). +- [claim-verified] diff_stat shows exactly 14 changed files and no changes to curl recipes, serving_video.py, or entrypoints, matching the PR's 'keep model-specific curl recipes' and 'leave the public online serving handler unchanged' claims. +- [claim-verified] CI state via gh_ci_read: pre-commit, DCO, build 3.11/3.12, vllm-omni, npu-ci, release, docs pass; buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-intel-ci fail (states only, no logs available). +- [claim-refuted] PR body: "leave the public online serving handler unchanged" — true for vllm_omni/entrypoints (no files touched), but the online client behavior changed despite 'generalize the online Python clients' also being claimed; this is a user-visible client behavior change, not a serving handler change. +- [validated] LingBot model-side validation exists and covers the removed builders: request_utils.py:131-157 resolves T2I/TI2V/T2V from modalities+image and rejects image with T2I, and request_utils.py:402-405 rejects T2I num_frames != 1. +- [validated] Wan2.2 I2V consumes last_image (pipeline_wan2_2_i2v.py:470-472 `last_image = multi_modal_data.get("last_image", None)` and :590-597), so dropping the old single-image guard enables first/last-frame instead of regressing it; mask/reference_images remain unread (see comment). +- [validated] tests/examples/offline_inference/test_image_task_prompts.py:19-45 drives the real envelope builders through the expected None/''/str negative_prompt cases and PIL image multi_modal_data; it asserts the actual produced envelope, not a pre-transformed value. +- [validated] registry.py:281-286 reconstructs builder inputs with `prompt=str(prompt["prompt"])` and `negative_prompt=prompt.get("negative_prompt")`, so model-specific builders still receive str/None exactly as before; unit tests at tests/model_extras/test_model_extras.py:283-308, 426-430, 607-... assert full output. +- [validated] removed generic LingBot/Cosmos builders have zero remaining references: grep for default_text_to_image_prompt, default_image_to_video_prompt, build_lingbot_image_to_video_prompt, build_cosmos3_text_to_image_prompt all returned no matches. + +`vllm_omni/model_extras/registry.py:~316` [minor] — This diff removes default_image_to_video_prompt's single-image validation, so build_image_to_video_prompt now returns the CLI envelope unchanged for builderless models (registry.py:315-317). The shared CLI assembles image/last_image/mask/reference_images unconditionally (image_to_video.py:430-437), so a Wan2.2/LTX/Hunyuan/Cosmos3 run can silently carry VACE-only mask/reference_images keys; pipeline_wan2_2_i2v.py consumes image and last_image (lines 470-472) and drops the rest. If that silent drop is unintended, add light validation/key-stripping before the passthrough while still allowing last_image for Wan2.2. (evidence: registry.py:315 `builder: ImageToVideoPromptBuilder | None = spec.get("image_to_video_prompt_builder") if spec else None`; registry.py:316-317 `if builder is None: + return prompt`; registry.py:209 `"WanVACEPipeline": {` is the only spec with `"image_to_video_prompt_builder": build_vace_image_to_video_prompt`. examples/offline_inference/image_to_video/image_to_video.py:430-437 assembles `media_inputs["image"] = ...`, `media_inputs["last_image"] = ...`, `media_inputs["mask"] = ...`, `media_inputs["reference_images"] = [...]` with no model gate. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py:470-472 `last_image: PIL.Image.Image | torch.Tensor | None = None` / `last_image = multi_modal_data.get("last_image", None)` with no mask/reference_images reads.) + +`examples/online_serving/text_to_image/openai_chat_client.py:~126` [minor] — Confirmed: this diff removes default=0 from --seed (examples/online_serving/text_to_image/openai_chat_client.py:126), and generate_image only sends seed when seed is not None (line 75), so a bare `python openai_chat_client.py` no longer sends seed=0 and loses deterministic output. Keep severity minor; if intentional, document the non-reproducible bare-run behavior. (evidence: Changed by this diff, PR-time head: examples/online_serving/text_to_image/openai_chat_client.py:126 `parser.add_argument("--seed", type=int, help="Random seed")`; examples/online_serving/text_to_image/openai_chat_client.py:75 `if seed is not None:` and line 76 `payload["seed"] = seed`. Base (pre-PR) content: examples/online_serving/text_to_image/openai_chat_client.py:126 `parser.add_argument("--seed", type=int, default=0, help="Random seed")`.) + +`examples/online_serving/text_to_image/openai_chat_client.py:~122` [minor] — This diff generalizes openai_chat_client.py to omit height/width/steps/cfg-scale defaults and rebrands gradio_demo.py plus the online T2I README as shared. However gradio_demo.py still hardcodes Qwen-Image defaults (height=1024 at line 102, width=1024 at line 109, steps=100 at line 118, true_cfg_scale=4.0 at line 125), and generate_image always sends all four via extra_body (lines 36-40), so the Gradio path neither omits those values like the shared client nor honors the documented model defaults. Update the Gradio sliders to leave parameters unset/model-driven, or keep the Gradio path explicitly Qwen-specific in both the demo and README. (evidence: openai_chat_client.py:122 `parser.add_argument("--height", type=int, help="Image height; defaults to the model setting")` (no default; width/steps/cfg-scale follow at lines 123-125). gradio_demo.py:36-40 `extra_body = {` / `"height": height,` / `"width": width,` / `"num_inference_steps": steps,` / `"true_cfg_scale": cfg_scale,` — always sent. gradio_demo.py:102 `value=1024,` and gradio_demo.py:109 `value=1024,`; gradio_demo.py:118 `value=100,`; gradio_demo.py:125 `value=4.0,`. These gradio slider/value lines are unchanged by this diff but present in the PR-time tree; the diff only rebrands the demo's title/markdown, leaving the hardcoded Qwen-Image values intact.) + +`vllm_omni/model_extras/registry.py:~316` [minor] — This diff adds the builderless I2V passthrough at vllm_omni/model_extras/registry.py:316-317 (`if builder is None: return prompt`). tests/model_extras/test_model_extras.py only exercises build_image_to_video_prompt via the VACE helper (`_build_vace_prompt`, WanVACEPipeline at line 501); the T2I analog has test_unknown_pipeline_preserves_canonical_text_to_image_prompt, but there is no CPU test for build_image_to_video_prompt(None, {'prompt': ..., 'modalities': ['video'], 'multi_modal_data': {'image': image}}, ...) returning the same dict. tests/examples/offline_inference/test_image_task_prompts.py::test_image_to_video_builds_canonical_prompt passes only negative_prompt='flicker', so the example builder's negative_prompt=None arm is never driven. Add both cases (parametrize None/''/message); if builderless non-VACE models should reject multi-key media, assert that contract here. (evidence: vllm_omni/model_extras/registry.py:316 ` if builder is None:` and :317 ` return prompt` (inside build_image_to_video_prompt) — builderless I2V returns the envelope unchanged. tests/model_extras/test_model_extras.py:500 ` return build_image_to_video_prompt(` followed at :501 by ` "WanVACEPipeline",` — the only call in that file, always through the VACE builder path; no builderless/None-model passthrough test exists. tests/examples/offline_inference/test_image_task_prompts.py:36 ` result = build_image_to_video_prompt(` with :38 ` negative_prompt="flicker",` — the example builder test only drives the non-None negative_prompt arm.) + +`examples/online_serving/text_to_image/openai_chat_client.py:122` [minor] — This diff removes the openai_chat_client argparse defaults for --height/--width/--steps/--cfg-scale/--seed (openai_chat_client.py:118-122), and generate_image() omits None-valued fields (openai_chat_client.py:64-82), so a bare invocation sends no size/step/cfg/seed fields. The only in-tree tests that drive this changed no-default path are tests/examples/online_serving/test_text_to_image.py::test_api_calls_002 and test_lora_001, which are full_model + example + H100 (pytestmark at line 17). Please run `PYTHONPATH=$PWD pytest -q tests/examples/online_serving/test_text_to_image.py -m 'full_model and example and H100' --run-level full_model` (or an equivalent server smoke with all sizing params omitted) to verify the server/model defaults resolve for a bare client run. (evidence: examples/online_serving/text_to_image/openai_chat_client.py:120 `parser.add_argument("--height", type=int, help="Image height; defaults to the model setting")` — default removed in this diff; examples/online_serving/text_to_image/openai_chat_client.py:65 `if width is not None and height is not None:` and line 74 `if steps is not None:` / line 76 `if true_cfg_scale is not None:` / line 80 `if seed is not None:` — None-valued fields are omitted from the payload. Unchanged by this diff, present in the PR-time tree: tests/examples/online_serving/test_text_to_image.py:17 `pytestmark = [pytest.mark.full_model, pytest.mark.example, *hardware_marks(res={"cuda": "H100"})]`; tests/examples/online_serving/test_text_to_image.py:86-95 `str(T2I_ONLINE_CLIENT), "--prompt", "A beautiful landscape painting", "--output", str(out), "--server", ...` with no --height/--width/--steps/--cfg-scale/--seed, followed by `assert_image_valid(out)`.) + +`examples/online_serving/text_to_image/openai_chat_client.py:73` [minor] — examples/online_serving/text_to_image/openai_chat_client.py:73 changed `if negative_prompt:` to `if negative_prompt is not None:`, so an empty-string negative_prompt is now forwarded in the payload. The only test invoking this client, tests/examples/online_serving/test_text_to_image.py::test_api_calls_002, does not pass `--negative`, so this branch is unverified. Consider adding a payload-construction unit test for negative_prompt="" (or an example test with --negative ''). (evidence: examples/online_serving/text_to_image/openai_chat_client.py:73 ` if negative_prompt is not None:` — changed branch. examples/online_serving/text_to_image/openai_chat_client.py:74 ` payload["negative_prompt"] = negative_prompt` — forwards empty string when not None. tests/examples/online_serving/test_text_to_image.py:83-93 `run_cmd([sys.executable, str(T2I_ONLINE_CLIENT), "--prompt", "A beautiful landscape painting", "--output", str(out), "--server", f"http://{omni_server.host}:{omni_server.port}"])` — no `--negative`, so the client's negative_prompt remains None and the changed branch is not exercised.) + +`vllm_omni/model_extras/registry.py:319` [minor] — Add a unit test in tests/model_extras/test_model_extras.py that calls build_image_to_video_prompt with a non-Mapping multi_modal_data (e.g. a string) on a builder-backed pipeline and asserts the TypeError raised at registry.py:319-320. Current coverage only exercises the VACE builder path with dict media_inputs, so the raise branch is never asserted. (evidence: vllm_omni/model_extras/registry.py:319 ` if not isinstance(media_inputs, Mapping):` ; vllm_omni/model_extras/registry.py:320 ` raise TypeError("Canonical I2V prompt multi_modal_data must be a mapping.")` ; tests/ contains no reference to the string 'Canonical I2V prompt' or 'multi_modal_data must be a mapping' (grep over tests returned no matches), and the only build_image_to_video_prompt caller in tests/model_extras/test_model_extras.py is _build_vace_prompt at line 500, which always passes a dict media_inputs.) + +`.buildkite/intel/scripts/run-xpu-test.sh:~75` [minor] — CI gate: buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-intel-ci are FAILING. This PR adds tests/examples/offline_inference/test_image_task_prompts.py, whose module-scope imports (lines 9-10) pull in the offline T2I/I2V example entrypoints, each of which imports `vllm_omni.entrypoints.omni` at module scope. The Intel lane runs `.buildkite/intel/scripts/run-xpu-test.sh:75` `pytest -v -s -m "core_model and xpu and B60"` with no path, so pytest imports this new file during collection even though it is marked only core_model+cpu and is deselected. The AMD ready lane runs `pytest -sv tests/diffusion tests/model_executor ...` (.buildkite/amd/test-amd-ready.yml:19), and tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py:75 imports `get_extra_body_params` from vllm_omni.model_extras, which this PR touches (registry.py, lingbot_video.py). Check the actual CI logs to determine whether both failures pre-date main or are introduced by this PR before merge. (evidence: Outside this diff, present in PR-time tree: .buildkite/intel/scripts/run-xpu-test.sh:75 ` pytest -v -s -m "core_model and xpu and B60"` (no test path). In this diff: tests/examples/offline_inference/test_image_task_prompts.py:9 `from examples.offline_inference.image_to_video.image_to_video import build_image_to_video_prompt`; :10 `from examples.offline_inference.text_to_image.text_to_image import build_text_to_image_prompt`; :12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. In this diff's module targets: examples/offline_inference/text_to_image/text_to_image.py:18 `from vllm_omni.entrypoints.omni import Omni`; examples/offline_inference/image_to_video/image_to_video.py:61 `from vllm_omni.entrypoints.omni import Omni`. Outside this diff, present in PR-time tree: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; tests/diffusion/models/lingbot_video/test_pipeline_lingbot_video.py:75 ` from vllm_omni.model_extras import get_extra_body_params`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_extras/registry.py:270` [minor] Confirmed. vllm_omni/model_extras/registry.py changes build_text_to_image_prompt (line 270) and build_image_to_video_prompt (line 306) from a string `prompt` (with separate negative_prompt/media_inputs) to a `prompt: dic + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.trace.json.gz new file mode 100644 index 000000000..86d28f054 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6049.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.cost.json new file mode 100644 index 000000000..ba058813b --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1463.5, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr6079/run-20260815-041738-63ae5e", + "recorded_at": "2026-08-15T04:41:59", + "attempts": [ + { + "invocation_id": "inv-64640b3c", + "outcome": "run:run-20260815-041738-63ae5e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-041738-63ae5e", + "llm_calls": 106, + "input_tokens": 428589, + "output_tokens": 171937, + "usd": 0.449749, + "source": "spans" + } + ], + "attempt_usd_total": 0.449749, + "llm_calls": 106, + "input_tokens": 428589, + "output_tokens": 171937, + "usd": 0.449749, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-041738-63ae5e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4497, + "minutes": 24.35, + "ci_minutes": 0, + "input_tokens": 428589, + "output_tokens": 171937, + "tool_calls": 219, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7837, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 106, + "by_role": { + "planner": { + "usd": 0.000963, + "input_tokens": 1936, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.425485, + "input_tokens": 417837, + "output_tokens": 152552, + "calls": 103 + }, + "reducer": { + "usd": 0.023302, + "input_tokens": 8816, + "output_tokens": 18985, + "calls": 2 + } + }, + "cache_read_tokens": 5366656, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.35, + "gate": 1.72, + "review": 1456.0, + "report": 0.1 + }, + "time_to_first_result_s": 1461.11 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6079.trace.json.gz", + "trace_bytes": 25923 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.md new file mode 100644 index 000000000..a17a2f23e --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.md @@ -0,0 +1,57 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6079, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head c171be607d6d) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] New JSON is a valid single-case array; mark hardware_marks res {cuda:H100,npu:A3}, num_cards 1 plus core_model/omni, so both '-m core_model and cuda' and '-m core_model and npu and A3 and omni' select exactly one case. +- [claim-verified] Neither CI step passes --assert-baseline; grep finds --assert-baseline defined only at tests/dfx/conftest.py:756 and no consumer, so the baseline block does not gate. +- [claim-verified] 'same deploy config, same seed-tts dataset, same request body and percentile metrics' vs nightly twin tests/dfx/perf/tests/test_minicpmo_4_5.json: fields are identical. +- [claim-verified] 'Marked core_model instead of full_model' and sweep narrowed [1,4,8]→[1,4] with num_prompts [10,40]: matches diff; nightly num_prompts [32,64,128]. +- [claim-verified] 'Both steps run without --assert-baseline': neither command passes the flag; assert_result only checks result['completed'] == num_prompt (run_benchmark.py:225). +- [claim-verified] 'Mark selection resolves to exactly 1 case': one case in the file; tests/helpers/mark.py hardware_marks yields cuda/H100/npu/A3 plus core_model/omni, so both -m expressions select the single param. +- [claim-refuted] PR body says the CUDA step is gated on 'the runner', but the added source_file_dependencies (.buildkite/cuda/test-ready.yml:232-237) omits tests/dfx/perf/scripts/run_benchmark.py and tests/dfx/conftest.py; upload_pipeline._match_source_file (lines 300-315) drops steps when no dep matches changed files. +- [claim-refuted] 'gated by source_file_dependencies on ... the runner': .buildkite/cuda/test-ready.yml:233-236 lists perf JSON, deploy yaml, model dir, stage input processor but NOT tests/dfx/perf/scripts/run_benchmark.py or tests/dfx/conftest.py. +- [claim-refuted] Test Plan 'Full run on H100 (1x)' and 'Full run on A3 (1x)' are unchecked; Test Result says '(fill in wall-clock time and measured metrics from the first H100 / A3 runs)' — the ready step has not been executed. +- [validated] behavior item 3 (breaking defaults/API/protocol): diff changes no code default, API, protocol, or output format — only a new perf JSON and two CI steps; nothing to grep for stale in-repo consumers. +- [validated] behavior item 9 (blast radius): changed values are scoped to the new test config; vllm_omni/deploy/minicpmo_4_5.yaml is unchanged (devices "0", tensor_parallel_size 1) so mark num_cards 1 matches the run; no shared default/flag/dispatch table is touched. +- [validated] failure-path: server-crash fallback result_omni_template.json has "completed": 0, so assert_result(0 != num_prompt) fails rather than fabricating a pass — the step does gate on request failures as claimed. +- [sweep] Read tests/dfx/perf/tests/test_minicpmo_4_5_ready.json and nightly sibling test_minicpmo_4_5.json: server_params/deploy-config/extra_body/percentile-metrics identical; only mark full_model->core_model and sweeps [32,64,128]/[1,4,8]->[10,40]/[1,4] differ. +- [sweep] Read tests/dfx/perf/scripts/run_benchmark.py: test_performance_benchmark runs QPS sweep then concurrency sweep; with request_rate absent and num_prompts/max_concurrency [10,40]/[1,4], exactly 2 concurrency runs execute; assert_result asserts result['completed']==num_prompt. + +`.buildkite/cuda/test-ready.yml:232` [major] — This hunk adds the CUDA 'Omni · MiniCPM-o 4.5 Perf Test' ready step whose commands run `tests/dfx/perf/scripts/run_benchmark.py`, but its `source_file_dependencies` at .buildkite/cuda/test-ready.yml:232-236 list only `tests/dfx/perf/tests/test_minicpmo_4_5_ready.json`, `vllm_omni/deploy/minicpmo_4_5.yaml`, `vllm_omni/model_executor/models/minicpmo_4_5/`, and `vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py` — not the runner or `tests/dfx/conftest.py`, despite the PR description saying the step is gated on 'the runner'. `upload_pipeline` skips a step when changed files match no dependency, so a PR touching only the runner/conftest would silently skip the ready-gate perf coverage this gate is meant to exercise. Add the runner and its dfx helper/conftest imports to the dependency list. (evidence: Unchanged by this diff, present in the PR-time tree: .buildkite/common/scripts/upload_pipeline.py:276-281 `def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool: + for path in changed_files: + for prefix in prefixes: + normalized = prefix.rstrip("/") + if path == normalized or path.startswith(f"{normalized}/"): + return True + return False`; .buildkite/common/scripts/upload_pipeline.py:305 `if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):` (then logs 'skip ... (no changes under ...)' and continues). In the changed file: .buildkite/cuda/test-ready.yml:239 `- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'` and .buildkite/cuda/test-ready.yml:232-236 `source_file_dependencies:` listing `tests/dfx/perf/tests/test_minicpmo_4_5_ready.json`, `vllm_omni/deploy/minicpmo_4_5.yaml`, `vllm_omni/model_executor/models/minicpmo_4_5/`, `vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py` — neither `tests/dfx/perf/scripts/run_benchmark.py` nor `tests/dfx/conftest.py` is present.) + +`tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:55` [minor] — This diff adds `tests/dfx/perf/tests/test_minicpmo_4_5_ready.json` with `num_prompts [10, 40]` and `max_concurrency [1, 4]`, but its `baseline.H100.request_throughput` values `[0.7803, 1.1008]` are copied from `test_minicpmo_4_5.json`, whose first two sweep points ran `num_prompts [32, 64]`. The runner pairs concurrency and num_prompts positionally (`run_benchmark.py:269`), so sweep index 1 compares concurrency=4 at 40 prompts against a value measured at 64 prompts. The config is also marked for NPU A3 and wired into the A3 ready step, yet there is no A3 baseline bucket, and `run_benchmark.py` writes the resolved baseline into the per-hardware result JSON unconditionally. Since the CI steps do not pass `--assert-baseline` (and no code consumes that option), these thresholds gate nothing but produce misleading per-hardware baselines. Drop the baseline block, or re-measure and supply valid H100 and A3 buckets for the `[10, 40]` workload. (evidence: tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:31-34 `"num_prompts": [ 10, 40 ]`; tests/dfx/perf/tests/test_minicpmo_4_5.json:31-37 `"num_prompts": [ 32, 64, 128 ]` and its `request_throughput` baseline `[0.7803, 1.1008, 1.2027]`; tests/dfx/perf/scripts/run_benchmark.py:269 `for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):` — positional pairing; tests/dfx/perf/scripts/run_benchmark.py:729-735 `if baseline_config: result["baseline"] = resolve_baseline_for_sweep(...)` — unchanged by this diff, present in the PR-time tree; tests/dfx/conftest.py:756 `"--assert-baseline"` with no `assert_baseline` consumer found by repo grep.) + +`.buildkite/cuda/test-ready.yml:231` [minor] — This diff adds ready-gate steps with `timeout_in_minutes: 60` that boot the three-stage MiniCPM-o 4.5 server and run two seed-tts benchmark points, but the PR Test Plan leaves both 'Full run on H100 (1x)' and 'Full run on A3 (1x)' unchecked and the Test Result still says to fill in wall-clock time and measured metrics. The `omni_server_context` fixture docstring in `run_benchmark.py` warns that multi-stage initialization can take 10-20+ minutes before benchmark timing starts, and the CUDA step additionally serves the `seed-tts` dataset at concurrency 4. Before this becomes a per-PR gate, run the exact added commands — `pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'` and the matching NPU command — and record wall-clock plus result metrics, or raise the timeout with that evidence. (evidence: .buildkite/cuda/test-ready.yml:231 ` timeout_in_minutes: 60` — added by this diff; .buildkite/cuda/test-ready.yml:239 ` - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'`; .buildkite/npu/test-npu-ready.yml:28 ` - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'`; tests/dfx/perf/scripts/run_benchmark.py:122 ` Multi-stage initialization can take 10-20+ minutes.` — unchanged by this diff, present in the PR-time tree.) + +`.buildkite/cuda/test-ready.yml:230` [nit] — This PR adds CUDA/NPU perf-ready steps only. Repo-wide grep for `test_minicpmo_4_5_ready` matches only `tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:3` and the two added Buildkite command lines (`.buildkite/cuda/test-ready.yml:233/239`, `.buildkite/npu/test-npu-ready.yml:28`); `git diff --stat` confirms no AMD/Intel pipeline files changed. The AMD ready pipeline (`tests/diffusion tests/model_executor -m 'core_model and cpu'`) and the Intel pipeline (`bash .buildkite/intel/scripts/run-xpu-test.sh`) have no reference to the new perf test, so the diff's surface does not intersect those two failing lanes. Please confirm those lanes are already red on main before attributing the failures to this PR. (evidence: Anchor confirmed: `.buildkite/cuda/test-ready.yml:230` ` - label: "Omni · MiniCPM-o 4.5 Perf Test"`. Grep matches only the new config and two new commands: `tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:3` ` "test_name": "test_minicpmo_4_5_ready"`; `.buildkite/cuda/test-ready.yml:239` ` - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'`; `.buildkite/npu/test-npu-ready.yml:28` ` - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'`. Unchanged by this diff, present in the PR-time tree: `.buildkite/amd/test-amd-ready.yml:19` ` - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (no tests/dfx/perf reference) and `.buildkite/intel/pipeline-intel.yml:6` ` command: "bash .buildkite/intel/scripts/run-xpu-test.sh"`. `git diff --stat` (merge-base bdb1d996bee2) lists only `.buildkite/cuda/test-ready.yml`, `.buildkite/npu/test-npu-ready.yml`, and `tests/dfx/perf/tests/test_minicpmo_4_5_ready.json`. gh_ci_read for PR 6079 returned FAILURE for both buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-intel-ci.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.trace.json.gz new file mode 100644 index 000000000..4cdb4ac4b Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6079.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.cost.json new file mode 100644 index 000000000..1b6ed43db --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 448.1, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r1/runs/pr6141/run-20260815-041953-ebef2b", + "recorded_at": "2026-08-15T04:27:19", + "attempts": [ + { + "invocation_id": "inv-9ef0031c", + "outcome": "run:run-20260815-041953-ebef2b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-041953-ebef2b", + "llm_calls": 45, + "input_tokens": 161949, + "output_tokens": 53099, + "usd": 0.156816, + "source": "spans" + } + ], + "attempt_usd_total": 0.156816, + "llm_calls": 45, + "input_tokens": 161949, + "output_tokens": 53099, + "usd": 0.156816, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-041953-ebef2b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1568, + "minutes": 7.43, + "ci_minutes": 0, + "input_tokens": 161949, + "output_tokens": 53099, + "tool_calls": 108, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3198, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 45, + "by_role": { + "agent": { + "usd": 0.148875, + "input_tokens": 154002, + "output_tokens": 47865, + "calls": 43 + }, + "reducer": { + "usd": 0.007941, + "input_tokens": 7947, + "output_tokens": 5234, + "calls": 2 + } + }, + "cache_read_tokens": 2025216, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.23, + "gate": 1.74, + "review": 440.62, + "report": 0.06 + }, + "time_to_first_result_s": 445.64 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6141.trace.json.gz", + "trace_bytes": 20506 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.md b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.md new file mode 100644 index 000000000..762e2d71b --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6141, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head b031feef060f) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Prior concern — removed 'Diffusion Workload Division' owners get lost: @ZJY0516, @RuixiangMa, @wtomin are all on the `.github/CODEOWNERS` `/vllm_omni/diffusion/` umbrella (`@Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666 @xuechendi`), and @yuanheng-zhao is on `/vllm_omni/diffusion/offloader/` and `.../attention/parallel/`, so the migration to CODEOWNERS keeps the routing. +- [claim-verified] 'only remaining community/volunteers reference is the intentional redirect': repo-wide grep 'volunteers' -> only mkdocs.yml:77 `community/volunteers.md: community/contact_us.md`. +- [claim-verified] meetups references: repo-wide grep 'meetups' -> only mkdocs.yml:76 `community/meetups.md: community/meetings.md`; grep 'meetup' in docs -> no matches. +- [claim-verified] stale meeting schedules removed: grep in docs for 'tinyurl.com/vllm-omni-meeting', '19:30 PDT', 'Tuesday' -> no matches. +- [claim-verified] stale security guidance removed: grep in docs for 'Security Advisories', 'advisories', 'security` label' -> no matches. +- [claim-verified] reviewer-routing links: .github/CODEOWNERS exists with path-based owner rules; docs/design/index.md contains '## Module Design Documents' heading, so the `../design/index.md#module-design-documents` anchor resolves. +- [claim-verified] FAQ bug-report template: .github/ISSUE_TEMPLATE/400-bug-report.yml exists. +- [claim-verified] documentation-only: diff_stat shows only docs/.nav.yml, 5 docs/*.md files, and mkdocs.yml; no runtime code touched. +- [claim-verified] 'only remaining community/volunteers reference is the intentional redirect': grep volunteers -> only mkdocs.yml:77 `community/volunteers.md: community/contact_us.md`. +- [claim-verified] 'only remaining community/meetups reference is the redirect': grep meetups -> only mkdocs.yml:76 `community/meetups.md: community/meetings.md`. +- [claim-verified] Reconcile issue/community/security guidance: grep for `security label`, `Security Advisories`, `tinyurl.com/vllm-omni-meeting`, `19:30 PDT`, `bi-weekly`, issue `1057`, `volunteer` in docs -> no matches; stale guidance fully removed. +- [claim-verified] FAQ bug-report link template exists: .github/ISSUE_TEMPLATE/400-bug-report.yml is present. +- [validated] docs/community contains exactly contact_us.md, governance.md, meetings.md (list_dir), so replacing the `community/*` glob in docs/.nav.yml with explicit Contact Us + Community Meetings orphans no remaining page; Governance stays explicitly in nav (docs/.nav.yml:165-167). +- [validated] mkdocs redirects: targets exist (docs/community/meetings.md, docs/community/contact_us.md) and sources match the deleted files. + +`docs/usage/faq.md:17` [minor] — At docs/usage/faq.md:17, the new AWQ answer says 'AWQ is available through the inherited vLLM quantization registry' and directs readers to the quantization overview for validated combinations, but docs/user_guide/quantization/overview.md has no AWQ entry: the 'Quantization Modes' table lists only FP8 W8A8, Int8 W8A8, BitsAndBytes W4, MXFP8 W8A8, MXFP4 W4A4 / ModelOpt, AutoRound, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale. Add an AWQ row/note or an explicit 'inherited upstream methods such as AWQ/GPTQ are not covered here' note, or reword the FAQ sentence so AWQ readers aren't pointed at a page that does not cover AWQ. (evidence: docs/usage/faq.md:17 `A: AWQ is available through the inherited vLLM quantization registry, and vLLM-Omni supports several additional methods. Actual support and validation vary by model, component, hardware, and checkpoint. See the [quantization overview](../user_guide/quantization/overview.md) for validated combinations and method-specific guides.` docs/user_guide/quantization/overview.md:15 `| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, BitsAndBytes W4, MXFP8 W8A8, MXFP4 W4A4 |` docs/user_guide/quantization/overview.md:17 `| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |` — both overview tables list no AWQ method, confirming the reader is redirected to a page lacking AWQ coverage.) + +`docs/community/contact_us.md:~9` [minor] — At docs/community/contact_us.md:9, the private-report path says to DM a lead maintainer in vLLM Slack, but docs/community/governance.md#lead-maintainers lists the three lead maintainers (@Gaohan123, @hsliuustc0106, @ywang96) only as GitHub URLs with no Slack handle or member-directory link, so a reader cannot determine who to DM. Add Slack handles next to the GitHub handles in governance.md, or link a public Slack member directory / contact mapping from this sentence. (evidence: docs/community/contact_us.md:9 `Do not share vulnerability details in public GitHub issues, pull requests, discussions, or Slack channels. To arrange a private report, send a direct message to one of the [lead maintainers](governance.md#lead-maintainers) in [vLLM Slack](https://slack.vllm.ai/).` — and docs/community/governance.md:28-30 (unchanged by this diff, present in the PR-time tree) lists only GitHub URLs with no Slack handles: `- [@Gaohan123](https://github.com/Gaohan123)`, `- [@hsliuustc0106](https://github.com/hsliuustc0106)`, `- [@ywang96](https://github.com/ywang96)`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.trace.json.gz new file mode 100644 index 000000000..4a291c11e Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r1/pr6141.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/manifest.json b/eval/dataset/arms/copilot_v14_holdout3_r2/manifest.json new file mode 100644 index 000000000..3b50654d5 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v14_holdout3_r2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout3" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.cost.json new file mode 100644 index 000000000..5778f2274 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 958.4, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr5678/run-20260815-050649-d1e8fe", + "recorded_at": "2026-08-15T05:22:45", + "attempts": [ + { + "invocation_id": "inv-cced99e2", + "outcome": "run:run-20260815-050649-d1e8fe", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-050649-d1e8fe", + "llm_calls": 144, + "input_tokens": 1086310, + "output_tokens": 233510, + "usd": 1.017962, + "source": "spans" + } + ], + "attempt_usd_total": 1.017962, + "llm_calls": 144, + "input_tokens": 1086310, + "output_tokens": 233510, + "usd": 1.017962, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-050649-d1e8fe", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.018, + "minutes": 15.93, + "ci_minutes": 0, + "input_tokens": 1086310, + "output_tokens": 233510, + "tool_calls": 270, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8449, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 144, + "by_role": { + "agent": { + "usd": 0.990133, + "input_tokens": 1054575, + "output_tokens": 216038, + "calls": 142 + }, + "reducer": { + "usd": 0.027829, + "input_tokens": 31735, + "output_tokens": 17472, + "calls": 2 + } + }, + "cache_read_tokens": 17325824, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.77, + "gate": 1.62, + "review": 950.36, + "report": 0.08 + }, + "time_to_first_result_s": 955.8 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5678.trace.json.gz", + "trace_bytes": 49534 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.md new file mode 100644 index 000000000..ecd43218f --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5678, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head eeaa56144818) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Prior concern 'legacy builder mutated stage_config.engine_args to drop None TP': now detached copy (`_to_dict` + pop); test `test_legacy_engine_args_drop_none_tp_without_mutating_stage_config` pins non-mutation. +- [resolved] prior concern that engine_extras metadata (final_output/final_output_type/is_comprehension) would be rejected: resolved by _LEGACY_STAGE_METADATA_EXTRA_FIELDS exclusion at omni_config.py:930-935. +- [resolved] prior concern that has_sampling_extra_args would be lost in typed projection: resolved via finalize parameter stage_config.model_config.has_sampling_extra_args (stage_init_utils.py:1019-1024). +- [claim-verified] PR claim 'stable entry point preserves signature/delegates' verified at stage_init_utils.py:983-999. +- [claim-verified] PR claim 'production consumers remain on stable entry point' verified via grep of `build_engine_args_dict(`. +- [claim-verified] PR claim 'every registered pipeline parity' implemented by `test_typed_engine_args_match_current_registry_backend_semantics` (parametrized over OMNI_PIPELINES); callable-resolver pipelines skipped when resolve_pipeline_config returns None — a documented coverage gap. +- [claim-verified] PR claim 'stable build_engine_args_dict entry point preserves its existing signature and delegates to legacy': stage_init_utils.py:983-999 does exactly this. +- [claim-verified] PR claim that runtime/headless consumers remain on the legacy path holds: stage_runtime.py and serve.py diff hunks are comment-only additions. +- [claim-verified] Typed projection `build_engine_args_dict_from_omni_stage_config` added and legacy `build_engine_args_dict` now delegates to `build_legacy_engine_args_dict` (stage_init_utils.py:983-1022). +- [claim-verified] New file tests/engine/test_stage_engine_args.py:31 has `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, so CUDA `Simple · Engine&Entrypoints Test` selects it (`-m 'core_model and cpu'`); it is not a marker-deselected no-op. +- [claim-refuted] PR claim 'Every currently registered pipeline retaining all backend-consumed legacy fields' overstates coverage: test_typed_engine_args_match_current_registry_backend_semantics skips any pipeline whose resolver returns None without HF config (test_stage_engine_args.py:475-476). +- [claim-refuted] PR's implied 'engine inputs retained by concern-owned configs' misses StageDeployConfig fields max_generated_image_size/tts_max_instructions_length/auxiliary_text_encoder: present at stage_config.py:423/424/408 but absent from every owner set in omni_config.py. +- [claim-refuted] PR claim 'every currently registered pipeline retaining all backend-consumed legacy fields' is only tested for `_LLM_BACKEND_FIELDS`/`_DIFFUSION_BACKEND_FIELDS`; non-backend projected fields such as `custom_voice_dir` are not checked by the registry-wide test (tests/engine/test_stage_engine_args.py:494-509). +- [claim-refuted] `test_typed_engine_args_own_rocm_attention_default` only exercises `is_enabled=False`; the `ROCM_AITER_FA` branch is untested (and currently dead). + +`vllm_omni/config/omni_config.py:889` [major] — The new `_validate_stage_engine_override_ownership` (omni_config.py:889) raises for every override key outside the per-execution-type owner sets, while `_stage_engine_overrides` (omni_config.py:926-929) copies every non-None `_STAGE_DEPLOY_ENGINE_FIELDS` entry. Deploy fields `diffusion_quantization_config` (stage_config.py:390, consumed at async_omni_engine.py:1260 as a quantization fallback), `auxiliary_text_encoder` (stage_config.py:408), `max_generated_image_size` (stage_config.py:423), and `tts_max_instructions_length` (stage_config.py:424) are absent from `_COMMON_STAGE_ENGINE_FIELDS`, `_DIFFUSION_STAGE_ENGINE_FIELDS`, and the parallel-config owner fields, so any deploy stage setting them now hard-fails `VllmOmniConfig.from_pipeline_config` while the legacy `merge_pipeline_deploy` path accepts them. Map `diffusion_quantization_config` to the diffusion quantization owner, give the other three owners or explicitly exclude them like `_LEGACY_STAGE_METADATA_EXTRA_FIELDS`, and add a regression test for a deploy stage carrying each field. (evidence: ['Added by this diff — vllm_omni/config/omni_config.py:889-894: `unowned_fields = set(overrides) - owner_fields` ... `raise ValueError(f"Stage {stage_id} ({execution_type.value}) has explicit engine argument(s) with no structured config owner: {names}")`.', 'Present in the PR-time tree (context of this diff) — vllm_omni/config/omni_config.py:926-929: `for name in _STAGE_DEPLOY_ENGINE_FIELDS: value = getattr(stage_deploy, name); if value is not None: overrides[name] = _copy_value(value)`.', 'Unchanged by this diff, present in the PR-time tree — vllm_omni/config/stage_config.py:390 `diffusion_quantization_config: str | None = None`; :408 `auxiliary_text_encoder: str | None = None`; :423 `max_generated_image_size: int | None = None`; :424 `tts_max_instructions_length: int | None = None`.', 'Unchanged by this diff, present in the PR-time tree — vllm_omni/engine/async_omni_engine.py:1260 `quantization_config = kwargs.get("diffusion_quantization_config") or kwargs.get("quantization_config")`.']) + +`vllm_omni/config/omni_config.py:885` [minor] — `_validate_stage_engine_override_ownership` only validates nested `parallel_config` when it is a `Mapping` (line 885), so a non-None non-Mapping value passes as owned and is then silently discarded by `_mapping_or_empty` (line 914). Raise `ValueError` for non-None non-Mapping `parallel_config` so the strict ownership check cannot be bypassed. (evidence: vllm_omni/config/omni_config.py:885 ` if isinstance(parallel_config, Mapping):` — in the new `_validate_stage_engine_override_ownership` added by this diff; a non-None non-Mapping `parallel_config` skips recursion and is not flagged. vllm_omni/config/omni_config.py:914 ` return value if isinstance(value, Mapping) else {}` — unchanged by this diff, present in the PR-time tree; `_mapping_or_empty` discards non-Mapping values, which is the silent-drop path cited in the comment.) + +`vllm_omni/engine/stage_init_utils.py:376` [minor] — `_apply_rocm_attention_backend` assigns `engine_args["attention_backend"] = "ROCM_AITER_FA"` when `rocm_aiter_ops.is_enabled()` is true (stage_init_utils.py:369-370) and then unconditionally overwrites it with `"TRITON_ATTN"` on line 376, so the AITER branch is dead. The new test only mocks `is_enabled=lambda: False` (tests/engine/test_stage_engine_args.py:436) and therefore cannot exercise this path. Either remove the `ROCM_AITER_FA` assignment if TRITON_ATTN is the intended default, or guard with `if/else`/`return` and add an `is_enabled=True` case. Note: the dead branch is carried over from the legacy code this PR extracted, so this is a preservation of an existing bug rather than a new behavior regression. (evidence: PR-time tree, in this diff: vllm_omni/engine/stage_init_utils.py:369-370 ` if rocm_aiter_ops.is_enabled(): + engine_args["attention_backend"] = "ROCM_AITER_FA"`; vllm_omni/engine/stage_init_utils.py:376 ` engine_args["attention_backend"] = "TRITON_ATTN"` — the second assignment executes unconditionally, so the AITER value is always overwritten. tests/engine/test_stage_engine_args.py:436 ` types.SimpleNamespace(is_enabled=lambda: False),` — the only test stub returns False, so no test covers the dead branch.) + +`tests/engine/test_stage_engine_args.py:~475` [minor] — In test_typed_engine_args_match_current_registry_backend_semantics (tests/engine/test_stage_engine_args.py:473-476), the parametrization covers sorted(OMNI_PIPELINES) but skips whenever resolve_pipeline_config(model_type) returns None. Because qwen3_omni_moe is a callable resolver (vllm_omni/config/pipeline_registry.py:118) invoked without an HF config, it never exercises _legacy_and_typed_stages -> VllmOmniConfig.from_pipeline_config or the new ownership validation. Resolve callable pipelines with a representative HF config, or report the skip count so the every-registered-pipeline coverage claim is accurate. (evidence: tests/engine/test_stage_engine_args.py:475 `if pipeline is None:` ; tests/engine/test_stage_engine_args.py:476 `pytest.skip(f"Pipeline {model_type!r} requires an HF config to resolve")` ; vllm_omni/config/pipeline_registry.py:118 `"qwen3_omni_moe": resolve_qwen3_omni_pipeline,` (callable, so resolve_pipeline_config calls it with hf_config=None and the test skips on the resulting None)) + +`vllm_omni/engine/stage_init_utils.py:355` [nit] — `buildkite/vllm-omni-amd-ci` is FAILURE. The lane runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`, and `tests/diffusion` imports the changed `vllm_omni.engine.stage_init_utils` under `core_model`+`cpu`, so the PR surface overlaps the red lane. Please confirm whether the failure is pre-existing on main or introduced here, and record the attribution or re-run the lane before merge. (evidence: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — unchanged by this diff, present in the PR-time tree; vllm_omni/engine/stage_init_utils.py:352 `def _apply_rocm_attention_backend(` — added by this diff; tests/diffusion/test_prompt_update.py:37 `from vllm_omni.engine.stage_init_utils import StageMetadata` and tests/diffusion/test_prompt_update.py:43 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]` — unchanged by this diff, present in the PR-time tree; gh_ci_read(5678) reports `buildkite/vllm-omni-amd-ci` state FAILURE.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.trace.json.gz new file mode 100644 index 000000000..3bf32d673 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5678.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.cost.json new file mode 100644 index 000000000..35c254494 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1835.2, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr5691/run-20260815-050650-a3631b", + "recorded_at": "2026-08-15T05:37:21", + "attempts": [ + { + "invocation_id": "inv-4fc7a34f", + "outcome": "run:run-20260815-050650-a3631b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-050650-a3631b", + "llm_calls": 220, + "input_tokens": 1934006, + "output_tokens": 369994, + "usd": 1.786457, + "source": "spans" + } + ], + "attempt_usd_total": 1.786457, + "llm_calls": 220, + "input_tokens": 1934006, + "output_tokens": 369994, + "usd": 1.786457, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-050650-a3631b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.7865, + "minutes": 30.52, + "ci_minutes": 0, + "input_tokens": 1934006, + "output_tokens": 369994, + "tool_calls": 349, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3232, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 220, + "by_role": { + "agent": { + "usd": 1.692152, + "input_tokens": 1739048, + "output_tokens": 332179, + "calls": 217 + }, + "reducer": { + "usd": 0.094304, + "input_tokens": 194958, + "output_tokens": 37815, + "calls": 3 + } + }, + "cache_read_tokens": 31751168, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.18, + "gate": 1.56, + "review": 1824.44, + "report": 0.06 + }, + "time_to_first_result_s": 1831.23 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5691.trace.json.gz", + "trace_bytes": 118471 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.md new file mode 100644 index 000000000..38baa86b0 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.md @@ -0,0 +1,67 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5691, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0eee3b18645a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 9 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Prior concern 'ref2va rank0 must broadcast encoded audio rows': `_encode_video_audio_conditions` does broadcast `lengths` then `_broadcast_tensor(rows, ...)` — present at pipeline lines ~690-710. +- [resolved] Prior concern 'CFG-distilled must reject cfg_parallel_size': pipeline raises `ValueError('MiniMax-H3 is CFG-distilled...')` at __init__. +- [claim-verified] MiniMaxH3Pipeline registered in registry.py:86 and post-process func in registry.py:507. +- [claim-verified] docs/models/supported_models.md row and recipes/README.md entry both added. +- [claim-verified] CI: gate_report shows DCO, buildkite/vllm-omni, buildkite/vllm-omni-amd-ci failing; diff touches tests/entrypoints/openai_api/test_video_server.py and tests/config/test_omni_config.py which those lanes collect — attribution unknown, flagged as minor only. +- [claim-verified] PR body Test Plan section is empty; the only behavioral evidence is unit tests (L2 CPU/mock) plus a skipped Ulysses8 GPU smoke (needs 8 GPUs + env var); no real-checkpoint semantic parity evidence is in the PR. +- [claim-refuted] PR body Test Plan/Test Result are the unfilled template ('PLEASE FILL IN THE PR DESCRIPTION HERE') despite recipe accuracy + 4-GPU evidence claims. +- [claim-refuted] PR/doc table 'FL2VA, 209 frames' — code uses `int(round(duration*fps))` banker's rounding, so a duration whose product lands on .5 boundary (e.g. 5.0s -> 120) diverges from `round`; the doc's own 8.7*24=208.8 rounds to 209, which matches, but the 124-frame example in the test (`round(5.0*24)==120`) contradicts the recipe's implied reference flow. +- [validated] sigma schedule: 12*x/(1+11*x) at x=0.75 -> 9/9.25=0.97297297, x=0.5 -> 6/6.5=0.9230769 matches test_minimax_h3_contract.py expected values. +- [validated] frame/latent arithmetic via calc: align(round(8.7*24))=209, video_latent_t(209)=62, audio_latent_t(8.7)=348; align(round(5*24))=124, video_latent_t(124)=37, audio_latent_t(5)=200. +- [validated] _parse_video_form rejects mixing `input_references` with `input_reference`/image/video via `if input_references and any(item is not None for ...)` raising 400. +- [validated] `_cleanup_video_references` exists and is called in `_run_video_generation_job` finally and `create_video_sync` finally, so happy-path temp-file cleanup is wired. +- [validated] tests/diffusion/attention/test_flash_attn.py:332 test_packed_varlen_metadata_bypasses_mask_unpadding drives real impl.forward_cuda with packed extra keys (not a mocked pre-transformed value). +- [validated] packed metadata completeness test asserts the ValueError branch matches flash_attn.py:280-284. + +`vllm_omni/diffusion/attention/backends/utils/fa.py:~65` [major] — On Blackwell the shared fa module rebinds its module-global `flash_attn_varlen_func` to `flash_attn.cute.flash_attn_varlen_func` (fa.py:65), and two non-diffusion encoders import that exact symbol and call it with FA2/FA3 positional signatures — ming_flash_omni/audio_encoder.py:65 (7 positional args) and qwen3_tts/.../whisper_encoder.py:143 (dropout_p=0.0). The PR's FA4 tests only monkeypatch the dense `ring_kernels.fa4_attn_func`, so the shared varlen symbol's FA2/FA3-compatible signature is never exercised. Prove FA4's varlen signature against those call sites, or gate the FA4 rebind so the non-diffusion encoders keep a FA3/FA2/vLLM varlen backend. (evidence: IN DIFF: vllm_omni/diffusion/attention/backends/utils/fa.py:65 `from flash_attn.cute import flash_attn_func, flash_attn_varlen_func # noqa: F401` inside `if _is_blackwell():` — unconditionally rebinds the shared module globals for the whole repo. UNCHANGED BY THIS DIFF, present in PR-time tree: vllm_omni/model_executor/models/ming_flash_omni/audio_encoder.py:65 `attn_output = flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen)` and vllm_omni/model_executor/models/qwen3_tts/tokenizer_25hz/vq/whisper_encoder.py:143 `x = flash_attn_varlen_func(q, k, v, cu_seqlens, cu_seqlens, max_seqlen, max_seqlen, dropout_p=0.0)`; both import the symbol at line 18/25 via `from vllm_omni.diffusion.attention.backends.utils.fa import HAS_FLASH_ATTN, flash_attn_varlen_func`. IN DIFF: tests/diffusion/attention/test_ring_fa4.py:26 `monkeypatch.setattr(ring_kernels, "fa4_attn_func", fake_fa4)` — the only FA4 coverage monkeypatches the dense ring symbol, never the shared varlen symbol.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:19` [major] — The only MiniMax H3 generation test (tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:19-32) is skipped unless VLLM_TEST_MINIMAX_H3_FL2VA_MODEL is set AND >=8 GPUs exist (and carries local_model/cuda marks), so CI never executes real generation. MiniMaxH3Pipeline is additionally listed in EXCLUDED_MODELS (tests/model_tests/diffusion/test_alignment.py:35), leaving only CPU/mock contract and round-trip tests with no HF-reference output comparison. The recipe still asserts the single-GPU path 'matches the accuracy-qualified reference path' (recipes/MiniMaxAI/MiniMax-H3.md:56) and publishes a 'Validated four-GPU evidence' table while the PR body Test Plan/Test Result remain the unfilled template. Per the new-model guardrail (shape smoke + 0-missing/0-unexpected only proves plumbing), add a semantic parity matrix or mark the recipe accuracy/performance claims as unvalidated. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:19-20 `@pytest.mark.skipif( + not os.environ.get(MODEL_ENV),` and :31-32 `if torch.accelerator.device_count() < 8: + pytest.skip("MiniMax H3 Ulysses smoke requires eight GPUs")`; tests/model_tests/diffusion/test_alignment.py:35 ` "MiniMaxH3Pipeline",` (inside EXCLUDED_MODELS, added by this diff); recipes/MiniMaxAI/MiniMax-H3.md:56 `This matches the accuracy-qualified reference path and prevents the Qwen3-VL` and the recipe section `## Validated four-GPU evidence`; PR 5691 body begins with `PLEASE FILL IN THE PR DESCRIPTION HERE.` with empty `**vLLM Version:**` / `**vLLM-Omni Commit:**` fields.) + +`vllm_omni/diffusion/attention/parallel/ring.py:150` [major] — This PR adds an FA4 branch at ring.py:150 but does not update the availability guard at ring.py:116 (`elif not HAS_FA3 and not HAS_FLASH_ATTN and not HAS_AITER:`). On the FA4-only Blackwell install this PR explicitly supports (`flash-attn-4[cu13]`), all three flags are false, so `backend_pref` is forced to `"sdpa"` and `run_attention` returns via `ring_pytorch_attn_func` before reaching the FA4 branch — making the new FA4 path unreachable. Separately, when the branch is reachable (e.g. FA2/FA3 co-installed), it selects FA4 for every Blackwell ring attention including causal self-attention, yet no test exercises real FA4 output: `test_ring_prefers_fa4_on_blackwell` monkeypatches `ring_flash_attn_func` and only asserts `attn_type is AttnType.FA4`. Add `HAS_FA4` to the line-116 guard and demonstrate a real FA4 ring run for the causal path before enabling it unconditionally. (evidence: vllm_omni/diffusion/attention/parallel/ring.py:116 ` elif not HAS_FA3 and not HAS_FLASH_ATTN and not HAS_AITER:` (pre-existing guard, unchanged by this diff) with :119 ` backend_pref = "sdpa"` — it does not test HAS_FA4; vllm_omni/diffusion/attention/parallel/ring.py:150 ` if HAS_FA4 and query.is_cuda and torch.cuda.get_device_capability(query.device)[0] >= 10:` (added by this diff) — unreachable after the guard forces `backend_pref = "sdpa"` because the earlier `if backend_pref in {"sdpa", "torch", "torch_sdpa"}:` branch returns `ring_pytorch_attn_func(...)` first; tests/diffusion/attention/test_ring_fa4.py:63 ` monkeypatch.setattr(ring_flash_attn, "ring_flash_attn_func", fake_ring_flash_attn_func)` and :74 ` assert captured["attn_type"] is AttnType.FA4` — the routing test substitutes a fake and asserts nothing about real FA4 kernel output.) + +`vllm_omni/entrypoints/openai/api_server.py:~3118` [major] — This change makes `ReferenceVideo.data` carry temp file paths and feeds them into the shared `multi_modal_data["video"]` channel. Three other in-repo video pipelines treat a list of strings in that key as a list of IMAGE paths and open only the first frame: Wan2.2-VACE does `PIL.Image.open(f).convert("RGB")` (vace.py:143), Cosmos3's `_video_payload_to_frames` returns the str list unchanged and then `_pil_to_rgb(raw_video_frames[0])` opens one frame (pipeline_cosmos3.py:460), and Lance does `Image.open(im) if isinstance(im, str)` (pipeline_lance.py). Since `input_references` is accepted by the generic `/v1/videos` and `/v1/videos/sync` endpoints regardless of model, a VACE/Cosmos3/Lance V2V request now silently degrades to a single-frame reference with no error. Scope the path-list form to the MiniMax H3 pipeline (e.g. a dedicated multimodal key or a pipeline marker) and keep the existing decoded-frames list for the other consumers, or decode videos at the serving boundary. (evidence: vllm_omni/entrypoints/openai/serving_video.py:131 `input_video = None if reference_video is None else reference_video.data`; vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py:143 `source_video = [PIL.Image.open(f).convert("RGB") for f in source_video]`; vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:460 `raw_video = multi_modal_data.get("video")`.) + +`vllm_omni/diffusion/attention/parallel/ring.py:~116` [major] — The FA4 path added by this PR is unreachable in the exact configuration the PR documents. In `vllm_omni/diffusion/attention/parallel/ring.py:116`, `elif not HAS_FA3 and not HAS_FLASH_ATTN and not HAS_AITER:` forces `backend_pref = "sdpa"` whenever FA3/FA2/AITER are all absent, and it executes before the FA4 selection at line 150 (`if HAS_FA4 and query.is_cuda and torch.cuda.get_device_capability(query.device)[0] >= 10:`). In a `vllm-omni[fa4]`-only install, `HAS_FA4` is True but `HAS_FA3`/`HAS_FLASH_ATTN`/`HAS_AITER` are False, so ring attention silently falls back to SDPA and never reaches `AttnType.FA4`. Add `not HAS_FA4` to the availability check (or fold FA4 into it) and add a FA4-only test that leaves FA3/FA2/AITER False. (evidence: vllm_omni/diffusion/attention/parallel/ring.py:116 `elif not HAS_FA3 and not HAS_FLASH_ATTN and not HAS_AITER:` (unchanged by this diff, present in the PR-time tree) precedes and preempts vllm_omni/diffusion/attention/parallel/ring.py:150 `if HAS_FA4 and query.is_cuda and torch.cuda.get_device_capability(query.device)[0] >= 10:`; the SDPA fallback at line 117-118 is `backend_pref = "sdpa"` and the SDPA dispatch at lines 130-132 `if backend_pref in {"sdpa", "torch", "torch_sdpa"}: ... return ring_pytorch_attn_func(...)` returns before the FA4 selection is ever evaluated. HAS_FA4 is set independently at vllm_omni/diffusion/attention/backends/ring/ring_globals.py:29-36 (`try: from flash_attn.cute import flash_attn_func ... HAS_FA4 = True`), while HAS_FLASH_ATTN is set only from the FA2 import block at lines 9-14 (`from flash_attn.flash_attn_interface import _flash_attn_forward`), never from `flash_attn.cute`, so a FA4-only install leaves HAS_FA3/HAS_FLASH_ATTN/HAS_AITER all False. The added test masks this by keeping the FA3 branch alive: tests/diffusion/attention/test_ring_fa4.py:58 `monkeypatch.setattr(ring_parallel, "HAS_FA3", True)`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:9` [major] — This new file is the only real-weight MiniMax H3 test (the other minimax_h3 files are core_model/cpu mock tests), but it is marked `local_model` (no `core_model`), is env-gated on `VLLM_TEST_MINIMAX_H3_FL2VA_MODEL`, and no `.buildkite` lane selects `local_model` (`.buildkite/cuda/test-ready.yml:63` runs `tests/diffusion/models/ -m 'core_model and cuda'`). With `MiniMaxH3Pipeline` also in `EXCLUDED_MODELS`, no real-weight MiniMax H3 test runs in Buildkite CI. Wire it into the nightly local-model runner (or add a tiny-model builder per test_alignment.py:14 and drop the exclusion), and record the exact command/checkpoint. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:10 `pytest.mark.local_model,` and :11-13 `pytest.mark.cuda,` / `pytest.mark.diffusion,` / `pytest.mark.parallel,` (no core_model); :19-20 `@pytest.mark.skipif(` / ` not os.environ.get(MODEL_ENV),`; tests/model_tests/diffusion/test_alignment.py:35 ` "MiniMaxH3Pipeline",` (inside EXCLUDED_MODELS); .buildkite/cuda/test-ready.yml:63 (unchanged by this diff, PR-time tree) `pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"` — grep of `.buildkite` for `local_model` returns no matches.) + +`tests/diffusion/attention/test_ring_fa4.py:13` [minor] — DCO, buildkite/vllm-omni, and buildkite/vllm-omni-amd-ci are red at head. The diff adds cpu/core_model-marked tests (test_ring_fa4.py:12, test_flash_attn.py:21, test_minimax_h3_contract/packing/parallel) that collect on the CPU/core-model lanes, and pyproject.toml:101-103 adds a `[fa4]` extra (`flash-attn-4[cu13]==4.0.0b18`) that the AMD/Intel lanes will not install, so the diff surface intersects those lanes' coverage even though it is GPU-optional. Confirm from the failing job log whether the AMD/omni failure is pre-existing on main or introduced here, and fix the missing DCO sign-off on the commits. (evidence: tests/diffusion/attention/test_ring_fa4.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` — unchanged-by-context new test file marked cpu/core_model, so it collects on CPU/core lanes. tests/diffusion/attention/test_flash_attn.py:21 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]`. pyproject.toml:101-103 `fa4 = [` / ` "flash-attn-4[cu13]==4.0.0b18",` — CUDA-13/Blackwell-only extra, not installed on AMD/Intel lanes. CI state (not a repo file; from gh_ci_read): DCO=ACTION_REQUIRED, buildkite/vllm-omni=FAILURE, buildkite/vllm-omni-amd-ci=FAILURE — matching the comment's 'FAILING CHECKS (3)'.) + +`tests/diffusion/attention/test_ring_fa4.py:13` [minor] — This new test file and the new cpu-marked `tests/diffusion/models/minimax_h3` tests (contract, packing, parallel) land inside the `core_model and cpu` selection that both failing lanes run: `.buildkite/cuda/test-ready.yml:18` runs `tests/diffusion -m 'core_model and cpu'` and `.buildkite/amd/test-amd-ready.yml:19` runs the same selection against `tests/diffusion tests/model_executor`. Since attribution to this PR cannot be established without seeing main, please confirm whether `buildkite/vllm-omni` and `buildkite/vllm-omni-amd-ci` are also red on main or whether these additions break collection/selection. (evidence: tests/diffusion/attention/test_ring_fa4.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` (new in this PR's diff); tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` (new in this PR's diff); tests/diffusion/models/minimax_h3/test_minimax_h3_packing.py:7 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` (new in this PR's diff); tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:8 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` (new in this PR's diff); unchanged by this diff, present in the PR-time tree: .buildkite/cuda/test-ready.yml:18 `- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; unchanged by this diff, present in the PR-time tree: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; gh_ci_read(5691) returns `buildkite/vllm-omni` state FAILURE and `buildkite/vllm-omni-amd-ci` state FAILURE.) + +**Additional observations (beyond the comment budget):** +- `docs/source/features/attention_backends.md:170` [minor] In docs/user_guide/diffusion/attention_backends.md (not docs/source/features/...), restore the SAGE qualifier in the TRTLLM_ATTN row (line 17) or remove the now-unlinked 'TRTLLM_ATTN SAGE Quantization' section (lines 170 +- `vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:925` [minor] `diffuse` passes `imgvid_cond_num_frames=len(condition_shapes)` (a count of visual-condition blocks) into a parameter whose name/docstring describe a frame count; `condition_noise.py` uses it as `full_t = target_latent_t +- `tests/entrypoints/openai_api/test_video_server.py:3024` [minor] CI failures are real (DCO ACTION_REQUIRED, buildkite/vllm-omni FAILURE, buildkite/vllm-omni-amd-ci FAILURE), but the lane-coverage intersection with tests/entrypoints/openai_api/test_video_server.py and tests/config/test +- `vllm_omni/entrypoints/openai/api_server.py:2882` [minor] Confirmed: `_run_video_generation_job` returns early at api_server.py:2882-2884 when the job was already popped, before the `finally` block at api_server.py:2952 runs `_cleanup_video_references`. Since `_persist_uploaded + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.trace.json.gz new file mode 100644 index 000000000..161212e01 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5691.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.cost.json new file mode 100644 index 000000000..dbd73445b --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1294.6, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr5713/run-20260815-050649-0f1ec4", + "recorded_at": "2026-08-15T05:28:21", + "attempts": [ + { + "invocation_id": "inv-e0e19eb8", + "outcome": "run:run-20260815-050649-0f1ec4", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-050649-0f1ec4", + "llm_calls": 84, + "input_tokens": 665688, + "output_tokens": 169403, + "usd": 0.581796, + "source": "spans" + } + ], + "attempt_usd_total": 0.581796, + "llm_calls": 84, + "input_tokens": 665688, + "output_tokens": 169403, + "usd": 0.581796, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-050649-0f1ec4", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5818, + "minutes": 21.53, + "ci_minutes": 0, + "input_tokens": 665688, + "output_tokens": 169403, + "tool_calls": 157, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7973, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 84, + "by_role": { + "planner": { + "usd": 0.008038, + "input_tokens": 15, + "output_tokens": 7247, + "calls": 1 + }, + "agent": { + "usd": 0.529584, + "input_tokens": 637305, + "output_tokens": 129021, + "calls": 80 + }, + "reducer": { + "usd": 0.044174, + "input_tokens": 28368, + "output_tokens": 33135, + "calls": 3 + } + }, + "cache_read_tokens": 7989504, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.14, + "gate": 1.67, + "review": 1286.07, + "report": 0.07 + }, + "time_to_first_result_s": 1291.94 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5713.trace.json.gz", + "trace_bytes": 31559 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.md new file mode 100644 index 000000000..2f77d7d26 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5713, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6a464218a760) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] concern 're-marked pure-diffusion test may never run': refuted — .buildkite/cuda/test-merge.yml:68 runs `tests/entrypoints/ -m "advanced_model and cuda and L4"` and test-ready.yml:104 runs `-m 'core_model and cuda'`; the test carries advanced_model (module pytestmark) + core_model + cuda + L4. +- [claim-verified] No production code changes: diff_stat lists only tests/dfx/.../test_invalid_server_control.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/test_omni_sleep_mode.py. +- [claim-verified] Device cleanup once per module: `_module_device_cleanup` is module-scoped autouse and calls run_pre_test_cleanup/run_post_test_cleanup around the module (test_omni_sleep_mode.py:141-151). +- [claim-verified] #4473 Repro A moved off BAGEL: removed `test_level2_sleep_wake_raises`; added live `test_wakeup_after_level2_sleep_fails` and CPU `test_wakeup_route_propagates_not_implemented_error`. +- [claim-verified] Shared engines use class scope with loop_scope='class'; pyproject.toml pins pytest-asyncio==1.4.0 (loop_scope supported). +- [claim-verified] production wake_up raises NotImplementedError whose message contains both asserted substrings — vllm_omni/entrypoints/async_omni.py:1204 `"wake_up() after sleep(level=2) is not yet implemented: weights were "`. +- [claim-verified] 'No production vllm_omni/ code changes' — true: diff_stat and reads confirm only the three test files. +- [claim-refuted] 'diffusion TP=2 sleep/wake remains covered' is not equivalent: replacement is explicit-stage BagelPipeline (BAGEL), while the removed test exercised default AsyncOmni + qwen_image_random at TP=2 — see review comment. +- [validated] `test_wakeup_route_propagates_not_implemented_error` drives the real router with sleeping_stages={0} and a mock wake_up raising NotImplementedError; asserts re-raise and that sleeping set is not cleared (test_omni_sleep_wakeup.py:178-196). +- [validated] Level-2 HTTP 501 mapping: `omni_wakeup` route re-raises NotImplementedError (vllm_omni/entrypoints/openai/api_server.py:3500); live e2e asserts 501 with ('sleep(level=2)', 'not yet implemented'). +- [validated] indexed access `sleep_resp = sleep_resps[0]` (test_invalid_server_control.py:125) is safe: `send_omni_sleep_http_request` always returns `[resp]` (tests/helpers/runtime.py:1278-1298), and the test then asserts status_code==200. +- [validated] new branch `if current_omni_platform.is_rocm():` (test_omni_sleep_mode.py:163,176) — both arms reachable, guard identical to old code, no dead arm. +- [validated] `--enable-sleep-mode` does not affect the invalid-JSON cases: routes validate required `stage_ids` on `OmniSleepRequest`/`OmniWakeupRequest` before engine access (vllm_omni/entrypoints/openai/api_server.py:3469-3498), so 400s are unchanged. +- [validated] rename clean_gpu_envs→clean_device_envs: grep finds only in-file callers (test_omni_sleep_mode.py:32,164,176,454,511); no stale external consumers. + +`tests/entrypoints/test_omni_sleep_mode.py:~515` [major] — In tests/entrypoints/test_omni_sleep_mode.py the refactor removes the old `test_pure_diffusion_scenario` (parametrized tp_size=[1,2], model=MODEL_DIFF='riverclouds/qwen_image_random', default AsyncOmni config) and replaces it with `test_pure_diffusion_sleep_wake` hardcoded to tensor_parallel_size=1 on L4. TP=2 diffusion sleep/wake now runs only in `TestOmniDiffusionSleepMode` via `diffusion_engine`, which uses model=MODEL (ByteDance-Seed/BAGEL-7B-MoT) with `model_class_name: BagelPipeline`. This is not equivalent coverage of the default AsyncOmni + qwen_image_random TP=2 path. (evidence: tests/entrypoints/test_omni_sleep_mode.py:511 `async def test_pure_diffusion_sleep_wake():` ... :515 ` "tensor_parallel_size": 1,` — replacement hardcodes TP=1, unchanged in scope of removed TP=2 case. tests/entrypoints/test_omni_sleep_mode.py:98 `MODEL_DIFF = "riverclouds/qwen_image_random"` and :179-187 `engine = AsyncOmni(model=MODEL, stages=stages, init_timeout=600, enable_sleep_mode=True)` with :185 `"model_class_name": "BagelPipeline",` — the TP=2 diffusion path now uses BAGEL/BagelPipeline, not qwen_image_random default config. Merge-base confirmed the removed `test_pure_diffusion_scenario` parametrized `tp_size` [1,2] with `"model": MODEL_DIFF`.) + +`tests/entrypoints/test_omni_sleep_mode.py:115` [major] — Class-scoped shared engines require every test to leave the engine fully awake, but `_ensure_awake` (tests/entrypoints/test_omni_sleep_mode.py:111-116) is best-effort and swallows exceptions. If the full wake in a `finally` fails — the exact partial-wake/duplicate-wake regression surface these tests cover — the current test passes and the next shared-engine test starts partially asleep, so its failure is misattributed or masked. Make `_ensure_awake` re-raise (or assert) after logging, ideally without replacing an already-active exception, so the test that left the engine asleep fails. Calls: LLM `finally` at 232, 247, 261; diffusion `finally` at 283, 306, 385. (evidence: tests/entrypoints/test_omni_sleep_mode.py:111-116 `async def _ensure_awake(engine: AsyncOmni, stage_ids: list[int]) -> None:` / ` """Best-effort full wake so the next shared-engine test starts active."""` / ` try:` / ` await engine.wake_up(stage_ids=stage_ids)` / ` except Exception as e:` / ` logger.warning("ensure_awake failed (stage_ids=%s): %s", stage_ids, e)`; and its consumers in `finally`, e.g. tests/entrypoints/test_omni_sleep_mode.py:231-232 ` finally:` / ` await _ensure_awake(llm_engine, [0])` (also 247, 261), plus the shared fixture `@pytest_asyncio.fixture(scope="class", loop_scope="class")`.) + +`.buildkite/amd:1` [minor] — The AMD gate (`buildkite/vllm-omni-amd-ci`) is FAILURE on this PR's head, and available data exposes only check state (no step logs), so the failure cannot be attributed to main vs this PR. Confirm it is pre-existing before merge. Note: `.buildkite/amd` is a directory, not a file — anchor to the CI check name rather than `.buildkite/amd:1`. (evidence: unchanged by this diff, present in the PR-time tree: `.buildkite/amd` is a directory — `list_dir('.buildkite/amd')` returns `scripts/`, `test-amd-merge.yml`, `test-amd-ready.yml`, `test-template-amd-omni.j2`; `gh_ci_read(5713)` returns `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` with no step logs; `diff_stat` shows only `tests/dfx/reliability/invalid_param_test/test_invalid_server_control.py`, `tests/entrypoints/openai_api/test_omni_sleep_wakeup.py`, and `tests/entrypoints/test_omni_sleep_mode.py` changed.) + +`tests/entrypoints/test_omni_sleep_mode.py:~373` [minor] — tests/entrypoints/test_omni_sleep_mode.py:373 — the merged lifecycle test's `assert abs(vram_restored - vram_initial) < 3.0` is vacuous for the process=True diffusion stage: get_vram_info measures torch.cuda.memory_reserved() in the test process only, so it cannot observe the worker's weight reload after wake_up. Use get_device_global_memory_used_gib (driver view) or ACK freed_bytes for the restore check. (evidence: tests/entrypoints/test_omni_sleep_mode.py:46 `"""Per-**process** CUDA allocator stats (GiB). Does not see other processes' usage."""` — tests/entrypoints/test_omni_sleep_mode.py:181 `"runtime": {"process": True, "devices": "0,1", "max_batch_size": 1},` — tests/entrypoints/test_omni_sleep_mode.py:371 `vram_restored = get_vram_info(device_id)["reserved"]` — tests/entrypoints/test_omni_sleep_mode.py:373 `assert abs(vram_restored - vram_initial) < 3.0, "VRAM failed to restore to initial levels"` — tests/entrypoints/test_omni_sleep_mode.py:72-73 (unchanged helper docstring, present in PR-time tree): `**separate worker processes**; ``memory_reserved`` in the test process does not reflect child workers' allocations.`) + +`tests/entrypoints/test_omni_sleep_mode.py:148` [minor] — The diff drops `pytest.mark.usefixtures("clean_gpu_memory_between_tests")` and replaces it with `_module_device_cleanup` that only runs `run_pre_test_cleanup()` once at module start. The LLM and diffusion engines are now independent class-scoped fixtures, so the transition between `TestOmniLlmSleepMode` and `TestOmniDiffusionSleepMode` has only `engine.shutdown()` + `await asyncio.sleep(1.5)` (line 169) and no `wait_for_gpu_memory_to_clear`, unlike the old per-test fixture (`tests/helpers/fixtures/env.py:38`). If LLM worker processes lag releasing VRAM, the TP=2 diffusion fixture can OOM flakily. Add `run_pre_test_cleanup()` (or `wait_for_gpu_memory_to_clear`) before the `diffusion_engine` fixture, or scope the cleanup per class. (evidence: tests/entrypoints/test_omni_sleep_mode.py:148 `@pytest.fixture(scope="module", autouse=True)` — module-scoped cleanup replaces the removed per-test fixture; tests/entrypoints/test_omni_sleep_mode.py:168-169 ` engine.shutdown()` / ` await asyncio.sleep(1.5)` — the only wait after the class-scoped LLM fixture yields (this line is part of the diff); tests/helpers/fixtures/env.py:32-38 `def clean_gpu_memory_between_tests():` ... ` run_pre_test_cleanup()` — unchanged by this diff, present in the PR-time tree; this is the old per-test gate that ran before every test; tests/helpers/env.py:219-229 `def run_pre_test_cleanup() -> None:` ... ` wait_for_gpu_memory_to_clear(` / ` threshold_ratio=0.05,` / ` timeout_s=60,` — unchanged by this diff, present in the PR-time tree; this is the ≤5% / 60s wait the comment cites.) + +`tests/entrypoints/test_omni_sleep_mode.py:506` [minor] — Confirmed: removing `test_diffusion_model_sleep_tp` drops the only default auto-decomposed two-stage BAGEL sleep/wake path (`AsyncOmni(**engine_args, stage_init_timeout=1200)` with `model=MODEL`, no explicit stages, `sampling_params_list=[llm_sp, diff_sp]`). `TestOmniDiffusionSleepMode.diffusion_engine` is an explicit single diffusion stage (`model_class_name: "BagelPipeline"`, `parallel_config.tensor_parallel_size: 2`), and `test_pure_diffusion_sleep_wake` uses `MODEL_DIFF` (`riverclouds/qwen_image_random`), so neither covers the dropped default-config path. Suggest keeping a slim default-config smoke or correcting the 'TP=2 remains covered' wording. (evidence: PR-time tree tests/entrypoints/test_omni_sleep_mode.py:506 ` Diffusion TP=2 sleep/wake is covered by TestOmniDiffusionSleepMode` — the challenged description. PR-time tree tests/entrypoints/test_omni_sleep_mode.py:216 ` engine = AsyncOmni(model=MODEL, stages=stages, init_timeout=600, enable_sleep_mode=True)` — head diffusion_engine fixture builds an explicit single `stage_type: "diffusion"` stage with `model_class_name: "BagelPipeline"` and `parallel_config.tensor_parallel_size: 2`, not the default auto-decomposed path. PR-time tree tests/entrypoints/test_omni_sleep_mode.py:513 ` "model": MODEL_DIFF,` — the pure diffusion test uses `riverclouds/qwen_image_random`, not BAGEL. Removed by this diff (pre-PR merge-base tests/entrypoints/test_omni_sleep_mode.py, `test_diffusion_model_sleep_tp`): `engine = AsyncOmni(**engine_args, stage_init_timeout=1200)` after `engine_args = {"model": MODEL, "enable_sleep_mode": True, "tensor_parallel_size": tp_size, ...}` and `async for _ in engine.generate("test", sampling_params_list=[llm_sp, diff_sp]):` — the only default-config two-stage BAGEL coverage.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.trace.json.gz new file mode 100644 index 000000000..3d9ba0c06 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5713.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.cost.json new file mode 100644 index 000000000..7d2cecd62 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1498.0, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr5843/run-20260815-050649-444af8", + "recorded_at": "2026-08-15T05:31:44", + "attempts": [ + { + "invocation_id": "inv-871df35b", + "outcome": "run:run-20260815-050649-444af8", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-050649-444af8", + "llm_calls": 161, + "input_tokens": 1386824, + "output_tokens": 281735, + "usd": 1.120536, + "source": "spans" + } + ], + "attempt_usd_total": 1.120536, + "llm_calls": 161, + "input_tokens": 1386824, + "output_tokens": 281735, + "usd": 1.120536, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-050649-444af8", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.1205, + "minutes": 24.91, + "ci_minutes": 0, + "input_tokens": 1386824, + "output_tokens": 281735, + "tool_calls": 276, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0466, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 161, + "by_role": { + "agent": { + "usd": 1.084535, + "input_tokens": 1367675, + "output_tokens": 253741, + "calls": 159 + }, + "reducer": { + "usd": 0.036002, + "input_tokens": 19149, + "output_tokens": 27994, + "calls": 2 + } + }, + "cache_read_tokens": 16155008, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.52, + "gate": 1.6, + "review": 1489.35, + "report": 0.06 + }, + "time_to_first_result_s": 1494.52 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5843.trace.json.gz", + "trace_bytes": 41764 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.md new file mode 100644 index 000000000..78cf74caa --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5843, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head d6ba7d145815) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] prior concern 'nonfinite validation only at CLI': OmniDiffusionConfig.__post_init__ (data.py:950-954) validates the same invariant, covering YAML/programmatic config construction paths. +- [resolved] Concern: new max_num_running_reqs<=1 gate changes old max_batch=1 wait path? No observable change — old loop broke immediately when waiting>=1 and busy loop only enters admission wait with has_requests() true. +- [resolved] Concern 'docs hook will NameError on _nonneg_finite_float' resolved at docs/mkdocs/hooks/generate_argparse.py:126 `"_nonneg_finite_float": float,` (readthedocs SUCCESS). +- [claim-verified] PR body claim 'Added unit tests in test_diffusion_scheduler' is true (diff_stat: +89 lines there). +- [claim-verified] PR says 'Added unit tests in test_diffusion_scheduler'; diff adds 4 RequestScheduler admission tests + 1 StepScheduler no-support test. +- [claim-verified] PR description 'Added unit tests in test_diffusion_scheduler' is true: test_admission_wait_* tests added. +- [claim-verified] 'All passed locally' cannot be confirmed here; gh_ci_read shows pre-commit/build/docs/Intel/NPU green and buildkite/vllm-omni-amd-ci FAILURE. +- [claim-verified] PR body 'Added unit tests in test_diffusion_scheduler' — grep confirms new tests at lines 251/262/281/295/977. +- [claim-refuted] PR body 'All passed locally' cannot be verified read-only, and gate_report shows buildkite/vllm-omni-amd-ci FAILURE (other lanes SUCCESS). +- [claim-refuted] PR body does not mention the TTS adapter/pre-commit budget changes, though commit timeline ('further fix unrelated TTS issues') shows they were bundled intentionally. +- [validated] docs/mkdocs/hooks/generate_argparse.py:123 adds `_nonneg_finite_float` to the sandbox globals so the static doc build doesn't NameError; readthedocs CI is SUCCESS. +- [validated] vllm_omni/diffusion/data.py:951 `if not math.isfinite(self.request_batch_max_wait_ms) or self.request_batch_max_wait_ms < 0` is backed by `import math` at data.py:5. +- [validated] pre-commit budgets: serving_speech.py has exactly 27 `_tts_model_type` ast.Compare nodes and tts_adapters/__init__.py has 0 LegacyDetector constructions, matching MAX_MODEL_TYPE_BRANCHES=27 / MAX_LEGACY_DETECTORS=0; pre-commit CI is SUCCESS. +- [validated] TTS removal preserves detection: MingFlashOmniTTSAdapter already existed at base with stage_keys={'ming_tts'} and default detect_priority 100 (< Ming dense 200), so detect_tts_model_type('ming_tts','MingTTSForConditionalGeneration') still returns 'ming_flash_omni_tts' (test_tts_detection.py:249). + +`tests/diffusion/test_diffusion_scheduler.py:282` [minor] — Confirmed: `test_admission_wait_disabled_while_wave_is_running` (tests/diffusion/test_diffusion_scheduler.py:281) initializes with `max_num_seqs=1`, so `max_num_running_reqs` is 1 and `get_admission_wait_decision` returns at the `max_num_running_reqs <= 1` guard; the `num_running_requests() > 0` branch is never reached and this test would stay green if that guard were deleted. Re-initialize with `max_num_seqs=2`, assert `should_wait is True` before scheduling, then schedule one request and assert `should_wait is False`. (evidence: tests/diffusion/test_diffusion_scheduler.py:284 `max_num_seqs=1,`; vllm_omni/diffusion/sched/base_scheduler.py:64 `self.max_num_running_reqs = max(1, int(max_num_seqs))`; vllm_omni/diffusion/sched/request_scheduler.py:49 `if max_wait_ms <= 0.0 or self.max_num_running_reqs <= 1:`; vllm_omni/diffusion/sched/request_scheduler.py:51 `if self.num_running_requests() > 0:`) + +`vllm_omni/diffusion/sched/base_scheduler.py:170` [minor] — DiffusionEngine._busy_loop now always calls _wait_for_admission_if_needed_locked() (diffusion_engine.py:437) and admission policy moved into BaseScheduler defaults should_wait=False / return True (base_scheduler.py:170,181). A custom scheduler injected through the documented scheduler= override inherits those no-op defaults, so request-batch coalescing silently stops even when request_batch_max_wait_ms>0; previously the engine performed the wait itself when `supports_request_batch or dp_concurrent`. Document this contract on BaseScheduler, or make the hooks abstract and give StepScheduler an explicit no-op override. (evidence: base_scheduler.py:170 ` return _AdmissionWaitDecision(should_wait=False)` — added by this diff; BaseScheduler's default admission decision never waits. base_scheduler.py:181 ` return True` — added by this diff; BaseScheduler's default should_end_admission_wait exits immediately, so a subclass must override both hooks for any coalescing. diffusion_engine.py:437 ` self._wait_for_admission_if_needed_locked()` — added by this diff; the diff removed the prior gate `if self.supports_request_batch or self.dp_concurrent:`, making the scheduler hooks the only admission-policy source.) + +`vllm_omni/diffusion/diffusion_engine.py:492` [minor] — diffusion_engine.py:491-498 now resets `stable_since` (`if waiting > last_waiting: stable_since = now`) before `should_end_admission_wait(...)`; the removed code checked the max-batch, stable-window, and deadline exits before resetting. Consequence: a request arriving in the poll where the previous stable window just expired re-arms the window and delays scheduling by up to another 50ms (0.3s for dp_concurrent) instead of scheduling immediately. The existing `test_request_batch_admission_exits_early_when_waiting_queue_stable` pre-populates the queue and does not cover a mid-wait arrival. If reset-on-arrival is intentional, document it and add a spaced-arrival regression test; otherwise restore the old check-before-reset order. (evidence: vllm_omni/diffusion/diffusion_engine.py:491-499 (PR-time tree): ` if waiting > last_waiting:` / ` stable_since = now` / ` last_waiting = waiting` / `` / ` if self.scheduler.should_end_admission_wait(` / ` decision,` / ` now=now,` / ` stable_since=stable_since,` / ` ):` / ` break` — the reset executes before the exit check. Removed pre-PR order (from pr_diff) evaluated the stable-window exit `if waiting > 0 and (now - stable_since) >= stable_window_s:` before `if waiting > last_waiting: stable_since = now`, so a new arrival at window expiry did not re-arm the window. The only engine-level stable-window test pre-adds both requests before invoking `_wait_for_admission_if_needed_locked()` (tests/diffusion/test_diffusion_engine.py, `test_request_batch_admission_exits_early_when_waiting_queue_stable`), so no mid-wait arrival path is exercised.) + +`vllm_omni/diffusion/sched/request_scheduler.py:49` [minor] — Add a direct unit test for the `self.max_num_running_reqs <= 1` guard in `RequestScheduler.get_admission_wait_decision` (request_scheduler.py:49). `test_admission_wait_disabled_while_wave_is_running` uses `max_num_seqs=1`, so it exits at that guard and never reaches `num_running_requests() > 0`; the `max_num_running_reqs <= 1` arm is therefore not independently pinned. (evidence: vllm_omni/diffusion/sched/request_scheduler.py:49 `if max_wait_ms <= 0.0 or self.max_num_running_reqs <= 1:`; tests/diffusion/test_diffusion_scheduler.py:287-290 `max_num_seqs=1,` / `request_batch_max_wait_ms=1000.0,` / `self.scheduler.add_request(_make_request("running"))` / `self.scheduler.schedule()` — the test config max_num_seqs=1 short-circuits the guard before the num_running_requests() > 0 check.) + +`vllm_omni/config/omni_config.py:~532` [minor] — The structured-config mirror still lacks the finite/non-negative constraint: `_DiffusionConfigProjection` declares `request_batch_max_wait_ms: float = 0.0` (vllm_omni/config/omni_config.py:532) and its `__post_init__` never validates it, so `nan`/`inf`/`-1` are accepted at construction and only rejected later when `enrich_config()` builds `OmniDiffusionConfig` (vllm_omni/config/omni_config.py:753), whose `__post_init__` runs the new check (vllm_omni/diffusion/data.py:951). The two representations therefore validate at different times, and the new tests do not cover the projection path. Add `Field(default=0.0, ge=0.0, allow_inf_nan=False)` to the projection field and a projection-level test. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/config/omni_config.py:532 ` request_batch_max_wait_ms: float = 0.0`. Deferred validation path: vllm_omni/config/omni_config.py:753 ` omni_diffusion_config = OmniDiffusionConfig(**kwargs)` inside `_DiffusionConfigProjection.enrich_config()`. The rejection only lives in the mirror's later-constructed config: vllm_omni/diffusion/data.py:951 ` if not math.isfinite(self.request_batch_max_wait_ms) or self.request_batch_max_wait_ms < 0:`.) + +`vllm_omni/diffusion/sched/base_scheduler.py:162` [nit] — The new public scheduler hooks `BaseScheduler.get_admission_wait_decision` and `BaseScheduler.should_end_admission_wait` (base_scheduler.py:162/172) expose the private `_AdmissionWaitDecision` type (interface.py:35) in their signatures, so any custom scheduler overriding these hooks must import a `_`-prefixed internal type. Consider renaming the dataclass to a public `AdmissionWaitDecision` (updating engine/tests imports), or documenting in the method docstrings that overriders must import the private type. (evidence: vllm_omni/diffusion/sched/interface.py:35 `class _AdmissionWaitDecision:` — private (underscore-prefixed) dataclass used in public hook signatures. vllm_omni/diffusion/sched/base_scheduler.py:162 `def get_admission_wait_decision(` and :167 `) -> _AdmissionWaitDecision:`; :172 `def should_end_admission_wait(` and :175 `decision: _AdmissionWaitDecision,` — base class exposes the private type as part of its API surface.) + +`vllm_omni/diffusion/diffusion_engine.py:437` [nit] — The diff replaces the `if self.supports_request_batch or self.dp_concurrent:` gate with an unconditional `self._wait_for_admission_if_needed_locked()` call in `DiffusionEngine._busy_loop` (diffusion_engine.py:437) and modifies `tests/diffusion/test_diffusion_engine.py`, `test_diffusion_scheduler.py`, and `test_diffusion_engine_rpc_routing.py`. The gate report shows `buildkite/vllm-omni-amd-ci` FAILURE, and the AMD lane command `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=...` exercises those directories. Please confirm whether the failure is pre-existing on main (and link the job log) or introduced by this refactor before merging. (evidence: gh_ci_read(5843): `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`; diff_stat: `vllm_omni/diffusion/diffusion_engine.py | 53 +++++--------`) + +`tools/pre_commit/check_tts_adapter.py:34` [nit] — The diff lowers the TTS adapter ratchet budgets `MAX_MODEL_TYPE_BRANCHES` 29→27 and `MAX_LEGACY_DETECTORS` 1→0 and empties `LEGACY_TTS_DETECTORS`, which is correct ratchet alignment (MingFlashOmniTTSAdapter owns the `ming_tts` stage key) but unrelated to the stated diffusion admission-policy refactor. Consider splitting this ride-along into its own PR or adding a one-line note in the PR description that it is pre-commit ratchet alignment from merging main. (evidence: tools/pre_commit/check_tts_adapter.py:34 `MAX_MODEL_TYPE_BRANCHES = 27` and tools/pre_commit/check_tts_adapter.py:35 `MAX_LEGACY_DETECTORS = 0` — vllm_omni/entrypoints/openai/tts_adapters/__init__.py:34 `LEGACY_TTS_DETECTORS: tuple[LegacyDetector, ...] = ()` — vllm_omni/entrypoints/openai/tts_adapters/ming_flash_omni_tts.py:19 `stage_keys = frozenset({"ming_tts"})` (present in PR-time tree, unchanged by this diff; proves the `ming_tts` stage key is owned by MingFlashOmniTTSAdapter). PR title/body states only the request scheduler admission refactor.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.trace.json.gz new file mode 100644 index 000000000..40a4598d0 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5843.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.cost.json new file mode 100644 index 000000000..97e1470fe --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1292.5, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr5853/run-20260815-050649-bc3be6", + "recorded_at": "2026-08-15T05:28:19", + "attempts": [ + { + "invocation_id": "inv-b6036728", + "outcome": "run:run-20260815-050649-bc3be6", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-050649-bc3be6", + "llm_calls": 185, + "input_tokens": 1811959, + "output_tokens": 230892, + "usd": 1.322999, + "source": "spans" + } + ], + "attempt_usd_total": 1.322999, + "llm_calls": 185, + "input_tokens": 1811959, + "output_tokens": 230892, + "usd": 1.322999, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-050649-bc3be6", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.323, + "minutes": 21.49, + "ci_minutes": 0, + "input_tokens": 1811959, + "output_tokens": 230892, + "tool_calls": 337, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0466, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 185, + "by_role": { + "agent": { + "usd": 1.292072, + "input_tokens": 1779970, + "output_tokens": 210663, + "calls": 183 + }, + "reducer": { + "usd": 0.030927, + "input_tokens": 31989, + "output_tokens": 20229, + "calls": 2 + } + }, + "cache_read_tokens": 21473664, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.26, + "gate": 1.76, + "review": 1282.45, + "report": 0.08 + }, + "time_to_first_result_s": 1289.51 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5853.trace.json.gz", + "trace_bytes": 53607 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.md new file mode 100644 index 000000000..d5c33d949 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5853, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head a24ff8f2ad80) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] prior concern 'once hint cleared after first request': test_refresh_rearms_force_refresh_hint verifies refresh re-passes the hint for repeated requests. +- [resolved] prior concern that `high` errors without startup Cache-DiT: quality_policy.py installs the H3 profile regardless of startup backend, matching docs/cache_dit.md table. +- [claim-verified] speedup arithmetic: calc(85.49/63.36)=1.3493, pair reductions 25.2%/26.0%/25.9% match the PR table. +- [claim-verified] PR latency table is internally consistent: 85.49/63.36 ≈ 1.349× (reported 1.35×) and median latency reduction ≈ 25.9%. +- [claim-verified] diff_stat: 28 files, 1183 insertions / 18 deletions, matches the supplied squashed diff; no dropped or extra files. +- [claim-verified] diff_stat confirms 28 files changed, 1183 insertions / 18 deletions; matches the stated scope. +- [claim-verified] gh_ci_read confirms only `buildkite/vllm-omni-amd-ci` is FAILURE; all other checks SUCCESS, matching gate_report. +- [claim-verified] PR median numbers self-consistent: 85.49/63.36 = 1.349× ≈ documented 1.35×. +- [claim-refuted] PR body 'Without startup Cache-DiT ... high returns an error' contradicts quality_policy.py:149 (`if quality == "high":` installs the profile) and docs table 'no Cache-DiT | high | Install H3's conservative Cache-DiT profile'. +- [claim-refuted] PR body says 'Without startup Cache-DiT capability ... high returns an error', but quality_policy.py has no `_configured_backend` gate on `quality == 'high'` and the added docs table says 'no Cache-DiT | high | Install H3's conservative Cache-DiT profile' — code/docs disagree with the description. +- [claim-refuted] PR body says without startup Cache-DiT 'high returns an error', but quality_policy.py resolves quality=='high' to a spec unconditionally and docs/cache_dit.md says no-Cache-DiT + high installs the profile. +- [validated] `OmniDiffusionSamplingParams` is a dataclass (data.py:197-198) and the new `__post_init__` rejects unsupported quality levels. +- [validated] only MiniMaxH3 opts in: grep shows `adopt_cache_dit_backend` implemented only in minimax_h3/pipeline_minimax_h3.py:546 plus the interface protocol. +- [validated] sched/base_scheduler.py:32 and request_scheduler.py:23 build keys from dataclass fields, so `quality` is auto-included; no stale builder was missed. + +`vllm_omni/diffusion/cache/cachedit/backend.py:~244` [major] — backend.py:244 stores `_default_get_pipeline_transformer(pipeline)` for the custom-enabler branch, but custom enablers such as Wan22 (model_specific.py:90-92) and Krea2 (via `enable_cache_for_dit` with an explicit BlockAdapter) install hooks on a BlockAdapter. `disable()` at backend.py:280 therefore calls `cache_dit.disable_cache` on the raw transformer, not the enabled adapter, leaving hooks installed for those pipelines. Currently unreachable for H3 (the only request-scoped adopter is not a custom enabler), but this is now the generic disable path. Store the exact target each enabler installed (or have the enabler return it) and add a custom-enabler disable test. (evidence: backend.py:244 `self._cache_targets = [_default_get_pipeline_transformer(pipeline)]` — custom-enabler branch stores the raw transformer as the only disable target; backend.py:280 `cache_dit.disable_cache(target)` — disable() disables each stored target. Custom enablers install on a different target, unchanged by this diff and present in the PR-time tree: model_specific.py:90-92 `cache_dit.enable_cache(` / `BlockAdapter(` (Wan22 path), and Krea2's `enable_cache_for_krea2` passes an explicit `BlockAdapter` to `enable_cache_for_dit`, whose `cache_target = transformer if block_adapter is None else block_adapter` means the enabled object is the BlockAdapter, not the raw transformer.) + +`tests/diffusion/test_diffusion_model_runner.py:165` [major] — `test_request_scoped_cache_dit_lifecycle_is_pipeline_opt_in` at tests/diffusion/test_diffusion_model_runner.py:165 carries only the module-level `pytest.mark.diffusion`. CUDA `test-ready.yml:18` (`pytest -sv tests/diffusion -m 'core_model and cpu'`), CUDA `test-ready.yml:68` (`pytest -sv tests/diffusion/*.py -m 'core_model and cuda'`), and AMD `test-amd-ready.yml:19` (`pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`) all filter on `core_model`, so this test is deselected in every lane that collects the file and the new `interface.py` opt-in contract gets no CI coverage. Add `@pytest.mark.core_model` and `@pytest.mark.cpu` and verify with `pytest -m 'core_model and cpu' tests/diffusion/test_diffusion_model_runner.py::test_request_scoped_cache_dit_lifecycle_is_pipeline_opt_in --collect-only`. (evidence: tests/diffusion/test_diffusion_model_runner.py:16 `pytestmark = [pytest.mark.diffusion]`; tests/diffusion/test_diffusion_model_runner.py:165 `def test_request_scoped_cache_dit_lifecycle_is_pipeline_opt_in():` (immediately preceded by a blank line — no `@pytest.mark` decorators); unchanged by this diff, present in the PR-time tree: .buildkite/cuda/test-ready.yml:18 `"timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; unchanged by this diff: .buildkite/cuda/test-ready.yml:68 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"`; unchanged by this diff: .buildkite/amd/test-amd-ready.yml:19 `"timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`) + +`vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149` [minor] — PR body says 'Without startup Cache-DiT capability, omitted and lossless are no-ops, high returns an error.', but quality_policy.py:149 handles quality == 'high' before any startup-backend check and returns an H3 high-quality CacheDiTRequestSpec. Update the PR description to say high installs H3's conservative Cache-DiT profile even without startup Cache-DiT. (evidence: vllm_omni/diffusion/models/minimax_h3/quality_policy.py:149 ` if quality == "high":` ; quality_policy.py:150-151 ` base_key = MINIMAX_H3_HIGH_CACHE_KEY` / ` base_config = _high_quality_cache_config()` ; quality_policy.py:164-165 ` return MiniMaxH3QualityPlan(` / ` cache_dit=CacheDiTRequestSpec(` — the `high` branch never reads `self._configured_backend`, so it installs the high profile without startup Cache-DiT. Contradicted PR body sentence (via gh_pr_view): `Without startup Cache-DiT capability, omitted and lossless are no-ops, high returns an error.`) + +`vllm_omni/entrypoints/openai/api_server.py:2814` [minor] — api_server.py:2814 `quality=req.quality or 'default'` populates VideoResponse.quality (videos.py:349), but no test in tests/entrypoints/openai_api asserts the response field. Add a direct test covering omitted, `lossless`, and `high` request quality in the video-response metadata. (evidence: Added by this diff: vllm_omni/entrypoints/openai/api_server.py:2814 `quality=req.quality or "default",` — writes request quality into VideoResponse. Present in PR-time tree (unchanged by this diff): vllm_omni/entrypoints/openai/protocol/videos.py:349 `quality: str = Field(default="default", description="Requested quality level for generation.")` — the response field targeted. Repo search of tests/entrypoints/openai_api found no assertion on `VideoResponse.quality`: no matches for `.quality`, `response.quality`, or `resp.quality`; existing `quality` assertions target `captured.quality` (sampling params) and `req.quality is None` (request validation).) + +`tests/diffusion/cache/test_cache_dit_request_runtime.py:13` [minor] — .buildkite/amd/test-amd-ready.yml:19 runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py ...`; the new tests/diffusion/cache/test_cache_dit_request_runtime.py (line 14) and tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py (line 11) both carry `core_model` + `cpu` marks and are not ignored, so they are collected by the failing `buildkite/vllm-omni-amd-ci` lane. Confirm from the failed log and a main run whether this is pre-existing or introduced before dismissing the red gate. (evidence: ['unchanged by this diff, present in the PR-time tree: .buildkite/amd/test-amd-ready.yml:19 `"timeout 40m pytest -sv tests/diffusion tests/model_executor -m \'core_model and cpu\' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`', 'new file in this diff: tests/diffusion/cache/test_cache_dit_request_runtime.py:14 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`', 'new file in this diff: tests/diffusion/models/minimax_h3/test_minimax_h3_quality_policy.py:11 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`']) + +`tests/diffusion/cache/test_cache_backends.py:~119` [minor] — Confirmed. No unit test exercises the real `cache_dit.disable_cache`: a repo-wide search shows the only test reference to `disable_cache` is the mocked assertion at tests/diffusion/cache/test_cache_backends.py:119, and tests/diffusion/cache/test_cache_dit_request_runtime.py replaces `CacheDiTBackend` with a recording fake. Consider a non-mocked enable→disable smoke (e.g. in tests/diffusion/cache/test_cache_dit.py) to prove the pinned cache-dit 1.3.0 uninstall path can remove hooks. (evidence: tests/diffusion/cache/test_cache_backends.py:119 `assert [call.args[0] for call in mock_cache_dit.disable_cache.call_args_list] == installed_adapters` — the only test occurrence of `disable_cache` (grep across `tests/`); this test is decorated with `@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")`. New file tests/diffusion/cache/test_cache_dit_request_runtime.py (line ~50) `monkeypatch.setattr(runtime_module, "CacheDiTBackend", FakeBackend)` — replaces the backend entirely. Production path added by this diff: vllm_omni/diffusion/cache/cachedit/backend.py (disable method) `cache_dit.disable_cache(target)`; pinned dependency requirements/common.txt:19 `cache-dit==1.3.0`. Therefore no test executes the real `cache_dit.disable_cache`.) + +`recipes/MiniMaxAI/MiniMax-H3.md:592` [minor] — This diff adds a benchmark block stating the table was 'measured on 4× NVIDIA H200 with SP4, text-encoder TP4, 1344×768, 124 frames, 24 FPS, and 50 inference steps', but the PR body attributes the identical median numbers (85.49 s / 63.36 s / 1.35× / 0.9709 / 34.98 dB) to '4x NVIDIA L20X'. H200 and L20X are different SKUs, so at least one attribution is wrong. Correct line is 592, not 601. Align the hardware label (or re-measure) in both the recipe and the PR description. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:592 (added by this diff): `The following result was measured on 4× NVIDIA H200 with SP4, text-encoder` and recipes/MiniMaxAI/MiniMax-H3.md:600: `| `high` | 63.36 s | 1.35× | 0.9709 | 34.98 dB | Faster with measured same-seed deviation |`. PR body (outside the diff, read via gh_pr_view): `The online server was exercised on 4x NVIDIA L20X with SP4, text-encoder TP4, 1344x768, 124 frames, 24 FPS, and 50 steps.` and its median row `| Median | 85.49 s | 63.36 s | 25.9% | 0.9709 | 34.98 dB |` — same figures, different GPU label.) + +`vllm_omni/entrypoints/openai/serving_video_output_stream.py:538` [minor] — The `quality` assignment in serving_video_output_stream.py:538 and serving_video.py:203 lacks the `and request.quality is not None` guard used by the sibling num_inference_steps lines. Since `provided_fields = request.model_fields_set` (pydantic v2) includes fields explicitly set to null, a client sending `quality: null` overwrites a copied server default (`OmniDiffusionSamplingParams(quality="high")`) with None, so the request no longer follows the configured default. Add the guard at both sites. (evidence: vllm_omni/entrypoints/openai/serving_video_output_stream.py:536-539 ` if "num_inference_steps" in provided_fields and request.num_inference_steps is not None: + gen_params.num_inference_steps = request.num_inference_steps + if "quality" in provided_fields: + gen_params.quality = request.quality` — the quality branch lacks the `request.quality is not None` guard; `provided_fields = request.model_fields_set` (line 535). vllm_omni/entrypoints/openai/serving_video.py:201-204 ` if "num_inference_steps" in provided_fields and request.num_inference_steps is not None: + gen_params.num_inference_steps = request.num_inference_steps + if "quality" in provided_fields: + gen_params.quality = request.quality` — identical unguarded assignment. requirements/common.txt:31 `pydantic>=2.1.0` — pydantic v2 `model_fields_set` includes explicitly-provided `None`, so explicit JSON `null` triggers the overwrite.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.trace.json.gz new file mode 100644 index 000000000..32955c9a2 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5853.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.cost.json new file mode 100644 index 000000000..d95f9007d --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 767.7, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr5857/run-20260815-052247-d02907", + "recorded_at": "2026-08-15T05:35:32", + "attempts": [ + { + "invocation_id": "inv-58bb340f", + "outcome": "run:run-20260815-052247-d02907", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-052247-d02907", + "llm_calls": 75, + "input_tokens": 379716, + "output_tokens": 127423, + "usd": 0.391048, + "source": "spans" + } + ], + "attempt_usd_total": 0.391048, + "llm_calls": 75, + "input_tokens": 379716, + "output_tokens": 127423, + "usd": 0.391048, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-052247-d02907", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.391, + "minutes": 12.75, + "ci_minutes": 0, + "input_tokens": 379716, + "output_tokens": 127423, + "tool_calls": 129, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5516, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 75, + "by_role": { + "agent": { + "usd": 0.37268, + "input_tokens": 369353, + "output_tokens": 113303, + "calls": 73 + }, + "reducer": { + "usd": 0.018368, + "input_tokens": 10363, + "output_tokens": 14120, + "calls": 2 + } + }, + "cache_read_tokens": 5494784, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.42, + "gate": 2.34, + "review": 759.38, + "report": 0.1 + }, + "time_to_first_result_s": 765.18 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5857.trace.json.gz", + "trace_bytes": 25495 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.md new file mode 100644 index 000000000..537f1da8e --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5857, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4aa748dc54f4) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 5 finding(s) below | + +**Validated:** +- [claim-verified] PR-body claim 'options available on main, no SM120 kernel': all flags defined on head serve.py; recipe is BF16 + CUDNN_ATTN with no fp8/Quack/SM120-gated flag (the only in-repo sm120 reference is tests/quantization/test_quack_fp8_gate.py, unrelated). +- [claim-verified] PR-body claim 'uses serving options available on main, no SM120 dependency': recipe contains no FLASHINFER_SM120_ATTN/SM120 flag; all flags present in serve.py at head. +- [claim-verified] PR-body claim about benchmark harness: PR #5852 exists (draft) and adds examples/offline_inference/minimax_h3/*; recipe numbers don't depend on it being merged. +- [claim-refuted] 'two-GPU leaves about 34.7 GiB below 73,415 MiB': 73415/1024=71.69 GiB and 71.69-36.38=35.31 GiB (four/eight-GPU figures 4.1/25.9 are correct) -> finding 2. +- [claim-refuted] two-GPU headroom 'about 34.7 GiB' is wrong: 73,415 MiB/1024=71.694 GiB; 71.694−36.38=35.31 GiB (not 34.7). The 4-GPU 4.1 and 8-GPU 25.9 figures are correct. +- [validated] All recipe flags exist on head in vllm_omni/entrypoints/cli/serve.py: --usp/--ring (460/478), --enable-distributed-layerwise-offload (636), --dlo-no-use-allgather (655), --dlo-resident-layers (663), --text-encoder-tp-size, --vae-patch-parallel-size, --vae-parallel-mode (choices include tile), --vae-use-tiling, --diffusion-attention-backend. +- [validated] CUDNN_ATTN is a registered backend enum member (vllm_omni/diffusion/attention/backends/registry.py:63). +- [validated] '50 denoise steps -> 49 updates' matches code: denoise_loop.py sets num_steps=len(sigmas_video)-1 and time_request.py _time_shift_sigmas(num_steps=50) yields 50 points. +- [validated] '124 frames' for duration 5.0 matches time_request.py:94 align_frame_count (round(5.0*24)=120 snapped to 17n+5 boundary = 124). +- [validated] Per-step latency recomputes: 504.69/49=10299.8ms, 541.90/49=11059.2ms, 278.73/49=5688ms, 299.85/49=6119ms, 159.40/49=3253ms, 167.25/49=3413ms. +- [validated] text_encoder_tp_size 2/4/8 are legal: pipeline_minimax_h3.py:546-556 requires <= dit_world and divides 64 heads/8 KV heads; VAE patch-parallel size equals the full DiT group size (2/4/8) on each route. +- [validated] All CLI flags in the three command blocks exist in vllm_omni/entrypoints/cli/serve.py: --omni, --num-gpus, --tensor-parallel-size, --usp/--ulysses-degree, --ring, --text-encoder-tp-size, --vae-patch-parallel-size, --vae-parallel-mode (choices include 'tile'), --vae-use-tiling, --diffusion-attention-backend, --enable-distributed-layerwise-offload, --dlo-no-use-allgather, --dlo-resident-layers. +- [validated] two-GPU combo --dlo-resident-layers 20 + --dlo-no-use-allgather is the only valid path: vllm_omni/diffusion/offloader/base.py:108-113 'dlo_resident_layers currently requires --dlo-no-use-allgather'. +- [validated] '50 denoise steps executes 49 denoise updates': denoise_loop.py:218 'num_steps = len(sigmas_video) - 1' and time_request.py _time_shift_sigmas returns linspace(1.0,0.0,num_steps)=50 points. + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:177` [minor] — Line 177 states the two-GPU route leaves "about 34.7 GiB" below the reported 73,415 MiB device capacity (the '73,415 MiB' text is line 178). 73,415 MiB = 71.694 GiB and table line 169 lists the two-GPU peak as 36.38 GiB, so headroom is 71.694 − 36.38 = 35.31 GiB (~35.3 GiB); the same formula yields 4.11 GiB and 25.86 GiB for the four- and eight-GPU routes. Fix 34.7 → ~35.3, or cite a peak that yields 34.7. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:177 "`nvidia-smi`. The two-GPU route leaves about 34.7 GiB below the reported" ; recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:178 "73,415 MiB device capacity, the four-GPU route leaves about 4.1 GiB, and the" ; recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:169 "| 2 | T2VA | TP1 x Ulysses2, DLO resident 20 | 515.57 | 1.19 | 504.69 | 8.79 | 10,300 | 36.38 |") + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:16` [minor] — The table claims `200 GiB minimum` available system RAM (line 16) while the two-GPU route runs `--tensor-parallel-size 1 --dlo-no-use-allgather` (lines 64, 73). In this mode base.py forces `dp_size=1` and streams the standard loader's rank-local tensors with H2D only, so each rank pins full (TP1, unsharded) DiT weights to host memory — two ranks on one host each retain a full DiT copy. Please re-derive the two-GPU host-RAM minimum (or publish measured host RSS) instead of carrying a figure from another parallelism route; the `384 GiB` recommendation may still be right. (evidence: ['recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:16 (added by this diff): `| Available system RAM | 200 GiB minimum |`', 'recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:64 (added by this diff): ` --tensor-parallel-size 1 \\`', 'recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:73 (added by this diff): ` --dlo-no-use-allgather \\`', 'vllm_omni/diffusion/offloader/base.py:100-102 (unchanged by this diff, present in the PR-time tree): ` # With dlo_use_allgather=False, do not add another DP shard. Each rank` / ` # streams the tensors produced by the standard loader, which may` / ` # already be TP-local shards. This avoids AllGather synchronization`', 'vllm_omni/diffusion/offloader/base.py:114-116 (unchanged by this diff, present in the PR-time tree): ` # If dlo_use_allgather=False, force dp_size=1 (each rank independent)` / ` if enable_distributed_layerwise_offload and not dlo_use_allgather:` / ` dp_size = 1`']) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:~122` [minor] — The three serving commands invoke `numactl` (lines 57, 92, and 122), but the recipe has no prerequisite/installation note for it, so hosts without numactl will fail before `vllm serve` starts. Add `numactl` to the Capacity/Prerequisites section (the canonical MiniMax-H3 recipe lists ffmpeg/ffprobe as prerequisites), or make the NUMA binding optional. (evidence: ['recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:57 `numactl --cpunodebind=0 --membind=0 \\`', 'recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:92 `numactl --cpunodebind=0 --membind=0 \\`', 'recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:122 `numactl --interleave=0,1 \\`', 'grep for `numactl` in recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md returns exactly the three command lines above; the file has no `Prerequisites` heading and no non-command numactl mention.']) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:~57` [minor] — The two- and four-GPU commands bind all allocations to NUMA node 0 (`--cpunodebind=0 --membind=0`, lines 57 and 92), but the capacity table (lines 16-17) only gives total system RAM (200 GiB min / 384 GiB recommended). The two-GPU route uses `--dlo-no-use-allgather` (line 73), which keeps rank-local weights in host memory and, per MiniMax-H3.md:72, needs at least 200 GiB available system RAM. With a strict node-0 bind, that memory must fit on one NUMA node; a symmetric 384 GiB dual-socket host has only ~192 GiB per node. Please add the per-node RAM requirement (or the validated host's per-node RAM) next to the capacity table/numactl lines. (evidence: new in this diff: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:57 `numactl --cpunodebind=0 --membind=0 \` (two-GPU section); recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:92 `numactl --cpunodebind=0 --membind=0 \` (four-GPU section); recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:16 `| Available system RAM | 200 GiB minimum |` and :17 `| Recommended system RAM | 384 GiB |`. Unchanged by this diff, present in the PR-time tree: recipes/MiniMaxAI/MiniMax-H3.md:72 `worker retains its standard-loader rank-local weights in host memory, including` followed by `Use at least 200 GiB available system RAM before starting the two-GPU recipe; a 384 GiB host is recommended`.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:~50` [minor] — This new recipe reports FL2VA/Ref2VA results and instructs a Ref2VA restart at lines 143-144, but it omits the HF checkpoint access/download steps, the ffmpeg/ffprobe PATH prerequisite, and FL2VA/Ref2VA request examples, so a reader cannot complete the flow from this page alone. Add a pointer to recipes/MiniMaxAI/MiniMax-H3.md (which has the download steps, prerequisites, and full T2VA/FL2VA/Ref2VA examples) near the top of the Capacity requirements section. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:143-144 `For Ref2VA, stop the FL2VA server and restart the same command with` / `MODEL="${MODEL_ROOT}/Ref2VA".`; recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:182 `## T2VA request example` (the only request example in the new file — no FL2VA/Ref2VA example). Unchanged by this diff, present in the PR-time tree: recipes/MiniMaxAI/MiniMax-H3.md:33 `hf download MiniMaxAI/MiniMax-H3 --local-dir "${MODEL_ROOT}"`; recipes/MiniMaxAI/MiniMax-H3.md:54 `` `ffmpeg` and `ffprobe` must be available on `PATH`. They are used for ``; recipes/MiniMaxAI/MiniMax-H3.md:415 `### 2. FL2VA: first frame to video and audio`; recipes/MiniMaxAI/MiniMax-H3.md:454 `### 3. Ref2VA: image-only, image/audio, or mixed references`.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-5000.md:1` [nit] — The new recipe is absent from the `recipes/README.md` 'Available Recipes' index: lines 48-49 list only `MiniMaxAI/MiniMax-H3.md` and `MiniMaxAI/MiniMax-H3-MUSA.md`, while this new RTX-PRO-5000 file and the existing `-5090.md` / `-NPU.md` variants are unlisted. Add a row for `MiniMax-H3-RTX-PRO-5000.md` so hardware variants stay discoverable, or document the convention if variants are intentionally excluded. (evidence: unchanged by this diff, present in the PR-time tree — recipes/README.md:48 `| [\`MiniMaxAI/MiniMax-H3.md\`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |` and recipes/README.md:49 `| [\`MiniMaxAI/MiniMax-H3-MUSA.md\`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |`; directory listing shows recipes/MiniMaxAI contains `MiniMax-H3-5090.md`, `MiniMax-H3-NPU.md`, and `MiniMax-H3-RTX-PRO-5000.md`, none of which appear in the README table.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.trace.json.gz new file mode 100644 index 000000000..056222fa2 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr5857.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.cost.json new file mode 100644 index 000000000..720365167 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 625.9, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr6045/run-20260815-052821-4322c9", + "recorded_at": "2026-08-15T05:38:45", + "attempts": [ + { + "invocation_id": "inv-4699f4a8", + "outcome": "run:run-20260815-052821-4322c9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-052821-4322c9", + "llm_calls": 62, + "input_tokens": 385564, + "output_tokens": 86163, + "usd": 0.3353, + "source": "spans" + } + ], + "attempt_usd_total": 0.3353, + "llm_calls": 62, + "input_tokens": 385564, + "output_tokens": 86163, + "usd": 0.3353, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-052821-4322c9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3353, + "minutes": 10.39, + "ci_minutes": 0, + "input_tokens": 385564, + "output_tokens": 86163, + "tool_calls": 131, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4739, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 62, + "by_role": { + "agent": { + "usd": 0.321531, + "input_tokens": 373243, + "output_tokens": 76704, + "calls": 60 + }, + "reducer": { + "usd": 0.01377, + "input_tokens": 12321, + "output_tokens": 9459, + "calls": 2 + } + }, + "cache_read_tokens": 5052544, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.25, + "gate": 2.17, + "review": 616.94, + "report": 0.06 + }, + "time_to_first_result_s": 623.41 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6045.trace.json.gz", + "trace_bytes": 27303 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.md new file mode 100644 index 000000000..ae80b4d9b --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6045, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head b112c7fdeed2) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] Weight-loading defaults match code: vllm_omni/entrypoints/cli/serve.py:635-644 defines `--disable-multithread-weight-load` (action=store_false, dest=enable_multithread_weight_load, default=True) and `--num-weight-load-threads` (default=4); omni_config.py:343-344 has enable=True / threads=Field(default=4, ge=1); diffusers_loader.py:245-253 gates on enable + safetensors and calls multi_thread_safetensors_weights_iterator with max_workers=num_threads. +- [claim-verified] Startup benchmark arithmetic: 168/27=6.222→6.2x and 283/56=5.054→5.1x; table labels them reference results. +- [claim-verified] Weight-loading anchor preserved: diffusion_features.md still contains `## Multi-Thread Weight Loading` redirecting to startup_and_loading.md. +- [claim-verified] Pipeline Parallelism was already absent from the user-guide nav at base (file_at_base docs/.nav.yml parallelism list had no pipeline_parallel.md); the diff only documents the existing deferral. +- [claim-verified] Pipeline Parallelism is absent from the User Guide nav at head and the guide file remains directly linked from features/README.md:60-61 (matches PR body's 'defer' intent). +- [validated] HunyuanImage3 step-execution claim: pipeline_hunyuan_image3.py:322 `supports_step_execution: ClassVar[bool] = True` and :1846-1853 raises for grouped DiT batching unless backend == TORCH_SDPA when max_num_seqs > 1 — matches note 5 and execution_modes.md#step-execution. +- [validated] Helios single-request step-execution claim: pipeline_helios.py:172 `supports_step_execution: ClassVar[bool] = True`; prepare_encode raises on len(req.prompts)>1 and denoise_step raises on len(states)!=1 — matches the VideoGen table note. +- [validated] HSDP compatibility correction matches config validation: omni_config.py:487-490 and data.py:298-303 reject tensor_parallel_size>1 or data_parallel_size>1 with use_hsdp, while sequence_parallel_size/cfg_parallel_size are folded into the HSDP world-size math — so Ulysses-SP/CFG-Parallel are allowed and TP is rejected. +- [validated] CPU-offload note 4 target exists: docs/user_guide/diffusion/cpu_offload.md:182 `## Distributed Layerwise Offloading` (anchor `#distributed-layerwise-offloading` resolves). +- [validated] startup_and_loading.md:13-14 defaults/flags — serve.py:635-640 defines --disable-multithread-weight-load (store_false, default True) and --num-weight-load-threads (default 4); vllm_omni/diffusion/data.py:731-732 defaults enable_multithread_weight_load=True, num_weight_load_threads=4. +- [validated] startup_and_loading.md benchmark arithmetic — calc 168/27=6.222 (6.2x) and 283/56=5.054 (5.1x); matches the doc's speedup column. +- [validated] HunyuanImage3 step-execution TORCH_SDPA constraint — pipeline_hunyuan_image3.py:1850-1853 raises for grouped batching without TORCH_SDPA; execution_modes.md:98-100 states the same. +- [validated] Helios single-request step execution — execution_modes.md:104 'Helios supports single-request step execution only: use --step-execution --max-num-seqs 1 for Helios.' +- [validated] HSDP+SP and HSDP+CFG are structurally permitted and TP/DP rejected — data.py:298-302 raises for TP/DP with use_hsdp, and world_size computation includes cfg_parallel_size; user_guide/diffusion/parallelism/hsdp.md documents 'Combined Mode' (Ulysses-SP, CFG-Parallel). + +`docs/user_guide/quantization/overview.md:~21` [major] — docs/.nav.yml removes the GGUF entry from the Quantization subtree, but the PR-time tree still promotes it: docs/user_guide/quantization/overview.md:21 keeps a GGUF Hardware-Support column and lines 49/66/82 keep GGUF rows linking to gguf.md, and the changed docs/user_guide/diffusion_features.md:95 still lists `[GGUF](quantization/gguf.md)`. The file docs/user_guide/quantization/gguf.md still exists. Either restore `- GGUF: user_guide/quantization/gguf.md` in the nav, or demote GGUF consistently (mark it OOT-plugin-only/direct-link in the overview, remove the GGUF column/rows there, and remove the GGUF row from diffusion_features.md). (evidence: docs/user_guide/quantization/overview.md:21 `| Device | FP8 W8A8 | Int8 W8A8 | ModelOpt | MXFP8 W8A8 | MXFP4 W4A4 | GGUF | AutoRound | msModelSlim |` (unchanged by this diff, present in PR-time tree); docs/user_guide/quantization/overview.md:49 `| GGUF | [GGUF](gguf.md) | Pre-quantized transformer weights | Qwen-Image | Validated where a model-specific GGUF adapter exists |` (unchanged); docs/user_guide/diffusion_features.md:95 `| **[GGUF](quantization/gguf.md)** | `quantization="gguf"` | Native GGUF transformer-only weights (Q4, Q8, etc.) | Memory reduction on consumer GPUs |` (present in PR-time tree; this diff keeps it); docs/.nav.yml Quantization subtree ends at `- AutoRound: user_guide/quantization/autoround.md` / `- msModelSlim: user_guide/quantization/msmodelslim.md` with no GGUF entry (diff removed `- GGUF: user_guide/quantization/gguf.md`); docs/user_guide/quantization/gguf.md exists in the tree.) + +`docs/features/README.md:60` [minor] — At docs/features/README.md:60-62, the Pipeline Parallelism deferral sentence omits the RFC #5919 link that the PR body itself provides ('...while its public contract is tracked in [RFC #5919](https://github.com/vllm-project/vllm-omni/issues/5919).'). Add that link so readers can find where the public contract is tracked. (evidence: docs/features/README.md:60-62 `The [Pipeline Parallelism guide](../user_guide/diffusion/parallelism/pipeline_parallel.md) +remains available by direct link, but it is not promoted in the primary User +Guide navigation until its user-facing support and placement are settled.` — no RFC link present. PR #6045 body (per gh_pr_view): `...defer Pipeline Parallelism from primary User Guide navigation while its public contract is tracked in [RFC #5919](https://github.com/vllm-project/vllm-omni/issues/5919).`) + +`docs/user_guide/diffusion_features.md:217` [minor] — Reword the note to: 'Multi-device Distributed Layerwise Offload's topology is documented in the [CPU Offloading guide](diffusion/cpu_offload.md#distributed-layerwise-offloading); its parallelism compatibility matrix is in the [design doc](../../design/feature/distributed_layerwise_offload.md#parallelism-compatibility).' (evidence: docs/user_guide/diffusion_features.md:217 ` compatibility matrix in the [CPU Offloading guide](diffusion/cpu_offload.md#distributed-layerwise-offloading).` — added by this PR. docs/user_guide/diffusion/cpu_offload.md:194-195 `For the implementation design and compatibility matrix, see` / `[Distributed Layerwise Offload](../../design/feature/distributed_layerwise_offload.md).` — unchanged by this diff, present in the PR-time tree. docs/design/feature/distributed_layerwise_offload.md:91 `## Parallelism compatibility` — unchanged by this diff, the section containing the actual matrix.) + +`docs/user_guide/diffusion_features.md:210` [minor] — Note 1 at docs/user_guide/diffusion_features.md:210-212 says 'other HSDP combinations in the table remain unverified', but the matrix itself gives definite HSDP verdicts for VAE Patch Parallel (✅ at line 203) and CPU Offloading (Layerwise) (❌ at line 201). Reword the note to acknowledge those specific verdicts instead of the blanket 'unverified' claim. (evidence: docs/user_guide/diffusion_features.md:210-212 (changed by this diff): `1. HSDP can be combined with Ulysses-SP or CFG-Parallel. Tensor Parallel` / ` and HSDP are not compatible; other HSDP combinations in the table` / ` remain unverified.` Contradicted in the same PR-time file by: docs/user_guide/diffusion_features.md:203 `| **💾VAE Patch Parallel** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | | | |` (HSDP column = ✅) and docs/user_guide/diffusion_features.md:201 `| **💾CPU Offloading (Layerwise)** | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | | | | | |` (HSDP column = ❌). The two matrix rows are outside the diff but present in the PR-time tree.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.trace.json.gz new file mode 100644 index 000000000..ffc644369 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6045.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.cost.json new file mode 100644 index 000000000..72ff4652d --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1043.0, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr6049/run-20260815-052823-f15a73", + "recorded_at": "2026-08-15T05:45:44", + "attempts": [ + { + "invocation_id": "inv-2edb99fd", + "outcome": "run:run-20260815-052823-f15a73", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-052823-f15a73", + "llm_calls": 155, + "input_tokens": 949685, + "output_tokens": 225768, + "usd": 0.829994, + "source": "spans" + } + ], + "attempt_usd_total": 0.829994, + "llm_calls": 155, + "input_tokens": 949685, + "output_tokens": 225768, + "usd": 0.829994, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-052823-f15a73", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.83, + "minutes": 17.35, + "ci_minutes": 0, + "input_tokens": 949685, + "output_tokens": 225768, + "tool_calls": 291, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8141, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 155, + "by_role": { + "agent": { + "usd": 0.800194, + "input_tokens": 928795, + "output_tokens": 203840, + "calls": 153 + }, + "reducer": { + "usd": 0.029799, + "input_tokens": 20890, + "output_tokens": 21928, + "calls": 2 + } + }, + "cache_read_tokens": 12045696, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.68, + "gate": 1.79, + "review": 1035.41, + "report": 0.06 + }, + "time_to_first_result_s": 1040.92 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6049.trace.json.gz", + "trace_bytes": 41533 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.md new file mode 100644 index 000000000..02dfc3f21 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.md @@ -0,0 +1,64 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6049, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0db924bc5444) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] prior concern 'LingBot invalid-media validation removed': resolved at vllm_omni/diffusion/models/lingbot_video/request_utils.py:156-177 (resolve_lingbot_mode still raises for multi-image/reference modalities), with coverage in tests/diffusion/models/lingbot_video/test_request_utils.py:56-76. +- [claim-verified] diff_stat shows exactly 14 files; no entrypoints/openai serving file changed, matching the 'public online serving handler unchanged' claim. +- [claim-verified] base I2V run_server.sh never contained BOUNDARY_RATIO/FLOW_SHIFT vars; the README removal fixes a pre-existing stale doc, not a script behavior change. +- [claim-verified] diff_stat: no entrypoints/serving files touched, so 'public online serving handler unchanged' holds. +- [claim-verified] diff_stat: run_curl_*.sh and serving_video.py absent from the diff, matching 'curl recipes unchanged'. +- [claim-verified] gh_ci_read: pre-commit SUCCESS matches the PR's pre-commit claim. +- [claim-verified] registry.py still wires Bagel/MammothModa2/Ming/VACE builders; read bagel.py, ming_flash_omni.py, mammothmodal2_preview.py, vace.py. +- [claim-verified] cosmos3.py and lingbot_video.py now keep only frozensets; registry specs dropped their builder entries. +- [claim-refuted] PR claim 'keep model-specific validation and normalization in the model pipeline' is false for Wan2.2/Hunyuan I2V: they accept but silently drop mask/reference-image conditioning. +- [validated] Claim 'leave the public online serving handler unchanged': diff_stat shows no vllm_omni/entrypoints files touched. +- [validated] Claim 'remove duplicated generic LingBot and Cosmos prompt builders': lingbot_video.py and cosmos3.py now contain only the extra-param constants. +- [validated] Claim 'preserve existing model-specific curl recipes': run_curl_image_to_video.sh and run_curl_text_to_image.sh are absent from diff_stat. +- [validated] LingBot T2I validation moved to pipeline: request_utils.py:218-220 rejects input images for modalities=["image"] and :403-404 requires num_frames=1. +- [validated] LingBot I2V num_frames dropped from the prompt dict is still resolved from sampling params: request_utils.py:395-396 `if explicit_num_frames is None and sampling.num_frames not in (None, 1)`. + +`vllm_omni/model_extras/registry.py:~316` [major] — The diff removes default_image_to_video_prompt and makes builder-less build_image_to_video_prompt return the envelope unchanged (registry.py:316-317), so multi-key multi_modal_data now flows through for Wan2.2, LTX2, Cosmos3, and HunyuanVideo-1.5. HunyuanVideo-1.5 reads only multi_modal_data['image'] (pipeline_hunyuan_video_1_5_i2v.py:496-497), so extra image inputs are silently ignored instead of the previous fast-fail ValueError. Restore a single-PIL-image validator for builder-less models or add per-model builders, and add a regression test with multi_modal_data={'image': img, 'last_image': img}. (evidence: PR-time tree vllm_omni/model_extras/registry.py:316-317: "if builder is None:\n return prompt" — builder-less I2V fallback forwards the envelope unchanged. The diff removed the previous guard from default_image_to_video_prompt: "if set(media_inputs) != {"image"} or not isinstance(media_inputs["image"], Image.Image):" raising ValueError("This model only supports a single --image input in the shared image-to-video example."). Unchanged by this diff but present in the PR-time tree, vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py:496-497: "multi_modal_data = req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else {}" / "raw_image = multi_modal_data.get("image", None)" — only the 'image' key is consumed, so extra media keys are silently ignored.) + +`.buildkite/amd/test-amd-ready.yml:~19` [minor] — Both buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-intel-ci are FAILURE at this head. AMD's core command at .buildkite/amd/test-amd-ready.yml:19 selects only tests/diffusion and tests/model_executor with 'core_model and cpu' (repeated at line 30), and Intel's command at .buildkite/intel/scripts/run-xpu-test.sh:75 selects 'core_model and xpu and B60'. The PR's new/changed tests are tests/examples/offline_inference/test_image_task_prompts.py (core_model+cpu) and tests/model_extras/test_model_extras.py (all tests core_model+cpu), so those selectors do not run them. Please confirm the failures are pre-existing on main or link the lane logs showing they are unrelated; do not merge while the gates are unexplained. (evidence: PR-time tree, unchanged by this diff: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/intel/scripts/run-xpu-test.sh:75 ` pytest -v -s -m "core_model and xpu and B60"`. New in this diff: tests/examples/offline_inference/test_image_task_prompts.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. Modified in this diff: tests/model_extras/test_model_extras.py:25 `@pytest.mark.core_model` and :26 `@pytest.mark.cpu` (all marked tests in the file follow this core_model+cpu pattern). gh_ci_read(pr=6049) returns `buildkite/vllm-omni-amd-ci`: FAILURE and `buildkite/vllm-omni-intel-ci`: FAILURE.) + +`vllm_omni/model_extras/registry.py:316` [minor] — registry.py:316-317 is a builder-less passthrough, and the PR deletes the old default_image_to_video_prompt guard that required exactly {"image"} and raised for any other key. Wan22I2VPipeline consumes multi_modal_data["last_image"] (pipeline_wan2_2_i2v.py:471-472), so this now silently enables first/last-frame conditioning that the old guard rejected, and unsupported keys such as mask/reference_images pass through and are silently ignored. HunyuanVideo-1.5 I2V likewise reads only multi_modal_data["image"] (pipeline_hunyuan_video_1_5_i2v.py:497). Add per-model builders/validators or update the PR description/docs to document the new Wan2.2 conditioning and the permissive passthrough. (evidence: Changed by this PR — vllm_omni/model_extras/registry.py:316-317: ` if builder is None:` / ` return prompt` (inside build_image_to_video_prompt; returns the example envelope unchanged when no per-model builder exists). Removed by this PR — old default_image_to_video_prompt in the same file: `if set(media_inputs) != {"image"} or not isinstance(media_inputs["image"], Image.Image):` / `raise ValueError("This model only supports a single --image input in the shared image-to-video example.")`. Unchanged by this diff, present in PR-time tree — vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py:471-472: `if multi_modal_data is not None:` / `last_image = multi_modal_data.get("last_image", None)` (Wan2.2 consumes last_image, enabling first/last-frame conditioning now that the guard is gone). Unchanged by this diff, present in PR-time tree — vllm_omni/diffusion/models/hunyuan_video/pipeline_hunyuan_video_1_5_i2v.py:497: `raw_image = multi_modal_data.get("image", None)` (Hunyuan I2V reads only image).) + +`examples/online_serving/image_to_video/README.md:~26` [minor] — The diff documents `PORT` default `8099` at examples/online_serving/image_to_video/README.md:26, but Basic Start at line 13 and all inline curl examples (lines 78, 100, 149, 168, 179-180, 188, 201, 209, 248, 254, 260, 266, 273) still hard-code `8091`. Update them to `8099` or explicitly note that the override applies only to `run_server.sh`. (evidence: diff changed examples/online_serving/image_to_video/README.md:26 to `- `PORT` (default: `8099`)`. Unchanged by this diff, present in the PR-time tree: examples/online_serving/image_to_video/README.md:13 `vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers --omni --port 8091`; examples/online_serving/image_to_video/README.md:78 `curl -X POST http://localhost:8091/v1/videos/sync \`; examples/online_serving/image_to_video/README.md:188 `curl -X POST http://localhost:8091/v1/videos \`.) + +`examples/online_serving/text_to_image/openai_chat_client.py:~65` [minor] — Confirmed at openai_chat_client.py:63-67. Because --height/--width now default to None (lines 122-123), the single-dimension branches are reachable: e.g. `--width 512` sends `size=512x512` instead of omitting size and using server/model defaults. Recommend omitting `size` unless both dimensions are supplied (or rejecting a lone dimension). (evidence: Changed by this diff: examples/online_serving/text_to_image/openai_chat_client.py:122 ` parser.add_argument("--height", type=int, help="Image height; defaults to the model setting")` (no default => None; same for --width at 123). Unchanged by this diff, present in the PR-time tree: examples/online_serving/text_to_image/openai_chat_client.py:63-68 ` if width is not None and height is not None: + payload["size"] = f"{width}x{height}" + elif width is not None: + payload["size"] = f"{width}x{width}" + elif height is not None: + payload["size"] = f"{height}x{height}"` — with height/width now defaulting to None, the single-dimension square-size branches are reachable.) + +`vllm_omni/model_extras/registry.py:~271` [minor] — The diff changes the exported build_text_to_image_prompt (registry.py:270) and build_image_to_video_prompt (registry.py:306) to accept a canonical envelope dict and removes the separate negative_prompt/media_inputs parameters. Both names are exported via vllm_omni/model_extras/__init__.py __all__ (lines 7 and 19), so external example/notebook code using the previous (model_class_name, prompt, negative_prompt, ...) signature now raises a TypeError with no deprecation path. Please add a small str|dict compatibility shim for the exported builders or explicitly flag this breaking change in the release notes. (evidence: vllm_omni/model_extras/registry.py:270-275 `def build_text_to_image_prompt(\n model_class_name: str | None,\n prompt: dict[str, Any],\n height: int | None = None,\n width: int | None = None,\n) -> dict[str, Any]:` — no separate `negative_prompt` parameter; vllm_omni/model_extras/registry.py:306-312 `def build_image_to_video_prompt(\n model_class_name: str | None,\n prompt: dict[str, Any],\n height: int | None = None,\n width: int | None = None,\n num_frames: int | None = None,\n) -> dict[str, Any]:` — no `media_inputs` parameter; vllm_omni/model_extras/__init__.py:7 ` build_text_to_image_prompt,` and :19 ` "build_text_to_image_prompt",` — both builders remain in the package import and `__all__`, so the changed signatures are publicly exposed.) + +`examples/online_serving/text_to_image/gradio_demo.py:~82` [minor] — The diff relabels gradio_demo.py as a generic 'Shared text-to-image Gradio demo' and the README now lists `num_inference_steps`/`true_cfg_scale` defaults as 'model', but the Gradio UI still hard-codes Qwen-Image values (height/width 1024, steps 100, cfg_scale 4.0) and generate_image() always puts `true_cfg_scale` into `extra_body` (gradio_demo.py:40). Running this against the now-generic MODEL override can send Qwen-specific knobs to models that do not support them. Make the sliders omit unset values like openai_chat_client.py does, or keep this demo explicitly Qwen-specific. (evidence: examples/online_serving/text_to_image/gradio_demo.py:40 `"true_cfg_scale": cfg_scale,` (unchanged by this diff, present in the PR-time tree); examples/online_serving/text_to_image/gradio_demo.py:118 `value=100,`; examples/online_serving/text_to_image/gradio_demo.py:125 `value=4.0,`; examples/online_serving/text_to_image/gradio_demo.py:102 `value=1024,`; examples/online_serving/text_to_image/gradio_demo.py:109 `value=1024,`; examples/online_serving/text_to_image/README.md:232 `| `num_inference_steps` | int | model | Number of denoising steps |`; examples/online_serving/text_to_image/README.md:233 `| `true_cfg_scale` | float | model | True CFG scale when supported |`; examples/online_serving/text_to_image/openai_chat_client.py:81 `if true_cfg_scale is not None:` followed by :82 `payload["true_cfg_scale"] = true_cfg_scale` — sibling client in the same diff omits unset values.) + +`tests/examples/offline_inference/test_image_task_prompts.py:33` [minor] — The I2V canonical-envelope test at `tests/examples/offline_inference/test_image_task_prompts.py:32` only exercises `negative_prompt="flicker"`, while the shared runner forces `negative_prompt=""` for non-LTX models (`examples/offline_inference/image_to_video/image_to_video.py:538-541`), so the empty-string branch of the I2V builder is never covered. Parametrize this test with `(None, None), ("", ""), ("flicker", "flicker")` and assert key presence/absence accordingly. (evidence: tests/examples/offline_inference/test_image_task_prompts.py:32-37 `def test_image_to_video_builds_canonical_prompt() -> None:` / ` result = build_image_to_video_prompt(` / ` prompt="the fox turns toward the camera",` / ` negative_prompt="flicker",` — I2V test only exercises the non-empty string case. tests/examples/offline_inference/test_image_task_prompts.py:14-16 `@pytest.mark.parametrize(` / ` ("negative_prompt", "expected_negative_prompt"),` / ` [(None, None), ("", ""), ("blurry", "blurry")],` — the T2I test already parametrizes None/''/non-empty, so the gap is I2V-specific. examples/offline_inference/image_to_video/image_to_video.py:538-541 (unchanged by this diff, present in PR-time tree) ` negative_prompt = args.negative_prompt` / ` if negative_prompt is None and not is_ltx2:` / ` # Preserve the historical empty-prompt behavior for non-LTX examples.` / ` negative_prompt = ""` — the shared runner forces the empty string for non-LTX models, meaning the untested empty-string branch is the path those models actually take.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.trace.json.gz new file mode 100644 index 000000000..6fa45d049 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6049.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.cost.json new file mode 100644 index 000000000..ba9e12865 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1536.0, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr6079/run-20260815-053146-841149", + "recorded_at": "2026-08-15T05:57:20", + "attempts": [ + { + "invocation_id": "inv-24b78e8b", + "outcome": "run:run-20260815-053146-841149", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-053146-841149", + "llm_calls": 104, + "input_tokens": 509382, + "output_tokens": 200356, + "usd": 0.502994, + "source": "spans" + } + ], + "attempt_usd_total": 0.502994, + "llm_calls": 104, + "input_tokens": 509382, + "output_tokens": 200356, + "usd": 0.502994, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-053146-841149", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.503, + "minutes": 25.56, + "ci_minutes": 0, + "input_tokens": 509382, + "output_tokens": 200356, + "tool_calls": 205, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8254, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 104, + "by_role": { + "planner": { + "usd": 0.00522, + "input_tokens": 16, + "output_tokens": 4694, + "calls": 1 + }, + "agent": { + "usd": 0.475689, + "input_tokens": 500042, + "output_tokens": 177908, + "calls": 101 + }, + "reducer": { + "usd": 0.022085, + "input_tokens": 9324, + "output_tokens": 17754, + "calls": 2 + } + }, + "cache_read_tokens": 5372928, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.91, + "gate": 1.6, + "review": 1528.02, + "report": 0.07 + }, + "time_to_first_result_s": 1533.59 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6079.trace.json.gz", + "trace_bytes": 27167 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.md new file mode 100644 index 000000000..897b2fd64 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.md @@ -0,0 +1,76 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6079, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head c171be607d6d) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] A3 perf-run concern: gh_ci_read(6079) shows buildkite/vllm-omni-npu-ci SUCCESS, so the new NPU ready perf step passed at head. +- [claim-verified] mark core_model (vs nightly full_model) and concurrency [1,4]/num_prompts [10,40] vs nightly [1,4,8]/[32,64,128] — matches PR body. +- [claim-verified] NPU step env and mark convention match the sibling MiniCPM-o steps in .buildkite/npu/test-npu-ready.yml. +- [claim-verified] Neither new command passes --assert-baseline; assert_result only checks result['completed'] == num_prompt (run_benchmark.py:180). +- [claim-verified] 'same as existing nightly perf jobs' — nightly MiniCPM-o perf (test-nightly.yml:125) also lacks --assert-baseline, but DOES upload artifacts. +- [claim-verified] Deploy config, dataset_name/path, backend, endpoint, extra_body, percentile-metrics all match nightly test_minicpmo_4_5.json. +- [claim-verified] Mark change full_model->core_model and sweep [1,4,8]->[1,4], num_prompts [32,64,128]->[10,40] are as described. +- [claim-verified] Both steps omit --assert-baseline; run_benchmark.py:161 assert_result only checks completed == num_prompt. +- [claim-verified] source_file_dependencies drops unmatched steps (.buildkite/common/scripts/upload_pipeline.py:306). +- [claim-verified] gh_ci_read(6079): cuda/npu/build/pre-commit/DCO pass; amd and intel fail. +- [claim-refuted] PR body says CUDA step is gated on 'the runner', but source_file_dependencies (cuda/test-ready.yml:233-236) omits tests/dfx/perf/scripts/run_benchmark.py and tests/dfx/conftest.py. +- [claim-refuted] 'Full run on H100 (1x)' and 'Full run on A3 (1x)' remain unchecked and wall-clock is 'fill after' in the merged PR body. +- [claim-refuted] PR body says CUDA step is gated on the runner, but .buildkite/cuda/test-ready.yml:232-235 list omits tests/dfx/perf/scripts/run_benchmark.py. +- [claim-refuted] 'same as existing nightly perf jobs': nightly MiniCPM-o perf uploads result JSON (.buildkite/cuda/test-nightly.yml:127); ready steps do not upload anything. + +`.buildkite/cuda/test-ready.yml:~239` [major] — The new CUDA/NPU ready perf steps run `pytest ... run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json ...` without `--assert-baseline` and never upload `tests/dfx/perf/results/*.json` (nightly does at .buildkite/cuda/test-nightly.yml:127). The runner's only hard assertion is `assert result["completed"] == num_prompt` (tests/dfx/perf/scripts/run_benchmark.py:161), and `--assert-baseline` (tests/dfx/conftest.py:756, default=False) has no consumer, so a TTFT/e2el/throughput regression that completes all requests passes green and its JSONs are discarded. The PR body's opening purpose says this makes per-PR performance regressions visible, which the wiring section partly concedes ('gate on request failures rather than baseline deltas'). Either assert/upload baselines (and add an A3 bucket for NPU) or relabel/reword the step as request-failure smoke coverage. (evidence: .buildkite/cuda/test-ready.yml:239 `- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'` — the new ready step has no `--assert-baseline` and no `buildkite-agent artifact upload`; tests/dfx/perf/scripts/run_benchmark.py:161 `assert result["completed"] == num_prompt, "Request failures exist"` — the runner's only hard assertion, unchanged by this diff; tests/dfx/conftest.py:756 `"--assert-baseline",` with `default=False` and repo-wide grep finding no consumer, unchanged by this diff; .buildkite/cuda/test-nightly.yml:127 `buildkite-agent artifact upload "tests/dfx/perf/results/*.json"` — unchanged by this diff, present in the PR-time tree, showing the nightly upload the ready steps lack.) + +`.buildkite/cuda/test-ready.yml:~233` [minor] — The CUDA step at .buildkite/cuda/test-ready.yml:232 gates only on the perf JSON, deploy yaml, model dir, and stage input processor, while its command invokes tests/dfx/perf/scripts/run_benchmark.py. Since upload_pipeline.py drops a step when no changed file matches source_file_dependencies, edits to the benchmark runner or the pytest helper tests/dfx/conftest.py (which registers --test-config-file) will not trigger this gate. Add tests/dfx/perf/scripts/run_benchmark.py and tests/dfx/conftest.py to source_file_dependencies. (evidence: .buildkite/cuda/test-ready.yml:232-239 (unchanged by this diff, present in the PR-time tree): `source_file_dependencies:` / `- tests/dfx/perf/tests/test_minicpmo_4_5_ready.json` / `- vllm_omni/deploy/minicpmo_4_5.yaml` / `- vllm_omni/model_executor/models/minicpmo_4_5/` / `- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py` / `commands:` / `- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'` — the command references the runner, but the dependency list omits it. .buildkite/common/scripts/upload_pipeline.py:300,305-307: `deps = step.get("source_file_dependencies")`; `if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):` / `_log(f"skip {_get_step_label(step)!r} (no changes under {deps})")` / `continue` — a step is dropped when no changed file matches its dependencies.) + +`tests/dfx/conftest.py:756` [minor] — Remove the unused `--assert-baseline` option registered at `tests/dfx/conftest.py:756` or implement a consumer for it. It defaults to False and no code reads `getoption('assert_baseline')`, so passing the option would not enforce baseline comparisons. (Pre-existing dead code, unrelated to this PR's diff.) (evidence: tests/dfx/conftest.py:756 `"--assert-baseline",` — registered inside `pytest_addoption` followed by `action="store_true",` and `default=False,`; repo-wide grep for `assert_baseline` and `assert-baseline` returns only this registration and no `getoption("assert_baseline")` consumer. This line is unchanged by the diff but present in the PR-time tree.) + +`.buildkite/npu/test-npu-ready.yml:~28` [minor] — The new NPU perf step (.buildkite/npu/test-npu-ready.yml:20-28) has timeout_in_minutes: 60 but no source_file_dependencies, unlike the CUDA twin (.buildkite/cuda/test-ready.yml:232-236). tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:53-54 has only an H100 baseline, and .buildkite/npu/test-npu-nightly.yml contains no MiniCPM-o perf step, so the NPU run has no A3 baseline comparison. Add source_file_dependencies (or justify the unconditional A3 cost) and add an A3 baseline bucket if NPU regression tracking is intended. (evidence: Added by this PR, .buildkite/npu/test-npu-ready.yml:20-28 has no source_file_dependencies: + - label: "Omni · MiniCPM-o 4.5 Perf Test" + timeout_in_minutes: 60 + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni' +CUDA twin added by this PR, .buildkite/cuda/test-ready.yml:232-236: + source_file_dependencies: + - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json + - vllm_omni/deploy/minicpmo_4_5.yaml + - vllm_omni/model_executor/models/minicpmo_4_5/ + - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py +tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:53-54 contains only H100: + "baseline": { + "H100": { +grep 'minicpmo' .buildkite/npu/test-npu-nightly.yml returns only functional/accuracy steps (lines 49, 60-61, 74-76); the only run_benchmark.py steps there are qwen3_omni (lines 22, 38) and tts (line 95), no MiniCPM-o perf step.) + +`tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:~31` [minor] — The new ready JSON sets `num_prompts: [10, 40]` with `max_concurrency: [1, 4]`, but `baseline.H100` carries `[0.7803, 1.1008]` etc., which are the first two sweep-index values from the nightly config whose first two sweep points run `num_prompts: [32, 64]`. Since `run_benchmark.py` pairs `max_concurrency` and `num_prompts` by zip and resolves baseline lists by sweep index, these thresholds describe (1,32)/(4,64), not the (1,10)/(4,40) workload this config actually runs. Re-measure the baseline at [10,40], or drop the baseline block from the ready config. (evidence: tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:31-34 `"num_prompts": [ 10, 40 ],` `"max_concurrency": [ 1, 4 ],`; tests/dfx/perf/tests/test_minicpmo_4_5_ready.json:55-58 `"request_throughput": [ 0.7803, 1.1008 ],`; tests/dfx/perf/tests/test_minicpmo_4_5.json:31-35 `"num_prompts": [ 32, 64, 128 ],` `"max_concurrency": [ 1, 4, 8 ],`; tests/dfx/perf/tests/test_minicpmo_4_5.json:56-59 `"request_throughput": [ 0.7803, 1.1008, 1.2027 ],`; tests/dfx/perf/scripts/run_benchmark.py:262-274 (concurrency sweep) `for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):` ... `baseline_config=params.get("baseline"), sweep_index=sweep_index,`; tests/dfx/conftest.py:577-581 `def resolve_baseline_value(...): ... return baseline_raw[sweep_index]` — index-resolved baseline unchanged by this diff but present in the PR-time tree.) + +`.buildkite/npu/test-npu-ready.yml:~21` [minor] — Both new ready-gate perf steps set `timeout_in_minutes: 60` (.buildkite/cuda/test-ready.yml:231 and .buildkite/npu/test-npu-ready.yml:21), while the PR body leaves `- [ ] Full run on H100 (1x)` and `- [ ] Full run on A3 (1x)` unchecked and says `(fill in wall-clock time and measured metrics from the first H100 / A3 runs)`. The nightly MiniCPM-o perf step (.buildkite/cuda/test-nightly.yml:118-120) gets `timeout_in_minutes: 180`. The 60-minute budget for the trimmed ready sweep is therefore an unvalidated assumption; run the exact ready commands once per hardware and record wall-clock before relying on this gate, otherwise ready-label PRs may hard-fail on timeout. (evidence: PR-time tree, added by this diff: `.buildkite/npu/test-npu-ready.yml:21` `timeout_in_minutes: 60` (new 'Omni · MiniCPM-o 4.5 Perf Test' step, label at line 20). PR-time tree, added by this diff: `.buildkite/cuda/test-ready.yml:231` `timeout_in_minutes: 60` (same step, label at line 230). Unchanged by this diff, present in PR-time tree: `.buildkite/cuda/test-nightly.yml:120` `timeout_in_minutes: 180` under `.buildkite/cuda/test-nightly.yml:118` `- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"`. PR body (gh_pr_view): `- [ ] Full run on H100 (1x)`, `- [ ] Full run on A3 (1x)`, and `(fill in wall-clock time and measured metrics from the first H100 / A3 runs)`.) + +`.buildkite/cuda/test-ready.yml:230` [nit] — This diff only adds MiniCPM-o 4.5 perf steps to `.buildkite/cuda/test-ready.yml` and `.buildkite/npu/test-npu-ready.yml` plus a new `tests/dfx/perf/tests/test_minicpmo_4_5_ready.json`; it does not touch AMD or Intel lanes. The failing checks are `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-intel-ci`; the AMD ready lane runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py ...` and the Intel lane runs `bash .buildkite/intel/scripts/run-xpu-test.sh`, neither of which references this diff's YAMLs or perf JSON. Confirm those failures are pre-existing on main; no change required in this PR for them. (evidence: Anchor: `.buildkite/cuda/test-ready.yml:230` `- label: "Omni · MiniCPM-o 4.5 Perf Test"` (the comment cited line 229; actual line is 230). Diff scope (from diff_stat): only `.buildkite/cuda/test-ready.yml`, `.buildkite/npu/test-npu-ready.yml`, `tests/dfx/perf/tests/test_minicpmo_4_5_ready.json`. AMD lane (unchanged by this diff, present in PR-time tree): `.buildkite/amd/test-amd-ready.yml:19` `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`. Intel lane (unchanged by this diff, present in PR-time tree): `.buildkite/intel/pipeline-intel.yml:6` `command: "bash .buildkite/intel/scripts/run-xpu-test.sh"`; the script's test body `.buildkite/intel/scripts/run-xpu-test.sh:85-88` runs `pytest -v -s -m "core_model and xpu and B60"`, `pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py`, `pytest -v -s -m "advanced_model and xpu and B60"`, `pytest -v -s -m "omni and xpu and B60"` — no reference to `test_minicpmo_4_5_ready.json` or the MiniCPM-o perf step. CI metadata: gh_ci_read(6079) shows `buildkite/vllm-omni-amd-ci` = FAILURE and `buildkite/vllm-omni-intel-ci` = FAILURE.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.trace.json.gz new file mode 100644 index 000000000..71a04fe34 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6079.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.cost.json b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.cost.json new file mode 100644 index 000000000..8daddd58b --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 575.1, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_holdout3_r2/runs/pr6141/run-20260815-053535-5092f4", + "recorded_at": "2026-08-15T05:45:08", + "attempts": [ + { + "invocation_id": "inv-15122bcb", + "outcome": "run:run-20260815-053535-5092f4", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-053535-5092f4", + "llm_calls": 38, + "input_tokens": 146189, + "output_tokens": 74342, + "usd": 0.168373, + "source": "spans" + } + ], + "attempt_usd_total": 0.168373, + "llm_calls": 38, + "input_tokens": 146189, + "output_tokens": 74342, + "usd": 0.168373, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-053535-5092f4", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1684, + "minutes": 9.55, + "ci_minutes": 0, + "input_tokens": 146189, + "output_tokens": 74342, + "tool_calls": 110, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3783, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 38, + "by_role": { + "agent": { + "usd": 0.151981, + "input_tokens": 138517, + "output_tokens": 61358, + "calls": 36 + }, + "reducer": { + "usd": 0.016392, + "input_tokens": 7672, + "output_tokens": 12984, + "calls": 2 + } + }, + "cache_read_tokens": 1745408, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.4, + "gate": 2.52, + "review": 566.68, + "report": 0.06 + }, + "time_to_first_result_s": 572.64 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6141.trace.json.gz", + "trace_bytes": 20491 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.md b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.md new file mode 100644 index 000000000..6d71166d3 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6141, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head b031feef060f) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | 2 finding(s) below | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] 'only remaining community/volunteers reference is the intentional redirect': repo-wide grep 'volunteer' matches only mkdocs.yml:77 redirect. +- [claim-verified] meetups removal: grep 'meetup' matches only mkdocs.yml:76 redirect; no dangling links to the deleted pages. +- [claim-verified] superseded claims absent: grep for '19:30 PDT', 'tinyurl.com/vllm-omni-meeting', 'Security Advisories', 'bi-weekly', 'issues/165', 'issues/1057' -> no matches in docs. +- [claim-verified] docs-only scope: diff_stat shows only docs/** and mkdocs.yml touched; no runtime code. +- [claim-verified] docs build/pre-commit: CI docs/readthedocs.org:vllm-omni SUCCESS and buildkite/omni-release SUCCESS corroborate 'mkdocs build --strict'; pre-commit CI SUCCESS corroborates pre-commit claim. +- [claim-verified] repo-wide grep `volunteers` -> only mkdocs.yml:77 (the intentional redirect); the PR's 'only remaining community/volunteers reference' statement holds. +- [claim-verified] repo-wide grep `meetups` -> only mkdocs.yml:76 (the intentional redirect); no stale meetups references remain. +- [claim-verified] docs grep for `19:30`, `tinyurl`, `bi-weekly` -> no matches; duplicated/expired meeting schedules were fully removed. +- [validated] docs/community/governance.md:24 `### Lead Maintainers` exists -> contact_us.md `governance.md#lead-maintainers` anchor resolves. +- [validated] .github/CODEOWNERS exists and is path-based -> governance 'Reviewer Routing' GitHub link is valid. +- [validated] docs/design/index.md:71 `## Module Design Documents` exists -> governance `../design/index.md#module-design-documents` anchor resolves. +- [validated] docs/serving/video_stream_api.md exists and its `/v1/video/chat/stream` endpoint is real (vllm_omni/entrypoints/openai/api_server.py:1675) -> FAQ streaming-video link is code-backed. +- [validated] docs/user_guide/diffusion/execution_modes.md:107 `## Streaming Output` exists -> FAQ `#streaming-output` anchor resolves. +- [validated] docs/README.md:55 `Full-duplex realtime serving with streaming audio input and output (experimental)` supports the FAQ's full-duplex claim. + +`docs/community/contact_us.md:~9` [minor] — This diff replaces the GitHub Security Advisories route with 'send a direct message to one of the lead maintainers in vLLM Slack' (docs/community/contact_us.md:9), but the linked lead-maintainer roster (docs/community/governance.md:28-30) lists only GitHub profile URLs for @Gaohan123, @hsliuustc0106, and @ywang96 — no Slack handle or alias — so the primary DM instruction is not directly actionable. Add each lead maintainer's Slack handle/display name next to the roster entries or inline here, or make the existing detail-free sig-omni request the primary route. (evidence: docs/community/contact_us.md:9 (added by this diff): "To arrange a private report, send a direct message to one of the [lead maintainers](governance.md#lead-maintainers) in [vLLM Slack](https://slack.vllm.ai/)." The linked roster is unchanged by this diff but present in the PR-time tree: docs/community/governance.md:28 "- [@Gaohan123](https://github.com/Gaohan123)"; docs/community/governance.md:29 "- [@hsliuustc0106](https://github.com/hsliuustc0106)"; docs/community/governance.md:30 "- [@ywang96](https://github.com/ywang96)" — no Slack handle, username, or alias appears anywhere in the Lead Maintainers section.) + +`docs/usage/faq.md:17` [minor] — Confirmed: docs/usage/faq.md:17 sends AWQ readers to the quantization overview 'for validated combinations and method-specific guides', but docs/user_guide/quantization/overview.md contains no AWQ entry at all (grep for 'AWQ' returns no matches). The Quantization Modes table covers only Online quantization, Runtime attention quantization, and Pre-quantized checkpoints, so an AWQ seeker gets no AWQ-specific validation or guide. Add an AWQ row/note scoped to upstream-inherited AR language-model stages and marked not validated, or reword the FAQ sentence to say AWQ is resolved via the upstream vLLM registry but is not evaluated or documented in vLLM-Omni's validation matrix. (evidence: docs/usage/faq.md:17 (changed by this diff) `A: AWQ is available through the inherited vLLM quantization registry, and vLLM-Omni supports several additional methods. Actual support and validation vary by model, component, hardware, and checkpoint. See the [quantization overview](../user_guide/quantization/overview.md) for validated combinations and method-specific guides.`; docs/user_guide/quantization/overview.md:15 (unchanged by this diff, present in the PR-time tree) `| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, BitsAndBytes W4, MXFP8 W8A8, MXFP4 W4A4 |`; docs/user_guide/quantization/overview.md:17 (unchanged by this diff) `| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |`; grep for literal `AWQ` in docs/user_guide/quantization/overview.md returns no matches.) + +`docs/community/contact_us.md:9` [minor] — Add a repo-root SECURITY.md that points to the Slack-DM route in docs/community/contact_us.md#security-disclosures (or link an explicit follow-up issue). The PR removes the old public security-advisory path but adds no conventional SECURITY.md at the repository root. (evidence: docs/community/contact_us.md:9 `Do not share vulnerability details in public GitHub issues, pull requests, discussions, or Slack channels. To arrange a private report, send a direct message to one of the [lead maintainers](governance.md#lead-maintainers) in [vLLM Slack](https://slack.vllm.ai/).` — in-diff line confirming the Slack-DM route the comment anchors to. PR-time tree absence (outside the diff): `list_dir('.')` and `list_dir('.github')` return no `SECURITY.md`, and `grep 'SECURITY.md'` returns no matches.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.trace.json.gz b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.trace.json.gz new file mode 100644 index 000000000..bbb8c627a Binary files /dev/null and b/eval/dataset/arms/copilot_v14_holdout3_r2/pr6141.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_smoke/manifest.json b/eval/dataset/arms/copilot_v14_smoke/manifest.json new file mode 100644 index 000000000..d2dc77eec --- /dev/null +++ b/eval/dataset/arms/copilot_v14_smoke/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v14_smoke", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_smoke/pr5009.cost.json b/eval/dataset/arms/copilot_v14_smoke/pr5009.cost.json new file mode 100644 index 000000000..e52cd9d4f --- /dev/null +++ b/eval/dataset/arms/copilot_v14_smoke/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1691.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_smoke/runs/pr5009/run-20260815-021542-ef0eda", + "recorded_at": "2026-08-15T02:43:51", + "attempts": [ + { + "invocation_id": "inv-e5eba6ee", + "outcome": "run:run-20260815-021542-ef0eda", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-021542-ef0eda", + "llm_calls": 181, + "input_tokens": 1329633, + "output_tokens": 307088, + "usd": 1.117099, + "source": "spans" + } + ], + "attempt_usd_total": 1.117099, + "llm_calls": 181, + "input_tokens": 1329633, + "output_tokens": 307088, + "usd": 1.117099, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-021542-ef0eda", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.1171, + "minutes": 28.16, + "ci_minutes": 0, + "input_tokens": 1329633, + "output_tokens": 307088, + "tool_calls": 365, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0968, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 181, + "by_role": { + "agent": { + "usd": 1.072655, + "input_tokens": 1296541, + "output_tokens": 274826, + "calls": 178 + }, + "reducer": { + "usd": 0.044444, + "input_tokens": 33092, + "output_tokens": 32262, + "calls": 3 + } + }, + "cache_read_tokens": 15566720, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.62, + "gate": 1.85, + "review": 1682.81, + "report": 0.06 + }, + "time_to_first_result_s": 1689.33 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 43635 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_smoke/pr5009.md b/eval/dataset/arms/copilot_v14_smoke/pr5009.md new file mode 100644 index 000000000..eb1d23650 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_smoke/pr5009.md @@ -0,0 +1,62 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] prior concern 'global flip will break Cosmos3 native-only': resolved — registry override reapplies native after the platform default at vllm_omni/diffusion/worker/diffusion_worker.py:176-177. +- [resolved] prior concern 'nn.RMSNorm breaks W4A16 weight loading': no evidence of breakage — both vllm RMSNorm and nn.RMSNorm expose a `weight` parameter and the PR claims `test_qwen_image_autoround_w4a16_load/_generate` pass (tests exist at tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py). +- [resolved] dependency floor: nn.RMSNorm is a restore of #4074 (not a new external call); vLLM 0.24-era torch baseline is >=2.4, so nn.RMSNorm availability is not a new lower-bound issue. +- [resolved] oink monkeypatch: platform.py imports 'import vllm.envs as envs' and reads envs.VLLM_USE_OINK_OPS at call time, so the test's monkeypatch.setattr on the same module object is effective. +- [resolved] removed import leaves no dangling reference — `import torch.nn as nn` is still present (qwen_image_transformer.py:10) and no other qwen_image file references the removed vllm RMSNorm. +- [resolved] quant routing unaffected by the norm class switch — vllm_omni/quantization/component_config.py:56-69 `safe_quant_config` only propagates INCConfig to norm layers and routes by prefix, not by RMSNorm module type. +- [claim-verified] PR says pre-#4709 CUDA default was always vllm_c: `search_history(using_inductor, vllm_omni/platforms/cuda/platform.py)` returns a560ed18 (#4709) as the commit that added the branch and b73e0355 as the one removing it. +- [claim-verified] PR says #3588 accidentally reverted #4074: `search_history('from vllm.model_executor.layers.layernorm import RMSNorm', ...qwen_image_transformer.py)` shows 99234881 (#3588) re-introducing the import and b5589481 (this PR) removing it; 35ee3c75 (#4074) had switched the same sites to nn.RMSNorm. +- [claim-verified] PR says models needing native-only have per-model overrides: vllm_omni/diffusion/registry.py:545 maps `Cosmos3OmniDiffusersPipeline` to `get_cosmos3_ir_op_priority_func`, and vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:621-622 sets `priority_kwargs["rms_norm"]=["native"]` and `fused_add_rms_norm=["native"]`. +- [claim-verified] PR arithmetic via calc: 1.7263->1.2613 = -26.94%; 1288->1080 = -16.15%; 2.4960->2.4502 = -1.83%; 2.4502 vs 2.2847 = +7.24%; issue table 4351.35 vs 2421.5 = -79.70% — all match the description. +- [claim-verified] 'pre-#4709 always preferred vllm_c' — search_history shows `using_inductor` added by a560ed18 (#4709 rebase) and removed by b73e0355 (this PR); no earlier introduction. +- [claim-verified] 'only Cosmos3 has a native-only per-model override' — registry.py:545-548 confirms the only override key. +- [claim-verified] '#3588 accidentally reverted #4074 nn.RMSNorm' — search_history for `self.norm_q = nn.RMSNorm` shows 35ee3c75 (#4074) added it, 99234881 (#3588) removed it, b5589481 (this PR) re-added it. +- [claim-verified] Qwen-Image P1+P2 −26.9% (1.7263→1.2613) computes to 26.94% and Part 2 isolation −10.1% computes to 10.14%. + +`vllm_omni/platforms/cuda/platform.py:253` [major] — platform.py now returns ["vllm_c","native"] unconditionally (line 253), but the same PR converts every vLLM RMSNorm in qwen_image_transformer.py to torch.nn.RMSNorm (lines 557/558/595/596/1009) and drops the vllm RMSNorm import. The merged tree therefore has no vLLM RMSNorm left in Qwen-Image, so the restored rms_norm/fused_add_rms_norm priority has no Qwen-Image layer to dispatch — Part 1's stated Qwen-Image mechanism (_C::rms_norm) is not delivered by this tree. The global default flip still affects all other IR op keys and the ~14 diffusion families that still import vLLM RMSNorm; of those, only FLUX.1-dev has a local A/B in the PR body. Please run Part 2 in isolation on tests/e2e/accuracy/test_qwen_image.py::test_qwen_image_matches_diffusers and the nightly test_qwen_image_single_device* perf tests, and either drop Part 1, scope it via _DIFFUSION_IR_OP_PRIORITY_FUNCS (registry.py), or add accuracy/perf evidence for the other inheritors. (evidence: Changed by this diff: vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]` — unconditional after the `using_inductor` branch was removed. Changed by this diff: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; :558 `self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; :595 `self.norm_added_q = nn.RMSNorm(head_dim, eps=eps)`; :596 `self.norm_added_k = nn.RMSNorm(head_dim, eps=eps)`; :1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)` — no vLLM RMSNorm remains in Qwen-Image. Unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/worker/diffusion_worker.py:175 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)` — sole in-repo consumer. Unchanged by this diff: vllm_omni/diffusion/registry.py:545 `_DIFFUSION_IR_OP_PRIORITY_FUNCS = {` — only `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func"` is registered. Unchanged by this diff: vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py `priority_kwargs["rms_norm"] = ["native"]` and `priority_kwargs["fused_add_rms_norm"] = ["native"]` — Cosmos3 pins only those two keys, leaving other keys to the new default. Unchanged by this diff: grep confirms 14 files under vllm_omni/diffusion/models import `from vllm.model_executor.layers.layernorm import RMSNorm`, e.g. vllm_omni/diffusion/models/flux2/flux2_transformer.py:18.) + +`vllm_omni/platforms/rocm/platform.py:~185` [minor] — The inductor→native-only IR priority branch was removed only in CudaOmniPlatform; RocmOmniPlatform.get_default_ir_op_priority still has it at vllm_omni/platforms/rocm/platform.py:184-185, and XPUOmniPlatform at vllm_omni/platforms/xpu/platform.py:116-117. If the #4964 regression also applies to ROCm `vllm_c` kernels under inductor, mirror the CUDA change or document why ROCm/XPU intentionally stay native-only under inductor. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/rocm/platform.py:184 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; vllm_omni/platforms/rocm/platform.py:185 `default = ["native"] if using_inductor else ["vllm_c", "native"]`; vllm_omni/platforms/xpu/platform.py:116 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; vllm_omni/platforms/xpu/platform.py:117 `default = ["native"] if using_inductor else ["xpu_kernels", "native"]` — while the diff removes the identical branch only from vllm_omni/platforms/cuda/platform.py (replaced with `default = ["vllm_c", "native"]`).) + +`tests/platforms/test_cuda_ir_op_priority.py:7` [minor] — tests/platforms/test_cuda_ir_op_priority.py:7 imports CudaOmniPlatform at module scope, and the file carries only core_model/cpu markers. run-xpu-test.sh:54 runs whole-tree `pytest -v -s -m "core_model and xpu and B60"`, so pytest executes this module-level import during collection before marker deselection; if that import fails on a non-CUDA torch build, collection aborts the Intel gate. Move the import into the test functions and add `@pytest.mark.skipif(not current_platform.is_cuda(), ...)` as tests/diffusion/attention/test_sage_attn3.py does (import at :93/:120, skipif at :86/:114). AMD/NPU merge specs only run tests/diffusion and tests/model_executor for the cpu marker and do not collect this file; CI state confirms the intel/amd/npu gates are red but does not by itself prove this diff is the cause. (evidence: In-diff new file tests/platforms/test_cuda_ir_op_priority.py:7 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (module scope) and :9 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` with no is_cuda skipif. Unchanged by this diff, present in the PR-time tree: .buildkite/scripts/hardware_ci/run-xpu-test.sh:54 ` pytest -v -s -m "core_model and xpu and B60"` (whole tree, no path). Contrast, unchanged by this diff: tests/diffusion/attention/test_sage_attn3.py:86 `@pytest.mark.skipif(not current_platform.is_cuda(), reason="sage_attn3 tests require CUDA platform")` and :93 ` from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (inside the test function).) + +`tests/platforms/test_cuda_ir_op_priority.py:7` [minor] — This PR adds tests/platforms/test_cuda_ir_op_priority.py, and buildkite/vllm-omni-amd-ci, intel-ci, and npu-ci are FAILURE at this head. The diff intersects those lanes' coverage: .buildkite/test-amd-merge.yml:44 runs `timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' ...` (line 21 is the 1/2 step covering only tests/diffusion), and the changed vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py is imported by tests/diffusion/distributed/test_sp_plan_hooks.py:474 in TestModelSpPlans.test_qwen_image_transformer_sp_plan. The new tests/platforms/ file is outside those directories and is unlikely to be the direct trigger. Please confirm whether these failures are pre-existing on main or introduced by this PR. (evidence: gh_ci_read(5009) returned buildkite/vllm-omni-amd-ci FAILURE, buildkite/vllm-omni-intel-ci FAILURE, buildkite/vllm-omni-npu-ci FAILURE at this head. .buildkite/test-amd-merge.yml:44 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (line 21 is `- "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`, tests/diffusion only). Consumer, unchanged by this diff, present in the PR-time tree: tests/diffusion/distributed/test_sp_plan_hooks.py:474 `from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (`. Anchor: tests/platforms/test_cuda_ir_op_priority.py:7 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform`.) + +`tests/platforms/test_cuda_ir_op_priority.py:25` [minor] — The regression test asserts only `rms_norm` and `fused_add_rms_norm` (tests/platforms/test_cuda_ir_op_priority.py:26-28, 36-37), while the changed production line `default = ["vllm_c", "native"]` (vllm_omni/platforms/cuda/platform.py:252) flows through `IrOpPriorityConfig.with_default(default, ...)` (:262) and is not covered. Add `assert priority.default == ["vllm_c", "native"]` in both tests. (evidence: PR-time tree, tests/platforms/test_cuda_ir_op_priority.py:26-28: `assert priority.rms_norm == ["vllm_c", "native"]` / `assert priority.fused_add_rms_norm == ["vllm_c", "native"]` (no `priority.default` assertion); vllm_omni/platforms/cuda/platform.py:252 `default = ["vllm_c", "native"]`; vllm_omni/platforms/cuda/platform.py:262 `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)`.) + +`tests/platforms/test_cuda_ir_op_priority.py:25` [minor] — Test coverage gap: the new tests only call CudaOmniPlatform.get_default_ir_op_priority directly (test_cuda_ir_op_priority.py:25,34) and never exercise the producer->consumer handoff through _resolve_ir_op_priority (diffusion_worker.py:174) that writes vllm_config.kernel_config.ir_op_priority (diffusion_worker.py:290). The existing wiring test test_ir_op_priority_hook_receives_platform_default mocks current_omni_platform (test_diffusion_plugin_hooks.py:136), so a future regression in the real CudaOmniPlatform default flowing through the worker would not be caught. Consider adding an integration assertion that patches diffusion_worker.current_omni_platform to CudaOmniPlatform and checks _resolve_ir_op_priority(SimpleNamespace(model_class_name="QwenImagePipeline"), _vllm_config(mode=CompilationMode.VLLM_COMPILE)).rms_norm == ["vllm_c", "native"]. (evidence: tests/platforms/test_cuda_ir_op_priority.py:25 ` priority = CudaOmniPlatform.get_default_ir_op_priority(_vllm_config(mode=mode))` — the new test asserts only the classmethod return value (lines 26-27 `assert priority.rms_norm == ["vllm_c", "native"]` / `assert priority.fused_add_rms_norm == ["vllm_c", "native"]`). tests/diffusion/test_diffusion_plugin_hooks.py:136 ` mock_platform.get_default_ir_op_priority.return_value = default_priority` and :139 ` assert _resolve_ir_op_priority(od_config, vllm_config) is merged_priority` — the only existing wiring test mocks current_omni_platform, so it cannot catch a real CudaOmniPlatform default regression. Unchanged by this diff but present in the PR-time tree, the missed handoff is real: vllm_omni/diffusion/worker/diffusion_worker.py:174 ` ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)` and vllm_omni/diffusion/worker/diffusion_worker.py:290 ` vllm_config.kernel_config.ir_op_priority = _resolve_ir_op_priority(self.od_config, vllm_config)`.) + +`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:621` [minor] — Update `get_cosmos3_ir_op_priority_func` in `vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py` to override every IR op key that must remain native-only, not only `rms_norm` and `fused_add_rms_norm`. With the CUDA default now `["vllm_c", "native"]`, Cosmos3's remaining op keys inherit vllm_c, so the PR-body claim that native-only models are already protected by per-model overrides is false. (evidence: [claim-refuted] PR claim 'models that need native-only already have per-model overrides (e.g. Cosmos3)': Cosmos3 override only pins rms_norm/fused_add_rms_norm to native and leaves the remaining keys to the new vllm_c default.) + +`vllm_omni/platforms/cuda/platform.py:252` [minor] — PR-body FLUX.1-dev table: Δ vs A is (0.5402−0.6166)/0.6166 = −12.4%, not −14.1%; −14.1% is (0.5402−0.6166)/0.5402, i.e. vs B. Fix the figure or the denominator. (Anchor platform.py:252 is fine; this is a description-only correction.) (evidence: PR #5009 description (outside the diff, via gh_pr_view): `| A — post-#4709 / current main | ["native"] only | 0.6166s | — |` and `| B — Part 1 (this PR) | ["vllm_c", "native"] | 0.5402s | **−14.1%** |`. Arithmetic: (0.6166−0.5402)/0.6166 = 0.1239 = −12.4% vs A; −14.1% = (0.5402−0.6166)/0.5402 = −0.1414, which uses B as denominator. The comment anchor vllm_omni/platforms/cuda/platform.py:252 is the changed function's docstring: `"""Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops."""`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:595` [minor] These hunks switch `norm_q`/`norm_k`/`norm_added_q`/`norm_added_k`/`txt_norm` from `vllm.RMSNorm` to `nn.RMSNorm` in the shared `QwenImageTransformer2DModel` (lines 557, 558, 595, 596, 1009), changing output numerics for + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_smoke/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v14_smoke/pr5009.trace.json.gz new file mode 100644 index 000000000..437ce3873 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_smoke/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_smoke/pr5863.cost.json b/eval/dataset/arms/copilot_v14_smoke/pr5863.cost.json new file mode 100644 index 000000000..4b30aeb69 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_smoke/pr5863.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 723.7, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_smoke/runs/pr5863/run-20260815-020235-c36fe2", + "recorded_at": "2026-08-15T02:14:37", + "attempts": [ + { + "invocation_id": "inv-24a7e3bd", + "outcome": "run:run-20260815-020235-c36fe2", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-020235-c36fe2", + "llm_calls": 82, + "input_tokens": 774300, + "output_tokens": 121749, + "usd": 0.504162, + "source": "spans" + } + ], + "attempt_usd_total": 0.504162, + "llm_calls": 82, + "input_tokens": 774300, + "output_tokens": 121749, + "usd": 0.504162, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-020235-c36fe2", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5042, + "minutes": 12.03, + "ci_minutes": 0, + "input_tokens": 774300, + "output_tokens": 121749, + "tool_calls": 142, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5811, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 82, + "by_role": { + "agent": { + "usd": 0.485953, + "input_tokens": 761349, + "output_tokens": 108374, + "calls": 80 + }, + "reducer": { + "usd": 0.018209, + "input_tokens": 12951, + "output_tokens": 13375, + "calls": 2 + } + }, + "cache_read_tokens": 5969536, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.96, + "gate": 2.29, + "review": 715.39, + "report": 0.06 + }, + "time_to_first_result_s": 721.69 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5863.trace.json.gz", + "trace_bytes": 29762 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_smoke/pr5863.md b/eval/dataset/arms/copilot_v14_smoke/pr5863.md new file mode 100644 index 000000000..62aa118c3 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_smoke/pr5863.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5863, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0a4c96151148) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Commit 9537c6c0 'Corrected Client E2E measurement for four GPUs': the corrected 172.32 s is internally consistent (stage sum 171.565 s, 0.755 s outside profiled stages), so the correction matches the doc's 0.75 s note. +- [resolved] Prior concern 'does --task-type actually select a single partition': resolved at pipeline_minimax_h3.py:125-140 — `_minimax_h3_partition_for_task` maps t2va/fl2va→FL2VA and ref2va→Ref2VA, matching the doc. +- [claim-verified] PR body 'include config guide for 2x 4x 8x for 6000' → 2/4/8-GPU serve blocks present at lines 73-160. +- [claim-verified] t2va explicit aspect_ratio requirement: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:348 `raise OmniClientError("t2va requires an explicit aspect_ratio")`. +- [claim-verified] ref2va default 16:9: pipeline_minimax_h3.py returns `MINIMAX_H3_SUPPORTED_ASPECT_RATIOS["16:9"]` when value is None and task != t2va. +- [claim-verified] cfg_parallel_size must be 1: pipeline_minimax_h3.py:552 raises `MiniMax-H3 is CFG-distilled and has no negative branch; cfg_parallel_size must be 1`. +- [claim-verified] SM120 does not get TRTLLM_ATTN default: vllm_omni/platforms/cuda/platform.py gates `trtllm_attn_default_ok` on `compute_capability.major == 10`, and Blackwell with cuDNN>=90500 falls through to CUDNN_ATTN. +- [claim-verified] CUDNN_ATTN is a valid backend enum: vllm_omni/diffusion/attention/backends/registry.py:63. +- [claim-verified] FP8 H3 not compatible with layerwise offload: docs/user_guide/quantization/fp8.md MiniMax H3 row says `not compatible with layerwise offload`. +- [claim-verified] VLLM_OMNI_VIDEO_SYNC_TIMEOUT exists: vllm_omni/entrypoints/openai/api_server.py:2969 (default 600.0). +- [claim-verified] X-Stage-Durations / X-Peak-Memory-MB headers exist: api_server.py:3510-3511. +- [claim-verified] --num-gpus/--usp/--ring/--text-encoder-tp-size/--vae-patch-parallel-size are real CLI flags in vllm_omni/entrypoints/cli/serve.py. +- [claim-refuted] PR body 'its only take 4 min to create video when 8*6000 started' → the doc's eight-GPU Client E2E is 90.48 s (~1.5 min), not 4 min; the description overstates the diff's own number. +- [validated] All serve flags used exist: serve.py registers --usp/--ulysses-degree (:461), --ring (:~520), --text-encoder-tp-size (:711 area), --vae-patch-parallel-size (:711), --vae-parallel-mode choices tile|spatial_shard_height|spatial_shard_width, --vae-use-tiling, --diffusion-attention-backend, --num-gpus, --task-type, --cfg-parallel-size (default 1), --enable-diffusion-pipeline-profiler. + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201` [major] — The two-server section tells readers to use `--cpunodebind=0 --membind=0` and `--cpunodebind=1 --membind=1` as server flags, but those are numactl options and `vllm serve` does not accept them; `vllm_omni/entrypoints/cli/serve.py` defines no such options, and the only repo occurrences of `cpunodebind`/`membind` are this doc. Wrap each launch as `numactl --cpunodebind=0 --membind=0 -- vllm serve ...` (and `...=1 ...` for the second server), or drop the flags since this layout was not measured. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201 `Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and`; recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:203 ``--cpunodebind=1 --membind=1`, and `--port 8092` for the Ref2VA server.` — both lines are in the PR's added file (in-diff). Outside the diff, unchanged in the PR-time tree: grep of `vllm_omni/entrypoints/cli/serve.py` for `cpunodebind|membind|numactl` returns no matches, so the serve parser defines no such options; repo-wide grep for `cpunodebind`/`membind` matches only these two doc lines.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:203` [major] — The two-server section says each server uses 'the four-GPU configuration', whose block sets `export MODEL="${MODEL_ROOT}/FL2VA"` (line 113), and only changes CUDA_VISIBLE_DEVICES, NUMA flags, and port for the Ref2VA server (lines 201-203). Readers following it start two FL2VA servers. Add `export MODEL="${MODEL_ROOT}/Ref2VA"` (or `--task-type ref2va`) to the second server command, mirroring 'Shared serving notes' (lines 215-216). (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:195-197 `An eight-GPU host can instead run one FL2VA server and one Ref2VA server side by side, each pinned to its own NUMA node with the four-GPU configuration.` — the referenced four-GPU block defines recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:113 `export MODEL="${MODEL_ROOT}/FL2VA"`. The two-server overrides only: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201-203 `Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and / `--port 8091` for the FL2VA server. Use `CUDA_VISIBLE_DEVICES=4,6,5,7`, / `--cpunodebind=1 --membind=1`, and `--port 8092` for the Ref2VA server.` — no MODEL change. The single-server note states the required Ref2VA switch: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:215-216 `For Ref2VA on a single-server layout, stop the FL2VA server and restart the / same command with `MODEL="${MODEL_ROOT}/Ref2VA"`.`) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:249` [minor] — PR description says 'its only take 4 min to create video when 8*6000 started', but this file's eight-GPU Client E2E is 90.48 s (~1.5 min). Update the description to match the measured result; anchor this comment at line 249 (the Client E2E table row), not line 160 (the eight-GPU command block's closing fence). (evidence: PR 5863 body (gh_pr_view): "its only take 4 min to create video when 8*6000 started". recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:249 `| Client E2E | 284.76 s | 172.32 s | 90.48 s |` — 90.48 s is ~1.5 min, contradicting the body's ~4 min claim. The cited line 160 in the comment is the eight-GPU code fence close, not the measurement row.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:328` [minor] — The recipe's profiler instructions tell readers to read peak memory from `X-Peak-Memory-MB`, but the table and methodology state peak HBM is `nvidia-smi --query-gpu=memory.used` sampled at 1 Hz; `X-Peak-Memory-MB` is populated from `current_omni_platform.max_memory_reserved()` (`diffusion_model_runner.py:355-358`), a per-process reserved-memory high-water mark that is not the same as device-wide used memory. State which metric the table uses, or note that the header is not comparable to the table's `nvidia-smi` peak. (evidence: Unchanged by this diff (recipe-only PR), present in the PR-time tree: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:328 `X-Peak-Memory-MB` from the saved headers. vs recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:254-256 'Peak memory is the maximum of `nvidia-smi --query-gpu=memory.used` sampled at 1 Hz across every device for the duration of the measured request.' vs vllm_omni/entrypoints/openai/api_server.py:3511 `"X-Peak-Memory-MB": f"{peak_memory_mb:.3f}"` vs vllm_omni/diffusion/worker/diffusion_model_runner.py:355-358 `peak_reserved_bytes = current_omni_platform.max_memory_reserved()` ... `peak_memory_mb = peak_reserved_bytes / (1024**2)`.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:186` [minor] — The 'Headroom variant: TP4 with Ulysses2' section (line 186: 'headroom, not as the default. Replace the parallel flags above with:') recommends TP4 as a fallback, but the validation table at line 244 measures only TP2 × USP1/2/4 and 'Not yet measured' (lines 296-302) lists only Ref2VA and the eight-GPU device-order comparison. Add a sentence marking TP4×Ulysses2 as not measured on this host, or list it under 'Not yet measured'. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:186 `headroom, not as the default. Replace the parallel flags above with:` — recommendation presented without a measured/unmeasured caveat. recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:244 `| Topology | TP2 × USP1 | TP2 × USP2 | TP2 × USP4 |` — no TP4 row in the measured table. recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:296 `### Not yet measured` with lines 298-302: `Ref2VA latency and memory have not been measured on this host. ... The eight-GPU device-order comparison in the section above was also not run.` — TP4×Ulysses2 is absent. All quoted lines are inside the added file in this PR's diff.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:4` [minor] — Intro at line 4 grounds the two-GPU minimum against an 'RTX PRO 5000 profile', but no such recipe exists in the repo — a repo-wide grep for 'RTX PRO 5000' matches only this line. Link the actual recipe if it lives elsewhere/upstream, or rephrase against the in-repo RTX 5090/four-GPU profiles. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:4 `The additional HBM over the RTX PRO 5000 profile removes the four-GPU` — repo-wide grep for `RTX PRO 5000` returns only this line, and `recipes/MiniMaxAI/` contains only `MiniMax-H3.md`, `MiniMax-H3-5090.md`, `MiniMax-H3-MUSA.md`, `MiniMax-H3-NPU.md`, `MiniMax-H3-RTX-PRO-6000.md` (no RTX PRO 5000 recipe).) + +`recipes/README.md:~48` [minor] — The PR adds `recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md` but does not add it to the recipes index; `recipes/README.md` still lists only `MiniMax-H3.md` (line 48) and `MiniMax-H3-MUSA.md` (line 49). Add a row for `MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md` (e.g. T2VA on RTX PRO 6000 Blackwell, 2/4/8 GPU TP+Ulysses profiles) so the guide is discoverable. (evidence: recipes/README.md:48 `| [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |`; recipes/README.md:49 `| [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |`. These are the only MiniMaxAI rows in the index; `git diff --stat merge-base..HEAD` shows only `recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md` added (346 insertions), so recipes/README.md is unchanged by this diff and still lacks a row for the new file.) + +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340` [nit] — The new Ref2VA example sends the contents of `/root/audio_ref.json` as the `audio_reference` form field but never documents the file's expected shape. The sync endpoint parses the field as JSON (`vllm_omni/entrypoints/openai/api_server.py:3186`) into `UrlAudioReference`, which forbids extra keys and requires `audio_url` (`vllm_omni/entrypoints/openai/protocol/videos.py:89`); `decode_audio_url` then rejects any value that is not an http(s) or data URL with a 400 (`vllm_omni/entrypoints/openai/video_api_utils.py:298`). Show the expected JSON (e.g. `{"audio_url": "http://..."}`) or link the main MiniMax-H3 recipe's audio_reference documentation. (evidence: recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:340 ` -F "audio_reference= DiffusionOutput:` ... `return super().execute_model(req, kv_prefetch_job=kv_prefetch_job)` — the subclass override does not accept the new kwarg.) + +`vllm_omni/diffusion/diffusion_kv/metadata.py:24` [minor] — The three DTOs at vllm_omni/diffusion/diffusion_kv/metadata.py:16-56 (DiffusionKVContextMetadata, DiffusionKVSequenceMetadata, DiffusionKVMetadata) are plain dataclasses with no __post_init__, so they accept malformed local values such as negative num_tokens, empty request_id, or a sequence context_ids entry not present in contexts. The sibling DiffusionKVRequest/DiffusionKVContext in request.py already enforces these local invariants, and the new worker validators only check envelope/metadata request identity, so field-level garbage is not rejected before forwarding. Add __post_init__ validation to the three DTOs (or document the per-field deferral explicitly). (evidence: vllm_omni/diffusion/diffusion_kv/metadata.py:23-26 (new in this diff; no __post_init__ appears anywhere in the file) ` context_id: str`, ` cache_role: str`, ` num_tokens: int`, ` block_ids: tuple[list[int], ...]`. Sibling validation, unchanged by this diff and present in the PR-time tree: vllm_omni/diffusion/diffusion_kv/request.py:29-34 ` if not self.context_id: + raise ValueError("context_id must be non-empty")` / ` if self.num_tokens <= 0: + raise ValueError(f"num_tokens must be positive, got {self.num_tokens}")`; vllm_omni/diffusion/diffusion_kv/request.py:77-81 ` if prefix_len + target_len > seq_len: + raise ValueError( + "prefix_len + target_len must not exceed seq_len: " + f"prefix_len={prefix_len}, target_len={target_len}, seq_len={seq_len}" + )`; vllm_omni/diffusion/diffusion_kv/request.py:82-84 ` context_ids = [context.context_id for context in contexts] + if len(context_ids) != len(set(context_ids)): + raise ValueError(f"kv_contexts must have unique context_id values, got {context_ids}")`.) + +`vllm_omni/diffusion/diffusion_kv/metadata.py:26` [minor] — The new allocation-snapshot DTOs in vllm_omni/diffusion/diffusion_kv/metadata.py are declared mutable @dataclass (metadata.py:16,30,46) and their block_ids fields are tuple[list[int], ...] (metadata.py:27,42) with no __post_init__ defensive copy. The inner lists are mutable and not frozen, so a later Scheduler/native-cache mutation of the same list objects would silently alter the snapshot already handed to the worker. Mirror the sibling DiffusionKVContext (request.py:13 @dataclass(frozen=True); request.py:35 converts block_hashes to a tuple in __post_init__): make all three metadata dataclasses frozen and convert block_ids to tuple[tuple[int, ...], ...] (or deep-copy on construction). (evidence: vllm_omni/diffusion/diffusion_kv/metadata.py:16 `@dataclass` (no frozen); metadata.py:27 ` block_ids: tuple[list[int], ...]`; metadata.py:30 `@dataclass`; metadata.py:42 ` block_ids: tuple[list[int], ...]`; metadata.py:46 `@dataclass`. Sibling pattern (unchanged by this diff, present in PR-time tree): vllm_omni/diffusion/diffusion_kv/request.py:13 `@dataclass(frozen=True)` and request.py:35 ` object.__setattr__(self, "block_hashes", tuple(self.block_hashes))`.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:471` [minor] — multiproc_executor.py:471's `and not has_diffusion_kv_metadata` makes any metadata-bearing wave fall through to the per-request `collective_rpc('execute_model', args=(req, ...), unique_reply_rank=0, exec_all_ranks=True)` loop. With `enable_distributed_layerwise_offload` + `dlo_use_allgather` + `data_parallel_size>1`, the worker receives a single non-list `req`, so `is_batch` is False, every rank computes the same request, only rank 0 replies, and N requests become N serial all-rank waves. The new `make_executor` sets `enable_distributed_layerwise_offload=False`, so this cell is untested. Add a guard that raises for metadata under the DLO broadcast precondition, or implement a metadata-aware DP path, and cover it with a metadata + DLO + DP>1 test. (evidence: multiproc_executor.py:462 ` has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)` ; multiproc_executor.py:471 ` and not has_diffusion_kv_metadata` ; multiproc_executor.py:544-550 ` args: tuple = (req, self.od_config, scheduler_output.kv_prefetch_job)` / ` if new_req.diffusion_kv_metadata is not None:` / ` args += (new_req.diffusion_kv_metadata,)` / ` result = self.collective_rpc(` / ` "execute_model",` / ` args=args,` / ` unique_reply_rank=0,` / ` exec_all_ranks=True,` ; diffusion_worker.py:444 ` is_batch = isinstance(req, list)` (unchanged by this diff, present in the PR-time tree); tests/diffusion/diffusion_kv/test_worker_contract.py:63 ` enable_distributed_layerwise_offload=False,`) + +`PR description:0` [minor] — Remove or mark as out-of-scope the PR description bullet 'Remove the obsolete Stage KV planner digest, physical-layout mirror, Scheduler-owned Worker bootstrap, and initialization acknowledgement types' — it is not represented in this diff (10 files, 622 insertions, 13 deletions; no deleted files, and no planner/bootstrap/acknowledgement terms under vllm_omni/diffusion/diffusion_kv). (evidence: PR #5550 description, 'Main changes' section (verbatim): '- Remove the obsolete Stage KV planner digest, physical-layout mirror, Scheduler-owned Worker bootstrap, and initialization acknowledgement types.' The PR-time tree `git diff --stat merge-base..HEAD` reports '10 files changed, 622 insertions(+), 13 deletions(-)' and every listed file is ADDED or MODIFIED, none DELETED. Grep of the PR-time tree `vllm_omni/diffusion/diffusion_kv` for `planner`, `bootstrap`, and `acknowledgement` returned no matches.) + +`vllm_omni/diffusion/diffusion_engine.py:190` [minor] — The new init-failure cleanup (`try: self._init_scheduler(...) ... except: scheduler.close(); self.executor.shutdown()`, diffusion_engine.py:191-207) only covers failures after `self._init_executor(od_config)` at line 190. If executor construction raises after worker processes have started — `_launch_workers` starts all workers (multiproc_executor.py:326) and raises on EOFError (multiproc_executor.py:337) before `_init_executor` assigns `self._processes` (multiproc_executor.py:112) and before the shutdown cleaner/finalizer is installed — the partially started workers are neither joined nor terminated, and the engine's new except handler never runs because `self.executor` was never assigned. Move `_init_executor` into the guarded region and shut down via `getattr(self, "executor", None)`, or make `MultiprocDiffusionExecutor._init_executor` self-clean when `_launch_workers` raises. (evidence: vllm_omni/diffusion/diffusion_engine.py:190 ` self._init_executor(od_config)` — executes before the added guarded block; vllm_omni/diffusion/diffusion_engine.py:191 ` try:` — the new cleanup starts only after executor init; vllm_omni/diffusion/diffusion_engine.py:205 ` self.executor.shutdown()` — only reachable if `self.executor` was assigned. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/executor/multiproc_executor.py:112 ` self._processes = processes` — assigned only after `_launch_workers` returns; vllm_omni/diffusion/executor/multiproc_executor.py:326 ` process.start()` and vllm_omni/diffusion/executor/multiproc_executor.py:337 ` except EOFError:` (then `processes[i].join()` and `raise`) — workers started before the failure, and only rank i is joined.) + +`vllm_omni/diffusion/sched/interface.py:191` [minor] — The new `diffusion_kv_metadata` field is read directly on every scheduled envelope: `validate_new_request_data_identity` (sched/interface.py:191), `has_diffusion_kv_metadata = any(...)` (multiproc_executor.py:462), and the serial args build (`if new_req.diffusion_kv_metadata is not None`, multiproc_executor.py ~545). Custom schedulers returning duck-typed `request_id` + `req` envelopes now raise `AttributeError` before model work. Use `getattr(new_req, "diffusion_kv_metadata", None)` at all three sites. (Note: the mock is at tests/diffusion/test_diffusion_engine.py:114 and emits only `request_id`, so it already lacked `req`; the clean regression case is an envelope with `request_id` + `req` and no new field.) (evidence: vllm_omni/diffusion/sched/interface.py:191 `metadata = new_req.diffusion_kv_metadata` (in diff); vllm_omni/diffusion/executor/multiproc_executor.py:462 `has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs)` (in diff); unchanged by this diff, present in the PR-time tree: tests/diffusion/test_diffusion_engine.py:114 `batch.append(SimpleNamespace(request_id=req.request_id))`.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:~463` [minor] — DiffusionWorker.execute_model now forwards diffusion_kv_metadata to the model runner (diffusion_worker.py:462-465), but no test exercises that non-None branch on a real DiffusionWorker: test_worker_wrapper_only_extends_paged_request_rpc drives WorkerWrapperBase with a fake Worker (test_worker_contract.py:247-253), and test_dense_worker_call_does_not_extend_model_runner_signature calls worker.execute_model(req, SimpleNamespace()) with no metadata (test_worker_contract.py:288). Add a test constructing DiffusionWorker via object.__new__ with a stub model_runner that records kwargs and asserts diffusion_kv_metadata is forwarded. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:462-465 ` kwargs: dict[str, Any] = {"kv_prefetch_job": kv_prefetch_job}\n if diffusion_kv_metadata is not None:\n kwargs["diffusion_kv_metadata"] = diffusion_kv_metadata\n output = self.model_runner.execute_model(req, **kwargs)`; tests/diffusion/diffusion_kv/test_worker_contract.py:277-280 ` class DenseModelRunner:\n def execute_model(self, req, kv_prefetch_job=None):\n calls.append((req, kv_prefetch_job))\n return DiffusionOutput(output=None)`; tests/diffusion/diffusion_kv/test_worker_contract.py:282 ` worker = object.__new__(DiffusionWorker)`; tests/diffusion/diffusion_kv/test_worker_contract.py:288 ` worker.execute_model(req, SimpleNamespace())`; tests/diffusion/diffusion_kv/test_worker_contract.py:290 ` assert calls == [(req, None)]`; tests/diffusion/diffusion_kv/test_worker_contract.py:247-253 ` class Worker:\n def execute_model(self, req, od_config, **kwargs): ... wrapper = object.__new__(WorkerWrapperBase)\n wrapper.worker = Worker()` — all lines are present in the PR diff (worker branch in the changed diffusion_worker.py hunk; test lines in the new test_worker_contract.py file).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_smoke2/pr5550.trace.json.gz b/eval/dataset/arms/copilot_v14_smoke2/pr5550.trace.json.gz new file mode 100644 index 000000000..d0203836b Binary files /dev/null and b/eval/dataset/arms/copilot_v14_smoke2/pr5550.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/manifest.json b/eval/dataset/arms/copilot_v14_train_r1/manifest.json new file mode 100644 index 000000000..79060dc05 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v14_train_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4804.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4804.cost.json new file mode 100644 index 000000000..b59e14187 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1976.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4804/run-20260815-024433-183f6c", + "recorded_at": "2026-08-15T03:17:26", + "attempts": [ + { + "invocation_id": "inv-70f0f840", + "outcome": "run:run-20260815-024433-183f6c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-024433-183f6c", + "llm_calls": 194, + "input_tokens": 2386698, + "output_tokens": 326592, + "usd": 1.655879, + "source": "spans" + } + ], + "attempt_usd_total": 1.655879, + "llm_calls": 194, + "input_tokens": 2386698, + "output_tokens": 326592, + "usd": 1.655879, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-024433-183f6c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.6559, + "minutes": 32.89, + "ci_minutes": 0, + "input_tokens": 2386698, + "output_tokens": 326592, + "tool_calls": 329, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3248, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 194, + "by_role": { + "agent": { + "usd": 1.619785, + "input_tokens": 2332333, + "output_tokens": 307158, + "calls": 192 + }, + "reducer": { + "usd": 0.036094, + "input_tokens": 54365, + "output_tokens": 19434, + "calls": 2 + } + }, + "cache_read_tokens": 24156288, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.58, + "gate": 1.73, + "review": 1967.08, + "report": 0.06 + }, + "time_to_first_result_s": 1973.43 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 68959 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4804.md b/eval/dataset/arms/copilot_v14_train_r1/pr4804.md new file mode 100644 index 000000000..91f4b17b3 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4804.md @@ -0,0 +1,71 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 7 finding(s) below | + +**Validated:** +- [resolved] CUDA-graph seeding concern: moss_tts sets no `talker_mtp_graph_safe`, so `_init_talker_mtp` leaves talker_mtp unwrapped and `_talker_mtp_forward` forces `CUDAGraphMode.NONE` (gpu_model_runner.py, `if not isinstance(self.talker_mtp, current_omni_platform.get_graph_wrapper_cls()): _cudagraph_mode = CUDAGraphMode.NONE`) — the per-request generator runs eagerly and is not discarded by graph replay. +- [claim-verified] patch.py `from vllm.multimodal.audio import AudioResampler` needs no new dependency floor: AudioResampler is already imported in qwen3_tts/voxcpm2/soulx_singer (repo-wide grep). +- [claim-refuted] PR body reports 'Maximum request concurrency: 1' but 'Peak concurrent requests: 2.00' — self-contradictory; calc(20/41.54)=0.481 req/s matches the reported 0.48, and calc(20*2070.86/1000/41.54)=0.997 is consistent with serial concurrency-1 execution, so peak 2.00 cannot be true under the stated max. +- [validated] seed rename is repo-consistent: grep shows zero remaining `qwen3_tts_request_seed` references; serving_speech.py:3715 writes `tts_local_seed` and gpu_model_runner.py:1826 reads `tts_local_seed`. +- [validated] talker_mtp None-codes guard is a real fix: moss_tts_local talker_mtp returns `(input_embeds_out, None)`, and gpu_model_runner.py:1898 now guards `if code_predictor_codes is not None` before merging. +- [sweep] serving_speech.py:3711 `sampling_params_list[0]` — guarded by `if request.seed is not None and sampling_params_list:` (non-empty), so the indexed access is safe. +- [sweep] audio_tokenizer_v2.py `next(iter(self.parameters())).device` and `wav_list[0]`/`codes_list[0]` — empty lists raise ValueError before indexing; inference-only vendored module always has parameters; acceptable. +- [sweep] audio_tokenizer_v2.py RingKVCache `self.cache[0]/[1]` and `modules[0](x)` — cache is a fixed 2-element tensor from __init__, len(modules)==1 branch guarded; safe. +- [sweep] seq_token_counts producer census: GPU real path (gpu_generation_model_runner.py:316) and GPU dummy path (:799) set it; NPU real path (npu_generation_model_runner.py:299) sets it; NPU `_dummy_run` (def at :587) sets only `runtime_additional_information` (:780-781) and does NOT set `seq_token_counts` — hard break for the now-required contract. +- [sweep] on_requests_finished producer census: gpu_ar_model_runner.py:942-943 (unconditional), npu_ar_model_runner.py:419-420, gpu_generation_model_runner.py:80-82 (added by PR, but only invoked under the `async_chunk and num_scheduled_tokens` gate at :157-158); npu_generation_model_runner.py has no such call. +- [sweep] PR body reports 'Streaming continuity OK rate: 0.00%' with mean AUDIO_UNDERRUN 0.67s vs mean duration 3.99s (~17%, calc(0.67/3.99)=0.168) — the headline streaming feature is reported with zero continuity-OK and a large underrun; harness semantics unverified, but this contradicts the PR purpose and needs clarification. +- [sweep] slot exhaustion: deploy yaml sets `codec_stream_slots: 8` equal to stage-1 `max_num_seqs: 8`, so the buffering/exhaustion path is defensive in this config; however `_decode_streaming_batch` drops buffered codes when a request finishes before a slot frees (warning-only), which under RFC #4676's 'observable' requirement is a weak, truncated-audio outcome rather than an explicit reject/fallback. +- [sweep] dependency floor: requirements/common.txt pins `transformers >= 5.5.3`; audio_tokenizer_v2.py imports `PreTrainedAudioTokenizerBase` from transformers.modeling_utils, with no other in-repo usage — availability at the oldest allowed transformers (5.5.3) could not be verified from the tree. +- [sweep] vendored file provenance: audio_tokenizer_v2.py has a Copyright header but no 'Vendored from @ ' line, contrary to the repo vendoring convention. + +`PR #4804 (codec selection):?` [blocker] — Legacy and v2 config classes share `model_type = "moss-audio-tokenizer"` (configuration_moss_audio_tokenizer_v2.py:68, audio_tokenizer.py:37), so `MossAudioTokenizerV2Config.from_pretrained(codec_path)` at modeling_moss_tts_codec.py:729 accepts a legacy v1 config.json — the v2 `__init__` accepts every legacy key and pops `model_type` — and the legacy fallback in `_build_codec` is effectively dead. With the new strict weight-load check (modeling_moss_tts_codec.py:679-681 raises on missing/skipped/shape-mismatched weights), a v1 checkpoint routed through the v2 path can hard-fail the shared delay/realtime codec instead of falling back. (evidence: vllm_omni/model_executor/models/moss_tts/audio_tokenizer.py:37 (legacy; unchanged by this diff, present in the PR-time tree): `model_type = "moss-audio-tokenizer"` +vllm_omni/model_executor/models/moss_tts/configuration_moss_audio_tokenizer_v2.py:68: `model_type = "moss-audio-tokenizer"` +vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:729: `codec_cfg = MossAudioTokenizerV2Config.from_pretrained(codec_path)` +vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:733: `except Exception:` — legacy fallback only runs on an exception +vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:679-681: `if missing or skipped or shape_mismatches:` / `raise RuntimeError(` / `"MOSS Audio Tokenizer weights were not fully loaded: "`) + +`vllm_omni/worker/gpu_generation_model_runner.py:81` [major] — `on_requests_finished` cleanup is added inside `GPUGenerationModelRunner._update_request_states`, but that method only runs when `async_chunk and num_scheduled_tokens` (line 157). A finished/aborted request step with zero scheduled tokens returns at line 161 before cleanup, leaking the `_MossCodecStreamSession` slot and buffered codes. Call `self.model.on_requests_finished(scheduler_output.finished_req_ids)` unconditionally in `execute_model` after `_update_states`, as `gpu_ar_model_runner.py:942-943` does. (evidence: Diff-added: vllm_omni/worker/gpu_generation_model_runner.py:81-82 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)`. Unchanged by this diff, present in the PR-time tree: vllm_omni/worker/gpu_generation_model_runner.py:157-161 `if self.model_config.async_chunk and num_scheduled_tokens:` / `self._update_request_states(scheduler_output)` / `deferred_state_corrections_fn = self._update_states(scheduler_output)` / `if not scheduler_output.total_num_scheduled_tokens:` / `return self.attach_omni_connector_output(EMPTY_MODEL_RUNNER_OUTPUT)`. Unchanged by this diff, present in the PR-time tree: vllm_omni/worker/gpu_ar_model_runner.py:942-943 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)` (executes before the zero-token early return).) + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:~780` [major] — Confirmed. The PR makes `MossTTSCodecDecoder.forward` raise when `seq_token_counts` is absent and supplies the key only in the GPU generation runner's `_dummy_run`. The NPU generation runner's `_dummy_run` (vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:780-782) sets only `runtime_additional_information` and never `seq_token_counts`, so its profile/cudagraph-capture path will hit `RuntimeError("MossTTS codec requires seq_token_counts; ...")` for moss_tts_local. Add `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]` in this block, mirroring the GPU twin. (evidence: Changed by this PR: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:260 `token_counts = self._normalize_seq_token_counts(kwargs.get("seq_token_counts"))` and :262-264 `if token_counts is None:` / `raise RuntimeError(` / `"MossTTS codec requires seq_token_counts; otherwise concatenated "`. Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:780-782 `if hasattr(self.model, "get_dummy_runtime_additional_information"):` / `runtime_addi = self.model.get_dummy_runtime_additional_information(num_reqs)` / `model_kwargs["runtime_additional_information"] = runtime_addi` — no `seq_token_counts` assignment; the only occurrence of that key in the file is the real path at :299 `model_kwargs["seq_token_counts"] = tokens`.) + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:~48` [major] — The diff adds `on_requests_finished` cleanup only in `GPUGenerationModelRunner._update_request_states` (gpu_generation_model_runner.py:80-82). The NPU generation twin `NPUGenerationModelRunner._update_request_states` (npu_generation_model_runner.py:48) still only removes finished requests from `input_batch` and never calls `self.model.on_requests_finished(...)`, so an aborted/disconnected MOSS streaming request on NPU leaks its `_MossCodecStreamSession` slot and pending codes. `NPUARModelRunner` already calls the hook at npu_ar_model_runner.py:419-420, confirming this is the generation-twin drift. Add the same finished-request cleanup in `NPUGenerationModelRunner._update_request_states` (or in its `execute_model` finished-req path). (evidence: vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:48-51 (unchanged by this diff, present in PR-time tree): ` def _update_request_states(self, scheduler_output: SchedulerOutput):` / ` # remove requests` / ` for req_id in scheduler_output.finished_req_ids:` / ` self.input_batch.remove_request(req_id)` — no `on_requests_finished` call. vllm_omni/worker/gpu_generation_model_runner.py:80-82 (added by this diff): ` if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / ` self.model.on_requests_finished(scheduler_output.finished_req_ids)`. vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:419-420 (unchanged by this diff): ` if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / ` self.model.on_requests_finished(scheduler_output.finished_req_ids)`.) + +`vllm_omni/deploy/moss_tts_local.yaml:~13` [major] — No test under tests/ references `moss_tts_local`, `MOSS_TTS_LOCAL_PIPELINE`, or `vllm_omni/deploy/moss_tts_local.yaml`; the codec streaming/session path added in this PR (`codec_streaming`, `codec_stream_slots`, `_MossCodecStreamSession`, `talker2codec_raw_async_chunk`) has no automated coverage. (evidence: vllm_omni/deploy/moss_tts_local.yaml:13 `codec_streaming: true`; vllm_omni/deploy/moss_tts_local.yaml:20 `codec_stream_slots: 8`; vllm_omni/model_executor/models/moss_tts/pipeline.py:83 `MOSS_TTS_LOCAL_PIPELINE = PipelineConfig(` — negative proof (not a quoted line, but the decisive absence): `grep -r 'moss_tts_local' tests/` returns no matches, `grep -r 'MOSS_TTS_LOCAL_PIPELINE' tests/` returns no matches, and `grep -r 'moss_tts_local.yaml' .` returns no matches; the only MOSS-TTS test hits are e2e suites for v1.5-delay, realtime, nano, and expansion variants, none of which load this config/pipeline.) + +`requirements/common.txt:12` [minor] — requirements/common.txt:12 allows `transformers >= 5.5.3`; the new `audio_tokenizer_v2.py` imports `PreTrainedAudioTokenizerBase` from `transformers.modeling_utils` (its only in-repo usage), so verify the symbol exists at transformers 5.5.3 or raise the dependency floor. (evidence: requirements/common.txt:12 `transformers >= 5.5.3`; vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:27 `from transformers.modeling_utils import PreTrainedAudioTokenizerBase`; vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1293 `class MossAudioTokenizerPreTrainedModel(PreTrainedAudioTokenizerBase):` — repo-wide grep for `PreTrainedAudioTokenizerBase` returns only these two occurrences in this new file, so the minimum-version availability is not evidenced anywhere in the PR-time tree.) + +`vllm_omni/worker/gpu_generation_model_runner.py:479` [minor] — In `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py` (`_decode_streaming_batch`, ~L475-481), a streaming request that finishes while no codec stream slot is free has its buffered codes dropped after a warning-only log; consider an explicit reject or offline-decode fallback so the exhaustion outcome is observable to the client per RFC #4676. Note: the draft cites `vllm_omni/worker/gpu_generation_model_runner.py`, but the code lives in `modeling_moss_tts_codec.py`. (evidence: Within the PR diff, vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:479-484: `"MOSS codec stream request %s finished before a codec stream slot became available; "` / `"dropping buffered codes instead of offline-decoding a non-delta waveform.",` followed by `self._finish_stream_request(request_id, session, None)` — the request's buffered codes are dropped with only a log warning and no explicit rejection or offline decode.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3715` [minor] — Confirmed. `_prepare_speech_generation` (serving_speech.py:3714-3715) now writes `extra_args["tts_local_seed"] = request.seed` unconditionally after `stage0_params.seed = request.seed` instead of only for `qwen3_tts`. `gpu_model_runner.py` `_explicit_talker_seed` (line 1826) reads that key for all talker_mtp decode requests, and FishSpeechSlowARForConditionalGeneration.talker_mtp forwards the resulting `generator` kwarg into `self.fast_ar`. Seeded fish_tts requests therefore now get deterministic Fast-AR sampling. Gate on qwen3_tts/moss_tts_local or document the fish_speech behavior change. (evidence: serving_speech.py:3715 ` stage0_params.extra_args["tts_local_seed"] = request.seed` — reached by `if request.seed is not None and sampling_params_list:` with no `_tts_model_type` gate (changed by this diff). +gpu_model_runner.py:1826 ` seed = extra_args.get("tts_local_seed")` — `_explicit_talker_seed` now reads the renamed key for all talker_mtp requests (changed by this diff). +Unchanged by this diff, present in the PR-time tree: fish_speech_slow_ar.py `talker_mtp` (def at line 653) ends its `self.fast_ar(...)` call with `seed=seed,` and `generator=kwargs.get("generator"),` — so a runner-supplied generator reaches fish_speech Fast-AR.) + +**Additional observations (beyond the comment budget):** +- `PR #4804 (codec/v2 config):None` [minor] Stale docstrings in the codec and v2 config should be updated. configuration_moss_audio_tokenizer_v2.py:36-46 documents defaults sampling_rate=24000, downsample_rate=1920, quantizer_type='rvq', but the constructor defaul +- `tests/worker/test_omni_gpu_model_runner.py:230` [minor] The only test-file change in this PR renames the extra_args key to tts_local_seed (tests/worker/test_omni_gpu_model_runner.py:278,322,326). The new codec streaming/session code — _MossCodecStreamSession (vllm_omni/model_ +- `vllm_omni/benchmarks/data_modules/seed_tts_eval.py:497` [minor] Audio save only runs for successful requests with non-empty pcm (gates at lines 555 and 569); saved_audio and save_audio_failed are updated consistently and audio_path is propagated into per-item rows. Logic and counters +- `vllm_omni/benchmarks/patch/patch.py:137` [minor] WER capture conversion (patch.py:140-161) is sound; the try/except fallback to raw captured PCM at patch.py:1243-1251 is safe. + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4804.trace.json.gz new file mode 100644 index 000000000..888dc22f2 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4817.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4817.cost.json new file mode 100644 index 000000000..1c5aea4b5 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4817.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1164.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4817/run-20260815-024433-bbbfb0", + "recorded_at": "2026-08-15T03:03:55", + "attempts": [ + { + "invocation_id": "inv-caae9d04", + "outcome": "run:run-20260815-024433-bbbfb0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-024433-bbbfb0", + "llm_calls": 70, + "input_tokens": 314827, + "output_tokens": 147301, + "usd": 0.327656, + "source": "spans" + } + ], + "attempt_usd_total": 0.327656, + "llm_calls": 70, + "input_tokens": 314827, + "output_tokens": 147301, + "usd": 0.327656, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-024433-bbbfb0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3277, + "minutes": 19.35, + "ci_minutes": 0, + "input_tokens": 314827, + "output_tokens": 147301, + "tool_calls": 139, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6483, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 70, + "by_role": { + "planner": { + "usd": 0.001132, + "input_tokens": 2563, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.275847, + "input_tokens": 291911, + "output_tokens": 105880, + "calls": 66 + }, + "reducer": { + "usd": 0.050677, + "input_tokens": 20353, + "output_tokens": 41021, + "calls": 3 + } + }, + "cache_read_tokens": 2985984, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.81, + "gate": 1.71, + "review": 1155.64, + "report": 0.06 + }, + "time_to_first_result_s": 1161.21 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4817.trace.json.gz", + "trace_bytes": 26355 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4817.md b/eval/dataset/arms/copilot_v14_train_r1/pr4817.md new file mode 100644 index 000000000..22b01273b --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4817.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4817, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head fc423e5a9e86) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] rename `_is_blackwell` -> `_is_quack_capable` leaves no stale callers (grep found only the definition and test). +- [resolved] test is hardware-free: quack_fp8 imports only os/torch/vllm.logger; quack/cutlass import is deferred to `_load_quack`, and install returns before it when the gate is False. +- [resolved] Multi-GPU heterogeneity: `torch.cuda.get_device_capability()` probes the current device, but vLLM workers run one GPU per process with per-rank CUDA_VISIBLE_DEVICES, so the per-process gate is correct; not raised as a finding. +- [claim-verified] pre-PR gate was `>= 10`: file_at_base quack_fp8.py:18-21 `_is_blackwell` returned `torch.cuda.get_device_capability()[0] >= 10`; introduced by b5352a58 (#4241) per search_history. +- [claim-verified] new test would fail on the old gate and pass on the new one: parametrized (12,0) expects False while old `>= 10` evaluates True. +- [claim-verified] escape hatch unchanged: quack_enabled checks env before `_is_quack_capable`; `_TRUTHY` unchanged at :14. +- [claim-verified] sm_100.x unchanged: `== 10` keeps (10,0)/(10,1)/(10,3) True. +- [claim-verified] Escape hatch preserved: quack_fp8.py:38-40 checks `VLLM_OMNI_USE_QUACK_FP8` before the capability probe; docs now document `=1` force-on. +- [claim-verified] sm_100 unchanged / sm_120 off: quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`. +- [claim-verified] PR's described per-GEMM fallback exists: quack_fp8.py:119-122 `try: out = quack_scaled_fp8_mm(...) ... except Exception: return original(self, ...)`. +- [claim-refuted] PR body says the test 'runs on CI without a Blackwell GPU', but no .buildkite job targets tests/quantization and the file has no pytestmark, so marker-based lanes deselect it (test-ready.yml:29). +- [claim-refuted] PR body root-cause lists sm_110a as a tcgen05 target, but the `== 10` gate excludes cc 11.x; sibling platform.py:80-81 treats major 11 as Blackwell — body/docstring/doc are inconsistent on sm_110. +- [claim-refuted] PR test plan says the new test 'runs on CI without a Blackwell GPU': tests/quantization/test_quack_fp8_gate.py has no pytestmark and .buildkite/test-merge.yml:29/.buildkite/test-ready.yml:29 select `-m 'core_model and cpu'`, so the unmarked test is collected then deselected; no .buildkite lane references tests/quantization (grep no matches). +- [validated] central gate read in situ: vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`; env override still checked first at :38-40; install early-returns at :102 so sm_120 skips quack. + +`tests/quantization/test_quack_fp8_gate.py:21` [major] — This diff adds `tests/quantization/test_quack_fp8_gate.py` as the regression guard for the `>= 10` -> `== 10` gate, but the module has no `pytestmark`. The Simple lanes select tests with `-m 'core_model and cpu'` (`.buildkite/test-merge.yml:29`, `.buildkite/test-ready.yml:29`), so the unmarked guard is collected then deselected — CI stays green while the regression guard never executes, and the PR's 'runs on CI without a Blackwell GPU' claim is not met. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` at module level and verify with `pytest -m 'core_model and cpu' tests/quantization/test_quack_fp8_gate.py --collect-only` that all nodes are selected. (evidence: Unchanged by this diff, present in the PR-time tree: `.buildkite/test-merge.yml:29` `- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (same command at `.buildkite/test-ready.yml:29`). Changed by this diff: `tests/quantization/test_quack_fp8_gate.py:17-21` `import pytest\nimport torch\n\nfrom vllm_omni.quantization import quack_fp8\n\n_ENV = "VLLM_OMNI_USE_QUACK_FP8"` — no module-level `pytestmark`; `grep pytestmark tests/quantization` returns no matches, and repo-wide grep for `pytest_collection_modifyitems`/`add_marker` returns no matches, so no hook auto-adds `core_model`/`cpu`. `pyproject.toml:234` registers `markers = [...]` with `"core_model: L1&L2 tests (run in each PR)"` and `"cpu: Tests that run on CPU"` but does not auto-assign them.) + +`vllm_omni/quantization/quack_fp8.py:32` [minor] — Gate changed to `== 10` and the docstring enumerates only `sm_100a/sm_101a/sm_103a` (10.x), but the PR body itself says `tcgen05.mma` targets `sm_110a` (11.x). A device reporting (11, 0) would now silently disable quack. Reconcile: gate `major in (10, 11)` if quack-kernels ships sm_110, or document why 11.x is intentionally excluded. (evidence: PR body: 'PTX `tcgen05.mma` targets `sm_100a / sm_101a / sm_103a / sm_110a`; `ptxas` rejects `tcgen05` on `sm_120`.' Changed by this diff — vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`; vllm_omni/quantization/quack_fp8.py:21 `(``sm_100a`` / ``sm_101a`` / ``sm_103a``, compute capability ``10.x``).`; unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/cuda/platform.py:83 `is_blackwell = major in (10, 11, 12)`.) + +`vllm_omni/quantization/quack_fp8.py:32` [minor] — The e2e numbers (>400 s → 9.7 s) and the native FlashInfer 'compute_120f' support appear only in the PR body and are absent from the PR-time repo tree (grep for '9.7' and 'compute_120f' found no matches), so they are not reproducible from this repo. Please mark them as vendor-provided/external in the PR body or add a reproducible benchmark for the sm_120 fallback path. (evidence: vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10` — the cited line is the capability gate and does not substantiate the e2e figures; PR body states '512×512 / 50 steps | > 400 s → client timeout (unusable) | 9.7 s' and 'working native FlashInfer FP8/NVFP4 kernels (`compute_120f`, cf. CUTLASS #3096, vLLM #31085)'; repo-wide grep for '9.7' and 'compute_120f' returned no matches in the PR-time tree, confirming those measurements/claims are external to the repo and only present in the PR body.) + +`tests/quantization/test_quack_fp8_gate.py:36` [nit] — The parametrization omits sm_101 even though the docstring names sm_100a/sm_101a/sm_103a. Add `((10, 1), True), # sm_101 datacenter Blackwell` so every claimed datacenter SKU is exercised by the gate test. (evidence: tests/quantization/test_quack_fp8_gate.py:6 `exists only on datacenter Blackwell (``sm_100a`` / ``sm_101a`` / ``sm_103a``,`; tests/quantization/test_quack_fp8_gate.py:35 `((10, 0), True), # sm_100 datacenter Blackwell — tcgen05 present`; tests/quantization/test_quack_fp8_gate.py:36 `((10, 3), True), # sm_103 datacenter Blackwell` — no `(10, 1)` case present; vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10` (unchanged by this diff aside from the rename to `_is_quack_capable`) means `(10, 1)` would be accepted but is untested.) + +`vllm_omni/quantization/quack_fp8.py:~32` [nit] — Please confirm whether `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` failures are pre-existing on main. The diff (3 files) does not intersect what those lanes execute: NPU runs only `tests/e2e/online_serving/test_qwen3_tts_customvoice.py` (`.buildkite/test-npu-ready.yaml:21`), AMD runs `tests/diffusion`/`tests/model_executor` plus unrelated e2e/engine tests, and no `.buildkite` lane references `tests/quantization` or `quack_fp8`. The changed gate is behind `if not torch.cuda.is_available(): return False` (`quack_fp8.py:30-31`), which is False on NPU, and AMD MI325 reports a non-10 capability, so `>= 10` -> `== 10` causes no behavior delta for either lane. (evidence: Changed by this diff: vllm_omni/quantization/quack_fp8.py:30-32 ` if not torch.cuda.is_available():` / ` return False` / ` return torch.cuda.get_device_capability()[0] == 10` — the `== 10` gate is exactly the snippet the comment anchors. Unchanged by this diff, present in the PR-time tree: .buildkite/test-npu-ready.yaml:21 `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; .buildkite/test-amd-merge.yml:20 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`. gh_ci_read(4817) states: `{"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`, `{"name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}`, `{"name":"buildkite/vllm-omni","state":"SUCCESS"}`. Grep of `.buildkite` for `quack_fp8` and `tests/quantization` returns no matches.) + +`tests/quantization/test_quack_fp8_gate.py:~31` [nit] — This diff pins auto-enable to `get_device_capability()[0] == 10` (vllm_omni/quantization/quack_fp8.py:31), which also disables cc 11.x (`sm_110a`), while the parametrize matrix at tests/quantization/test_quack_fp8_gate.py:35-40 has no 11.x case and the PR body lists `sm_110a` among `tcgen05.mma` targets. Add `((11, 0), False)` with a comment stating cc 11.x is intentionally excluded until CuteDSL ships `sm_110a`, so a future refactor back to `>= 10` cannot silently re-enable it. (evidence: tests/quantization/test_quack_fp8_gate.py:35 ` ((10, 0), True), # sm_100 datacenter Blackwell — tcgen05 present`; tests/quantization/test_quack_fp8_gate.py:36 ` ((10, 3), True), # sm_103 datacenter Blackwell`; tests/quantization/test_quack_fp8_gate.py:37 ` ((12, 0), False), # sm_120 workstation Blackwell — the regression`; tests/quantization/test_quack_fp8_gate.py:38 ` ((12, 1), False), # sm_121 consumer Blackwell`; tests/quantization/test_quack_fp8_gate.py:39 ` ((9, 0), False), # Hopper — CUTLASS already fuses bias, quack unused`; tests/quantization/test_quack_fp8_gate.py:40 ` ((8, 9), False), # Ada`; vllm_omni/quantization/quack_fp8.py:31 ` return torch.cuda.get_device_capability()[0] == 10`. No `(11,` parametrize case exists in the file; PR body text states PTX `tcgen05.mma` targets `sm_100a / sm_101a / sm_103a / sm_110a`.) + +`docs/user_guide/quantization/fp8.md:54` [nit] — docs/user_guide/quantization/fp8.md:54 advertises `VLLM_OMNI_USE_QUACK_FP8=1` as a generic force-on. On sm_120/sm_121 that env override bypasses `_is_quack_capable` (quack_fp8.py:39) and re-enters the per-GEMM FlashInfer fallback the PR's own docstring calls 'catastrophically slow' (quack_fp8.py:24-26). Add the same caveat the test carries: force-on is for 'once CuteDSL ships sm_120a support'. (evidence: docs/user_guide/quantization/fp8.md:54 `VLLM_OMNI_USE_QUACK_FP8=1` to force quack on, or `VLLM_OMNI_USE_QUACK_FP8=0` to force` — changed by this diff, no sm_120 caveat. vllm_omni/quantization/quack_fp8.py:39 `return override.lower() in _TRUTHY` — unchanged by this diff but present in the PR-time tree; truthy env bypasses the capability gate for every arch. vllm_omni/quantization/quack_fp8.py:24-26 ` ``12.x``, e.g. RTX PRO 6000 / RTX 50-series) lacks ``tcgen05``, so quack can` / ` never run there — CuteDSL rejects the arch and every GEMM falls back to` / ` FlashInfer one call at a time, which is catastrophically slow. Those GPUs` — changed by this diff. tests/quantization/test_quack_fp8_gate.py:63 ` # sm_120 auto-disables, but an explicit truthy override forces quack on` — new file in this diff, matching the requested caveat.) + +`vllm_omni/quantization/quack_fp8.py:~136` [nit] — This diff makes quack off by default on sm_120/121 via `quack_enabled()`, but `warmup_quack_fp8()` (quack_fp8.py:136) bypasses that gate and calls `_load_quack()` directly — so a user on sm_120 who follows the documented 'Compile cache and warmup' snippet with quack installed still triggers quack's CuteDSL arch rejection (uncaught) instead of staying on the FlashInfer path the new default selects. Make `warmup_quack_fp8()` return early when `quack_enabled()` is False (the env override still allows forcing warmup on). (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/quantization/quack_fp8.py:131-136 `def warmup_quack_fp8(...) -> None:` ... ` if _load_quack() is None:` ` return` — no `quack_enabled()` check. Contrast vllm_omni/quantization/quack_fp8.py:101-104 `def install_quack_fp8_patch() -> None:` ` if not quack_enabled():` ` return` ` if _load_quack() is None:` ` return`. Also vllm_omni/quantization/quack_fp8.py:96 `gemm.gemm_tuned(a, b, out, bias=bias, alpha=alpha)` has no try/except, while the sm_120 gate is vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4817.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4817.trace.json.gz new file mode 100644 index 000000000..decbc9d4a Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4817.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4859.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4859.cost.json new file mode 100644 index 000000000..f8c1e21bc --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4859.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1165.6, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4859/run-20260815-030916-ec25a9", + "recorded_at": "2026-08-15T03:28:39", + "attempts": [ + { + "invocation_id": "inv-3938bbc2", + "outcome": "run:run-20260815-030916-ec25a9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-030916-ec25a9", + "llm_calls": 96, + "input_tokens": 872158, + "output_tokens": 225507, + "usd": 0.684292, + "source": "spans" + } + ], + "attempt_usd_total": 0.684292, + "llm_calls": 96, + "input_tokens": 872158, + "output_tokens": 225507, + "usd": 0.684292, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-030916-ec25a9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6843, + "minutes": 19.39, + "ci_minutes": 0, + "input_tokens": 872158, + "output_tokens": 225507, + "tool_calls": 249, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8006, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 96, + "by_role": { + "agent": { + "usd": 0.657282, + "input_tokens": 851852, + "output_tokens": 205974, + "calls": 94 + }, + "reducer": { + "usd": 0.02701, + "input_tokens": 20306, + "output_tokens": 19533, + "calls": 2 + } + }, + "cache_read_tokens": 7435264, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.52, + "gate": 2.28, + "review": 1157.32, + "report": 0.07 + }, + "time_to_first_result_s": 1163.16 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4859.trace.json.gz", + "trace_bytes": 40781 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4859.md b/eval/dataset/arms/copilot_v14_train_r1/pr4859.md new file mode 100644 index 000000000..d66b7061a --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4859.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4859, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6ff4ce59ed87) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] removed `_audio_pcm_format`/`_AUDIO_SAMPLE_RATE_ENV`/`_AUDIO_CHANNELS_ENV` from patch.py: grep for `_audio_pcm_format` and `_AUDIO_SAMPLE_RATE_ENV` returns zero matches — no dangling caller. +- [resolved] No stale include_language caller remains anywhere in the repo (grep '.' for include_language: no matches). +- [resolved] removed `include_language` kwarg leaves no stale in-repo callers — grep `include_language` across vllm_omni returns no matches. +- [claim-verified] diff_stat: 6 files changed, 59 insertions / 45 deletions, no test files touched — matches sweep_targets. +- [claim-verified] PR test plan commands are benchmark-only (`vllm bench serve ... --seed-tts-wer-eval`); no semantic unit test is shown for any changed path. +- [claim-verified] PR body's `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100` test plan is present in pr_context. +- [claim-verified] diff_stat confirms the changed-file list matches the PR scope exactly (the 6 files in the diff). +- [claim-refuted] PR claims Seed-TTS EN WER 1.83% using VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100, but patch.py already writes tts_output_pcm_bytes at 24 kHz mono (_pcm_s16le_to_seed_tts_wer_bytes docstring 'Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER'), and seed_tts_eval.py:189/201 now resamples that PCM as if it were 44100 -> target_len = len(raw)*16000/44100 instead of /24000; the reported WER is computed on sped-up/truncated audio. +- [claim-refuted] PR body reports mean WER 0.0183 with env VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100; at merged head the consumer seed_tts_eval.py:189 re-reads that env and resamples the already-normalized 24 kHz `tts_output_pcm_bytes` (patch.py:427/1215) as 44.1 kHz, so the 1.83% figure is computed on time-distorted audio and is not trustworthy. +- [claim-refuted] PR's headline 'Mean WER 0.0183' is not valid evidence under the PR's own command: test plan sets VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100, but patch.py:1215 `_pcm_s16le_to_seed_tts_wer_bytes` already normalizes capture to `target_sr=24000`, then seed_tts_eval.py:616 `_pcm_s16le_to_f32_16k(pcm)` re-applies 44100 → 0.544x-duration double resample (calc: 24000/44100=0.5442). +- [validated] patch.py:1163 and seed_tts_eval.py:189 both call the same `defs.stream_pcm_format_from_env()`; capture and WER-decode now share one rate/channel source. +- [validated] tests/model_executor/models/ming_tts/test_ming_shared_modules.py:88-98: the existing instruction-parser test does not pass `include_language`, so it still passes after the removal; no test asserts language->方言. +- [validated] patch.py:1215 (openai-audio-speech) and patch.py:948/957 (openai-chat-omni) both produce tts_output_pcm_bytes as 24 kHz mono; the two producers agree, and seed_tts_eval.py:586/616 consumes that field. +- [validated] patch.py:1163 `sample_rate, channels = defs.stream_pcm_format_from_env()` replaces the removed `_audio_pcm_format`; grep for `_audio_pcm_format` across vllm_omni returns no remaining references. + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:141` [major] — audio_vae.py:131 builds `self.encoder = Qwen2Model(config)` with the encoder-depth config; audio_vae.py:141 then mutates the SAME object in place (`config.num_hidden_layers = 4`) before audio_vae.py:142 builds `self.aggregator = Qwen2Model(config)`. Transformers keeps the config by reference, so the encoder's actual layers are at the original depth while `self.encoder.config.num_hidden_layers` reads 4, and encoder/aggregator share one mutable config. Use a fresh copy for the aggregator (e.g. the removed `_qwen2_config({**encoder_args, "num_hidden_layers": 4})`) instead of mutating the consumed object. (evidence: audio_vae.py:130 ` config = _qwen2_config(encoder_args)`; audio_vae.py:131 ` self.encoder = Qwen2Model(config)` (unchanged context); audio_vae.py:141 ` config.num_hidden_layers = 4` (added by this diff); audio_vae.py:142 ` self.aggregator = Qwen2Model(config)` (changed by this diff). The removed diff line `aggregator_config = _qwen2_config({**encoder_args, "num_hidden_layers": 4})` proves the old code constructed a separate copy and never touched the encoder's config. Since Qwen2Model builds its decoder layers when line 131 executes and then keeps `config` by reference, line 141 leaves `self.encoder.config.num_hidden_layers == 4` while the encoder module's layers were built at the original depth.) + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:189` [major] — Confirmed: seed_tts_eval.py:616 calls `_pcm_s16le_to_f32_16k(pcm)` with no arguments, and the diff-added branch at seed_tts_eval.py:189 reads `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`/`VLLM_OMNI_BENCH_AUDIO_CHANNELS` for any missing argument. That `pcm` is `out.tts_output_pcm_bytes`, already normalized to 24 kHz mono by `_pcm_s16le_to_seed_tts_wer_bytes` (patch.py:1215). With the PR test plan exporting `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100`, the eval path resamples 24 kHz PCM as 44.1 kHz, producing `len(raw)*16000/44100` samples — 24000/44100 ≈ 0.544x the correct 16 kHz duration — which corrupts the reported 1.83% Seed-TTS WER. Pass `pcm_sample_rate=24000, channels=1` at the call site (or do not consult the env in the eval path) and rerun the benchmark. (evidence: ['seed_tts_eval.py:189 (added by this diff): `env_sample_rate, env_channels = stream_pcm_format_from_env()` — inside `if pcm_sample_rate is None or channels is None:`, so the no-arg call path reads the benchmark env vars.', 'seed_tts_eval.py:616 (unchanged by this diff, present in PR-time tree): `wav_16k = _pcm_s16le_to_f32_16k(pcm)` — passes no sample rate or channels.', 'patch.py:1215 (unchanged by this diff, present in PR-time tree): `output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(bytes(pcm_capture), sample_rate=sample_rate, channels=channels)` — the audio-speech path normalizes captured PCM via the helper below.', 'patch.py:112-119 (unchanged by this diff, present in PR-time tree): `def _pcm_s16le_to_seed_tts_wer_bytes(...)` with docstring `Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER.` and `resampler = AudioResampler(target_sr=24000)` — the bytes stored in `tts_output_pcm_bytes` are 24 kHz mono before reaching seed_tts_eval.py.']) + +`.buildkite/test-amd-merge.yml:32` [major] — Resolve or document the failing `buildkite/vllm-omni-amd-ci` gate before merge. `.buildkite/test-amd-merge.yml:32` runs `pytest -sv tests/model_executor -m 'core_model and cpu'`, which collects `tests/model_executor/models/ming_tts/test_ming_shared_modules.py` (marked core_model+cpu); that file imports the PR-changed `serving_speech` and `audio_vae` modules, so the lane's collection surface intersects this diff. The diff alone cannot prove the failure was introduced by this PR — pull the lane logs and either fix the red lane or document the pre-existing failure. (evidence: Unchanged by this diff, present in PR-time tree — .buildkite/test-amd-merge.yml:32 `- "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`. Test file unchanged by this diff — tests/model_executor/models/ming_tts/test_ming_shared_modules.py:7-8 `from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech` / `from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAEConfig`, with line 27 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.tts]` so `-m 'core_model and cpu'` collects it. diff_stat shows this PR modifies vllm_omni/entrypoints/openai/serving_speech.py and vllm_omni/model_executor/models/common/ming/audio_vae.py (plus ming_tts/patch_emission.py), so the collected test imports modules touched by the PR. gh_ci_read for PR 4859 returns buildkite/vllm-omni-amd-ci state FAILURE.) + +`vllm_omni/entrypoints/openai/serving_speech.py:2623` [major] — This diff deletes the only `include_language` plumbing in `_parse_ming_instruction_fields` and stops `_parse_ming_instruction` from passing `include_language=True`, so the OpenAI `language` field is silently ignored for Ming dense/MoE instruction payloads while examples/online_serving/text_to_speech/ming_tts/README.md:101 and openai_speech_client.py:192 still advertise it. `_conditioning_cache_salt` (serving_speech.py:276) still hashes `request.language`. Either keep the old mapping behind a deprecation path or update the docs/client and salt; add a regression test for non-Auto `language` behavior. (evidence: PR-time serving_speech.py:2623 ` include_voice=False,` — the signature now lacks `include_language`; the diff deletes `if include_language and request.language not in (None, "", "Auto"):` and `instruction_dict["方言"] = request.language` (repo grep for `include_language` returns no matches, and grep for `方言` under serving_speech.py returns nothing). serving_speech.py:2648-2651 ` return self._parse_ming_instruction_fields( + request, + include_voice=True, + plain_text_passthrough=True, + )` — no `include_language=True`. Unchanged by this diff but present in the PR-time tree: serving_speech.py:276 ` request.language,` inside `_conditioning_cache_salt`; examples/online_serving/text_to_speech/ming_tts/README.md:101 `| `language` | Ming `方言` control |`; examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py:192 ` parser.add_argument("--dialect", default=None, help="Maps to Ming `方言`")`.) + +`vllm_omni/model_executor/models/ming_tts/patch_emission.py:130` [minor] — This diff changes both `_validate_ming_decode_window` (patch_emission.py:130) and `_resolve_ming_stop_decision` (patch_emission.py:148) from `min_stop_step + 1` to `min_stop_step + 2`. With `STOP_HEAD_MIN_STEPS = 3`, a request with `ming_max_decode_steps == 4` previously passed (`max(3+1, 0) == 4`) and now raises at forward time because `max_steps < max(3+2, 0)`. Since `serving_speech.py:2679` passes `request.max_new_tokens` into `ming_max_decode_steps`, small user `max_new_tokens` values regress at this boundary. Please add a boundary regression test at `min_stop_step + 1` vs `min_stop_step + 2` (e.g. `pytest tests/model_executor/models/ming_tts/test_ming_shared_modules.py -m 'core_model and cpu'`) and state the HF-parity evidence for the +2 offset. (evidence: patch_emission.py:130 `min_required = max(min_stop_step + 2, min_steps)` (changed from `min_stop_step + 1` in the diff); patch_emission.py:148 `min_required_decode_steps = max(min_stop_step + 2, min_decode_steps)`; patch_emission.py:133-136 `if max_steps < min_required: ... raise ValueError(...)`; constants.py:51 `STOP_HEAD_MIN_STEPS = 3` (unchanged by this diff, present in PR-time tree); serving_speech.py:2679 `runtime_controls[KEY_MAX_DECODE_STEPS] = request.max_new_tokens` (unchanged by this diff, present in PR-time tree).) + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:351` [minor] — This diff changes `AudioVAE.encode_latent` from deterministic mean-only output to posterior sampling (`mean + std * torch.randn_like(mean)`) while the method is under `@torch.inference_mode()` (audio_vae.py:328) and no caller seeds a generator — grep of `vllm_omni/model_executor/models` found no `manual_seed`/`generator`. Identical reference audio now yields different prompt latents across requests and cached voice presets (prompt_encoder.py:137 and voice_presets.py:257, both unchanged by this diff), so any reported WER number carries unquantified run-to-run variance. Confirm this matches the HF posterior for both Ming dense and flash; if sampling is intended, add seed control and report variance (or run WER with a fixed seed), and add a unit test that seeds `torch.manual_seed` and asserts reproducibility/finiteness. (evidence: Changed by this diff — vllm_omni/model_executor/models/common/ming/audio_vae.py:351 `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)`. Unchanged but present in the PR-time tree — audio_vae.py:328 `@torch.inference_mode()`; vllm_omni/model_executor/models/ming_tts/prompt_encoder.py:137 `latents, _ = encoder.encode_latent(waveform, waveform_length)`; vllm_omni/model_executor/models/ming_flash_omni/voice_presets.py:257 `prompt_wav_lat, _ = self._audio_vae.encode_latent(`. Grep for `manual_seed|randn_like|generator` under `vllm_omni/model_executor/models` returned no matches, so no seed/generator is supplied on these paths.) + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:116` [minor] — `_save_seed_tts_eval_audio` still writes `pcm_s16le_mono_to_wav_bytes(pcm, sample_rate=24000)` at seed_tts_eval.py:116, and `pcm_s16le_mono_to_wav_bytes` always writes mono. Since the new env-driven `stream_pcm_format_from_env` is only used in the WER decode path (`_pcm_s16le_to_f32_16k`), saved eval WAVs remain 24 kHz mono even when `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`/`VLLM_OMNI_BENCH_AUDIO_CHANNELS` are set. Update the save path to use the resolved sample rate/channels and add a matching test. (evidence: vllm_omni/benchmarks/data_modules/seed_tts_eval.py:116 ` path.write_bytes(pcm_s16le_mono_to_wav_bytes(pcm, sample_rate=24000))` — unchanged by this diff, present in the PR-time tree; vllm_omni/benchmarks/data_modules/seed_tts_eval.py:86 `def pcm_s16le_mono_to_wav_bytes(pcm: bytes, *, sample_rate: int = 24000) -> bytes:` — always writes mono; vllm_omni/benchmarks/data_modules/seed_tts_eval.py:189 ` env_sample_rate, env_channels = stream_pcm_format_from_env()` — env parsing is consumed only in the WER decode path, not the save path.) + +`vllm_omni/metrics/definitions.py:317` [minor] — Add a unit test for `stream_pcm_format_from_env` (vllm_omni/metrics/definitions.py:317–337). Repo-wide grep shows no test references — only benchmark callers at seed_tts_eval.py:64/189 and patch.py:1163. Cover env-var parsing, invalid int warning/fallback, and the `max(..., 1)` clamp. (evidence: vllm_omni/metrics/definitions.py:317 `def stream_pcm_format_from_env(` ... vllm_omni/metrics/definitions.py:337 ` return max(sample_rate, 1), max(channels, 1)`; repo-wide grep for `stream_pcm_format_from_env` returns only vllm_omni/metrics/definitions.py:317, vllm_omni/benchmarks/data_modules/seed_tts_eval.py:64 and :189, and vllm_omni/benchmarks/patch/patch.py:1163 — no `tests/` or test-file matches.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4859.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4859.trace.json.gz new file mode 100644 index 000000000..b2872c1fe Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4859.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4870.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4870.cost.json new file mode 100644 index 000000000..95d5e65d4 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1871.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4870/run-20260815-024433-bd24c1", + "recorded_at": "2026-08-15T03:15:41", + "attempts": [ + { + "invocation_id": "inv-b2d7def4", + "outcome": "run:run-20260815-024433-bd24c1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-024433-bd24c1", + "llm_calls": 192, + "input_tokens": 2313193, + "output_tokens": 274841, + "usd": 1.507309, + "source": "spans" + } + ], + "attempt_usd_total": 1.507309, + "llm_calls": 192, + "input_tokens": 2313193, + "output_tokens": 274841, + "usd": 1.507309, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-024433-bd24c1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.5073, + "minutes": 31.14, + "ci_minutes": 0, + "input_tokens": 2313193, + "output_tokens": 274841, + "tool_calls": 338, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2588, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 192, + "by_role": { + "agent": { + "usd": 1.482384, + "input_tokens": 2295520, + "output_tokens": 256558, + "calls": 190 + }, + "reducer": { + "usd": 0.024924, + "input_tokens": 17673, + "output_tokens": 18283, + "calls": 2 + } + }, + "cache_read_tokens": 21497088, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.92, + "gate": 1.77, + "review": 1861.85, + "report": 0.07 + }, + "time_to_first_result_s": 1868.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 41922 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4870.md b/eval/dataset/arms/copilot_v14_train_r1/pr4870.md new file mode 100644 index 000000000..d50b6d03b --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4870.md @@ -0,0 +1,57 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Existing `test_prepare_speech_generation_treats_audio_as_streaming` still expects `coerce(..., True)`: with `async_chunk` unset the new gate's `not bool(async_chunk)` is False, so `qwen3_full_payload=False` and the old DELTA coercion is preserved. +- [resolved] Prior #4527 non-async pooler-starvation concern is not regressed: gpu_ar_model_runner.py:1795-1804 still ships the full payload to the next stage for `async_chunk=False`; this PR only changes the split length. +- [claim-verified] PR diff evidence contains exactly the four stated files (serving_speech.py, gpu_ar_model_runner.py, test_serving_speech.py, test_gpu_ar_model_runner.py); the probe file is not in the diff. +- [claim-verified] PR body E2E table shows the only real-serving validation used `stream=False` (`test_voice_001_nonstream_concurrent_no_async_chunk[no_async_chunk]`); the stream=True + async_chunk=False path is covered only by the coercion unit test. +- [claim-verified] diff_stat confirms exactly the 4 files the PR body claims ("four files changed"), 99 insertions/13 deletions. +- [claim-verified] NPUARModelRunner is the NPU AR runner (platforms/npu/worker/npu_ar_worker.py:19 `self.model_runner = NPUARModelRunner(...)`) and Qwen3-TTS talker is patched on NPU (platforms/npu/_310p/patch/qwen3_tts.py), so the NPU sibling reaches the same full-payload path. +- [claim-verified] diff_stat shows exactly the 4 claimed files changed and no remote validation probe file was included, matching the PR body. +- [claim-verified] the PR body's H20 E2E numbers and the quoted pytest/ruff results are remote claims; they cannot be reproduced from this read-only checkout. +- [claim-verified] PR description states the probe file 'is not included in this PR diff' — diff_stat confirms only the 4 files above. +- [claim-refuted] PR body says 'Reproduced the real serving failure on remote H20 before the fix' and 'Verified the same real serving E2E path... after the fix' — the probe file is explicitly not in the diff, so the claim is not reproducible from this PR's committed tests. +- [validated] Worker regression test genuinely drives the tail-only case: `hidden_states=torch.tensor([[1.0]])` (shape [1,1]) vs `codes` shape [3,16] with `total_num_scheduled_tokens=3` and 1 token per request, asserting `codes[0:1]`/`codes[1:2]`/`codes[2:3]` slices. +- [validated] New serving tests exercise the qwen3 FINAL_ONLY coercion (False) and the non-qwen3 DELTA coercion (True) paths through the real `_prepare_speech_generation` entry point with the real `_tts_model_type` detection. +- [validated] gpu_ar_model_runner.py:1734 now computes `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` and passes it at line 1790; to_payload_element (mm_outputs.py:131) slices dim0==seq_len tensors per request, so the batched codes.audio path is repaired. +- [validated] serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; coerce_param_message_types (entrypoints/utils.py:759-780) converts CUMULATIVE->DELTA only when is_streaming, so keeping False preserves FINAL_ONLY for qwen3 no-async-chunk streaming. + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:~1144` [major] — The GPU runner now slices the multimodal payload with mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) (gpu_ar_model_runner.py:1734, :1787), but the NPU twin still passes seq_len = hidden_states.shape[0] (npu_ar_model_runner.py:905) into to_payload_element at :1144. For Qwen3-TTS, requires_full_prefix_cached_hidden_states=False makes hidden states tail-only, so the mm_outputs.py:113 slicing gate fails and the whole batched codes.audio tensor is cloned into every request. Mirror the one-line GPU change in NPUARModelRunner.sample_tokens, or document Qwen3-TTS no-async-chunk as unsupported on NPU. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905 `seq_len = hidden_states.shape[0]`; vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:1144 `seq_len=seq_len,`; vllm_omni/utils/mm_outputs.py:113 `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len:` (returns element[start:end].contiguous()) with fallback `elif isinstance(element, torch.Tensor): ... return element.clone()`; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:311 `self.requires_full_prefix_cached_hidden_states = False`. The GPU side this diff changed: vllm_omni/worker/gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` and :1787 `seq_len=mm_seq_len`.) + +`vllm_omni/entrypoints/openai/serving_speech.py:~3509` [minor] — Verified at serving_speech.py:3508-3509: `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` and `is_streaming_request = request.is_streaming() and not qwen3_full_payload` keep FINAL_ONLY for streaming qwen3_tts with async_chunk=False. The PR's only real-serving E2E case is stream=False, and the added unit test asserts only `coerce_param_message_types(..., False)`, so it does not exercise `_generate_audio_chunks`/`_generate_audio_sse_events` on a real stream=True endpoint. Minor: run one stream=True (raw audio or SSE) E2E with --no-async-chunk to prove the final waveform is emitted once. (evidence: serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`) + +`vllm_omni/entrypoints/openai/serving_speech.py:~3507` [minor] — serving_speech.py:3507 `async_chunk = getattr(self.model_config, "async_chunk", True)` differs from the existing read at 3823 `async_chunk = bool(getattr(self.engine_client.model_config, "async_chunk", False))` and from the OmniModelConfig default `async_chunk: bool = False` (vllm_omni/config/model.py:121). Align both sites on a single accessor/default (e.g. `self.engine_client.model_config` / `False`) and make the new test set the same object production reads (`self.model_config` vs `engine_client.model_config`). (evidence: PR-time tree, added by this diff: vllm_omni/entrypoints/openai/serving_speech.py:3507 `async_chunk = getattr(self.model_config, "async_chunk", True)`. Pre-existing in the same file: vllm_omni/entrypoints/openai/serving_speech.py:3823 `async_chunk = bool(getattr(self.engine_client.model_config, "async_chunk", False))`. Repo config default, unchanged by this diff: vllm_omni/config/model.py:121 `async_chunk: bool = False`. New test in the diff: tests/entrypoints/openai_api/test_serving_speech.py:3454 `qwen3_tts_server.engine_client.model_config.async_chunk = False`.) + +`vllm_omni/entrypoints/openai/serving_speech.py:~3508` [minor] — `_prepare_speech_generation` is shared by HTTP raw-audio, HTTP SSE, and WebSocket streaming (serving_speech.py:3733, 4069, 4089; serving_speech_stream.py:273). The new `qwen3_full_payload` gate therefore switches qwen3_tts with `async_chunk=False` to FINAL_ONLY in every one of those paths, while the only added tests exercise the raw-audio qwen3 path and the voxtral keep-DELTA path. Add or point to qwen3 SSE/WebSocket coverage and one more non-qwen3 `async_chunk=False` TTS type, or narrow the comment to the validated surface. (evidence: serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload` (both changed by this diff, inside the shared `_prepare_speech_generation`). Unchanged-by-this-diff callers present in the PR-time tree: serving_speech.py:3733 `request_id, generator, _ = await self._prepare_speech_generation(request)` (inside `_iter_pcm_audio_bytes` at :3731); serving_speech_stream.py:273 `request_id, generator, _ = await self._speech_service._prepare_speech_generation(request)` (WebSocket handler); serving_speech.py:4069 and :4089 (HTTP raw-audio and SSE paths in `create_speech`). The PR diff adds only `test_prepare_speech_generation_no_async_chunk_stream_uses_final_only` (qwen3_tts, `stream_format="audio"`) and `test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3` (voxtral).) + +`.buildkite/test-amd.yaml:~1` [minor] — The buildkite/vllm-omni-amd-ci gate is FAILURE at this head. .buildkite/test-amd.yaml runs diffusion, Qwen2.5/Qwen3-Omni, tests/engine/test_async_omni_engine_abort.py (line 92), and tests/entrypoints/test_omni_sleep_mode.py (line 118) — but grep over .buildkite finds no reference to the changed test files test_serving_speech/test_gpu_ar_model_runner, so this lane does not exercise the changed paths. Confirm the AMD failure is pre-existing on main and do not treat this gate as coverage for the PR. (evidence: ['unchanged by this diff, present in the PR-time tree: .buildkite/test-amd.yaml:79 ` - pytest -s -v tests/diffusion/test_diffusion_worker.py`', 'unchanged by this diff, present in the PR-time tree: .buildkite/test-amd.yaml:105 ` - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py`', 'unchanged by this diff, present in the PR-time tree: .buildkite/test-amd.yaml:118 ` - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"`', 'grep `.buildkite` for `test_serving_speech|test_gpu_ar_model_runner`: no matches — the changed test files are absent from the lane config.', 'diff_stat: this PR changes tests/entrypoints/openai_api/test_serving_speech.py, tests/worker/test_gpu_ar_model_runner.py, vllm_omni/entrypoints/openai/serving_speech.py, and vllm_omni/worker/gpu_ar_model_runner.py.']) + +`tests/e2e/online_serving/test_qwen3_tts_base_expansion.py:~114` [minor] — This diff changes the payload split length to `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` (vllm_omni/worker/gpu_ar_model_runner.py:1734) and the serving coercion in `_prepare_speech_generation`, but none of the committed tests drive the real producer path with concurrent `no_async_chunk` requests: the worker test monkeypatches `_process_additional_information_updates`/`_resolve_pooler_payload_req_ids` and calls `_build_omni_model_runner_output_from_snapshot` directly, and the serving test only asserts `coerce_param_message_types(..., False)`. Add (or explicitly track) a hardware-gated regression that runs issue #4851's `test_response_format_001[no_async_chunk]` (Base) and a concurrent CustomVoice E2E case, or extend the worker test to drive `sample_tokens` with a tail-only hidden state and real scheduler layout. (evidence: Production change (in diff): vllm_omni/worker/gpu_ar_model_runner.py:1734 ` mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`. Serving test (in diff): tests/entrypoints/openai_api/test_serving_speech.py:3471 ` mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, False)`. Worker test (in diff, tests/worker/test_gpu_ar_model_runner.py hunk @@ -439,6 +435,51 @@) shows only a static-builder call with mocks: `+ monkeypatch.setattr(GPUARModelRunner, "_resolve_pooler_payload_req_ids", lambda self, req_ids: ("latent", req_ids),)` and `+ monkeypatch.setattr(GPUARModelRunner, "_process_additional_information_updates", lambda *args, **kwargs: None)` then `+ output = GPUARModelRunner._build_omni_model_runner_output_from_snapshot(`. diff_stat reports only 4 changed files, none of which is tests/e2e/online_serving/test_qwen3_tts_base_expansion.py, so the E2E path the comment asks for is not committed.) + +`vllm_omni/worker/gpu_ar_model_runner.py:~1734` [nit] — Add a comment (and preferably an assert) in _build_omni_model_runner_output_from_snapshot near `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` documenting that split-length multimodal tensors are aligned to the scheduled-token total: when `element.shape[0] != seq_len`, `to_payload_element` silently returns `element.clone()` and would emit the whole batch for that request. (evidence: vllm_omni/worker/gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` (changed by this diff; feeds `seq_len` to `_build_omni_pooler_payload` -> `to_payload_element`). Unchanged by this diff, present in the PR-time tree: vllm_omni/utils/mm_outputs.py:131 `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len: return element[start:end].contiguous()`; fallback vllm_omni/utils/mm_outputs.py:151 `return element.clone()` (whole tensor). Therefore slicing only occurs when the tensor's first dim exactly equals `total_num_scheduled_tokens`; any mismatch silently clones the full tensor.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4870.trace.json.gz new file mode 100644 index 000000000..a79b14d42 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4923.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4923.cost.json new file mode 100644 index 000000000..f3a9ba7b9 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1483.5, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4923/run-20260815-024434-7e486d", + "recorded_at": "2026-08-15T03:09:14", + "attempts": [ + { + "invocation_id": "inv-91d39050", + "outcome": "run:run-20260815-024434-7e486d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-024434-7e486d", + "llm_calls": 125, + "input_tokens": 1630707, + "output_tokens": 285419, + "usd": 1.080564, + "source": "spans" + } + ], + "attempt_usd_total": 1.080564, + "llm_calls": 125, + "input_tokens": 1630707, + "output_tokens": 285419, + "usd": 1.080564, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-024434-7e486d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0806, + "minutes": 24.66, + "ci_minutes": 0, + "input_tokens": 1630707, + "output_tokens": 285419, + "tool_calls": 254, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0298, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 125, + "by_role": { + "agent": { + "usd": 1.032723, + "input_tokens": 1603667, + "output_tokens": 248627, + "calls": 122 + }, + "reducer": { + "usd": 0.047841, + "input_tokens": 27040, + "output_tokens": 36792, + "calls": 3 + } + }, + "cache_read_tokens": 12085632, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.64, + "gate": 1.86, + "review": 1473.03, + "report": 0.06 + }, + "time_to_first_result_s": 1479.57 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 38446 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4923.md b/eval/dataset/arms/copilot_v14_train_r1/pr4923.md new file mode 100644 index 000000000..75926186e --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4923.md @@ -0,0 +1,66 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] Prior concern 'nested inner code-predictor graphs on GPU under outer full graph' is resolved: qwen3_tts_code_predictor_vllm.py:39 now passes `use_cuda_graphs=current_omni_platform.is_npu()`, so `_capture_cuda_graphs` is skipped on CUDA (qwen3_code_predictor.py:552). +- [resolved] NPU consistency: PIECEWISE has no full cudagraphs, so talker_mtp is not outer-wrapped and per-row generators stay enabled, while use_cuda_graphs=is_npu() keeps the code predictor's internal NPU graphs — matches the PR's platform split. +- [resolved] Async output vs prefix cache mutual exclusion: gpu_ar_model_runner.py `_should_use_async_omni_output` returns False when `self.omni_prefix_cache is not None`; yaml now sets stage 0 enable_prefix_caching=false so the async path activates. +- [claim-verified] PR body claims the #4970 `seed: 42` removal is NOT included; grep for `seed` in qwen3_tts.yaml returns no matches. +- [claim-verified] `talker_mtp_graph_safe` consumers: gpu_model_runner.py:208 (wrap decision), npu_ar_model_runner.py:184 (capture error text), fish_speech_slow_ar.py:198 — Qwen3-TTS now opts in consistently. +- [claim-verified] cited serving benchmark arithmetic checks out: 32 req / 4.56 s = 7.017 req/s; 179.44 audio-s / 4.56 s = 39.35 audio-s/s. +- [claim-verified] PR is MERGED and files list matches diff_stat surface (3 files). +- [claim-refuted] PR body says 'Qwen3-TTS talker did not set ... omni_pooler_payload_include_hidden = False', but file_at_base already contains `self.omni_pooler_payload_include_hidden = False` (~line 317); the diff adds a duplicate assignment at line 353. +- [claim-refuted] PR body 'restore a single batched talker_mtp call per decode step under full cudagraphs' is only true for unseeded batches: gpu_model_runner.py:1844-1865 still enters the scalar-per-row fallback whenever any row has tts_local_seed and talker_mtp_accepts_per_row_generators is False. +- [claim-refuted] PR claim 'single batched talker_mtp call per decode step under full cudagraphs' only holds for unseeded batches: gpu_model_runner.py:1852 `decode_batch_size > 1 and any(generator is not None ...) and not getattr(self.model, 'talker_mtp_accepts_per_row_generators', False)` still takes the scalar fallback for seeded rows, and the captured graph at gpu_ar_model_runner.py:570 is captured with no generator kwarg, so the seed is ignored. +- [claim-refuted] test coverage claim: the only related test, test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models, injects `talker_mtp_accepts_per_row_generators=True` directly (tests/worker/test_omni_gpu_model_runner.py:374), so the diff's new branch could be inverted and CI would stay green. +- [claim-refuted] perf isolation: PR body asserts `talker_mtp 39ms -> 9ms` and `RTF 0.54 -> 0.19` but the pasted Test Result block contains no baseline; the same block reports `Maximum request concurrency: 8` yet `Peak concurrent requests: 17.00`, and `Streaming continuity OK rate: 21.88%` is unexplained. +- [validated] qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu()` matches sibling Qwen3-Omni (qwen3_omni_moe_code_predictor_mtp.py:21); on CUDA `_setup_compile` takes the torch.compile-only branch (qwen3_code_predictor.py:552 `if self._wrapper_config.use_cuda_graphs:` false). +- [validated] Model gating (qwen3_tts_talker.py:346-350) and runner wrap decision (gpu_model_runner.py:209 `if cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe):`) use the same predicate, so per-row-generator disablement and outer graph wrap cannot drift. + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~350` [major] — Under full cudagraphs this diff sets talker_mtp_accepts_per_row_generators=False, but OmniGPUModelRunner._talker_mtp_forward still builds row_generators from tts_local_seed and, when any seeded row exists, takes the scalar-per-row fallback because the flag is False (gpu_model_runner.py:1854 -> 1870). The claimed single batched talker_mtp call per decode step is therefore false for seeded batches, and the seed is still discarded by graph replay. Short-circuit generator construction when the flag is False (build no row_generators, skip the fallback) and log once that tts_local_seed is ignored. (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:349 `self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs()`; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`; unchanged by this diff, present in the PR-time tree: vllm_omni/worker/gpu_model_runner.py:1854 `and not getattr(self.model, "talker_mtp_accepts_per_row_generators", False)`; vllm_omni/worker/gpu_model_runner.py:1870 `self._talker_mtp_forward([req_id], inputs_embeds, row_offsets)`) + +`vllm_omni/deploy/qwen3_tts.yaml:~48` [minor] — This diff retunes stage 0 max_num_batched_tokens from 512 to 32768 and both stages' max_num_seqs from 10 to 64, but two in-file comments still assert the old tuning: line 6 documents `max_num_batched_tokens=512` and line 69 says CustomVoice/VoiceDesign are `TTFA-optimal at 1`. Update both comments to match the new values and state the new rationale (or revert the retune if accidental). (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/deploy/qwen3_tts.yaml:6 `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk` vs vllm_omni/deploy/qwen3_tts.yaml:48 ` max_num_batched_tokens: 32768`; vllm_omni/deploy/qwen3_tts.yaml:69 ` # Tuned for Base voice clone; CustomVoice / VoiceDesign are TTFA-optimal at 1.` vs vllm_omni/deploy/qwen3_tts.yaml:70 ` max_num_seqs: 64`.) + +`vllm_omni/deploy/qwen3_tts.yaml:~46` [minor] — Stage 0 sets `enable_prefix_caching: false`, but the comment directly above still says 'Prefix caching on by default ... net throughput win ... Disable explicitly if memory headroom is tight.' Rewrite it: prefix caching is disabled here because `_should_use_async_omni_output` returns False whenever `omni_prefix_cache` is set; re-enabling it silently disables this PR's async-output fast path. (evidence: vllm_omni/deploy/qwen3_tts.yaml:41 `# Prefix caching on by default. The HS opt-out (talker only consumes the` ... :45 `# Disable explicitly if memory headroom is tight on a single-GPU deploy.` followed by :46 `enable_prefix_caching: false` — the comment advertises on-by-default while the value is false. vllm_omni/worker/gpu_ar_model_runner.py:1540-1541 (unchanged by this diff, present in PR-time tree): `if self.omni_prefix_cache is not None:` / `return False` — this gates `_should_use_async_omni_output`, so disabling prefix caching is required for the PR's async-output fast path to activate.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~353` [minor] — Confirmed duplicate: `self.omni_pooler_payload_include_hidden = False` is already set at line 317 (pre-existing) and is added again at line 353. Remove the line-353 assignment; the PR body's claim that this flag was missing should also be corrected. (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:317 (unchanged by this diff, present in the PR-time tree): `self.omni_pooler_payload_include_hidden = False`; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:353 (added by this diff): `self.omni_pooler_payload_include_hidden = False`. The base version also contained the line-317 assignment before the PR.) + +`vllm_omni/deploy/qwen3_tts.yaml:~99` [minor] — buildkite/vllm-omni-npu-ci and buildkite/vllm-omni-amd-ci are both FAILURE at this head. This diff adds `platforms.npu.stages[*].compilation_config.cudagraph_mode: PIECEWISE` (vllm_omni/deploy/qwen3_tts.yaml:99,102) and changes the GPU code-predictor graph path to `use_cuda_graphs=current_omni_platform.is_npu()` (qwen3_tts_code_predictor_vllm.py:36), which alters the graph path AMD executes. Please confirm whether these failures are pre-existing on main or introduced by this change. (evidence: vllm_omni/deploy/qwen3_tts.yaml:99 `cudagraph_mode: PIECEWISE` (and :102) under `platforms: npu:` added by this diff — unchanged outside the diff's cited config surface; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:36 `use_cuda_graphs=current_omni_platform.is_npu(),` (changed from `True`) — the non-NPU/GPU graph path; gh_ci_read(4923) returns state FAILURE for `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci`.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — This diff adds a behavior-changing branch (per-row MTP generators disabled when `talker_mtp_graph_safe` and `cudagraph_mode.has_full_cudagraphs()` are true, so per-request `tts_local_seed` reproducibility is dropped), but no test exercises the model-side gate. Add a test that drives `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:348-350` (full cudagraphs -> flag False; PIECEWISE/None -> flag True), or extract the gating into a pure helper and test it — today an inverted gate would silently change seeded-batch behavior while CI stays green. (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:348-350 `talker_mtp_graph_wrapped = ( + self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs() + ) + self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped` (in the PR diff). tests/worker/test_omni_gpu_model_runner.py:374 `talker_mtp_accepts_per_row_generators=True,` injects the final flag directly instead of exercising this gate (file unchanged by this diff, present in PR-time tree). vllm_omni/worker/gpu_model_runner.py:1854 `and not getattr(self.model, "talker_mtp_accepts_per_row_generators", False)` is the only non-test consumer (file unchanged by this diff, present in PR-time tree).) + +`vllm_omni/deploy/qwen3_tts.yaml:37` [minor] — Confirmed against the PR-time tree: the deploy retune is in the same PR as the graph/async-output rework, so the single before/after benchmark (RTF 0.54→0.19, talker_mtp 39ms→9ms) is not isolated. vllm_omni/deploy/qwen3_tts.yaml:38 changes max_num_seqs 10→64 and :48 changes max_num_batched_tokens 512→32768, while qwen3_tts_talker.py and qwen3_tts_code_predictor_vllm.py change graph capture/async output. The pasted run also shows Maximum request concurrency 8 vs Peak concurrent requests 17.00 and Streaming continuity OK rate 21.88%, and talker_mtp timing appears only in prose/screenshots, not in the result block. Please A/B the config retune alone versus the graph/async rework (or provide per-change result JSON) and explain the concurrency/continuity numbers. (evidence: vllm_omni/deploy/qwen3_tts.yaml:38 ` max_num_seqs: 64` (diff: 10→64); vllm_omni/deploy/qwen3_tts.yaml:48 ` max_num_batched_tokens: 32768` (diff: 512→32768); vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py (PR diff hunk @@ -324,13 +324,33 @@) `+ self.talker_mtp_graph_safe = True` — graph/async-output rework in the same PR; tests/dfx/perf/tests/test_tts.json:49-53 `"max_concurrency": [ 8, 16, 64 ]` (qwen3_tts_base throughput cell); PR #4923 body Test Result block: `Maximum request concurrency: 8` and `Peak concurrent requests: 17.00` and `Streaming continuity OK rate: 21.88%`; `talker_mtp: 39ms -> 9ms` appears only in prose/screenshots, not in the pasted result block.) + +`docs/design/qwen3_omni_tts_performance_optimization.md:510` [minor] — Update docs/design/qwen3_omni_tts_performance_optimization.md:510 from 512 to 32768 to match vllm_omni/deploy/qwen3_tts.yaml:48; either propagate the new value to glm_tts.yaml:40, indextts2.yaml:20, aura_omni.yaml:65, qwen3_tts_high_concurrency.yaml:54 or document why the divergence is intentional. (evidence: docs/design/qwen3_omni_tts_performance_optimization.md:510 ` max_num_batched_tokens: 512` (in the Qwen3-TTS stage-0 Talker key-config example). vllm_omni/deploy/qwen3_tts.yaml:48 ` max_num_batched_tokens: 32768` (changed by this diff). Sibling yamls unchanged by this diff, present in the PR-time tree: vllm_omni/deploy/glm_tts.yaml:40 ` max_num_batched_tokens: 512`, vllm_omni/deploy/indextts2.yaml:20 ` max_num_batched_tokens: 512`, vllm_omni/deploy/aura_omni.yaml:65 ` max_num_batched_tokens: 512`, vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:54 ` max_num_batched_tokens: 512`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:36` [minor] At PR head this line is vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39. On GPU `current_omni_platform.is_npu()` is False, so `use_cuda_graphs=False`, and qwen3_code_predictor.py:477 only en +- `docs/user_guide/examples/online_serving/text_to_speech.md:334` [minor] Confirmed: this diff changes stage 1 max_num_seqs from 10 to 64, so docs/user_guide/examples/online_serving/text_to_speech.md:334 ('ships stage 1 (Code2Wav) at max_num_seqs: 10') and docs/serving/speech_api.md:667 ('benc + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4923.trace.json.gz new file mode 100644 index 000000000..537a1b4e2 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4926.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4926.cost.json new file mode 100644 index 000000000..cdd8c37ab --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1934.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4926/run-20260815-030752-cfcef7", + "recorded_at": "2026-08-15T03:40:04", + "attempts": [ + { + "invocation_id": "inv-f231dcfe", + "outcome": "run:run-20260815-030752-cfcef7", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-030752-cfcef7", + "llm_calls": 166, + "input_tokens": 939562, + "output_tokens": 329239, + "usd": 0.966636, + "source": "spans" + } + ], + "attempt_usd_total": 0.966636, + "llm_calls": 166, + "input_tokens": 939562, + "output_tokens": 329239, + "usd": 0.966636, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-030752-cfcef7", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9666, + "minutes": 32.2, + "ci_minutes": 0, + "input_tokens": 939562, + "output_tokens": 329239, + "tool_calls": 349, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1027, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 166, + "by_role": { + "agent": { + "usd": 0.931098, + "input_tokens": 912490, + "output_tokens": 303615, + "calls": 164 + }, + "reducer": { + "usd": 0.035537, + "input_tokens": 27072, + "output_tokens": 25624, + "calls": 2 + } + }, + "cache_read_tokens": 12992256, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.14, + "gate": 1.79, + "review": 1925.92, + "report": 0.07 + }, + "time_to_first_result_s": 1931.91 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 49845 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4926.md b/eval/dataset/arms/copilot_v14_train_r1/pr4926.md new file mode 100644 index 000000000..fa466be27 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4926.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] Prior concern: piecewise path with only varlen func would crash — fixed at head; flash_attn_hub.py:117-123 `_flash_wrapper` falls back to `_run_varlen_dense` when `attn_func` is None. +- [resolved] Prior concern: FLASH_ATTN_3_HUB pre-Hopper gate — present at head; cuda/platform.py:140-145 falls back to FLASH_ATTN_HUB when major < 9. +- [resolved] Prior concern: strict execution assertions — present at head; tests/diffusion/attention/test_kernels_hub.py:88 and :103 assert `max_diff < 1e-2`. +- [resolved] concern 'constructor raises only when both funcs None and dense path then calls varlen': _forward_varlen_dense is only reached when flash_attn_func is None, in which case the constructor guard guarantees varlen_func is not None — safe; residual risk is only in _forward_varlen_masked (comment). +- [resolved] New backend classes omit abstract get_metadata_cls/get_builder_cls like existing flash_attn.py/sdpa.py; backends are never instantiated (selector._load_backend_cls returns the class, layer.py uses get_impl_cls), so ABC instantiation enforcement is not triggered. +- [claim-verified] Non-CUDA fallback to local FLASH_ATTN: rocm/platform.py:91-97, musa/platform.py:81-89, npu/platform.py:110-118, xpu/platform.py:45-53 all map HUB backends to FLASH_ATTN. +- [claim-verified] CUDA missing-kernels fallback: cuda/platform.py:124-138 imports kernels and falls back to FLASH_ATTN for both HUB names on ImportError; test_kernels_hub_platform_fallback monkeypatches sys.modules['kernels']=None and asserts the FLASH_ATTN path. +- [claim-verified] Non-CUDA fallback: rocm/musa/npu/xpu selectors each map HUB names to FLASH_ATTN with a warning; default selected_backend=None paths are untouched in all five platform files. +- [claim-verified] No existing default flipped: selected_backend=None paths in all five platform selectors are unchanged; additive enum members are backwards-compatible for existing callers. +- [claim-verified] CLI/env do not restrict backend names (serve.py:510 has no choices; parse_attention_config/build_attention_config perform no enum validation), so the new names reach the platform selector. +- [claim-refuted] 'perfect numerical consistency' claim: only test is batch=1/seq_len=32/bf16/causal=False/no mask vs SDPA at 1e-2; e2e evidence is two images with no quantitative parity metric. +- [claim-refuted] PR claim 'All unit tests pass successfully' is a local CUDA result: test_kernels_hub_platform_fallback is selected by the AMD lane 'core_model and cpu' but is CUDA-specific (comment), and test_kernels_hub_execution is selected by no Buildkite lane (grep .buildkite for 'attention' = no matches). +- [claim-refuted] PR claim 'E2E test ... kernels FA3': the automated FA3 branch is skipped on the only requested hardware (L4 sm_89) via 'if major < 9: pytest.skip', and no H100 lane runs tests/diffusion/attention. +- [claim-refuted] PR's 'eliminates numerical drift / perfect numerical consistency' claim is weakened by the unpinned version=1/2/default get_kernel chain (flash_attn_hub.py:96-103,275-282). + +`tests/diffusion/attention/test_kernels_hub.py:57` [major] — Diff adds `test_kernels_hub_execution` marked `@hardware_test(res={"cuda": "L4"}, num_cards=1)` at tests/diffusion/attention/test_kernels_hub.py:57, but no Buildkite lane selects `tests/diffusion/attention` with a CUDA marker: the recursive lane is `-m 'core_model and cpu'` (hardware_test adds cuda/gpu/L4 marks, not cpu, so this test is deselected) and the only CUDA glob is `tests/diffusion/*.py`, which does not recurse into subdirectories. Additionally, the FA3 half self-skips on L4 (`pytest.skip(...)` at line 93 because `major < 9`). Wire `tests/diffusion/attention` into a collected CUDA step and split the FA3 half onto an H100-marked test (or request H100), otherwise the PR's FA2/FA3 execution evidence is never enforced by CI. (evidence: tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` (added by this diff). tests/diffusion/attention/test_kernels_hub.py:93 `pytest.skip("FLASH_ATTN_3_HUB execution requires Hopper-class GPU (compute capability >= 9.0)")` (added by this diff). .buildkite/test-ready.yml:13 `timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml` (unchanged by this diff, present in the PR-time tree; hardware_test adds no cpu mark). .buildkite/test-ready.yml:145 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"` (unchanged by this diff, present in the PR-time tree; glob does not recurse into tests/diffusion/attention). tests/helpers/mark.py:14 `test_resource = pytest.mark.L4` (unchanged by this diff; hardware_test returns gpu/cuda/L4 marks for {"cuda": "L4"}, never cpu).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~108` [major] — Confirmed: flash_attn_hub.py:108 (and the duplicated guard at :287) uses `if self.flash_attn_func is None and self.flash_attn_varlen_func is None`, so a hub module exposing only `flash_attn_func` passes construction while `flash_attn_varlen_func` stays None. Both `_forward_varlen_masked` (:147/:326) and `_forward_varlen_dense` (:169/:348) then call `self.flash_attn_varlen_func(...)` unguarded, so any masked-attention request (`attention_mask` containing a False entry) crashes with `TypeError: 'NoneType' object is not callable`. Fix by changing `and` to `or` (require both functions) or by explicitly guarding/falling back in the varlen paths, and add a masked-input test covering both Hub impls. (evidence: flash_attn_hub.py:106 `self.flash_attn_varlen_func = getattr(hub_module, "flash_attn_varlen_func", None)`; flash_attn_hub.py:108 `if self.flash_attn_func is None and self.flash_attn_varlen_func is None:`; flash_attn_hub.py:147 `out_unpad = self.flash_attn_varlen_func(`; flash_attn_hub.py:169 `return _run_varlen_dense(` with the body calling `varlen_func` unguarded (flash_attn_hub.py:36-45); same pattern duplicated for FA3 at flash_attn_hub.py:285-287, :326, :348. The `and` guard passes when only `flash_attn_func` exists, leaving `flash_attn_varlen_func` None and the masked/dense varlen paths then call it without a None check.) + +`vllm_omni/diffusion/models/ltx2/ltx2_transformer.py:~397` [major] — Diff adds HUB backends whose get_name() returns FLASH_ATTN_HUB/FLASH_ATTN_3_HUB (flash_attn_hub.py:64 and 243), but ltx2_transformer.py:397 only converts the attention mask to a 2D padding mask when the backend name is exactly FLASH_ATTN. Under either HUB backend an LTX2 mask remains 4D [B, heads, q, k] and reaches FlashAttention*HubImpl.forward_cuda; any mask containing a masked entry takes the masked branch and hits flash_attn_hub.py:141 `assert attention_mask.ndim == 2`, so LTX2 masked attention crashes under the new backends. Extend the condition to include the two HUB names, or key the decision off backend mask capability. (evidence: vllm_omni/diffusion/models/ltx2/ltx2_transformer.py:397 `if attn.attn.attn_backend.get_name().upper() == "FLASH_ATTN":` (unchanged by this diff, present in the PR-time tree; this is the only branch calling _to_padding_mask). vllm_omni/diffusion/attention/backends/flash_attn_hub.py:64 `return "FLASH_ATTN_HUB"`. vllm_omni/diffusion/attention/backends/flash_attn_hub.py:243 `return "FLASH_ATTN_3_HUB"`. vllm_omni/diffusion/attention/backends/flash_attn_hub.py:141 `assert attention_mask.ndim == 2, "attention_mask must be 2D, (batch_size, seq_len)"`.) + +`vllm_omni/platforms/cuda/platform.py:124` [major] — Confirmed: the HUB branch at `cuda/platform.py:124-138` only falls back when `kernels` is missing. With `kernels` installed on a pre-Ampere GPU (capability < 8.0), `FLASH_ATTN_HUB` skips the `flash_attn_supported` gate at line 149 (which only matches `FLASH_ATTN`) and returns `FlashAttentionHubBackend`, moving the failure inside the Hub kernel instead of falling back to `TORCH_SDPA`. Add the same compute-capability gate for `FLASH_ATTN_HUB`, and re-check after the `FLASH_ATTN_3_HUB -> FLASH_ATTN_HUB` fallback at line 147. (evidence: vllm_omni/platforms/cuda/platform.py:124 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):` (added by this diff) — enters a branch that only replaces backend_upper with "FLASH_ATTN" in `except ImportError` when `kernels` is unavailable; if `kernels` imports, backend_upper stays as a HUB name. vllm_omni/platforms/cuda/platform.py:149 `if backend_upper == "FLASH_ATTN" and not flash_attn_supported:` (unchanged by this diff, present in PR-time tree) — the compute-capability fallback to `DiffusionAttentionBackendEnum.TORCH_SDPA.get_path()` only applies when backend_upper == "FLASH_ATTN", so FLASH_ATTN_HUB bypasses it. vllm_omni/platforms/cuda/platform.py:80 `compute_supported = capability >= 80` and vllm_omni/platforms/cuda/platform.py:95 `flash_attn_supported = compute_supported and packages_available` (unchanged by this diff, present in PR-time tree) — the local FLASH_ATTN gate that HUB lacks. vllm_omni/platforms/cuda/platform.py:147 `backend_upper = "FLASH_ATTN_HUB"` (added by this diff) — FA3-to-FA2_HUB fallback occurs after the import branch with no subsequent capability re-check.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~96` [minor] — flash_attn_hub.py:96 and :275 call `get_kernel` inside the impl `__init__` with no module-level cache, and `layer.py:95` constructs one impl per attention layer, so the Hub resolution runs once per layer at startup. This is startup-overhead polish rather than a correctness defect (the external `kernels.get_kernel` may cache internally); consider memoizing the resolved module/funcs per backend and pinning the artifact/commit instead of relying only on the `version=1/2` fallback chain. (evidence: ['flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)` — executed in `FlashAttentionHubImpl.__init__` with no module-level/lru cache; the identical pattern recurs at flash_attn_hub.py:275 for flash-attn3.', 'layer.py:95 (unchanged by this diff, present in the PR-time tree) `self.attention = self.attn_impl_cls(` — constructs the impl for each Attention layer, so `__init__` (and the `get_kernel` call) runs once per layer.']) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:103` [minor] — flash_attn_hub.py's get_kernel(...) fallback chain (lines 95-103) has no outer guard: only the version=1 and version=2 attempts are caught, so the final default get_kernel call's exception propagates. cuda/platform.py:124-127 only guards importlib.import_module('kernels') with except ImportError, so when kernels is installed but the Hub fetch fails (offline cluster, rate limit, auth) model load raises instead of falling back to local FLASH_ATTN. Note the docs at attention_backends.md:231 only promise fallback when kernels is not available; either extend the fallback/error handling for Hub-fetch failures or narrow the doc claim accordingly. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:103 ` hub_module = get_kernel("kernels-community/flash-attn2")` — final fallback call, no enclosing try/except (the two except blocks catch only the version=1 and version=2 attempts). vllm_omni/platforms/cuda/platform.py:125 ` importlib.import_module("kernels")` and vllm_omni/platforms/cuda/platform.py:127 ` except ImportError:` — the selector only handles ImportError, not Hub-fetch failures. docs/user_guide/diffusion/attention_backends.md:231 `If the `kernels` library is not available in the environment, vLLM-Omni will log a warning and fall back gracefully to the corresponding local backend implementations (`FLASH_ATTN`).` — documents fallback only for a missing kernels library.) + +`tests/diffusion/attention/test_kernels_hub.py:17` [minor] — Diff adds `test_kernels_hub_platform_fallback` with `@pytest.mark.cpu` (tests/diffusion/attention/test_kernels_hub.py:19), but the test body calls `CudaOmniPlatform.get_diffusion_attn_backend_cls`, which unconditionally runs `torch.backends.cudnn.version() or 0` before the selected-backend branch. On non-CUDA builds (e.g. the ROCm/NPU `core_model and cpu` lanes) this errors instead of skipping. Add `@pytest.mark.skipif(not current_platform.is_cuda(), ...)` like the sibling CUDA-platform test, monkeypatch the cudnn probe, or mark the test `cuda` instead of `cpu`. (evidence: tests/diffusion/attention/test_kernels_hub.py:42 'backend_path = CudaOmniPlatform.get_diffusion_attn_backend_cls("FLASH_ATTN_HUB", head_size=64)'; vllm_omni/platforms/cuda/platform.py:100 'cudnn_version = torch.backends.cudnn.version() or 0'; .buildkite/test-amd-merge.yml:21 '"timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"'; tests/diffusion/attention/test_sage_attn3.py:86 '@pytest.mark.skipif(not current_platform.is_cuda(), reason="sage_attn3 tests require CUDA platform")'.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:117` [minor] — Add the GQA guard from sdpa.py:88 (`self.requires_gqa = num_heads != num_kv_heads`) and sage_attn3's GQA SDPA fallback to the duplicated Hub masked/varlen machinery in flash_attn_hub.py; the current Hub impls drop that GQA awareness, so MQA/GQA attention shapes are not handled by the same fallback the sibling backends enforce. (evidence: [sweep] Sibling contrast: flash_attn_hub.py duplicates FlashAttentionImpl's masked/varlen machinery but drops the GQA awareness that sdpa.py:88 (`self.requires_gqa = num_heads != num_kv_heads`) and sage_attn3's GQA SDPA fallback enforce.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/platforms/npu/platform.py:110` [minor] `buildkite/vllm-omni-npu-ci` is FAILURE on this head (gh_ci_read). The only NPU file this PR touches is `vllm_omni/platforms/npu/platform.py`, and the change there is solely the `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` -> `FL +- `pyproject.toml:70` [minor] Dependency mismatch: runtime selects kernel versions via get_kernel(..., version=1/2) (vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96,100) and docs pin kernels==0.14.1 (docs/user_guide/diffusion/attention_ba +- `vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py:217` [minor] Diff registers new backend names `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` in `registry.py:65-66`, but `pipeline_diffusers_adapter.py` still dispatches only `case "FLASH_ATTN" | None:` (line 217). A user selecting either new g +- `tests/diffusion/attention/test_kernels_hub.py:88` [minor] PR adds only kernel-level accuracy assertions comparing vLLM Hub output to PyTorch SDPA with a 1e-2 tolerance (test_kernels_hub.py:88 and :104); this does not validate the documented 'perfect numerical consistency' betwe + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4926.trace.json.gz new file mode 100644 index 000000000..f07e1d1cd Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4950.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4950.cost.json new file mode 100644 index 000000000..f20cdc834 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4950.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 278.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4950/run-20260815-031728-7ed483", + "recorded_at": "2026-08-15T03:22:05", + "attempts": [ + { + "invocation_id": "inv-ad4b821c", + "outcome": "run:run-20260815-031728-7ed483", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-031728-7ed483", + "llm_calls": 12, + "input_tokens": 179927, + "output_tokens": 15755, + "usd": 0.082258, + "source": "spans" + } + ], + "attempt_usd_total": 0.082258, + "llm_calls": 12, + "input_tokens": 179927, + "output_tokens": 15755, + "usd": 0.082258, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-031728-7ed483", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0823, + "minutes": 4.6, + "ci_minutes": 0, + "input_tokens": 179927, + "output_tokens": 15755, + "tool_calls": 26, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2042, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 12, + "by_role": { + "agent": { + "usd": 0.082258, + "input_tokens": 179927, + "output_tokens": 15755, + "calls": 12 + } + }, + "cache_read_tokens": 605440, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.44, + "gate": 2.21, + "review": 270.07, + "report": 0.07 + }, + "time_to_first_result_s": 275.76 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4950.trace.json.gz", + "trace_bytes": 10149 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4950.md b/eval/dataset/arms/copilot_v14_train_r1/pr4950.md new file mode 100644 index 000000000..49ad5a8d5 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4950.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4950, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 48d4fd8a5e98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] Top-level vs extra_body routing: vllm_omni/entrypoints/openai/serving_chat.py:373 'chat_template_kwargs=request.chat_template_kwargs' and :381 'merged_template_kwargs = self._effective_chat_template_kwargs(request)' read only the typed field in the non-diffusion path, so nested extra_body.chat_template_kwargs is ignored for MiniCPM-o 4.5 — matches the PR fix. +- [claim-verified] SDK flattening: vllm_omni/entrypoints/openai/serving_chat.py image path comment 'When sending request via openai client Python library, `extra_body` is flattented and merged into the payload's root' supports the new SDK note. +- [claim-verified] Silent-audio mechanism: vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py only sets tts_token_ids/tts_hidden_states when tts_bos_idx is found; minicpmo_4_5_omni_tts.py:406 'if tts_token_ids is None or tts_hidden_states is None: ... return self._dummy_hidden_states(...)' means no waveform is generated — consistent with 'silent audio, not text-only'. The exact final HTTP shape (silent WAV vs error) was not re-run live. +- [claim-verified] Two-choice response shape: minicpmo_4_5/pipeline.py stage0 final_output_type='text' (line 44) and stage1 final_output_type='audio' (line 70); chat_completion_full_generator appends _create_text_choice then _create_audio_choice, so two choices entries result. +- [sweep] Diff surface: only recipes/OpenBMB/MiniCPM-o-4_5.md is modified (pr_view files + diff); no code/config/test files touched. +- [sweep] grep use_tts_template repo-wide: hits are recipes/OpenBMB/MiniCPM-o-4_5.md, examples/online_serving/minicpmo/gradio_demo.py:223 and examples/online_serving/minicpmo/README.md:58; both demo paths go through the OpenAI SDK, so neither is stale under the new curl/SDK distinction. +- [sweep] CI gate: gh_ci_read(4950) shows buildkite/vllm-omni-npu-ci FAILURE while buildkite/vllm-omni, amd, intel and readthedocs pass; the diff touches no NPU platform/config path, so the docs change cannot exercise that lane. + +`recipes/OpenBMB/MiniCPM-o-4_5.md:106` [minor] — This PR changes only a markdown recipe, yet buildkite/vllm-omni-npu-ci is FAILING at head while buildkite/vllm-omni, amd, intel and readthedocs all pass. The diff does not touch any NPU platform/config file that lane loads, so this is very likely pre-existing on main — please confirm on main and note it, otherwise the merge carries a red check with no in-diff mechanism. (evidence: gh_ci_read(4950) lists buildkite/vllm-omni-npu-ci state FAILURE; pr_view files=[recipes/OpenBMB/MiniCPM-o-4_5.md]; diff_stat-equivalent changed set is only that recipe markdown.) + +`examples/online_serving/minicpmo/README.md:~58` [nit] — This sibling README still presents `extra_body.chat_template_kwargs.use_tts_template=True` without the curl-vs-SDK distinction this PR adds to the recipe. It is only correct because the Gradio demo goes through the OpenAI SDK, which flattens `extra_body` into the request root; add the same one-line clarification here so a curl reader does not re-introduce the exact bug this PR fixes. (evidence: examples/online_serving/minicpmo/README.md:58 '`extra_body.chat_template_kwargs.use_tts_template=True`, which appends'; examples/online_serving/minicpmo/gradio_demo.py:223 '"chat_template_kwargs": {"use_tts_template": True},' inside build_extras() and passed as kwargs['extra_body'] to client.chat.completions.create.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4950.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4950.trace.json.gz new file mode 100644 index 000000000..61c0677b1 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4950.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4970.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4970.cost.json new file mode 100644 index 000000000..85c81e6a0 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4970.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 459.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4970/run-20260815-031544-15f0c0", + "recorded_at": "2026-08-15T03:23:21", + "attempts": [ + { + "invocation_id": "inv-6fbe63a6", + "outcome": "run:run-20260815-031544-15f0c0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-031544-15f0c0", + "llm_calls": 7, + "input_tokens": 101297, + "output_tokens": 31536, + "usd": 0.071717, + "source": "spans" + } + ], + "attempt_usd_total": 0.071717, + "llm_calls": 7, + "input_tokens": 101297, + "output_tokens": 31536, + "usd": 0.071717, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-031544-15f0c0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0717, + "minutes": 7.61, + "ci_minutes": 0, + "input_tokens": 101297, + "output_tokens": 31536, + "tool_calls": 34, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2833, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 7, + "by_role": { + "agent": { + "usd": 0.071717, + "input_tokens": 101297, + "output_tokens": 31536, + "calls": 7 + } + }, + "cache_read_tokens": 358400, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.64, + "gate": 1.59, + "review": 451.32, + "report": 0.07 + }, + "time_to_first_result_s": 456.59 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4970.trace.json.gz", + "trace_bytes": 11549 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4970.md b/eval/dataset/arms/copilot_v14_train_r1/pr4970.md new file mode 100644 index 000000000..3cb899707 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4970.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4970, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4992e10b7367) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] fleet-exposure concern from the review checklist: both qwen3_tts deploy profiles are now seed-free, so no bundled qwen3_tts default injects tts_local_seed anymore. +- [claim-verified] sibling `vllm_omni/deploy/qwen3_tts_high_concurrency.yaml` is already seedless under both `default_sampling_params` blocks (file read). +- [claim-verified] seed propagation is qwen3_tts-gated and reads only stage 0: serving_speech.py:3696 `if self._tts_model_type == "qwen3_tts" and sampling_params_list:`, :3698 `default_seed = getattr(stage0_params, "seed", None)`, :3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`. +- [claim-verified] 'no test references the deploy-level seed': grep seed in tests/test_config_factory.py -> 0; tests/e2e/offline_inference/test_qwen3_tts_base.py -> 0; tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -> 0. The only seed-42 test is tests/entrypoints/openai_api/test_serving_speech.py:3347, which uses a synthetic `SimpleNamespace(seed=42)` not the yaml. +- [claim-verified] nightly perf harness uses the default profile with no per-request seed and quality disabled: tests/dfx/perf/tests/test_tts.json `server_params` has only `"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base"` (no stage-config path), quality phase `"enabled": false`, and no `seed` in any `benchmark_params`. +- [validated] per-request determinism survives: serving_speech.py request.seed branch sets `stage0_params.seed = request.seed` and `stage0_params.extra_args["tts_local_seed"] = request.seed`, and the default-seed block uses `setdefault`, so a request seed wins over any deploy default. +- [validated] stage-1 seed removal is behavior-neutral: stage 1 is greedy (`temperature: 0.0`) and the propagation code only reads `sampling_params_list[0]` (stage 0), so stage 1's seed never fed tts_local_seed. +- [sweep] read vllm_omni/deploy/qwen3_tts.yaml at head in full: both `seed: 42` lines are absent; adjacent `min_tokens: 2` comment block (lines 57-59) is still accurate and makes no seed claim. +- [sweep] sibling `seed: 42` census: after this PR, zero bundled qwen3_tts deploy profiles expose the value (qwen3_tts_forced_aligner.yaml has no stages). ~50 non-qwen3 deploy yamls still ship `seed: 42`, but they are not exposed to the tts_local_seed cliff because propagation is gated on `_tts_model_type == "qwen3_tts"`. +- [sweep] docs sweep for stale seed references: grep seed in docs/serving/speech_api.md, docs/user_guide/examples/online_serving/text_to_speech.md, recipes/Qwen/Qwen3-TTS.md -> only 'seed-tts' dataset names; no default-seed claim to update. +- [sweep] config-factory round-trip: tests/test_config_factory.py:1270 loads qwen3_tts.yaml but grep seed in that file -> 0, so no assertion pins the removed value. +- [sweep] red-gate coverage intersection: AMD lane .buildkite/test-amd-merge.yml:192 runs test_qwen3_tts_base.py and NPU lane .buildkite/test-npu-ready.yaml:21 runs test_qwen3_tts_customvoice.py, both loading qwen3_tts.yaml via get_deploy_config_path — config-only is not inert for these lanes. +- [sweep] merge/rebase damage: commit timeline shows merge 4992e10b; head tree read matches the 2-line net diff exactly, no duplicated or conflicting hunks in the file. +- [sweep] mode/variant matrix: qwen3_tts.yaml has async_chunk: true and stage-0 cudagraph default; the restored unseeded path is the pre-07-04 hot path (no per-row generators), and the seeded per-request path remains on #4889's per-row generator path — no new cell broken by this deletion. + +`vllm_omni/deploy/qwen3_tts.yaml:60` [minor] — This diff deletes `seed: 42` from stage 0 (between `min_tokens: 2` and `repetition_penalty: 1.05`) and stage 1 (between `max_tokens: 65536` and `repetition_penalty: 1.0`) but leaves no in-file note explaining why the default seed must stay absent. That absence is now load-bearing: for qwen3_tts, `_prepare_speech_generation` copies any deploy-level stage-0 seed into `extra_args["tts_local_seed"]`, which forces the per-row multinomial path (~40% audio_throughput cliff). Please add a tombstone comment in the file's existing style next to `default_sampling_params` (e.g. 'do not set a default seed: it is propagated to tts_local_seed and forces the seeded per-row path; pass seed per request for determinism'). (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3696-3706: `if self._tts_model_type == "qwen3_tts" and sampling_params_list: ... default_seed = getattr(stage0_params, "seed", None) ... stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`; grep `seed: 42` in vllm_omni/deploy still matches ~50 non-qwen3 yamls, so a user copying a sibling's `seed: 42` back here silently reintroduces the cliff.) + +`vllm_omni/deploy/qwen3_tts.yaml:90` [minor] — This diff removes the default seed from both stages, but `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are failing at head and both lanes load this exact config. Please confirm these failures are pre-existing on main rather than introduced here: the AMD lane runs `tests/e2e/online_serving/test_qwen3_tts_base.py`, which calls `get_deploy_config_path("qwen3_tts.yaml")`, and the NPU lane runs `tests/e2e/online_serving/test_qwen3_tts_customvoice.py` (same config load). (evidence: .buildkite/test-amd-merge.yml:192 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'`; .buildkite/test-npu-ready.yaml:21 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; tests/e2e/online_serving/test_qwen3_tts_base.py:49 `stage_config_path=get_deploy_config_path("qwen3_tts.yaml")`; tests/e2e/online_serving/test_qwen3_tts_customvoice.py:31 `get_deploy_config_path("qwen3_tts.yaml")`.) + +`vllm_omni/deploy/qwen3_tts.yaml:91` [minor] — This diff's entire justification is the measured perf restoration quoted from #4963 (base c=16: TTFP 658->1134ms, RTF 0.365->0.635, audio_throughput 40.3->24.2), but the PR shows no run of the perf harness that exercises this default profile ('Test Result: Pending CI'; 'happy to run an H20 smoke on request'); CI's green main lane is functional, not perf. Please run `pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json` and attach the qwen3_tts_base / qwen3_tts_customvoice throughput cells confirming audio_throughput returns to the baselines already recorded in that JSON (c=16: 40.634 / TTFP 647.1272ms / RTF 0.3642). (evidence: tests/dfx/perf/tests/test_tts.json `test_qwen3_tts_base` baselines: `"audio_throughput": [23.501, 40.634, 43.0758]`, `"median_audio_ttfp_ms": [227.5653, 647.1272, ...]`, `"median_audio_rtf": [0.3313, 0.3642, ...]`; server_params has only `"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base"` (no stage-config path, so it resolves this default profile); quality phase `"enabled": false`; .buildkite/test-nightly.yml:470 runs `pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4970.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4970.trace.json.gz new file mode 100644 index 000000000..e9d6fe4db Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4970.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4977.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr4977.cost.json new file mode 100644 index 000000000..8f2fcaa3f --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 902.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr4977/run-20260815-030357-d39adc", + "recorded_at": "2026-08-15T03:18:58", + "attempts": [ + { + "invocation_id": "inv-1b37cee0", + "outcome": "run:run-20260815-030357-d39adc", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-030357-d39adc", + "llm_calls": 96, + "input_tokens": 326281, + "output_tokens": 147402, + "usd": 0.371233, + "source": "spans" + } + ], + "attempt_usd_total": 0.371233, + "llm_calls": 96, + "input_tokens": 326281, + "output_tokens": 147402, + "usd": 0.371233, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-030357-d39adc", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3712, + "minutes": 15.01, + "ci_minutes": 0, + "input_tokens": 326281, + "output_tokens": 147402, + "tool_calls": 192, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5897, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 96, + "by_role": { + "agent": { + "usd": 0.350391, + "input_tokens": 312889, + "output_tokens": 131780, + "calls": 94 + }, + "reducer": { + "usd": 0.020842, + "input_tokens": 13392, + "output_tokens": 15622, + "calls": 2 + } + }, + "cache_read_tokens": 4481280, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.38, + "gate": 1.72, + "review": 895.16, + "report": 0.06 + }, + "time_to_first_result_s": 900.31 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 30539 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4977.md b/eval/dataset/arms/copilot_v14_train_r1/pr4977.md new file mode 100644 index 000000000..f76ba9642 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr4977.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Prior concern trust_remote_code kwarg breaking kernels 0.13.x: resolved at head — `_load_hub_module` calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` with no extra kwarg (flash_attn_hub.py:33-35). +- [resolved] Cache poisoning: `_hub_modules[repo_id] = _load_hub_module(repo_id)` never executes on failure, so failed loads are retried rather than cached — correct conservative choice (flash_attn_hub.py:44-45). +- [resolved] Lock granularity: `_load_hub_module` runs inside `with _hub_lock`, serializing concurrent constructions; acceptable because the stated goal is exactly one load per process, and the lazy `from kernels import get_kernel` import cannot re-enter this module. +- [claim-verified] 'loaded once per process' — `_get_hub_module` (lines 42-47) holds `_hub_lock` and caches per repo_id; both constructors call it (lines 119, 284). +- [claim-verified] 'one backend per layer' — layer.py:94-104 constructs `self.attn_impl_cls(...)` per `Attention` module, so caching removes N redundant `get_kernel` calls. +- [claim-verified] "reduces redundant loading / speeds worker init": the diff removes the inline 3-step `get_kernel` call from both `__init__` methods and routes through a process-wide cache keyed by repo_id, eliminating per-layer redundant loads on the success path. +- [claim-verified] 'Diffusion models create one attention backend per layer': vllm_omni/diffusion/attention/layer.py Attention.__init__ constructs `self.attention = self.attn_impl_cls(...)` per layer, supporting the stated per-process caching benefit. +- [claim-verified] Fallback order v1→v2→default is preserved: `for version in (1, 2, None)` with `get_kernel(repo_id, version=version)` then `get_kernel(repo_id)` matches the base try/except chain. +- [claim-verified] "reduces redundant HuggingFace Hub traffic" is valid: kernels 0.14.1 `get_kernel` calls `_check_trust_remote_code` (HF API `get_organization_overview`) and `install_kernel`/`resolve_status`/`get_variants` on every call, so per-process caching avoids repeated network resolution. +- [claim-refuted] PR body claim 'adding `trust_remote_code=True` for more reliable Hub access' — final head has no trust_remote_code; show_commit 9947f414 removed it ('kernels 0.13.x get_kernel() does not accept trust_remote_code'); pyproject.toml:70 allows `"kernels<0.15.0"`. +- [claim-refuted] PR body says "adding `trust_remote_code=True` for more reliable Hub access", but `_load_hub_module` passes no `trust_remote_code` (`return get_kernel(repo_id, version=version)` / `return get_kernel(repo_id)`); commit timeline item 9947f414 is titled "Drop trust_remote_code from hub kernel loading". +- [claim-refuted] Test plan cites `test_kernels_hub_platform_fallback`; that test only monkeypatches `CudaOmniPlatform.get_diffusion_attn_backend_cls` and asserts `.get_path()` strings (test_kernels_hub.py:42-53), so it never imports `flash_attn_hub` or calls `_load_hub_module`/`_get_hub_module`. The only test constructing `FlashAttentionHubImpl` is `test_kernels_hub_execution` (test_kernels_hub.py:58), which is `@hardware_test(res={"cuda": "L4"})` and not selected by any Buildkite lane (grep `.buildkite` for `attention` and `test_kernels_hub` returns nothing; `tests/diffusion/*.py` at test-ready.yml:145 does not recurse into `tests/diffusion/attention/`). +- [claim-refuted] PR body promises `trust_remote_code=True`, but the merged head passes no such kwarg; commit 9947f414 says 'Drop trust_remote_code from hub kernel loading — kernels 0.13.x get_kernel() does not accept trust_remote_code'. +- [claim-refuted] PR description says "adding `trust_remote_code=True` for more reliable Hub access", but head calls `get_kernel` with no trust_remote_code (flash_attn_hub.py:34-35); commit 9947f414 is "Drop trust_remote_code from hub kernel loading"; base also lacked it (file_at_base). + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:119` [major] — Both `FlashAttentionHubImpl.__init__` and `FlashAttention3HubImpl.__init__` now call the new cached `_get_hub_module(...)` (flash_attn_hub.py:119 and :284), adding a process-wide cache and v1→v2→default fallback, but no CI job executes this new path. `test_kernels_hub_platform_fallback` (tests/diffusion/attention/test_kernels_hub.py:19) only monkeypatches `sys.modules['kernels'] = None` (:27) and asserts `CudaOmniPlatform.get_diffusion_attn_backend_cls(...)` routing strings — it never imports `flash_attn_hub` or constructs a `*HubImpl`. The only test that constructs the impls, `test_kernels_hub_execution` (:58), is marked `@hardware_test(res={"cuda": "L4"}, num_cards=1)` and is not selected by any Buildkite job: the CUDA job runs `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` (.buildkite/test-ready.yml:145), whose shell glob does not recurse into `tests/diffusion/attention`, and the CPU job runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` (.buildkite/test-ready.yml:13) and deselects `test_kernels_hub_execution` because it lacks the `cpu` marker. Add `tests/diffusion/attention` to a `core_model and cuda` job, or add a CPU-runnable test that monkeypatches `kernels.get_kernel` to assert exactly one load across two constructions of the same backend, the v1→v2→default fallback order, and that a failed load is not cached. (evidence: Changed by this diff: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:119 ` hub_module = _get_hub_module("kernels-community/flash-attn2")` and :284 ` hub_module = _get_hub_module("kernels-community/flash-attn3")`. Unchanged by this diff, present in the PR-time tree: tests/diffusion/attention/test_kernels_hub.py:19 `def test_kernels_hub_platform_fallback(monkeypatch: pytest.MonkeyPatch):` and :27 ` monkeypatch.setitem(sys.modules, "kernels", None)` (test body only asserts routing strings, never imports flash_attn_hub or constructs a HubImpl); :57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` and :58 `def test_kernels_hub_execution():`. .buildkite/test-ready.yml:13 `- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/test-ready.yml:145 `- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"`; grep for `tests/diffusion/attention` under .buildkite returns no matches.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~34` [minor] — Shipped `_load_hub_module` calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` without `trust_remote_code` (flash_attn_hub.py:34-35), while the PR body still advertises "adding `trust_remote_code=True` for more reliable Hub access". Update the PR body to match the shipped code or note the kwarg was dropped. (evidence: ['vllm_omni/diffusion/attention/backends/flash_attn_hub.py:34 `return get_kernel(repo_id, version=version)` and :35 `return get_kernel(repo_id)` — unchanged by the diff in the PR-time tree; the entire `_load_hub_module` function (lines 26-39) passes no `trust_remote_code` argument.', 'PR 4977 body (gh_pr_view): "adding `trust_remote_code=True` for more reliable Hub access" — PR metadata, not a file line, present at review time.']) + +`.buildkite/test-npu-ready.yaml:21` [minor] — Confirmed: .buildkite/test-npu-ready.yaml:21 runs only `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; that test contains no HUB-backend selection, and NPU's `get_diffusion_attn_backend_cls` (vllm_omni/platforms/npu/platform.py:110-116) rewrites `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` to `FLASH_ATTN` without importing flash_attn_hub, so this PR's changed file does not intersect the lane. Verify the red buildkite/vllm-omni-npu-ci gate against main to establish it is pre-existing/independent before merge. (evidence: .buildkite/test-npu-ready.yaml:21 ` - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'` — the lane's sole test. Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/platform.py:110-116 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"): ... backend_upper = "FLASH_ATTN"` — NPU falls back before importing the changed module. Grep of tests/e2e/online_serving/test_qwen3_tts_customvoice.py for `HUB` and `attention` returned no matches.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:26` [nit] — In the PR-time tree, `_load_hub_module(repo_id: str)` at line 26 and `_get_hub_module(repo_id: str)` at line 42 both lack return type annotations. Under `[tool.mypy]` `disallow_untyped_defs = true` (pyproject.toml:213), mypy raises `[no-untyped-def]` for each. Add return annotations, e.g. `-> object` (or the concrete module type). (evidence: pyproject.toml:213 `disallow_untyped_defs = true` (inside `[tool.mypy]`). flash_attn_hub.py:26 `def _load_hub_module(repo_id: str):` and flash_attn_hub.py:42 `def _get_hub_module(repo_id: str):` — both added by this diff, both module-level, both missing a `->` return annotation.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr4977.trace.json.gz new file mode 100644 index 000000000..c7d00ee26 Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr5009.cost.json b/eval/dataset/arms/copilot_v14_train_r1/pr5009.cost.json new file mode 100644 index 000000000..63c2f8527 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1399.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v14_train_r1/runs/pr5009/run-20260815-024433-e07c38", + "recorded_at": "2026-08-15T03:07:50", + "attempts": [ + { + "invocation_id": "inv-06df811b", + "outcome": "run:run-20260815-024433-e07c38", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-024433-e07c38", + "llm_calls": 143, + "input_tokens": 943237, + "output_tokens": 262786, + "usd": 0.818898, + "source": "spans" + } + ], + "attempt_usd_total": 0.818898, + "llm_calls": 143, + "input_tokens": 943237, + "output_tokens": 262786, + "usd": 0.818898, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-024433-e07c38", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8189, + "minutes": 23.27, + "ci_minutes": 0, + "input_tokens": 943237, + "output_tokens": 262786, + "tool_calls": 311, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9176, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 143, + "by_role": { + "agent": { + "usd": 0.768858, + "input_tokens": 912908, + "output_tokens": 224799, + "calls": 140 + }, + "reducer": { + "usd": 0.05004, + "input_tokens": 30329, + "output_tokens": 37987, + "calls": 3 + } + }, + "cache_read_tokens": 10191104, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.21, + "gate": 1.86, + "review": 1390.29, + "report": 0.07 + }, + "time_to_first_result_s": 1396.41 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 41830 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr5009.md b/eval/dataset/arms/copilot_v14_train_r1/pr5009.md new file mode 100644 index 000000000..9c8d72182 --- /dev/null +++ b/eval/dataset/arms/copilot_v14_train_r1/pr5009.md @@ -0,0 +1,62 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Prior concern that the CPU test wouldn't be selected: the file carries pytestmark=[core_model, cpu], so the CUDA merge lane `-m 'core_model and cpu'` does collect and run it. +- [resolved] prior concern 'CI never selects tests/platforms': refuted — .buildkite/test-ready.yml:29 and test-merge.yml:29 run repo-wide `pytest -m 'core_model and cpu'` with `testpaths=["tests"]` (pyproject.toml:227), so the new test is collected on CUDA lanes. +- [resolved] torch.nn.RMSNorm is not a new dependency floor: already used at vllm_omni/diffusion/models/hidream_image/hidream_image_transformer.py:236 and ltx2/ltx2_transformer.py:658; no new torch lower bound introduced by this diff. +- [claim-verified] PR arithmetic re-derived: (1.2613-1.7263)/1.7263=-26.9%, (1080-1288)/1288=-16.1%, (2.4502-2.4960)/2.4960=-1.8%, (2.4502-2.2847)/2.2847=+7.2%, (2.4960-2.2847)/2.2847=+9.2% — all match the body. +- [claim-verified] The new test would fail pre-fix: file_at_base cuda/platform.py had `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` then `default = ["native"] if using_inductor else ["vllm_c", "native"]`, so VLLM_COMPILE/STOCK_TORCH_COMPILE+inductor yielded ["native"] and the assertion `priority.rms_norm == ["vllm_c", "native"]` fails. +- [claim-verified] Cosmos3 per-model native override exists: registry.py:549 maps Cosmos3OmniDiffusersPipeline to get_cosmos3_ir_op_priority_func; pipeline_cosmos3.py:619-620 sets rms_norm/fused_add_rms_norm to ["native"]. +- [claim-verified] Only in-repo caller of get_default_ir_op_priority is diffusion_worker._resolve_ir_op_priority (diffusion_worker.py:175), so Part 1's blast radius is diffusion workers, not AR/generation workers. +- [claim-verified] PR latency arithmetic: (1.2613-1.7263)/1.7263 = -26.94% (claims -26.9%); (1080-1288)/1288 = -16.1% (claims -16%); (2.4502-2.4960)/2.4960 = -1.83% (claims -1.8%); (2.4502-2.2847)/2.2847 = +7.24% (claims +7.2%); (2.4960-2.2847)/2.2847 = +9.24% (claims +9.2%). +- [claim-verified] Cosmos3 native-only override exists as claimed: vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:619-621 set priority_kwargs["rms_norm"]=["native"] and fused_add_rms_norm=["native"]. +- [claim-verified] 'Models that need native-only already have per-model overrides (e.g. Cosmos3)': vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:612-624 `get_cosmos3_ir_op_priority_func` sets rms_norm=fused_add_rms_norm=['native']. +- [claim-verified] Only non-test caller of get_default_ir_op_priority is vllm_omni/diffusion/worker/diffusion_worker.py:175 inside `_resolve_ir_op_priority`, consumed at :290 `vllm_config.kernel_config.ir_op_priority = _resolve_ir_op_priority(...)`. +- [claim-verified] PR body contains per-part isolation evidence: Part 1 profiler 1288→1080 ms (-16%; aten::mean 2042→114) and FLUX.1-dev serving -14.1%; Part 2 isolation -10.1% latency and PSNR 29.52→31.09 — satisfies 'isolate each bundled change'. +- [claim-refuted] PR Part 1 framing ('Qwen-Image RMSNorm dispatch onto vllm_c') does not describe the merged head: Part 2 in the SAME diff switches those exact layers to torch.nn.RMSNorm (557/558/595/596/1009), which is not a vLLM IR op, so Part 1 no longer changes Qwen-Image's denoise-loop RMSNorm dispatch; the Part 1 profiler table describes a Part-1-only state that never exists at head. +- [claim-refuted] PR body FLUX.1-dev A/B headline math is inconsistent: the table quotes 0.6166s -> 0.5402s and '−14.1%', but calc gives (0.6166-0.5402)/0.6166*100 = 12.39%. + +`vllm_omni/platforms/cuda/platform.py:~253` [major] — Confirmed against head: `platform.py:253` now hard-codes `default = ["vllm_c", "native"]` for every CUDA diffusion worker, and `diffusion_worker.py:175` feeds that default into `kernel_config.ir_op_priority`. The same diff swaps Qwen-Image's qk/txt norms to `torch.nn.RMSNorm` (`qwen_image_transformer.py:557-558`, `595-596`, `1009`) and the directory has no remaining `vllm.model_executor.layers.layernorm import RMSNorm`, so Qwen-Image's RMSNorm calls no longer hit vllm_c. The flip therefore lands on models that still import vLLM RMSNorm (`ovis_image_transformer.py:29`, `sd3_transformer.py:13`, `flux2_transformer.py:18`, `hunyuan_video_15_transformer.py:20`, ...). Please scope via `_DIFFUSION_IR_OP_PRIORITY_FUNCS` (the Cosmos3 pattern `get_cosmos3_ir_op_priority_func`, `pipeline_cosmos3.py:619-620`) or add per-inheritor parity/perf evidence, and update the PR framing/table accordingly. (evidence: vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]` (in diff); vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (in diff); vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)` (in diff); unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/worker/diffusion_worker.py:175 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`; grep `vllm.model_executor.layers.layernorm import RMSNorm` under vllm_omni/diffusion/models/qwen_image -> no matches, while unchanged inheritors still import it at vllm_omni/diffusion/models/ovis_image/ovis_image_transformer.py:29, vllm_omni/diffusion/models/sd3/sd3_transformer.py:13, vllm_omni/diffusion/models/flux2/flux2_transformer.py:18, vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py:20.) + +`vllm_omni/platforms/cuda/platform.py:~253` [major] — This hunk changes the base fill for every IrOpPriorityConfig field, not just RMSNorm: with_default(default, rms_norm=..., fused_add_rms_norm=...) fills all other dataclass fields with `default`. Removing the `using_inductor` branch means that for inductor-active modes (VLLM_COMPILE/STOCK_TORCH_COMPILE with backend inductor) every non-overridden field that previously resolved to ['native'] now resolves to ['vllm_c','native'], while the new test pins only rms_norm/fused_add_rms_norm (tests/platforms/test_cuda_ir_op_priority.py:26-27). Cosmos3's override (pipeline_cosmos3.py:619-622, unchanged by this diff) copies all fields from the platform default and resets only rms_norm/fused_add_rms_norm to ['native'], so its remaining IR ops silently change dispatch to vllm_c under inductor. Restrict the fix to the two RMSNorm fields (keep the upstream using_inductor branch for `default`) or enumerate every field and provide dispatch/perf evidence for each newly vllm_c-ified field. (evidence: platform.py:253 ` default = ["vllm_c", "native"]` — changed by this diff; the removed branch previously computed `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` and `default = ["native"] if using_inductor else ["vllm_c", "native"]`. platform.py:262 ` return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)` — `default` is the base for all non-overridden fields. Unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:619 ` priority_kwargs = {field.name: list(getattr(ir_op_priority, field.name)) for field in fields(ir_op_priority)}`, :620 ` priority_kwargs["rms_norm"] = ["native"]`, :621 ` priority_kwargs["fused_add_rms_norm"] = ["native"]`, :622 ` return IrOpPriorityConfig(**priority_kwargs)`. New in this diff: tests/platforms/test_cuda_ir_op_priority.py:26-27 ` assert priority.rms_norm == ["vllm_c", "native"]` / ` assert priority.fused_add_rms_norm == ["vllm_c", "native"]`.) + +`tests/platforms/test_cuda_ir_op_priority.py:7` [minor] — tests/platforms/test_cuda_ir_op_priority.py imports `CudaOmniPlatform` at module scope (line 7) and is marked cpu-only (line 9). That import pulls `vllm.platforms.cuda` (vllm_omni/platforms/cuda/platform.py:11 `from vllm.platforms.cuda import CudaPlatformBase`). The Intel XPU lane runs `pytest -v -s -m "core_model and xpu and B60"` with no path (.buildkite/scripts/hardware_ci/run-xpu-test.sh:54), so pytest collects and imports the whole tree before marker filtering, meaning this CUDA-platform import executes on XPU despite the file carrying no xpu marker. Existing tests avoid this by deferring the same import into guarded functions (tests/diffusion/attention/test_kernels_hub.py:24 inside the test body; tests/diffusion/attention/test_sage_attn3.py:93 inside a `skipif(not current_platform.is_cuda())` test). Move the import into the two test functions or add a CUDA-platform skip/importorskip guard so non-CUDA lanes can collect safely. (The buildkite AMD/Intel/NPU failure note could not be verified in this pass and should be checked against CI logs separately.) (evidence: tests/platforms/test_cuda_ir_op_priority.py:7 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (module scope; added by this PR). tests/platforms/test_cuda_ir_op_priority.py:9 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. vllm_omni/platforms/cuda/platform.py:11 `from vllm.platforms.cuda import CudaPlatformBase` (unchanged by this diff — the PR touched get_default_ir_op_priority, not this import). .buildkite/scripts/hardware_ci/run-xpu-test.sh:54 `pytest -v -s -m "core_model and xpu and B60"` (unchanged; no path argument, so the full tree is collected/imported). Repo pattern, unchanged by this diff: tests/diffusion/attention/test_kernels_hub.py:24 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (inside test function body); tests/diffusion/attention/test_sage_attn3.py:93 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (inside a function decorated `@pytest.mark.skipif(not current_platform.is_cuda(), ...)`).) + +`vllm_omni/platforms/cuda/platform.py:253` [minor] — Confirmed: vllm_omni/platforms/cuda/platform.py:253 now returns ["vllm_c", "native"] unconditionally; _resolve_ir_op_priority (diffusion_worker.py:174-178) applies this default to every CUDA diffusion pipeline, and _DIFFUSION_IR_OP_PRIORITY_FUNCS (registry.py:545-550) has only the Cosmos3 override, so the inductor-active switch from native-only to vllm_c is repo-wide. The PR body validates Qwen-Image/FLUX.1-dev and marks other RMSNorm-path models 'local A/B not run', so either run same-seed accuracy parity for the unvalidated models or scope the flip to validated models via _DIFFUSION_IR_OP_PRIORITY_FUNCS while keeping the default inductor-aware; also fix the FLUX percentage to -12.4% (0.6166s -> 0.5402s). (evidence: ['vllm_omni/platforms/cuda/platform.py:251-253 (changed by this diff): ` def get_default_ir_op_priority(cls, vllm_config: VllmConfig) -> IrOpPriorityConfig:\n """Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops."""\n default = ["vllm_c", "native"]` — the diff deletes the prior `using_inductor` branch.', 'vllm_omni/diffusion/worker/diffusion_worker.py:174-178 (unchanged by this diff, present in the PR-time tree): `def _resolve_ir_op_priority(od_config: OmniDiffusionConfig, vllm_config: VllmConfig) -> Any:\n ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)\n ir_op_priority_func = get_diffusion_ir_op_priority_func(od_config)\n if ir_op_priority_func is not None:\n ir_op_priority = ir_op_priority_func(ir_op_priority, vllm_config=vllm_config)\n return ir_op_priority`', 'vllm_omni/diffusion/registry.py:545-550 (unchanged by this diff, present in the PR-time tree): `_DIFFUSION_IR_OP_PRIORITY_FUNCS = {\n # arch: ir_op_priority_func\n # `ir_op_priority_func` function must be placed in {mod_folder}/{mod_relname}.py,\n # where mod_folder and mod_relname are defined and mapped using `_DIFFUSION_MODELS` via the `arch` key.\n "Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",\n}`', 'PR body (gh_pr_view): `| B — Part 1 (this PR) | ["vllm_c", "native"] | 0.5402s | **−14.1%** |` with A=0.6166s; calc (0.6166-0.5402)/0.6166*100 = 12.39, so the correct delta is -12.4%.']) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:~557` [minor] — Part 2 changes Qwen-Image's QK norm layers to torch.nn.RMSNorm (qwen_image_transformer.py:557-558), but the PR adds no test for this layer-type switch — the only new test covers Part 1 IR priority. The parity tests that would catch a re-revert are H100-gated benchmark tests (tests/e2e/accuracy/test_qwen_image.py:195-197 and :215-217) and do not run in PR CI. Given the cited revert history, add a small CPU unit test asserting isinstance(attn.norm_q, nn.RMSNorm) / isinstance(attn.norm_k, nn.RMSNorm) after constructing QwenImageCrossAttention or QwenImageTransformer2DModel, or provide the exact command + artifact for an H100 parity run. (evidence: PR-time tree, unchanged except as noted: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` and :558 `self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (changed by this diff). tests/e2e/accuracy/test_qwen_image.py:196 `@hardware_test(res={"cuda": "H100"}, num_cards=1)` / :197 `def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None:` and :216 `@hardware_test(res={"cuda": "H100"}, num_cards=1)` / :217 `def test_qwen_image_2512_matches_diffusers_pixelwise(accuracy_artifact_root: Path) -> None:` (unchanged by this diff). The diff's only added test file, tests/platforms/test_cuda_ir_op_priority.py, asserts `priority.rms_norm == ["vllm_c", "native"]` on `CudaOmniPlatform.get_default_ir_op_priority`, i.e. Part 1, not the nn.RMSNorm layer type.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:~557` [minor] — Part 1 alone leaves 2.4502s (+7.2% vs v0.23 2.2847s), and the PR body attributes the gap only to 'likely additional v0.24 rebase overhead outside RMSNorm' without an owner. Issue #4964 tracks >10% nightly regressions, so this un-owned residual should be named and tracked (new issue or checklist in #4964) rather than left as a hand-wave. Note: this is the Part 1-only table; the combined P1+P2 serving table uses a different step count and should not be conflated. (evidence: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (anchor exists, in diff). Decisive text is outside the diff, present in live PR #5009 body: `| Part 1 only (patched) | 2.4502s (**-1.8%** vs broken; **+7.2%** vs v0.23) |` and `likely additional v0.24 rebase overhead outside RMSNorm.` Issue #4964 body: `| Buildkite | Qwen/Qwen-Image | ... | e2e_latency_ms | 4351.3522 | 2421.5 | -79.70% | fail | 3 days |` (all listed rows worse than 10%).) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — AMD lane does exercise this changed file: .buildkite/test-amd-merge.yml:21 runs pytest over tests/diffusion, and tests/diffusion/.../test_qwen_image_rope_table_failure_case.py and tests/diffusion/distributed/test_sp_plan_hooks.py import qwen_image_transformer (the file modified by the nn.RMSNorm swap). Attribute the AMD failure to pre-existing main with a link/log comparison, or debug the qwen_image_transformer import/execution path in that lane before merge. (evidence: .buildkite/test-amd-merge.yml:21 `- "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (unchanged by this diff); tests/diffusion/models/qwen_image/test_qwen_image_rope_table_failure_case.py:10 `from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import QwenEmbedRope` (unchanged by this diff); tests/diffusion/distributed/test_sp_plan_hooks.py:474 `from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import (` (unchanged by this diff); vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (in-diff change).) + +`vllm_omni/platforms/rocm/platform.py:~185` [minor] — The PR removes the inductor-aware branch only from CudaOmniPlatform; the ROCm and XPU siblings still return `["native"]` RMSNorm dispatch when inductor is active (rocm/platform.py:185, xpu/platform.py:117, both unchanged by this diff), so they would retain the native-only path this PR fixes on CUDA. AMD and Intel CI lanes are already failing. Either apply the same default to RocmOmniPlatform and XPUOmniPlatform or document why this fix is CUDA-only. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/rocm/platform.py:185 `default = ["native"] if using_inductor else ["vllm_c", "native"]`; vllm_omni/platforms/xpu/platform.py:117 `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`; the diff changes only vllm_omni/platforms/cuda/platform.py to `default = ["vllm_c", "native"]`.) + +**Additional observations (beyond the comment budget):** +- `tests/platforms/test_cuda_ir_op_priority.py:25` [minor] #4964 is a kernel-dispatch regression, but this test only asserts the intermediate IrOpPriorityConfig list (lines 25-28) and never drives the consumer path at vllm_omni/diffusion/worker/diffusion_worker.py:175/290, so a + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v14_train_r1/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v14_train_r1/pr5009.trace.json.gz new file mode 100644 index 000000000..57a5f905a Binary files /dev/null and b/eval/dataset/arms/copilot_v14_train_r1/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/manifest.json b/eval/dataset/arms/copilot_v15_holdout4_r1/manifest.json new file mode 100644 index 000000000..803b83295 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v15_holdout4_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout4" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.cost.json new file mode 100644 index 000000000..41d7227a2 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 877.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5608/run-20260815-065026-8aea12", + "recorded_at": "2026-08-15T07:05:01", + "attempts": [ + { + "invocation_id": "inv-75afda2a", + "outcome": "run:run-20260815-065026-8aea12", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-065026-8aea12", + "llm_calls": 132, + "input_tokens": 572149, + "output_tokens": 274798, + "usd": 0.684999, + "source": "spans" + } + ], + "attempt_usd_total": 0.684999, + "llm_calls": 132, + "input_tokens": 572149, + "output_tokens": 274798, + "usd": 0.684999, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-065026-8aea12", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.685, + "minutes": 14.57, + "ci_minutes": 0, + "input_tokens": 572149, + "output_tokens": 274798, + "tool_calls": 252, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7055, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 132, + "by_role": { + "agent": { + "usd": 0.649418, + "input_tokens": 554311, + "output_tokens": 246871, + "calls": 130 + }, + "reducer": { + "usd": 0.035581, + "input_tokens": 17838, + "output_tokens": 27927, + "calls": 2 + } + }, + "cache_read_tokens": 8453376, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.36, + "gate": 1.5, + "review": 869.34, + "report": 0.06 + }, + "time_to_first_result_s": 874.26 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5608.trace.json.gz", + "trace_bytes": 38008 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.md new file mode 100644 index 000000000..5cc6c5cd3 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5608, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head de8517f6f2bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] 'fix ci' (de8517f6) monkeypatches the new rotary module into sys.modules so `_load_source_module` of qwen3_tts_tokenizer_v2 can import it under the faked vllm_omni package (test_310p_patches.py:492-499). +- [resolved] Non-4D caller concern: the only in-repo consumer of the helper is qwen3_tts_tokenizer_v2 (4D q/k), so no current caller breaks; residual is future-caller fragility noted as a nit. +- [claim-verified] Tokenizer decoder cos/sin are 3-D (B,S,D) so fallback `cos.unsqueeze(2)` is broadcast-correct: modeling_qwen3_tts_tokenizer_v2.py:311-313 builds `emb = torch.cat((freqs, freqs), dim=-1)`; docstring line 109 states cos shape [batch_size, seq_len, head_dim]. +- [claim-verified] Tokenizer decoder head_dim=64 fp16 (hidden_size 1024 / num_attention_heads 16, configuration_qwen3_tts_tokenizer_v2.py:77,99) → aligned branch holds; crash is B*N>S*8 only. +- [claim-verified] Unit test exercises the fallback arm: q=(1,2,3,2) fp32 → head_dim//2 % 8 != 0 → unsupported → cos unsqueezed at dim 2 asserted (test_310p_patches.py:547). +- [claim-verified] Sibling arithmetic: CodePredictor defaults heads=16/head_dim=128/num_code_groups=32 (configuration_qwen3_tts.py:198,200,215) → max_seq=33, B*16<=264 → B<=16; deploy stage-0 max_num_seqs=64 (qwen3_tts.yaml:44) exposes buckets 32/64. +- [claim-verified] buildkite/vllm-omni-npu-ci passed and runs tests/e2e/online_serving/test_qwen3_tts_customvoice.py (test-npu-ready.yml:34) — the real CustomVoice crash path. +- [claim-verified] Commit 504fa8cb extraction preserved helper semantics; only simplification was unconditional `.contiguous()` on the input transpose (a transpose is always non-contiguous) — equivalent. +- [claim-verified] `npu_rotary_mul_with_bsnd_fallback` has exactly one in-repo consumer: qwen3_tts_tokenizer_v2.py:31-32 (repo-wide grep). +- [claim-verified] PR 'concurrency >= 4' is consistent: Code2Wav heads=16, first ramp S=4 => 4*16=64 > 4*8=32; c=1,2 stay within the bound (calc). +- [claim-verified] Commit de8517f6 'fix ci' is exactly the test monkeypatch + assertion change shown in the net diff (show_commit). +- [claim-verified] PR benchmark arithmetic is internally consistent: 2884.76/5540=0.521 -> RTF 0.52 (c=1); 3324.27/5540=0.600 -> 0.60 (c=4); 4005.14/5580=0.718 -> 0.72 (c=8); 266.00/1.92=138.5s ~ 48*2.885s. +- [claim-verified] WER 0.982% and the c=4/8/16 hardware numbers are author-reported and not reproducible in this read-only review; treated as evidence, not CI-verified. +- [claim-refuted] The test's fallback is triggered by head_dim=2 fp32 (`head_dim//2 % (32//4)` = 1%8 != 0), not by the crash predicate `batch*num_heads > seq_len*8`; the motivating condition is therefore untested. + +`vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:~184` [major] — The diff adds `npu_rotary_mul_with_bsnd_fallback` but only wires it into `_apply_rotary_pos_emb_npu` (`vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py`). Two BNSD raw-op sites remain unchanged by this diff: `vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184-185` (`q/k = torch_npu.npu_rotary_mul(q, k, cos, sin)` after `cos/sin.unsqueeze(1)`) and `vllm_omni/model_executor/models/common/qwen3_code_predictor.py:260-261` (same pattern). With defaults num_attention_heads=16, head_dim=128, num_code_groups=32 (seq_len=33) and stage-0 `max_num_seqs: 64`, the supported bound `batch * 16 <= 33 * 8` is exceeded at batch 17+ (272 > 264), so these sites can still hit the CANN BNSD tiler error that the fallback avoids. Route both through `npu_rotary_mul_with_bsnd_fallback` or document the batch cap. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184 `q = torch_npu.npu_rotary_mul(q, cos, sin)` and :185 `k = torch_npu.npu_rotary_mul(k, cos, sin)`; vllm_omni/model_executor/models/common/qwen3_code_predictor.py:260 `q = torch_npu.npu_rotary_mul(q, cos, sin)` and :261 `k = torch_npu.npu_rotary_mul(k, cos, sin)`. Added by this diff: vllm_omni/platforms/npu/layers/rotary_embedding.py:25 `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8`. Defaults: vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py:215 `num_code_groups=32,` (seq_len = 33); with num_attention_heads=16, batch 17 gives 17*16=272 > 33*8=264, so the BNSD fast path is rejected and the raw sites have no fallback. Reachability: vllm_omni/deploy/qwen3_tts.yaml:44 `max_num_seqs: 64`.) + +`tests/platforms/npu/test_310p_patches.py:547` [minor] — The fallback assertion proves BSND dispatch happened, but with `q = torch.zeros(1, 2, 3, 2)` (fp32, head_dim=2) `head_dim // 2 % half_dim_alignment = 1 % 8 != 0` rejects the fast path via the alignment guard while `batch * num_heads <= seq_len * 8` (2 <= 24) passes, so the new short-sequence gate is untested. Add one case that only trips the B*N bound (q=(4,8,3,64) -> cos shape (4,3,1,64)) and one supported fast-path case (q=(1,2,3,128) -> cos shape (1,1,3,128)). (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:28 ` return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8`; tests/platforms/npu/test_310p_patches.py:538 ` q = torch.zeros(1, 2, 3, 2)`; tests/platforms/npu/test_310p_patches.py:547 ` assert rotary_calls[0][1].shape == (1, 3, 1, 2)` — with fp32 q, half_dim_alignment=32//4=8, so 1%8!=0 rejects the fast path while 1*2<=3*8 holds.) + +`.buildkite/amd/test-amd-merge.yml:~21` [minor] — Diff only touches NPU-scoped files (`vllm_omni/platforms/npu/**`, `tests/platforms/npu/test_310p_patches.py`). The AMD merge pipeline's Simple Test group runs only `tests/diffusion` and `tests/model_executor` with `-m 'core_model and cpu'` (.buildkite/amd/test-amd-merge.yml:21,32), and the changed NPU source is reachable only via `NPUOmniPlatform.__init__` (`vllm_omni/platforms/npu/platform.py:42-48`), so this PR should not drive the AMD failure. Please confirm on main that the failure is pre-existing. (evidence: Unchanged by this diff, present in PR-time tree: .buildkite/amd/test-amd-merge.yml:21 `"timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/amd/test-amd-merge.yml:32 `"timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; vllm_omni/platforms/npu/platform.py:42 `from vllm_omni.platforms.npu.models.qwen3_tts_tokenizer_v2 import (`; changed by this diff: vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:12 `from vllm_omni.platforms.npu.layers.rotary_embedding import (`) + +`tests/platforms/npu/test_310p_patches.py:547` [minor] — Add a unit test in `tests/platforms/npu/test_310p_patches.py` that exercises the `unsqueeze_dim != 1` branch of `npu_rotary_mul_with_bsnd_fallback` (the direct `torch_npu.npu_rotary_mul` short-circuit); the current test uses `q = torch.zeros(1, 2, 3, 2)` with the default `unsqueeze_dim=1`, whose head_dim=2 fails half-dim alignment and therefore only covers the BSND fallback path. (evidence: tests/platforms/npu/test_310p_patches.py:538 `q = torch.zeros(1, 2, 3, 2)` and :542 `q_out, k_out = tokenizer.apply_rotary_pos_emb(q, k, cos, sin)` — only unit invocation, default `unsqueeze_dim=1`. vllm_omni/platforms/npu/layers/rotary_embedding.py:44-47 `if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):` / `return torch_npu.npu_rotary_mul(hidden_states, cos.unsqueeze(unsqueeze_dim), sin.unsqueeze(unsqueeze_dim))`. vllm_omni/platforms/npu/layers/rotary_embedding.py:27 `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8` — for head_dim=2, `1 % 8 != 0`, so this shape takes the BSND fallback, not the short-circuit branch. A repo-wide grep for `npu_rotary_mul_with_bsnd_fallback` finds no test-file reference, confirming no unit test covers `unsqueeze_dim != 1`.) + +`tests/e2e/online_serving/test_qwen3_tts_customvoice.py:~96` [minor] — The NPU ready lane `pytest ... -m 'core_model and npu'` selects only the two text-to-audio tests in this file, and both are `skipif(current_omni_platform.is_npu(), reason="#issue 5479")`, so the new `npu_rotary_mul_with_bsnd_fallback` path gets no on-hardware coverage there; `qwen3_tts.yaml` keeps `codec_chunk_ramp` commented out. The diff's unit test in tests/platforms/npu/test_310p_patches.py checks the transposed shape in mock, but not a real NPU decode. Please re-enable/add an NPU e2e with the ramp enabled at concurrency >= 4, or add a dedicated -m npu regression that drives a short-ramp decode through the fallback. (evidence: Decisive lines are unchanged by this diff but present in the PR-time tree: +.buildkite/npu/test-npu-ready.yml:34 ` - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model'` +tests/e2e/online_serving/test_qwen3_tts_customvoice.py:95-97 `@pytest.mark.skipif( + current_omni_platform.is_npu(), + reason="#issue 5479", +)` +vllm_omni/deploy/qwen3_tts.yaml:35 ` # codec_chunk_ramp: [4, 4, 8, 16, 25]`) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:~57` [nit] — The fallback branch makes its input contiguous via `hidden_states.transpose(1, 2).contiguous()` (line 51) but returns `output_bsnd.transpose(1, 2)` (line 57), a non-contiguous view, whereas the fast path returns the fused op's tensor directly. The in-tree consumer `_apply_rotary_pos_emb_npu` (qwen3_tts_tokenizer_v2.py:30-33) does not currently require contiguity, so consider `return output_bsnd.transpose(1, 2).contiguous()` to keep both branches' memory contract consistent. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:51 `hidden_states_bsnd = hidden_states.transpose(1, 2).contiguous()`; vllm_omni/platforms/npu/layers/rotary_embedding.py:57 `return output_bsnd.transpose(1, 2)` — return is a non-contiguous view while the input was explicitly made contiguous; fast path at line 44-48 returns `torch_npu.npu_rotary_mul(...)` directly. Unchanged consumer present in PR-time tree: vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:30-33 passes the returned tensors through without requiring contiguity.) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:~51` [nit] — For unsqueeze_dim==1, any input with hidden_states.ndim != 4 returns False from _bnsd_rotary_shape_is_supported and falls through to the BSND fallback, which unconditionally does hidden_states.transpose(1, 2).contiguous(): a 3-D tensor is silently mis-transposed and a 2-D tensor raises IndexError. The current caller only passes 4-D q/k, so this is defensive; consider raising/asserting hidden_states.ndim == 4 in the fallback (and letting head_dim % 2 != 0 fail loudly) so future callers cannot get a wrong-layout op. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:19 `if hidden_states.ndim != 4:`; :20 `return False`; :44 `if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):`; :51 `hidden_states_bsnd = hidden_states.transpose(1, 2).contiguous()`) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:~28` [nit] — The helper hardcodes two CANN-derived thresholds — `half_dim_alignment = 32 // element_size` (line 25) and `batch * num_heads <= seq_len * 8` (line 28) — with no CANN source/version citation; the docstring only states 'CANN requires' without identifying the version. Add a comment citing the CANN source/version from which these thresholds come (or pin them with a unit test) so future CANN upgrades can re-verify the gate. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:25 `half_dim_alignment = 32 // element_size`; vllm_omni/platforms/npu/layers/rotary_embedding.py:28 `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8`; vllm_omni/platforms/npu/layers/rotary_embedding.py:14-16 `For the aligned-head-dimension branch, CANN requires ``batch * num_heads <= seq_len * 8``.` — no CANN version or source identifier is present anywhere in the file.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.trace.json.gz new file mode 100644 index 000000000..b01e37b4a Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5608.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.cost.json new file mode 100644 index 000000000..b8af78d63 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1678.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5720/run-20260815-065027-a54b29", + "recorded_at": "2026-08-15T07:18:22", + "attempts": [ + { + "invocation_id": "inv-0fd342f8", + "outcome": "run:run-20260815-065027-a54b29", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-065027-a54b29", + "llm_calls": 253, + "input_tokens": 2633962, + "output_tokens": 365646, + "usd": 1.923888, + "source": "spans" + } + ], + "attempt_usd_total": 1.923888, + "llm_calls": 253, + "input_tokens": 2633962, + "output_tokens": 365646, + "usd": 1.923888, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-065027-a54b29", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.9239, + "minutes": 27.91, + "ci_minutes": 0, + "input_tokens": 2633962, + "output_tokens": 365646, + "tool_calls": 401, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3144, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 253, + "by_role": { + "agent": { + "usd": 1.894225, + "input_tokens": 2601671, + "output_tokens": 346646, + "calls": 251 + }, + "reducer": { + "usd": 0.029663, + "input_tokens": 32291, + "output_tokens": 19000, + "calls": 2 + } + }, + "cache_read_tokens": 30018816, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.39, + "gate": 2.03, + "review": 1667.96, + "report": 0.06 + }, + "time_to_first_result_s": 1674.43 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5720.trace.json.gz", + "trace_bytes": 61689 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.md new file mode 100644 index 000000000..c7beba441 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.md @@ -0,0 +1,76 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5720, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head b4733f6bfd98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [claim-verified] PR claim 'By default ... initialize both FL2VA and Ref2VA's DiTs': default task_type None→partition 'combined', weights_sources append Ref2VA transformer and instantiate `transformers_ref` (pipeline:573-620). +- [claim-verified] tests/model_tests/diffusion/test_alignment.py:36 adds MiniMaxH3ModularPipeline; commit b4733f6b subject says 'exclude ... alias from alignment' — consistent with an exclusion/skip list (list name not read). +- [claim-verified] test_task_type_roundtrip uses StageConfigFactory.create_default_diffusion (real serialization path), not a hand-built dict. +- [claim-verified] PR body claim 'by default initialize both FL2VA and Ref2VA DiTs' — verified combined branch creates both MiniMaxH3DiTModel instances. +- [claim-verified] PR body claim 'task_type controls pipeline initialization' — verified _minimax_h3_partition_for_task and OmniDiffusionConfig.task_type propagation. +- [validated] `_refresh_func`→`_refresh_funcs`: grep finds references only in vllm_omni/diffusion/cache/cachedit/backend.py and tests/diffusion/cache/test_cache_backends.py; no stale external callers. +- [validated] test_cache_backends.py:113 set-comprehension indexes `mock_cache_dit.refresh_context.call_args_list` args[0]; the new test mocks one refresh per declared DiT, so arg[0] is the transformer. +- [validated] omni_snapshot_download deferral: test_omni_snapshot_download monkeypatches file_or_path_exists; MiniMaxH3 `_resolve_minimax_h3_model_root` re-downloads with task-specific allow_patterns (combined vs fl2va/ref2va). +- [validated] DLO resident skip `enabled=transformer is self.transformer` (diffuse, ~1472) prevents staging FL2VA resident blocks for combined Ref2VA; new test_distributed_layerwise_resident_blocks_can_be_skipped added. +- [validated] task_type end-to-end plumbing: serve.py drops CLI choices; async_omni_engine.py:1022 puts task_type into engine_args; stage_init_utils.build_diffusion_config -> OmniDiffusionConfig.from_kwargs(**engine_args_dict); test_task_type_roundtrip drives the real StageConfigFactory.create_default_diffusion path. +- [validated] MiniMaxH3 partition routing read in situ: _minimax_h3_partition_for_task (pipeline_minimax_h3.py:124-139) + _transformer_for_task (707-710) match the new tests; combined/single DiT counts asserted by test_pipeline_loads_task_selected_dits_and_shared_components_once. +- [validated] new test files carry pytestmark core_model+cpu (test_minimax_h3_contract.py:13, test_cache_backends.py:30, test_async_omni_diffusion_config.py:13, test_arg_utils.py:23, test_omni_snapshot_download.py:19) and are collected by .buildkite/cuda/test-merge.yml:18,23. +- [validated] gh_ci_read: buildkite/vllm-omni (CUDA merge/ready) SUCCESS while buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE; CUDA green implies the new CPU tests pass on CUDA. +- [validated] Qwen3-TTS startup validation covered by test_qwen3_tts_startup_task_type_is_validated (valid 'Base' accepted, 'fl2va' raises) via OmniModelConfig.from_vllm_model_config. + +`vllm_omni/config/model.py:215` [major] — The new `_validate_startup_task_type` guard uses `Qwen3TTSTalkerForConditionalGenerationARVLLM`, which is not the production Qwen3-TTS arch. Production stages are declared/registered as `Qwen3TTSTalkerForConditionalGeneration` (`vllm_omni/model_executor/models/qwen3_tts/pipeline.py:20`, `vllm_omni/model_executor/models/registry.py:157-160`) and `Qwen3TTSCode2Wav`. Since argparse `choices` on `--task-type` were removed, an invalid value such as `--task-type fl2va` now passes parse time and config init for the real Qwen3-TTS path. Change the guard to the production arch (covering `Qwen3TTSCode2Wav` if it inherits `task_type`), re-check the same arch string at `model.py:283`, and update `test_qwen3_tts_startup_task_type_is_validated` to use the production arch. (evidence: In-diff: vllm_omni/config/model.py:215 `if self.model_arch != "Qwen3TTSTalkerForConditionalGenerationARVLLM" or self.task_type is None:` — validation is skipped for every non-ARVLLM arch. Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/qwen3_tts/pipeline.py:20 `model_arch="Qwen3TTSTalkerForConditionalGeneration",` and vllm_omni/model_executor/models/registry.py:157 `"Qwen3TTSTalkerForConditionalGeneration": (` / registry.py:160 `"Qwen3TTSTalkerForConditionalGeneration",` — production Qwen3-TTS is declared under the non-ARVLLM arch, so the new validator never fires for it. In-diff: tests/engine/test_arg_utils.py:141-144 `model_arch="Qwen3TTSTalkerForConditionalGenerationARVLLM",` / `task_type="Base",` — the added test exercises only the dead arch. In-diff: vllm_omni/entrypoints/cli/serve.py defines `--task-type` with `type=str, default=None` and no `choices=`, so argparse no longer rejects `--task-type fl2va`.) + +`tests/diffusion/cache/test_cache_backends.py:256` [minor] — Update the HunyuanImage3 and DreamID cache-test mocks to exercise the declared-DiT branch: set `mock_pipeline._dit_modules = ['model']`. For DreamID, also set `mock_pipeline.model = mock_pipeline.transformer` (the real DreamIDOmniPipeline sets `self.transformer = self.model`); setting only `_dit_modules=['model']` would make the backend getter return an auto-created child Mock and break the existing `adapter_kwargs["transformer"] is mock_pipeline.transformer` assertion. HunyuanImage3 already has `mock_pipeline.model` plus `mock_pipeline.transformer = mock_pipeline.model`, so only `_dit_modules` is needed there. (evidence: tests/diffusion/cache/test_cache_backends.py:256 `mock_pipeline = Mock()` … :265-266 `# NOTE: pipe.transformer is pipe.model in HunyuanImage3Pipelines` / `mock_pipeline.transformer = mock_pipeline.model` — no `_dit_modules` assignment; tests/diffusion/cache/test_cache_backends.py:296-298 `mock_pipeline = Mock()` / `mock_pipeline.__class__.__name__ = "DreamIDOmniPipeline"` / `mock_pipeline.transformer = Mock()` — no `_dit_modules`. Backend (added by this diff) vllm_omni/diffusion/cache/cachedit/backend.py:38-40 `names = getattr(pipeline, "_dit_modules", None)` / `if not isinstance(names, (list, tuple)):` / ` names = ("transformer",)`. Real classes (unchanged by this diff, present in PR-time tree): vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py:383 `_dit_modules: ClassVar[list[str]] = ["model"]`; vllm_omni/diffusion/models/dreamid_omni/pipeline_dreamid_omni.py:104 `_dit_modules: ClassVar[list[str]] = ["model"]`.) + +`vllm_omni/entrypoints/omni_base.py:83` [minor] — At vllm_omni/entrypoints/omni_base.py:80-84, file_or_path_exists(model_id, "modular_model_index.json", revision=None) is guarded by an except that catches only GatedRepoError/RepositoryNotFoundError; any other exception (e.g. a transient HfHubHTTPError) propagates out of omni_snapshot_download before the full-download fallback. Note the fallback's own except only translates GatedRepoError/RepositoryNotFoundError/PermissionError, so a transient error would still abort there. (evidence: vllm_omni/entrypoints/omni_base.py:80-84 (changed code in this diff): +``` + try: + if file_or_path_exists(model_id, "modular_model_index.json", revision=None): + return model_id + except (huggingface_hub.errors.GatedRepoError, huggingface_hub.errors.RepositoryNotFoundError): + pass +``` +The only caught exceptions are the two hub errors; any other exception escapes the try block before the `download_weights_from_hf_specific` full-download fallback that follows.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:696` [minor] — Confirmed the contiguity check at pipeline_minimax_h3.py:696 raises on interleaved sources. DiffusersPipelineLoader.get_all_weights already emits sources in weights_sources order (`for source in sources: yield from ...`), so the ordering contract is explicit in code rather than assumed; the actual gap is a regression test pinning that the real loader emits all `transformer.*` weights before any `transformers_ref.*` weights for the combined H3 pipeline. Severity remains minor. (evidence: pipeline_minimax_h3.py:696 `raise ValueError(f"MiniMax-H3 weight source {prefix!r} is not contiguous")` (in this diff). Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/model_loader/diffusers_loader.py:297-299 `sources = self._get_weight_sources(model)` / `for source in sources:` / `yield from self._get_weights_iterator(source, model=model)` — emission follows weights_sources order.) + +`vllm_omni/entrypoints/cli/serve.py:216` [minor] — The diff removes `choices=["CustomVoice", "VoiceDesign", "Base"]` from `--task-type` (serve.py:213-218), so argparse accepts any string for every model. MiniMax-H3 is the only model that validates its own value at startup (pipeline_minimax_h3.py:551 → `_minimax_h3_partition_for_task`). The shared Qwen3-TTS validator in model.py:215 is gated on `Qwen3TTSTalkerForConditionalGenerationARVLLM`, but production Qwen3-TTS stages register `Qwen3TTSTalkerForConditionalGeneration` (pipeline.py:20), so that validator never fires there. Add central validation or document that unsupported startup task types are silently accepted outside MiniMax-H3. (evidence: vllm_omni/entrypoints/cli/serve.py:213-218 (in diff): `omni_config_group.add_argument( + "--task-type", + type=str, + default=None, + help="Model-defined startup task type. The selected model validates " + "supported values; for example, TTS models accept CustomVoice, "` — no `choices=` present. vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:133 (in diff): `raise ValueError(f"MiniMax-H3 task_type must be one of auto, t2va, fl2va, or ref2va; got {task_type!r}")`; :551 (in diff): `self.partition = _minimax_h3_partition_for_task( + getattr(od_config, "task_type", None), + str(od_config.model), + )` — H3-only validator. vllm_omni/config/model.py:215 (in diff): `if self.model_arch != "Qwen3TTSTalkerForConditionalGenerationARVLLM" or self.task_type is None: + return` — validator gated on ARVLLM arch. Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/qwen3_tts/pipeline.py:20 `model_arch="Qwen3TTSTalkerForConditionalGeneration",` — production Qwen3-TTS arch does not match the gate, so the shared validator never runs there.) + +`.buildkite/amd/test-amd-ready.yml:~19` [minor] — The CI anchor is accurate: at this head buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are FAILURE while buildkite/vllm-omni is SUCCESS. .buildkite/amd/test-amd-ready.yml runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` at lines 19 and 30, and the PR's new MiniMax H3/Cache-DiT tests declare core_model+cpu markers so they are collected; .buildkite/npu/test-npu-ready.yml:21 only runs MiniCPM-o tests that this diff does not touch. Pull both lane logs and attribute each failure to a specific step before merge. (evidence: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (same command at line 30). .buildkite/npu/test-npu-ready.yml:21 `- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'`. Unchanged by this diff but present in the PR-time tree: tests/diffusion/cache/test_cache_backends.py:30 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` and tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:13 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`.) + +`vllm_omni/entrypoints/omni_base.py:81` [minor] — Confirmed as a scope concern. `omni_snapshot_download` keys the download deferral on the mere presence of `modular_model_index.json` (omni_base.py:81-82), but the compensating `download_weights_from_hf_specific` is only implemented in MiniMaxH3Pipeline (pipeline_minimax_h3.py:154-160). The generic DiffusersAdapterPipeline loads via `DiffusionPipeline.from_pretrained` (pipeline_diffusers_adapter.py:116) with no eager snapshot download, so a future modular repo served through the adapter in `HF_HUB_OFFLINE`/`local_files_only` mode would fail on an empty cache. Restrict the deferral to the resolved MiniMax H3 pipeline or an explicit allowlist instead of keying on the index file alone. (evidence: Changed by this diff — vllm_omni/entrypoints/omni_base.py:81-82 `if file_or_path_exists(model_id, "modular_model_index.json", revision=None):` / `return model_id` — unconditional on pipeline. Unchanged by this diff, present in the PR-time tree — vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py:116 `self._pipeline = DiffusionPipeline.from_pretrained(model_id, **load_kwargs)` (no download_weights_from_hf_specific in this adapter). Changed by this diff — vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:154-160 `return Path(download_weights_from_hf_specific(model_name_or_path=model, cache_dir=None, allow_patterns=allow_patterns, revision=revision, require_all=True))`.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1477` [minor] — The diff routes denoising through self._transformer_for_task(task) and wraps it with self._resident_dit_layers_on_device(enabled=transformer is self.transformer), but the only added DLO-skip test (test_distributed_layerwise_resident_blocks_can_be_skipped) calls the context manager directly with enabled=False. Add a combined-pipeline test that runs diffuse(task='ref2va', ...) with a mock _dlo_residency_controller and asserts the controller is never loaded/offloaded and minimax_h3_denoise_loop receives transformers_ref; this covers the exact route where a stale resident-layer plan would stage the inactive FL2VA DiT. (evidence: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1477 ` with self._resident_dit_layers_on_device(enabled=transformer is self.transformer):` — present in this diff, exactly the cited anchor. tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:787 ` with pipeline._resident_dit_layers_on_device(enabled=False):` — present in this diff; the new test only exercises the context manager directly and never calls diffuse.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.trace.json.gz new file mode 100644 index 000000000..3fb88d20f Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5720.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.cost.json new file mode 100644 index 000000000..fb81ac73a --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 829.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5723/run-20260815-065026-794800", + "recorded_at": "2026-08-15T07:04:13", + "attempts": [ + { + "invocation_id": "inv-422f4376", + "outcome": "run:run-20260815-065026-794800", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-065026-794800", + "llm_calls": 93, + "input_tokens": 645185, + "output_tokens": 166077, + "usd": 0.504072, + "source": "spans" + } + ], + "attempt_usd_total": 0.504072, + "llm_calls": 93, + "input_tokens": 645185, + "output_tokens": 166077, + "usd": 0.504072, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-065026-794800", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5041, + "minutes": 13.78, + "ci_minutes": 0, + "input_tokens": 645185, + "output_tokens": 166077, + "tool_calls": 189, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6202, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 93, + "by_role": { + "agent": { + "usd": 0.468306, + "input_tokens": 628094, + "output_tokens": 137808, + "calls": 90 + }, + "reducer": { + "usd": 0.035766, + "input_tokens": 17091, + "output_tokens": 28269, + "calls": 3 + } + }, + "cache_read_tokens": 5451392, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.85, + "gate": 1.6, + "review": 821.38, + "report": 0.06 + }, + "time_to_first_result_s": 826.88 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5723.trace.json.gz", + "trace_bytes": 30599 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.md new file mode 100644 index 000000000..bf918535b --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.md @@ -0,0 +1,60 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5723, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 93c0112c114e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [claim-refuted] PR body 'mark MiniMax H3 AMD GPU supported' — docs/models/supported_models.md:47 AMD GPU cell is still empty; the H3 footnote at :101 is never referenced from the table. +- [claim-refuted] recipes/MiniMaxAI/MiniMax-H3.md:729 'current vLLM-Omni path takes exactly one image plus one audio reference, or one or more videos with no separate audio_reference' — contradicted by pipeline_minimax_h3.py:309-330 (accepts up to 9 images/3 videos/3 audios/12 total) and model_metadata.py:43-47 (supports_mixed_reference_inputs=True, max_multimodal_image_inputs=9). +- [claim-refuted] PR description 'gfx950 (single + 4-GPU)' validation: evidence table has only a 1x gfx950 row (line 481) and line 469 says text-encoder TP 'on gfx950 it was not exercised' — the 4-GPU gfx950 claim is not substantiated by the diff. +- [claim-refuted] PR goal 'mark MiniMax H3 AMD GPU supported': supported_models.md:47 AMD cell is empty and no `H3` appears in the table, so the footnote at line 102 is orphaned. +- [claim-refuted] '(12 files)' arithmetic: calc 9+3+3=15; code enforces total<=12 (pipeline_minimax_h3.py _validate_ref2va_reference_counts 'ref2va accepts at most 12 total references'). +- [claim-refuted] 'videos with no separate audio_reference' contradicts recipe:643 ('a separate `audio_reference`, or both'), recipe:663 (matrix lists video+audio), and pipeline_minimax_h3.py:309-326 which permits audio alongside video. +- [claim-refuted] MODEL_ROOT undefined: only appears inside a run-script env prefix (recipe:205); ROCm blocks at 426/446 expand an unset variable. +- [claim-refuted] PR description says gfx950 validated 'single + 4-GPU', but the evidence table lists only 1x gfx950 and the doc says text-encoder TP was 'not exercised' on gfx950. +- [validated] FLASH_ATTN resolves to AITER on ROCm: fa.py:20-28 imports aiter flash_attn_func/varlen_func when is_rocm; rocm/platform.py:139-143 gates on 90H3` footnote (docs/models/supported_models.md:101) asserts validated AMD support, but it is not attached to any table cell and the MiniMax H3 row's AMD GPU column (docs/models/supported_models.md:47) is blank. The legend at docs/models/supported_models.md:99 says an empty cell means 'not listed as supported', which contradicts the footnote. Attach `H3` to the MiniMax H3 row and put `✅︎` in its AMD GPU column, or reword the footnote/legend to make the unlisted status explicit. (evidence: docs/models/supported_models.md:47 `| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |` — MiniMax H3 AMD cell blank (unchanged by this diff; present in PR-time tree). docs/models/supported_models.md:99 `✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.` (unchanged legend; present in PR-time tree). docs/models/supported_models.md:101 `H3 MiniMax H3 AMD GPU support is validated on gfx942 (MI300X) and` (added by this diff) — the marker is never attached to any table cell, so the footnote asserts validated AMD support while the legend still says the blank AMD cell means 'not listed as supported'.) + +`recipes/MiniMaxAI/MiniMax-H3.md:~730` [major] — This rewritten limitation is wrong: the same page's API section and input matrix, and `_validate_ref2va_reference_counts`, all permit standalone `audio_reference` alongside videos (videos ≤3, audios ≤3, total ≤12, audio requires a visual reference). Rewrite the bullet accordingly: video requests may use the embedded soundtrack, a separate `audio_reference`, or both. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:730 `reference, or one or more videos with no separate `audio_reference` (it uses the` ; recipes/MiniMaxAI/MiniMax-H3.md:643 `A video may use its embedded soundtrack, a separate `audio_reference`, or both.` ; recipes/MiniMaxAI/MiniMax-H3.md:663 `| Ref2VA | image-only, image+image, image+video, video+audio, and mixed image/video/audio | images ≤9, videos ≤3, audios ≤3, total references ≤12; audio requires a visual reference |` ; unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:317 `if image_count + video_count == 0:` and :318 `raise OmniClientError("ref2va requires at least one image or video reference")` and :323 `if audio_count > 3:` and :325 `if image_count + video_count + audio_count > 12:` — the validator rejects only missing visual reference and count overruns, so video_count>0 with audio_count>0 is accepted.) + +`recipes/MiniMaxAI/MiniMax-H3.md:~426` [major] — Both ROCm serve blocks use `${MODEL_ROOT}/FL2VA` (lines 426 and 446), but the recipe never exports MODEL_ROOT; line 205 passes MODEL_ROOT only as a one-shot env prefix to a run script, and the prerequisites define MODEL, not MODEL_ROOT. Copying either ROCm block verbatim yields `export MODEL="/FL2VA"`. Define `export MODEL_ROOT=/path/to/MiniMax-H3` (ideally with a `huggingface-cli download ... --local-dir "${MODEL_ROOT}"` step, as in recipes/MiniMaxAI/MiniMax-H3-MUSA.md) before the ROCm blocks, or use the HF repo id. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:426 `export MODEL="${MODEL_ROOT}/FL2VA"`; recipes/MiniMaxAI/MiniMax-H3.md:446 `export MODEL="${MODEL_ROOT}/FL2VA"`; recipes/MiniMaxAI/MiniMax-H3.md:205 `MODEL_ROOT=/path/to/MiniMax-H3 \` (env prefix to a run script only, not an export); grep for MODEL_ROOT in recipes/MiniMaxAI/MiniMax-H3.md returns only lines 205, 426, and 446, so no `export MODEL_ROOT=` exists before the ROCm blocks. Unchanged sibling pattern present in the PR-time tree: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:31 `export MODEL_ROOT=/path/to/MiniMax-H3`.) + +`recipes/MiniMaxAI/MiniMax-H3.md:469` [major] — The ROCm four-GPU command (line 460, including `--text-encoder-tp-size 4`) is introduced as 'The best-practice CUDA four-GPU configuration works on ROCm with the changes above' (line 440), but line 468-469 states `--text-encoder-tp-size` is validated on gfx942 and 'on gfx950 it was not exercised', and the only gfx950 evidence row is single-GPU CPU offload (line 481). The PR description additionally claims gfx950 'single + 4-GPU' validation, which the diff does not substantiate. Add a gfx950 4-GPU evidence row matching the documented command, or scope the claim to gfx942 and drop '4-GPU' from the description's gfx950 validation sentence. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:440 `The best-practice CUDA four-GPU configuration works on ROCm with the changes above.`; recipes/MiniMaxAI/MiniMax-H3.md:460 ` --text-encoder-tp-size 4 \`; recipes/MiniMaxAI/MiniMax-H3.md:469 `gfx942; on gfx950 it was not exercised.`; recipes/MiniMaxAI/MiniMax-H3.md:481 `| T2VA, 832x480, ~4 s, 40 steps | 1x gfx950 (MI350), FLASH_ATTN, CPU offload | valid MP4 (H.264 + synced audio); ~0.73 s/denoise-step (~1.37 it/s), ~55 s client E2E incl. warmup |`; PR body (outside the diff, gh_pr_view 5723): `MiniMax H3 T2VA generates valid H.264 + 32 kHz audio on gfx950 (single + 4-GPU; FLASH_ATTN ~2× faster than SDPA)`) + +`recipes/MiniMaxAI/MiniMax-H3.md:~728` [minor] — The parenthetical '(12 files)' immediately follows three maxima that sum to 15, which reads as 9+3+3=12. The real contract is a total cap of 12 references (the per-type maxima cannot all be reached simultaneously). Rephrase to '(up to 12 total references; ≤9 images, ≤3 videos, ≤3 audios)'. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:728 ` to 9 images, 3 video clips, and 3 audio clips (12 files) per Omni Reference`; recipes/MiniMaxAI/MiniMax-H3.md:663 `| Ref2VA | image-only, image+image, image+video, video+audio, and mixed image/video/audio | images ≤9, videos ≤3, audios ≤3, total references ≤12; audio requires a visual reference |`; pipeline_minimax_h3.py:325-326 (unchanged by this diff, present in PR-time tree) ` if image_count + video_count + audio_count > 12: + raise OmniClientError("ref2va accepts at most 12 total references")`) + +`recipes/MiniMaxAI/MiniMax-H3.md:405` [minor] — The instruction to 'drop the CUDA-only FLASHINFER_DISABLE_VERSION_CHECK' is dangling: no CUDA command in this recipe sets that variable, and the engine sets it automatically via os.environ.setdefault on every platform. Either point out that the engine sets it internally (so ROCm users need do nothing) or drop the clause. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:405 `CUDA-only `FLASHINFER_DISABLE_VERSION_CHECK`, and install without the `[fa4]` extra` — this diff line is the only occurrence of FLASHINFER_DISABLE_VERSION_CHECK in the recipe, so no CUDA block exports it. Unchanged by this diff, present in the PR-time tree: vllm_omni/engine/stage_engine_core_proc.py:122 `os.environ.setdefault("FLASHINFER_DISABLE_VERSION_CHECK", "1")`.) + +`recipes/MiniMaxAI/MiniMax-H3.md:481` [minor] — The ROCm evidence table (recipes/MiniMaxAI/MiniMax-H3.md:477-481) has only FLASH_ATTN rows — two 4x gfx942 and one 1x gfx950 — and the file contains no 'SDPA' string, so the PR description's 'FLASH_ATTN ~2× faster than SDPA' claim is unsubstantiated in the diff. Add an SDPA baseline row (or otherwise document the comparison) or remove the claim from the description. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:481 `| T2VA, 832x480, ~4 s, 40 steps | 1x gfx950 (MI350), FLASH_ATTN, CPU offload | valid MP4 (H.264 + synced audio); ~0.73 s/denoise-step (~1.37 it/s), ~55 s client E2E incl. warmup |` — the final ROCm evidence row is FLASH_ATTN; the two preceding rows (lines 479-480, unchanged context above the cited line) are also `FLASH_ATTN`, and `grep SDPA` on the recipe returns no matches. PR body (gh_pr_view): 'MiniMax H3 T2VA generates valid H.264 + 32 kHz audio on gfx950 (single + 4-GPU; FLASH_ATTN ~2× faster than SDPA)' — the ~2× claim is only in the PR description, not in the diff.) + +`recipes/MiniMaxAI/MiniMax-H3.md:417` [minor] — The parenthetical '(for image+audio Ref2VA)' is unsupported: MiniMax H3 image references are decoded with PIL, and reference audio loads via torchaudio -> soundfile -> ffmpeg fallback; nothing under vllm_omni/diffusion/models/minimax_h3 imports torchcodec. Keep the bundled-package fact if true, but drop or correct the 'for image+audio Ref2VA' rationale. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:417 `the image bundles TorchCodec (for image+audio Ref2VA) and \`ffmpeg\` (for` (added by this diff); unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/reference_video.py:468-469 `def _soundfile_to_waveform(path: str) -> tuple[torch.Tensor, int]:\n import soundfile as sf`; same file:477-493 `def load_audio_file(path: str) -> tuple[torch.Tensor, int]:\n ...\n try:\n import torchaudio\n\n return torchaudio.load(path)\n except (ImportError, RuntimeError, OSError):\n pass\n try:\n return _soundfile_to_waveform(path)\n except Exception:\n ... ffmpeg ...`; grep for `torchcodec` under vllm_omni/diffusion/models/minimax_h3 returns no matches; same file:411 `from PIL import Image`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.trace.json.gz new file mode 100644 index 000000000..2e36b0e53 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5723.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.cost.json new file mode 100644 index 000000000..138724ed9 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1265.1, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5756/run-20260815-065026-6b2f6a", + "recorded_at": "2026-08-15T07:11:29", + "attempts": [ + { + "invocation_id": "inv-47aac078", + "outcome": "run:run-20260815-065026-6b2f6a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-065026-6b2f6a", + "llm_calls": 127, + "input_tokens": 1190961, + "output_tokens": 224916, + "usd": 0.904113, + "source": "spans" + } + ], + "attempt_usd_total": 0.904113, + "llm_calls": 127, + "input_tokens": 1190961, + "output_tokens": 224916, + "usd": 0.904113, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-065026-6b2f6a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9041, + "minutes": 21.04, + "ci_minutes": 0, + "input_tokens": 1190961, + "output_tokens": 224916, + "tool_calls": 221, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9093, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 127, + "by_role": { + "planner": { + "usd": 0.002858, + "input_tokens": 2429, + "output_tokens": 2002, + "calls": 1 + }, + "agent": { + "usd": 0.867773, + "input_tokens": 1160837, + "output_tokens": 199312, + "calls": 124 + }, + "reducer": { + "usd": 0.033481, + "input_tokens": 27695, + "output_tokens": 23602, + "calls": 2 + } + }, + "cache_read_tokens": 12412800, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.74, + "gate": 2.02, + "review": 1256.68, + "report": 0.07 + }, + "time_to_first_result_s": 1262.49 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5756.trace.json.gz", + "trace_bytes": 43782 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.md new file mode 100644 index 000000000..3bc3312e5 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5756, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 43592108ac79) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] mid-PR commit 2f9ebd04 removed the conflicting duration field; residual = H3 length is now only via num_frames and fps must be 24, but the node defaults are 16/41 (flagged as comment). +- [claim-verified] PR body 'two test cases' matches diff: VIDEO_REF2VA_IMAGE_AUDIO and VIDEO_REF2VA_MULTI_VIDEO added at tests/e2e/features/comfyui/test_comfyui_integration.py:390-439. +- [claim-verified] frame/references exclusivity is enforced in nodes.py:185 and again in api_client.py:276. +- [claim-verified] 'image' input renamed to 'frame' (nodes.py:168) and the example workflow JSON was updated to link frame; grep VLLMOmniGenerateVideo found no other stale GenerateVideo workflow. +- [claim-verified] README 'Task routing is automatic from which inputs you connect' matches client task selection fl2va/ref2va/t2va at api_client.py:301-329. +- [claim-verified] PR '20 passed' arithmetic: 4x2 image + 1x2 understanding + 2x2 tts + 2x2 video + 2 ref2va = 20 collected cases. +- [claim-verified] tests/e2e/features/comfyui/test_comfyui_integration.py has pytestmark=[core_model,cpu]; .buildkite/cuda/test-merge.yml:28 runs `tests/ -m 'core_model and cpu'`, so the new tests are selected in the merge gate. +- [claim-verified] PR body claim that frame/references are exclusive is enforced in two places (nodes.py VALIDATE_INPUTS and api_client.py:276). +- [claim-verified] MiniMax-H3 recipe (recipes/MiniMaxAI/MiniMax-H3.md:258-276) confirms image+audio uses `input_reference`+`audio_reference` and video-only uses repeated `input_references` — client matches. +- [claim-refuted] nodes.py:223 claims model_params['type'] 'is used in api_client', but api_client.py:359 pops it and api_client.py:361 routes via lookup_model_spec(model), so the field is dead. +- [validated] client multipart keys (input_reference, input_references, audio_reference, extra_params) match server _parse_video_form declarations at api_server.py:2981-3009. +- [validated] audio_to_base64 returns a data:audio URL (format.py:263-270) and decode_audio_url accepts data:audio URLs (video_api_utils.py:365-376). +- [validated] extra_params(task/audio_flow_shift) and top-level flow_shift reach sampling params; the real serving layer builds them before the mocked AsyncOmni.generate asserts them (test:390-439). +- [validated] extra_params is the correct /videos form key; grep found no extra_body Form in api_server.py's video path. + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:754` [major] — apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py: `VLLMOmniGenerateVideo` keeps `fps` default 16 (line 164) and forwards it via `generate_video(..., fps=fps)` (line 236) to `api_client.py` which sends `form.add_field("fps", str(fps))` (line 286). MiniMax-H3 therefore rejects the default at generation time in `MiniMaxH3Pipeline._resolve_shape` (pipeline_minimax_h3.py:399-401) because `MINIMAX_H3_FPS = 24`. Add a guard or force fps=24 in `VLLMOmniGenerateVideo.VALIDATE_INPUTS` when the model resolves to MiniMax-H3. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:164 (PR-time tree, unchanged by this diff): `"fps": ("INT", {"default": 16, "min": 1}),`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:236 (changed by this diff): `fps=fps,`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:286 (PR-time tree, unchanged by this diff): `form.add_field("fps", str(fps))`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:399-401 (outside this diff, present in PR-time tree): `fps = int(sampling.fps or MINIMAX_H3_FPS)` / `if fps != MINIMAX_H3_FPS:` / `raise ValueError(f"MiniMax H3 output fps is fixed at {MINIMAX_H3_FPS}")`, with `MINIMAX_H3_FPS = 24` defined earlier in the same file.) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:71` [minor] — `max_poll_duration` is raised from 300s to 1800s in the shared `VLLMOmniClient` constructor (api_client.py:71) and consumed by the generic `generate_video` polling deadline (api_client.py:387). Video nodes instantiate `VLLMOmniClient(url)` with no per-model override (e.g. nodes.py:227), so Wan and any other video model inherit the 30-minute ceiling too. The only ~30-minute-scale justification in-repo is MiniMax-H3 two-video Ref2VA (recipes/MiniMaxAI/MiniMax-H3.md:306 `784.394 s accounted model-stage mean`). Consider applying the longer duration only to the H3/long-running path or documenting the global trade-off. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:71 ` max_poll_duration: float = 60 * 30,`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:387 ` deadline = asyncio.get_running_loop().time() + self.max_poll_duration`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:227 ` client = VLLMOmniClient(url)` (unchanged by this diff, present in the PR-time tree; no max_poll_duration override). The diff hunk changes the constructor default from `60 * 5` to `60 * 30`. recipes/MiniMaxAI/MiniMax-H3.md:306 `| Two-video Ref2VA, 362 frames, 1344x768 | no offload, U4, VPP4 tile, regional compile | 784.394 s accounted model-stage mean |`.) + +`apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:1` [minor] — The rewritten example workflow's MarkdownNote still says 'The node will create corresponding payload depending on whether an input image is present or not', even though GenerateVideo's input is now `frame` (and `references` was added). Update the note text to match the current inputs. (evidence: apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:1 `"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."]` — the stale 'input image' phrase is present verbatim in the PR-time tree.) + +`tests/e2e/features/comfyui/test_comfyui_integration.py:390` [minor] — Add a MiniMax-H3 `t2va` (and optionally `fl2va`) integration case to exercise api_client.py:303/330; the new tests only cover the two `ref2va` cases, so the H3 `t2va`/`fl2va` task-routing branches are currently unselected in the e2e suite. (evidence: tests/e2e/features/comfyui/test_comfyui_integration.py:873 `SamplingCase(kind=SamplingKind.VIDEO_REF2VA_IMAGE_AUDIO, sampling_params=None),` and :879 `SamplingCase(kind=SamplingKind.VIDEO_REF2VA_MULTI_VIDEO, sampling_params=None),` are the only MiniMax-H3 parametrizations. In changed code, apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:303 `video_task = "fl2va"` and :330 `video_task = "t2va"` are the only H3 routing branches not reached by these tests.) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:223` [minor] — The `type` field set at nodes.py:777 (`params["type"] = "minimax_h3"`) is never read: api_client.py:359 pops it (`model_params.pop("type", None)`), api_client.py:361 selects the builder via `lookup_model_spec(model)`, and models.py:44 pops it again (`params.pop("type", None)`). The nodes.py:223 comment saying it is "used in api_client" is misleading — it is only discarded. Either select the MiniMax-H3 builder from `model_params['type'] == 'minimax_h3'` (which would also cover local checkpoint paths whose model string lacks `MiniMax-H3`) or drop the field and fix the comment. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:223 ` # model_params["type"] is also an internal field, but is used in api_client`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:777 ` params["type"] = "minimax_h3"`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:359 ` model_params.pop("type", None)`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:361 ` spec, _ = lookup_model_spec(model)`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:44 ` params.pop("type", None)`) + +`tests/e2e/features/comfyui/test_comfyui_integration.py:896` [minor] — The new MiniMax-H3 Ref2VA e2e cases call `node.generate(..., fps=VIDEO_FPS, num_frames=VIDEO_NUM_FRAMES, model='MiniMaxAI/MiniMax-H3')` (tests/e2e/features/comfyui/test_comfyui_integration.py:927) with `VIDEO_FPS = 8`, but `MiniMaxH3Pipeline._resolve_shape` raises `ValueError` unless fps equals `MINIMAX_H3_FPS` (24; pipeline_minimax_h3.py:401). Since `mock_async_omni` replaces `AsyncOmni`, the real pipeline is never instantiated, so these cases prove multipart/extra_params plumbing, not H3-compatible inputs. Set fps=24 and assert the submitted value (and optionally add an FL2VA case to cover `video_task='fl2va'`), or add a pipeline-level unit check against the fps guard. (evidence: ['tests/e2e/features/comfyui/test_comfyui_integration.py:86 (unchanged by this diff): `VIDEO_FPS = 8`', 'tests/e2e/features/comfyui/test_comfyui_integration.py:927 (added by this diff): ` fps=VIDEO_FPS,`', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:401 (outside this diff, present in PR-time tree): ` raise ValueError(f"MiniMax H3 output fps is fixed at {MINIMAX_H3_FPS}")`', "recipes/MiniMaxAI/MiniMax-H3.md:192 (unchanged by this diff): ` -F 'fps=24' \\`"]) + +`apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:~1` [minor] — The rewritten `example_workflows/vLLM-Omni Video Generation.json` adds node 21 `VLLMOmniVideoReferences`, but its `references` output has `links:null` and Generate Video node 16's `references` input has `link:null`; the `links` array contains no 21→16 edge, while node 16's `frame` input stays connected via link 29. Because `generate_video` routes a non-null `frame` to `fl2va` before checking `references`, the example as shipped runs FL2VA instead of the Ref2VA path its note advertises. Connect node 21's output to node 16's `references` input (and disconnect `frame` per the included note) so the r2v example is executable as documented. (evidence: apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:1 — node 16 `"inputs":[{"localized_name":"frame","name":"frame","shape":7,"type":"IMAGE","link":29},{"localized_name":"references","name":"references","shape":7,"type":"VIDEO_REFERENCES","link":null},...]`; node 21 `"outputs":[{"localized_name":"references","name":"references","type":"VIDEO_REFERENCES","links":null}]`; `"links":[[29,4,0,16,0,"IMAGE"],[30,8,0,16,2,"SAMPLING_PARAMS"],[32,16,0,13,0,"VIDEO"],[33,18,0,16,4,"VIDEO_PARAMS"],[34,23,0,21,0,"IMAGE"],[35,24,0,21,2,"AUDIO"]]` — no 21→16 edge and frame link 29 remains. Unchanged-by-this-diff consumer in the PR-time tree: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:301-303 `if frame is not None:` / `input_reference_image = frame` / `video_task = "fl2va"` — so the connected frame routes the example to FL2VA, not ref2va.) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:781` [minor] — VLLMOmniVideoReferences (nodes.py:781) exposes six optional inputs but no VALIDATE_INPUTS, so image-only, audio-only, or video-mixed-with-image/audio combinations pass graph configuration and are only rejected later in api_client.generate_video (api_client.py:312-327). Add a VALIDATE_INPUTS classmethod mirroring those supported modes — one or more videos only, or exactly one image plus one audio — so invalid reference combinations fail at graph-configuration time. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:781 `class VLLMOmniVideoReferences:` — the class body contains only `INPUT_TYPES` (line 783) and `get_references` (line 806), with no `VALIDATE_INPUTS` classmethod. apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:320 `elif len(images) == 1 and len(audios) == 1:` ... api_client.py:325-327 `else: + raise ValueError( + "Invalid references combination. Supported modes: " + "(1) one or more videos only, or (2) exactly one image and one audio." + )` — enforcement happens only at job-submission time inside generate_video, not at graph-configuration time.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.trace.json.gz new file mode 100644 index 000000000..f2e8c2109 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5756.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.cost.json new file mode 100644 index 000000000..6b0788385 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1713.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5779/run-20260815-065027-87bffe", + "recorded_at": "2026-08-15T07:18:57", + "attempts": [ + { + "invocation_id": "inv-eb6536f6", + "outcome": "run:run-20260815-065027-87bffe", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-065027-87bffe", + "llm_calls": 218, + "input_tokens": 2274853, + "output_tokens": 330314, + "usd": 1.576373, + "source": "spans" + } + ], + "attempt_usd_total": 1.576373, + "llm_calls": 218, + "input_tokens": 2274853, + "output_tokens": 330314, + "usd": 1.576373, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-065027-87bffe", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.5764, + "minutes": 28.5, + "ci_minutes": 0, + "input_tokens": 2274853, + "output_tokens": 330314, + "tool_calls": 353, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2371, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 218, + "by_role": { + "agent": { + "usd": 1.545073, + "input_tokens": 2253588, + "output_tokens": 307116, + "calls": 216 + }, + "reducer": { + "usd": 0.031301, + "input_tokens": 21265, + "output_tokens": 23198, + "calls": 2 + } + }, + "cache_read_tokens": 22178432, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.06, + "gate": 1.8, + "review": 1704.1, + "report": 0.06 + }, + "time_to_first_result_s": 1710.01 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5779.trace.json.gz", + "trace_bytes": 47218 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.md new file mode 100644 index 000000000..a60ddf142 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5779, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f092a92eb579) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] empty-trailing-sequence drop loop keeps a numel()>2 guard so cu_seqlens cannot shrink below 2 entries; single-sequence H3 cu=[0,used,seq_len] is untouched. +- [resolved] Arbitrary masks still rejected: trtllm_attn.py:322-325 raises 'TRTLLM_ATTN only supports prefix-valid packed attention masks' after torch.equal; residual checked: multi-batch prefix masks are also rejected (good), but empty-suffix trimming has no direct test. +- [resolved] Packed output shape restored to the physical [B,S,N,D] tensor with zero padding at trtllm_attn.py:398-405; residual: the restore path relies on output_tokens captured before trimming. +- [claim-verified] packed_sequence.py aligns used up to a multiple of 64 and builds cu=[0,used,seq_len]; PR's 58758-valid/58816-aligned description matches. +- [claim-verified] token refiner is short: denoise_loop builds refiner cu=[0,text_len,text_len] and transformer role minimax_h3.token_refiner; per_role override documented. +- [claim-verified] diff_stat confirms the PR touches only the 11 claimed files; recipe FLASH_ATTN -> default TRTLLM switch and docs updates are present. +- [claim-verified] Dependency check: no new external kwarg introduced; the SAGE call surface is unchanged from base and the only new call path uses existing trtllm_ragged_attention_deepseek. +- [claim-refuted] recipe claims dense TRTLLM vs FA4 'within 2%', but PR body reports only FA4 88.98s and says TRTLLM e2e latency 'is not reported' — no cited <2% measurement. +- [validated] trtllm_attn.py:263-269 incomplete packed-key detection raises before the HAS_FLASHINFER guard; test_packed_metadata_must_be_complete drives it. +- [validated] trtllm_attn.py:293-344 packed trimming validates prefix mask via torch.equal(flat_mask, expected_mask), requires a single Q/K boundary, slices q/k/v to valid_tokens and truncates cu_seqlens. +- [validated] trtllm_attn.py:398-406 zero-padding restores physical shape; out.reshape(physical_batch,q_len) is consistent because output_tokens==physical_batch*q_len. +- [validated] test_packed_metadata_trims_padding_mask drives the real forward_cuda entry (kernel mocked) and asserts q/k/v trimmed to 6 with the padded tail zeroed. +- [validated] test_bf16_packed_padding_matches_sdpa and test_sage_packed_non_aligned_length_matches_sdpa are GPU-gated (requires_trtllm_attn/requires_sage) and compare to _sdpa_ref over valid tokens only. +- [validated] test_sage_short_sequence_uses_dense_kernel proves the fallback by monkeypatching _sage_quantize_fn to fail; s=14= self.quant.k_block_size).item())`. Unchanged by this diff but present in the PR-time tree (producer contract the new code violates): vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:394-396 ` # max_seqlen is already the first (real) packed document length. Do` / ` # not read the CUDA cu_seqlens scalars here: this function runs once` / ` # per layer and .item() would serialize every attention launch.` and vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:417 ` "valid_kv_length": used,`.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:~356` [major] — The diff changes `_skip_factor` to `self._resolve_skip_factor(max_kv_len)` (trtllm_attn.py:356) and passes that one scalar as `skip_softmax_threshold_scale_factor` for the whole ragged batch (trtllm_attn.py:395). Since `SkipSoftmaxConfig.resolve_factor` returns `self.threshold * seqlen` (trtllm_attn.py:66) and the packed path computes `seq_lens = cu_seq_lens_kv[1:] - cu_seq_lens_kv[:-1]` (trtllm_attn.py:341) with potentially differing lengths, every sequence shorter than the max receives the max-length scale. H3's validated path trims to a single `[0, used]` sequence, so it is unaffected, but any heterogeneous packed batch under skip-softmax is silently mis-scaled. Either compute per-sequence factors if the kernel supports them, or reject `skip_softmax` when the packed batch has more than one distinct sequence length. (evidence: trtllm_attn.py:356 ` _skip_factor = self._resolve_skip_factor(max_kv_len)` (changed by this diff); trtllm_attn.py:341 ` seq_lens = (cu_seq_lens_kv[1:] - cu_seq_lens_kv[:-1]).to(dtype=torch.int32).contiguous()` (added by this diff, can yield differing per-sequence lengths); trtllm_attn.py:395 ` skip_softmax_threshold_scale_factor=_skip_factor,` (unchanged context in the PR-time tree, receives one scalar for all sequences); trtllm_attn.py:66 ` factor = self.threshold * seqlen` (unchanged context in the PR-time tree, proves the factor is sequence-length-dependent).) + +`tests/diffusion/attention/test_trtllm_attn.py:212` [major] — Confirmed. tests/diffusion/attention/test_trtllm_attn.py:14 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` marks all tests in the module `cuda` (not `cpu`), so the merge lane `pytest tests/diffusion -m 'core_model and cpu'` (.buildkite/cuda/test-merge.yml:18) deselects every test in the file, including the new CPU-only packed-metadata tests and the mask-free/denoise-progress assertions. No Buildkite lane references `tests/diffusion/attention` (grep over .buildkite returns zero matches), so these regressions are never exercised. Add `@pytest.mark.cpu`, split the CPU tests into a cpu-marked module, or add a lane that runs the attention directory with `core_model and cpu`. (evidence: tests/diffusion/attention/test_trtllm_attn.py:14 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]` — module-level mark applies `cuda` (not `cpu`) to every test, including the new CPU-only tests, e.g. tests/diffusion/attention/test_trtllm_attn.py:212 `def test_packed_metadata_trims_padding_mask(monkeypatch):`. .buildkite/cuda/test-merge.yml:18 `"timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` selects only `core_model and cpu` tests, so the module is deselected. Repo-wide grep of `.buildkite` for `tests/diffusion/attention` returns zero matches (CI YAML unchanged by this diff, present in the PR-time tree), so no Buildkite lane runs the module.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:307` [minor] — Add a direct test for the empty-trailing-sequence trim loop in vllm_omni/diffusion/attention/backends/trtllm_attn.py (while-condition lines 305-309, body 310-311): drive a packed metadata whose cu_seqlens ends in a repeated value (e.g. [0,6,6]) through forward_cuda and assert it is trimmed without shrinking the tensor below two entries. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:303-311 `# Drop trailing empty sequences retained by packed producers when\n # alignment adds no padding tokens.\n while (\n cu_seq_lens_q.numel() > 2\n and int(cu_seq_lens_q[-1].item()) == int(cu_seq_lens_q[-2].item())\n and int(cu_seq_lens_kv[-1].item()) == int(cu_seq_lens_kv[-2].item())\n ):\n cu_seq_lens_q = cu_seq_lens_q[:-1]\n cu_seq_lens_kv = cu_seq_lens_kv[:-1]` — trim loop present and uncovered; the PR-time test file's only cu_seqlens fixtures are [0,6,8] (tests/diffusion/attention/test_trtllm_attn.py:216,254), [0,s] (:273), [0,6] (:282), [0,used,s] (:394,444), none of which has equal trailing entries to trigger the loop.) + +`recipes/MiniMaxAI/MiniMax-H3.md:~38` [minor] — The sentence at lines 38-39 ('Install the optional dependency only for the four-GPU B300/GB200 `FLASH_ATTN` profile:') is stale: the four-GPU profile now uses dense BF16 `TRTLLM_ATTN` with no backend flag, and the command below it is the base `uv pip install -e .`. Reword to say FA4 is needed only when explicitly selecting `--diffusion-attention-backend FLASH_ATTN`. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:38 `FlashAttention-4. Install the optional dependency only for the four-GPU` (unchanged by this diff, present in the PR-time tree) and :39 `B300/GB200 `FLASH_ATTN` profile:`; recipes/MiniMaxAI/MiniMax-H3.md:44 `uv pip install -e .` (base install now shown under that sentence); recipes/MiniMaxAI/MiniMax-H3.md:209 `- dense BF16 `TRTLLM_ATTN`, with Ring and TP left at 1.` (the four-GPU profile now uses TRTLLM_ATTN, not FLASH_ATTN).) + +`recipes/MiniMaxAI/MiniMax-H3.md:247` [minor] — recipes/MiniMaxAI/MiniMax-H3.md:247 claims dense TRTLLM_ATTN and FA4 are 'within 2% of each other', but the PR body provides no TRTLLM latency/throughput measurement — only the FA4 88.98 s model-stage number, PSNR 27.10 dB / SSIM 0.8880 quality parity, and an explicit note that the TRTLLM run's end-to-end latency is not reported as steady-state backend performance. Cite the measurement that produced '<2%' or soften the sentence to the evidence shown. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:247 (added by this diff): "Stable measurements with the four-GPU profile above put dense `TRTLLM_ATTN` and FA4 within 2% of each other." PR body for PR 5779 (no repo file): "50-step FA4 baseline with the same prompt/seed and 1248x768, 209-frame shape: model stage 88.98 s"; "Encoded-video comparison against FA4: average PSNR 27.10 dB and SSIM 0.8880"; "This was a cold TRTLLM service run and included regional/VAE compilation plus slow CPU MP4 encoding, so its end-to-end latency is not reported as steady-state backend performance." No TRTLLM latency or throughput number exists in the PR body, so the '<2%' comparison is unsupported.) + +`vllm_omni/diffusion/attention/layer.py:~96` [minor] — Confirmed at vllm_omni/diffusion/attention/layer.py:96-100: when the resolved backend is TRTLLM_ATTN, skip_sequence_parallel is false, and allgather_degree > 1, Attention.__init__ raises ValueError instead of falling back. Because this PR sets MiniMaxH3Pipeline attention_mask_free=True (model_metadata.py:46) and that value drives allow_trtllm_default (layer.py:83), datacenter Blackwell + flashinfer now auto-selects TRTLLM_ATTN for H3 main attention, so `--allgather-degree 2` hits startup failure. Please add a focused test for this rejection and state whether H3+AllGather-KV is intentionally unsupported; if it should work, the selector should skip TRTLLM_ATTN rather than fail at init. (evidence: vllm_omni/diffusion/attention/layer.py:96 `if not skip_sequence_parallel and allgather_degree > 1 and attn_backend_cls.get_name() == "TRTLLM_ATTN":` — added by this diff; vllm_omni/diffusion/attention/layer.py:97-100 `raise ValueError("TRTLLM_ATTN does not support AllGather-KV sequence parallelism. " "Set --allgather-degree 1 or select another diffusion attention backend.")` — added by this diff; vllm_omni/diffusion/model_metadata.py:46 `attention_mask_free=True,` — added by this diff inside the MiniMaxH3Pipeline metadata; vllm_omni/diffusion/attention/layer.py:83 `allow_trtllm_default = get_diffusion_model_metadata(model_class_name).attention_mask_free` — unchanged by this diff, present in the PR-time tree, connects the H3 mask-free mark to TRTLLM_ATTN default selection.) + +`vllm_omni/diffusion/data.py:~1437` [nit] — `SkipSoftmaxSpec.enabled` (data.py:1438-1439) has no consumer in the PR-time tree: grep for `skip_softmax.enabled` returns no matches, and the runtime ring-attention gate reads `getattr(skip, 'configured', False)` on `SkipSoftmaxConfig` (layer.py:331, trtllm_attn.py:57). The property duplicates existing gating and is dead code in this PR — remove it or wire a consumer. (evidence: vllm_omni/diffusion/data.py:1438 `def enabled(self) -> bool:` and :1439 `return self.threshold is not None or self.target_sparsity is not None` — added by this diff, no consumer found (repo grep for `skip_softmax.enabled` returns no matches). Unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/attention/layer.py:331 `if skip is not None and getattr(skip, "configured", False):` and vllm_omni/diffusion/attention/backends/trtllm_attn.py:57 `def configured(self) -> bool:` — the runtime gate uses `configured`, not `enabled`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.trace.json.gz new file mode 100644 index 000000000..dc6e4bf6c Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5779.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.cost.json new file mode 100644 index 000000000..9a3d00f7e --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1330.3, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5801/run-20260815-070415-c95285", + "recorded_at": "2026-08-15T07:26:23", + "attempts": [ + { + "invocation_id": "inv-787330b2", + "outcome": "run:run-20260815-070415-c95285", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-070415-c95285", + "llm_calls": 115, + "input_tokens": 714006, + "output_tokens": 295588, + "usd": 0.700616, + "source": "spans" + } + ], + "attempt_usd_total": 0.700616, + "llm_calls": 115, + "input_tokens": 714006, + "output_tokens": 295588, + "usd": 0.700616, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-070415-c95285", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7006, + "minutes": 22.13, + "ci_minutes": 0, + "input_tokens": 714006, + "output_tokens": 295588, + "tool_calls": 241, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8544, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 115, + "by_role": { + "agent": { + "usd": 0.653742, + "input_tokens": 689566, + "output_tokens": 259040, + "calls": 112 + }, + "reducer": { + "usd": 0.046874, + "input_tokens": 24440, + "output_tokens": 36548, + "calls": 3 + } + }, + "cache_read_tokens": 6766208, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.65, + "gate": 1.64, + "review": 1321.41, + "report": 0.06 + }, + "time_to_first_result_s": 1327.75 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5801.trace.json.gz", + "trace_bytes": 37331 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.md new file mode 100644 index 000000000..054a24d36 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.md @@ -0,0 +1,72 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5801, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1ef0cb1562a6) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Merge commit 1ef0cb15 inspected — branch helper reconciled with main's dim==3 strip (no `>2 reshape` regression); no semantic conflict in the 6 changed files. +- [resolved] torch.set_default_dtype appears only as float32 inside try/finally restore in minicpmo_4_5 (batched_token2wav.py:112-116, minicpmo_4_5_code2wav.py:758-765); no global bf16 default, so explicit dtype=torch.float32 has no blast radius. +- [resolved] RotaryEmbeddingWan still overrides half_head_dim=False after super().__init__ (rope.py:220-222); Wan NPU path already passed self.half_head_dim at base, so this diff does not change Wan. +- [resolved] Existing RotaryEmbedding callers are safe: default half_head_dim=True preserves old behavior, and RotaryEmbeddingWan already passed half_head_dim at base (file_at_base confirms apply_rotary_emb_mindiesd already had the half_head_dim parameter). +- [claim-verified] PR claim 'configurable weight dtype + fp32 native accumulation' holds: norm.py:68 and norm.py:134. +- [claim-verified] PR claim 'full-head-dimension cos/sin support' holds: rope.py:92 half_head_dim param and rope.py:124 slicing. +- [claim-verified] gh_ci_read: buildkite/vllm-omni-amd-ci FAILURE; vllm-omni, -npu-ci, -intel-ci, pre-commit, DCO, builds all SUCCESS. +- [claim-verified] diff_stat confirms exactly 6 files changed; no hidden inclusions/removals beyond the supplied diff. +- [claim-refuted] PR body '-3.35%' does not match its own numbers: (491014-475070)/491014 = 3.247% (-3.25%), checked with calc. +- [claim-refuted] Perf baseline commit 5215e03a (show_commit) is an older main scheduler-refactor commit (#5461); head 1ef0cb15 merges main 9235b0ae, so the latency pair is not measured against the current merge-base. +- [claim-refuted] PR body says -3.35%: (491014-475070)/475070 = 3.356% (calc); vs baseline it is (491014-475070)/491014 = 3.247% — percentage computed against the optimized value, not the baseline. +- [claim-refuted] E2E latency numbers were measured on commit 5215e03a, but head 1ef0cb15 merges main (9235b0ae) which changes minimax_h3 pipeline/encoder/vae/offloader — numbers not re-verified at merged head. +- [claim-refuted] PR purpose 'preserving fp32 accumulation' is only true of the native fallback; it is not true of the CUDA fused path H3 actually runs. +- [claim-refuted] PR body '-3.35%': (491014-475070)/491014 = 3.247% (calc); the quoted numbers give -3.25% relative to baseline (or -3.36% if divided by 475070). + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177` [minor] — This hunk switches H3 `_norm` from `torch.nn.RMSNorm` to the Omni `RMSNorm` CustomOp (minimax_h3_transformer.py:177). On CUDA eager, `RMSNorm.forward_cuda` returns `_forward_fused` (norm.py:95), which calls `vllm._custom_ops.rms_norm`; the repo's own parity test documents that fused kernel as computing bf16 inputs in bf16 precision internally (test_norm.py:625), while the removed `nn.RMSNorm` path is documented to upcast to fp32 for variance reduction and the comment at :174 still asserts that. So H3's bf16 q_norm/k_norm/norm layers lose fp32 accumulation on the CUDA fused path, contradicting the PR's 'preserving fp32 accumulation' contract (the new test only exercises `forward_native`). Either route bf16 weights through `forward_native` as the compile path already does, or document the accepted precision delta and add a fused-vs-`nn.RMSNorm` parity test with realistic non-ones bf16 weights (the existing test uses 2e-2 tolerance for exactly this gap). (evidence: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177 (changed by this diff): ` return RMSNorm(size, eps=eps, dtype=dtype)` +vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:174 (unchanged by diff, now stale): ` # RMSNorm uses fp32 accumulation with bf16 inputs and outputs.` and ` # torch.nn.RMSNorm upcasts reduced-precision inputs for the variance` +vllm_omni/diffusion/layers/norm.py:95 (unchanged by diff): ` return self._forward_fused(x)` — this is `RMSNorm.forward_cuda`'s eager (non-compiling) return; `_forward_fused` calls `from vllm._custom_ops import rms_norm as fused_rms_norm; fused_rms_norm(out, x_2d, self.weight.data, self.variance_epsilon)`. +tests/diffusion/layers/test_norm.py:625 (unchanged by diff): ` (vllm._custom_ops.rms_norm) computes in bf16 precision internally,` — from `test_rmsnorm_fused_native_parity_custom_weight`, which allows bf16 atol 2e-2 for this gap.) + +`vllm_omni/diffusion/layers/rope.py:125` [minor] — vllm_omni/diffusion/layers/rope.py:125 — the neox branch of `_prepare_half_head_dim_cos_sin` slices only the first half of full-dim cos/sin, which is valid only because the sole `RotaryEmbedding(half_head_dim=False)` caller is H3 (`MiniMaxH3Attention.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)`), fed by `MiniMaxH3Rope.forward`'s duplicated-half layout (`torch.cat((half, half), dim=-1)`). Repo-wide grep shows no `is_neox_style=False, half_head_dim=False` caller reaches this helper, so the `cos[..., ::2]` interleaved arm is dead and untested. Document/assert the duplicated-half invariant on this public helper, and drop or cover the interleaved arm. (evidence: vllm_omni/diffusion/layers/rope.py:124-125: + if not self.half_head_dim: + cos = cos[..., ::2] if self.interleaved else cos[..., : cos.shape[-1] // 2] + +vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:227: + return torch.cat((half, half), dim=-1) # [S, 96] + +vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335: + self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False) + +PR-time grep for `half_head_dim=False` matches only this H3 instantiation, the test instantiation at tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:72 (`attention.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)`), and a direct non-helper call at tests/diffusion/layers/test_rope_broadcast.py:259 (`actual = apply_rotary_emb_mindiesd(x, cos, sin, interleaved=False, half_head_dim=False)`). No `RotaryEmbedding(is_neox_style=False, half_head_dim=False)` caller exists, so the helper's `cos[..., ::2]` arm is unexercised.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:73` [minor] — This test is named 'fused' but pins `_forward_method` to `forward_native` (test_minimax_h3_parallel.py:73), so it only proves the half_head_dim=False slicing for the native path. The actual fused CUDA path — `RotaryEmbedding.forward_cuda` calling `_prepare_half_head_dim_cos_sin` with half_head_dim=False (rope.py:137,126-127) and then the vLLM kernel — has no test; grep `half_head_dim=False` in tests/ returns only this pinned-native test and the mindiesd plumbing test (test_rope_broadcast.py:259). Add a CUDA parity test comparing `RotaryEmbedding(is_neox_style=True, half_head_dim=False).forward_cuda` against `forward_native` (or drive `MiniMaxH3Attention._apply_rope` through real dispatch), and rename this test to not claim 'fused'. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:72-73 `attention.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)` / `attention.rope._forward_method = attention.rope.forward_native`; vllm_omni/diffusion/layers/rope.py:136-137 `def forward_cuda(...): ... cos, sin = self._prepare_half_head_dim_cos_sin(cos, sin)`; vllm_omni/diffusion/layers/rope.py:126-127 `if not self.half_head_dim:` / `cos = cos[..., ::2] if self.interleaved else cos[..., : cos.shape[-1] // 2]`; grep tests/ for `half_head_dim=False` returns only tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:72 (pinned native) and tests/diffusion/layers/test_rope_broadcast.py:259 (mindiesd plumbing), so no CUDA-path test exists. All cited lines are present in the PR-time tree.) + +`PR description:0` [minor] — Update the benchmark section: the PR body quotes -3.35%, but its own numbers give (491014-475070)/491014 = 3.247% (-3.25%); the E2E latency pair is pinned to commit 5215e03a while PR head 1ef0cb15 merges main 9235b0ae (touching minimax_h3 pipeline/encoder/vae/offloader) without re-verification; and precision evidence is image/video URLs only, not a numerical parity artifact or result JSON. Re-run at 1ef0cb15 and attach numeric artifacts, or remove the unverified claims. (evidence: PR #5801 body (description, not a file): 'E2E Latency: 491014ms -> 475070ms. **-3.35%**'; '**vLLM-Omni Commit:**\n5215e03a91adecbb5ffece29aa74360a7569d0c5'; '**Precision:**\nori (left) vs opt (right)\n\nhttps://github.com/user-attachments/assets/3cbd06eb-8e77-4106-a42a-eba346f71873'. calc: (491014-475070)/491014*100 = 3.247157922177372 -> -3.25%. show_commit 1ef0cb1562a6: 'Merge: aaeaa000 9235b0ae' and 'Merge branch 'main' into h3_opt'; merge brings main-side changes to vllm_omni/diffusion/models/minimax_h3/encoder.py, vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, vllm_omni/diffusion/models/minimax_h3/vae.py, and vllm_omni/diffusion/offloader/*.) + +`tests/diffusion/layers/test_norm.py:322` [minor] — test_rmsnorm_npu_receives_gamma_with_configured_dtype's npu_rms_norm stub returns (x,) and only asserts gamma identity/dtype and epsilon, so it cannot fail for real NPU RMSNorm output behavior (shape/dtype/device/normalization). Replace the identity mock with a non-identity mock and assert against a reference computation. (evidence: tests/diffusion/layers/test_norm.py:331 ` return (x,)` (inside test_rmsnorm_npu_receives_gamma_with_configured_dtype, defined at line 322); tests/diffusion/layers/test_norm.py:337 ` assert norm.forward_npu(x) is x`; tests/diffusion/layers/test_norm.py:338 ` assert captured["gamma"] is norm.weight`. The identity stub means the test passes regardless of the tensor returned by a real NPU kernel; the real path under test is vllm_omni/diffusion/layers/norm.py:112 ` output = torch_npu.npu_rms_norm(x, gamma=self.weight, epsilon=self.variance_epsilon)[0]`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:64` [minor] — Add a GPU merge-gate test that runs the H3 RMSNorm/RoPE integration through real CUDA dispatch (the new test at tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:64 pins `forward_native`, and the file is `cpu`-marked). Nightly full_model H3 quantization_quality is the only automated end-to-end exercise, so a CUDA regression in the fused RMSNorm/RoPE integration would not be caught by merge CI. (evidence: PR-time tree, unchanged by this diff: tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:8 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`; PR-time tree, added by this diff: tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:64 `def test_h3_fused_rope_matches_reference_and_preserves_unrotated_dims():`; PR-time tree, added by this diff: tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:71 `attention.rope._forward_method = attention.rope.forward_native`; PR-time tree, outside the diff: .buildkite/cuda/test-nightly.yml:495 `- pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"`) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:362` [minor] — Confirmed: with half_head_dim=False, apply_rotary_emb_mindiesd forwards H3's 2D `[T, rot_dim]` cos/sin unchanged to mindiesd.rotary_position_embedding (vllm_omni/diffusion/layers/rope.py:66), although the function's own signature documents full-dim cos/sin as `(B, S, D)` (vllm_omni/diffusion/layers/rope.py:45) and the half_head_dim=True branch reshapes to 4D before the call. The new test monkeypatches the op and asserts only identity/kwargs, so it does not validate the real operator's rank expectation. Confirm whether `rotary_position_embedding` accepts 2D full-dim cos/sin with `rotated_mode="rotated_half"` and 3D `x`; if not, add the missing batch dim in the half_head_dim=False branch. (evidence: vllm_omni/diffusion/layers/rope.py:45 `half_head_dim: bool = True, # if true, size of sin and cos is (B, S, D/2), otherwise (B, S, D)`; vllm_omni/diffusion/layers/rope.py:66 `return rotary_position_embedding(x, cos, sin, rotated_mode="rotated_half", head_first=False, fused=True)`; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335 `self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)` (in diff); vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:362 `cos = torch.cos(freqs).to(x.dtype) # [T, rot_dim]` (in diff); tests/diffusion/layers/test_rope_broadcast.py:259 `actual = apply_rotary_emb_mindiesd(x, cos, sin, interleaved=False, half_head_dim=False)` — the test monkeypatches the real op and never exercises a real rank check.) + +`vllm_omni/diffusion/layers/rope.py:~157` [minor] — `buildkite/vllm-omni-amd-ci` is FAILING at this head (gh_ci_read: state=FAILURE), and this diff changes the ROCm dispatch path: forward_hip now runs `cos, sin = self._prepare_half_head_dim_cos_sin(cos, sin)` before `apply_rotary_emb_flash_attn`. Please confirm whether the AMD red is pre-existing on main or introduced here, and attach the failing log if introduced; do not merge with an unexplained platform-lane red. (evidence: vllm_omni/diffusion/layers/rope.py:157 (inside `forward_hip`, the ROCm dispatch path, changed by this diff): ` cos, sin = self._prepare_half_head_dim_cos_sin(cos, sin)`; gh_ci_read(5801) entry: `{"name": "buildkite/vllm-omni-amd-ci", "state": "FAILURE"}` (external CI check, not a file in the diff).) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.trace.json.gz new file mode 100644 index 000000000..1eb203d66 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5801.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.cost.json new file mode 100644 index 000000000..3f049a21c --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 614.2, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5833/run-20260815-070503-7b531f", + "recorded_at": "2026-08-15T07:15:15", + "attempts": [ + { + "invocation_id": "inv-c3442fb9", + "outcome": "run:run-20260815-070503-7b531f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-070503-7b531f", + "llm_calls": 68, + "input_tokens": 210164, + "output_tokens": 110551, + "usd": 0.237735, + "source": "spans" + } + ], + "attempt_usd_total": 0.237735, + "llm_calls": 68, + "input_tokens": 210164, + "output_tokens": 110551, + "usd": 0.237735, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-070503-7b531f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2377, + "minutes": 10.2, + "ci_minutes": 0, + "input_tokens": 210164, + "output_tokens": 110551, + "tool_calls": 143, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4263, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 68, + "by_role": { + "agent": { + "usd": 0.22004, + "input_tokens": 202665, + "output_tokens": 96343, + "calls": 66 + }, + "reducer": { + "usd": 0.017695, + "input_tokens": 7499, + "output_tokens": 14208, + "calls": 2 + } + }, + "cache_read_tokens": 2199424, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.73, + "gate": 1.67, + "review": 606.63, + "report": 0.06 + }, + "time_to_first_result_s": 612.08 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5833.trace.json.gz", + "trace_bytes": 20883 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.md new file mode 100644 index 000000000..296775345 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5833, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac67e8874d00) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Old nav listed design/feature/vae_parallel.md twice; the new nav lists it once — duplicate-entry cleanup confirmed. +- [claim-verified] diff_stat: only docs/.nav.yml and docs/design/index.md changed (2 files, 91 insertions/24 deletions) — matches the PR's 'only these two files' scope claim. +- [claim-verified] No design document content is added/modified: diff_stat shows no files under docs/design/feature, docs/design/module, or docs/design/metrics.md touched. +- [claim-verified] diff_stat: exactly docs/.nav.yml and docs/design/index.md changed (2 files, +91/-24) — matches the PR's 'only .nav.yml and index.md' claim. +- [claim-verified] No design .md files added/renamed/deleted — 'arrangement-only' and 'existing document URLs stay stable' hold. +- [validated] All new .nav.yml Design-doc targets exist (list_dir: docs/design, docs/design/feature, docs/design/feature/omni_connectors, docs/design/module, docs/user_guide/quantization) — no broken sidebar paths. +- [validated] Every docs/design/index.md link resolves: architecture_overview.md, all feature/*.md, 6 connector docs, ../user_guide/quantization/overview.md, metrics.md, qwen3_omni_tts_performance_optimization.md, all module/*.md. +- [validated] read docs/.nav.yml at head: new nested nav entries all resolve to existing files and no document from the old flat list is dropped. +- [validated] read docs/design/index.md at head: every new landing-page link resolves to an existing file. +- [validated] Every nav/index target exists: architecture_overview, all 6 omni_connector pages, async_chunk, async_diffusion_output, omni_async_output_materialization, prefix_caching, cfg/expert/hsdp/pipeline/sequence/tensor/vae_parallel, skip_softmax, cache_dit, teacache, diffusion_continuous_batching, metrics, qwen3_omni_tts_performance_optimization, module/*, user_guide/quantization/overview.md. +- [validated] H1 of newly surfaced pages matches nav labels (vae_parallel '# VAE Patch Parallelism', pipeline_parallel '# Pipeline Parallel', async_diffusion_output '# Async Diffusion Output'); metrics/qwen3 labels are shortened forms of their H1s. +- [sweep] file_at_base(docs/design/index.md): all pre-PR links are retained; the PR only regroups and expands — no existing doc link dropped. +- [sweep] mkdocs.yml uses the awesome-nav plugin; heading-only nested sections (e.g. 'Offline Inference:', 'Parallelism:') are an established pattern in .nav.yml, so the new section headers are structurally valid. +- [sweep] grep 'design/feature/omni_connectors/' → only .nav.yml matches; no repo doc links depend on the removed directory-level nav entry. + +`docs/design/index.md:49` [minor] — In docs/design/index.md, `#### Quantization` (line 49) is immediately followed by `- [Cache-DiT](feature/cache_dit.md)`, `- [TeaCache](feature/teacache.md)`, and `- [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)` (lines 53-55) with no intervening heading, so all three render inside the Quantization section. The sidebar (docs/.nav.yml:133-136) lists Quantization, Cache-DiT, TeaCache, and Diffusion Continuous Batching as siblings under Diffusion Acceleration, and these pages are cache/batching docs, not quantization docs. Insert a separate heading before Cache-DiT/TeaCache (e.g. `#### Cache acceleration`) and place Diffusion Continuous Batching outside the Quantization section so the landing page grouping matches the nav tree. (evidence: docs/design/index.md:49 `#### Quantization`; docs/design/index.md:50 `- [Quantization Overview](../user_guide/quantization/overview.md)`; docs/design/index.md:53 `- [Cache-DiT](feature/cache_dit.md)`; docs/design/index.md:54 `- [TeaCache](feature/teacache.md)`; docs/design/index.md:55 `- [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)` with no heading between lines 50 and 53. Nav tree unchanged by this diff, present in the PR-time tree: docs/.nav.yml:133 `- Quantization: user_guide/quantization/overview.md`, docs/.nav.yml:134 `- Cache-DiT: design/feature/cache_dit.md`, docs/.nav.yml:135 `- TeaCache: design/feature/teacache.md`, docs/.nav.yml:136 `- Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md` — these are sibling items under `Diffusion Acceleration`.) + +`docs/.nav.yml:133` [minor] — The diff adds `- Quantization: user_guide/quantization/overview.md` under the new `Diffusion Acceleration` group in docs/.nav.yml:133 and mirrors it as `#### Quantization` / `[Quantization Overview](../user_guide/quantization/overview.md)` in docs/design/index.md, but the linked page is the general quantization overview: it states the same `quantization_config` entrypoint is used across diffusion-only, multi-stage omni/TTS, and multi-stage diffusion models (docs/user_guide/quantization/overview.md:3-5, unchanged by this diff). Grouping it under Diffusion Acceleration mislabels its scope. Move this link out of `Diffusion Acceleration` or retitle it at a non-diffusion-specific level (e.g. `Quantization (User Guide)`), and mirror that choice in docs/design/index.md. (evidence: docs/.nav.yml:133 ` - Quantization: user_guide/quantization/overview.md` (under the `Diffusion Acceleration` group added by this diff); docs/design/index.md:51 `- [Quantization Overview](../user_guide/quantization/overview.md)` (under `#### Quantization` / `### Diffusion acceleration`); unchanged by this diff, present in the PR-time tree: docs/user_guide/quantization/overview.md:3-5 `vLLM-Omni exposes quantization through the unified `quantization_config` path. The same configuration entrypoint is used across diffusion-only models, multi-stage omni/TTS models, and multi-stage diffusion models, but each model type has a different quantization scope.`) + +`docs/design/ar_diffusion_pipeline_capability.md:1` [minor] — docs/design/ar_diffusion_pipeline_capability.md exists but is referenced nowhere and is absent from both docs/.nav.yml and docs/design/index.md (pre-existing orphan; this PR's reorganization continues to leave it undiscoverable). Either link it in the design section or delete the orphan. (evidence: docs/design/ar_diffusion_pipeline_capability.md:1 `# AR-Diffusion pipeline capability` — file exists in the PR-time tree. Repo-wide grep for `ar_diffusion_pipeline_capability` returns no matches. docs/design/index.md:66-72 (changed by this PR) list only `- [AR Module](module/ar_module.md)`, `- [DIT Module](module/dit_module.md)`, `- [Entrypoint Module](module/entrypoint_module.md)`, `- [AsyncOmni Architecture (Qwen3-Omni Example)](module/async_omni_architecture.md)` with no ar_diffusion entry; docs/.nav.yml design section likewise omits it.) + +`docs/.nav.yml:109` [nit] — The diff replaces the bare link with the explicit label `Ray-based Execution` (docs/.nav.yml:109; docs/design/index.md:16), but the target page's H1 is still `# Distributed utils` (docs/design/feature/ray_based_execution.md:1), so the sidebar/index label and the document title disagree. Align them: retitle the page to `Ray-based Execution` or change the nav/index label back to `Distributed utils`. (evidence: docs/.nav.yml:109 ` - Ray-based Execution: design/feature/ray_based_execution.md`; docs/design/index.md:16 `- [Ray-based Execution](feature/ray_based_execution.md)`; docs/design/feature/ray_based_execution.md:1 `# Distributed utils` (unchanged by this diff, present in the PR-time tree).) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.trace.json.gz new file mode 100644 index 000000000..6e3f41f28 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5833.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.cost.json new file mode 100644 index 000000000..0c170328b --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1525.1, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5864/run-20260815-071131-8d4e36", + "recorded_at": "2026-08-15T07:36:54", + "attempts": [ + { + "invocation_id": "inv-3c30c8ec", + "outcome": "run:run-20260815-071131-8d4e36", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-071131-8d4e36", + "llm_calls": 208, + "input_tokens": 1866296, + "output_tokens": 308710, + "usd": 1.492829, + "source": "spans" + } + ], + "attempt_usd_total": 1.492829, + "llm_calls": 208, + "input_tokens": 1866296, + "output_tokens": 308710, + "usd": 1.492829, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-071131-8d4e36", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.4928, + "minutes": 25.38, + "ci_minutes": 0, + "input_tokens": 1866296, + "output_tokens": 308710, + "tool_calls": 319, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1631, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 208, + "by_role": { + "agent": { + "usd": 1.461552, + "input_tokens": 1831637, + "output_tokens": 288825, + "calls": 206 + }, + "reducer": { + "usd": 0.031276, + "input_tokens": 34659, + "output_tokens": 19885, + "calls": 2 + } + }, + "cache_read_tokens": 24049920, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.74, + "gate": 1.73, + "review": 1517.12, + "report": 0.07 + }, + "time_to_first_result_s": 1522.64 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5864.trace.json.gz", + "trace_bytes": 60025 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.md new file mode 100644 index 000000000..ef024682a --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.md @@ -0,0 +1,64 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5864, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 07fc831e7a43) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] Config-mutation anti-pattern: head removed `od_config.request_batch_max_wait_ms = 500.0` mutation; residual: with the 0.0 default the admission wait returns early, so DLO DP concurrency now requires a caller-set batch wait (comment 1). +- [resolved] collect_rank_status change only affects unique_reply_rank=None AND exec_all_ranks=True; DiffusionEngine.collective_rpc passes exec_all_ranks=False for control RPCs, so rank-status envelopes for control-plane methods are unchanged. +- [claim-verified] PR premise: base `_launch_workers` only kept rank 0's result handle (`if i == 0: result_handle = data.get("result_handle")`) while workers already created per-worker queues — confirmed via file_at_base; head collects all handles. +- [claim-verified] CI: gh_ci_read shows all 6 checks SUCCESS; gate 'mergeable, no failing checks' holds. +- [claim-verified] Two default (None) num_inference_steps are now valid: multiproc_executor.py:467 'Two default (None) step counts are valid and resolve identically' + test_dlo_dp_allows_shared_default_denoise_steps. +- [claim-verified] PR body 'fails closed on unforeseen asymmetry' matches `_fail_closed_on_dp_wave_timeout` (sets `_is_failed`, shutdown, invokes failure callbacks) and the partial-reply timeout test. +- [claim-verified] PR scope statement 'result routing, IPC, and shared worker lifecycle behavior' covers the np.ndarray SHM support and inline-client shutdown reorder; no out-of-scope behavior change found beyond the removed batch-wait default. +- [claim-refuted] docs/user_guide/diffusion/cpu_offload_diffusion.md:299/309 still assert num_inference_steps=None is not allowed — now false (finding #3). +- [validated] multiproc_executor.py:495-523: DLO DP wave now uses collective_rpc Path 2 with `timeout=_DLO_DP_WAVE_TIMEOUT_S`, and `execute_request` calls `_fail_closed_on_dp_wave_timeout` on TimeoutError — verified in tree. +- [validated] multiproc_executor.py:662-677: `collect_rank_status`/`output_rank` change affects only multi_rank_reply (unique_reply_rank=None + exec_all_ranks=True); grep of all `collective_rpc(`/`unique_reply_rank` callers confirms execute_stepwise/execute_model_batch/single execute_model/profile/LoRA keep prior semantics. +- [validated] request_scheduler.py:24-31: `build_request_batch_sampling_params_key` is shared by scheduler and DP dispatch; `RequestBatchSamplingParamsKey` excludes extra_args, so the extra_args + prompt preflight in execute_request is the genuinely new gate. +- [validated] test_diffusion_ipc.py:64-148 `test_per_worker_result_queues_release_nested_numpy_shm_and_processes` is a real spawn+MessageQueue integration test that drives the production pump loop and asserts exitcode 0 + no leaked SHM segments. +- [validated] inline client clone: `OmniDiffusionSamplingParams.clone()` is `copy.deepcopy` (inputs/data.py:372), so shared params no longer mutate across requests; test asserts distinct objects. +- [validated] collective_rpc primary-rank formula dp_rank * replica_size matches RankGenerator order 'tp-sp-pp-cfg-dp' in vllm_omni/diffusion/distributed/parallel_state.py::initialize_model_parallel, so [0,2]/[0,4] step-mode expectations are correct. + +`vllm_omni/diffusion/diffusion_engine.py:~247` [major] — Confirmed. `_init_runtime_state` (diffusion_engine.py:247) no longer sets the DLO-DP 500ms admission default that the merge-base code applied when `request_batch_max_wait_ms == 0`. Since the config/CLI defaults remain 0.0 (arg_utils.py:190, data.py:845), `_wait_for_request_batch_admission_locked` returns immediately at diffusion_engine.py:478-480, so the first staggered DLO+AllGather DP request is scheduled as a single-request wave before its peers arrive, losing the dp_size concurrency the removed default accumulated. Existing DLO DP consumers that do not pass `request_batch_max_wait_ms` (only the new MiniMax-H3 example does) silently lose this batching. Recommend restoring a DLO-DP-only positive default without mutating the shared config, or warning when `dp_concurrent` is enabled with `request_batch_max_wait_ms == 0`. (evidence: ['vllm_omni/diffusion/diffusion_engine.py:247-249 (PR-time) `if _uses_dlo_dp_concurrency(self.od_config):` / `self.scheduler.max_num_running_reqs = dp_size` / `self.dp_concurrent = True` — no request_batch_max_wait_ms default is set here anymore.', 'vllm_omni/diffusion/diffusion_engine.py:478-480 (PR-time, unchanged by this diff) `max_wait_s = self.od_config.request_batch_max_wait_ms / 1000.0` / `if max_wait_s == 0:` / `return`.', 'vllm_omni/diffusion/data.py:845 (PR-time, unchanged by this diff) `request_batch_max_wait_ms: float = 0.0`.', 'vllm_omni/engine/arg_utils.py:190 (PR-time, unchanged by this diff) `request_batch_max_wait_ms: float = 0.0`.', "vllm_omni/diffusion/diffusion_engine.py at merge-base (removed by this diff): `if getattr(self.od_config, 'request_batch_max_wait_ms', 0) == 0:` / `self.od_config.request_batch_max_wait_ms = 500.0`."]) + +`vllm_omni/diffusion/executor/multiproc_executor.py:~47` [major] — The pre-dispatch guard `_is_empty_dp_prompt` only recognizes `None`, an empty `str`, and a dict containing `"prompt"` (`vllm_omni/diffusion/executor/multiproc_executor.py:47-49`). Other declared `OmniPromptType` shapes — e.g. `{"prompt_token_ids": []}` (TokensPrompt), empty EmbedsPrompt payloads, or `OmniCustomPrompt` — hit `return False` and enter the DLO AllGather collective wave, defeating the barrier this PR adds. Since `OmniDiffusionRequest.prompt` is `OmniPromptType` (`vllm_omni/diffusion/request.py:29`), these are supported prompt shapes, not exotic inputs. Handle all supported prompt shapes explicitly, or fail closed on unrecognized prompt types. (evidence: Changed by this PR: vllm_omni/diffusion/executor/multiproc_executor.py:47-49 — `if isinstance(prompt, dict) and "prompt" in prompt:` / ` return not prompt["prompt"]` / `return False`. Unchanged by this diff, present in the PR-time tree: vllm_omni/inputs/data.py:151 — `OmniPromptType: TypeAlias = PromptType | OmniTextPrompt | OmniTokensPrompt | OmniEmbedsPrompt | OmniCustomPrompt`; vllm_omni/diffusion/request.py:29 — `prompt: OmniPromptType`. Therefore a supported `TokensPrompt` such as `{"prompt_token_ids": []}` has no `"prompt"` key and falls through to `return False`, so it is not rejected before dispatch.) + +`vllm_omni/diffusion/inline_stage_diffusion_client.py:329` [minor] — This hunk reorders the inline client shutdown to call `self._engine.close()` before `self._executor.shutdown(wait=True, cancel_futures=True)`, then unconditionally sets `_shutdown_complete = True`. However, `DiffusionEngine.close()` can return early when the busy-loop thread is still alive after a 10s join (`vllm_omni/diffusion/diffusion_engine.py:1034-1035` logs "Worker thread did not terminate within 10s; scheduler and executor shutdown will be deferred." and returns). On that path the client still marks shutdown complete at line 329, so a later `shutdown()` returns at line 311 and never retries engine teardown while workers keep running. Make `close()` resolve or fail the in-flight RPC (or give it a timeout), and only mark the client shutdown-complete once the engine has actually torn down. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/diffusion_engine.py:1034-1035 `"Worker thread did not terminate within 10s; scheduler and executor shutdown will be deferred."` / ` return` — close() can return without scheduler/executor shutdown. Changed by this diff: vllm_omni/diffusion/inline_stage_diffusion_client.py:321 ` self._engine.close()`; vllm_omni/diffusion/inline_stage_diffusion_client.py:329 ` self._shutdown_complete = True`; vllm_omni/diffusion/inline_stage_diffusion_client.py:311 ` if self._shutdown_complete:` / 312 ` return`.) + +`vllm_omni/inputs/data.py:372` [minor] — `add_request_async` now unconditionally calls `sampling_params.clone()`, and `clone()` is `copy.deepcopy(self)` (vllm_omni/inputs/data.py:372-373). If callers reuse one `OmniDiffusionSamplingParams` whose `generator` is a non-None `torch.Generator`, both cloned requests start from identical RNG state; `OmniDiffusionRequest.__post_init__` (vllm_omni/diffusion/request.py:41) only reseeds when generator AND seed are both None, so the duplicated generator is not re-isolated and the two requests can draw identical noise. Reinitialize/reseed the generator in `clone()` or otherwise isolate per-request RNG state rather than deep-copying it. (evidence: vllm_omni/diffusion/inline_stage_diffusion_client.py:110 (changed by this diff): ` sampling_params = sampling_params.clone()`; unchanged by this diff, present in the PR-time tree: vllm_omni/inputs/data.py:372-373 ` def clone(self) -> "OmniDiffusionSamplingParams":` / ` return copy.deepcopy(self)`; vllm_omni/inputs/data.py:215 ` generator: torch.Generator | list[torch.Generator] | None = None`; vllm_omni/diffusion/request.py:41-42 ` if self.sampling_params.generator is None and self.sampling_params.seed is None:` / ` self.sampling_params.seed = random.randint(0, 2**31 - 1)`) + +`tests/diffusion/test_multiproc_engine_concurrency.py:346` [minor] — At tests/diffusion/test_multiproc_engine_concurrency.py:345-346 the mock `result_queue(global_rank)` returns `global_rank` as `dp_rank`, and the assertion at :354 compares against `expected_primary_ranks = dp_rank * replica_size`, so the test only verifies the executor's `dp_rank * replica_size` index arithmetic — not the real rank-to-(dp,sp,pp,cfg) mapping from `initialize_model_parallel`. The test also calls `collective_rpc("execute_model", unique_reply_rank=None, exec_all_ranks=True)` directly in `step_execution=True` mode, while production `execute_step` calls `collective_rpc("execute_stepwise", ..., unique_reply_rank=0, exec_all_ranks=True)` (multiproc_executor.py:637-641), so `multi_rank_reply` is false there and this step-execution multi-rank branch is unreached by production. Add an assertion on the actual `initialize_model_parallel` rank-to-(dp,sp,pp,cfg) mapping, or explicitly label this test plumbing-only. (evidence: tests/diffusion/test_multiproc_engine_concurrency.py:337 `"dp_rank": global_rank,` (mock returns each queue's own index as dp_rank; inside the diff). tests/diffusion/test_multiproc_engine_concurrency.py:354 `assert [result.error for result in results] == [str(rank) for rank in expected_primary_ranks]` (inside the diff). vllm_omni/diffusion/executor/multiproc_executor.py:637-641 (unchanged by this diff, present in PR-time tree): `result = self.collective_rpc(` / `"execute_stepwise",` / `args=(scheduler_output,),` / `unique_reply_rank=0,` / `exec_all_ranks=True,` — with `unique_reply_rank=0`, `multi_rank_reply = unique_reply_rank is None and exec_all_ranks` is False, so production `execute_step` never enters the multi-rank reply branch this test targets.) + +`vllm_omni/diffusion/inline_stage_diffusion_client.py:~110` [minor] — At `vllm_omni/diffusion/inline_stage_diffusion_client.py:110`, `add_request_async` now unconditionally calls `sampling_params.clone()`, which is `copy.deepcopy(self)` (`vllm_omni/inputs/data.py:372-373`). Any sampling params containing a non-deepcopyable object (e.g. a generator or arbitrary pipeline object in `extra_args`) now raise where the previous code shared the object. Consider reusing `clone_sampling_params` (`vllm_omni/entrypoints/openai/stage_params.py:18-30`), which falls back to the original object on deepcopy failure, or wrapping this clone in an equivalent guarded fallback. (evidence: Changed by this diff: vllm_omni/diffusion/inline_stage_diffusion_client.py:110 ` sampling_params = sampling_params.clone()`. Unchanged by this diff, present in the PR-time tree: vllm_omni/inputs/data.py:372 ` def clone(self) -> "OmniDiffusionSamplingParams":` and vllm_omni/inputs/data.py:373 ` return copy.deepcopy(self)`. Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/stage_params.py:18 `def clone_sampling_params(params: OmniSamplingParams) -> OmniSamplingParams:` and vllm_omni/entrypoints/openai/stage_params.py:26-30 ` try:` / ` return copy.deepcopy(params)` / ` except Exception as exc:` / ` logger.warning("Failed to deepcopy sampling params; reusing original object: %s", exc)` / ` return params`.) + +`docs/user_guide/diffusion/cpu_offload_diffusion.md:~299` [minor] — The PR changes DLO DP preflight to allow matching None `num_inference_steps` defaults (see the executor comment at vllm_omni/diffusion/executor/multiproc_executor.py:467 and test_dlo_dp_allows_shared_default_denoise_steps at tests/diffusion/test_multiproc_engine_concurrency.py:397), but the user guide still asserts the old contract at docs/user_guide/diffusion/cpu_offload_diffusion.md:299 (`num_inference_steps` must be specified explicitly; None is not allowed) and :309 (`num_inference_steps=None` is not allowed in DP multi-concurrency mode). Update both lines to state that matching None defaults are valid and resolve identically inside the same pipeline. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:467 `# and LoRA settings. Two default (None) step counts are valid and` / `# resolve identically inside the same pipeline.`; tests/diffusion/test_multiproc_engine_concurrency.py:397 `def test_dlo_dp_allows_shared_default_denoise_steps(self):`; unchanged by this diff, present in the PR-time tree: docs/user_guide/diffusion/cpu_offload_diffusion.md:299 `- `num_inference_steps` must be specified explicitly (None is not allowed)` and docs/user_guide/diffusion/cpu_offload_diffusion.md:309 `- `num_inference_steps=None` is not allowed in DP multi-concurrency mode`) + +`vllm_omni/diffusion/worker/diffusion_worker.py:958` [nit] — This hunk gates the worker result-queue drop in `WorkerProc.shutdown()` with `if self.result_mq is not None and (self._async_output_thread is None or not self._async_output_thread.is_alive())`. If the async output thread survives the 10s join, the branch is skipped and the `ShmRingBuffer.__del__` unlink promised by the preceding comment never runs before process exit; cleanup is left to process termination. That is a deliberate safe choice, but state it in the else branch or retry with a bounded second join so the comment's 'before multiprocessing.resource_tracker exits' guarantee is not silently void on the timeout path. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:954-957 `# The worker creates this queue's shared-memory ring buffer. + # Dropping the final creator reference invokes + # ShmRingBuffer.__del__, which closes and unlinks it before + # multiprocessing.resource_tracker exits.` and :958-960 `if self.result_mq is not None and ( + self._async_output_thread is None or not self._async_output_thread.is_alive() + ):` — the drop/unlink only executes when the thread is not alive, so the 10s join timeout path (`_ASYNC_OUTPUT_THREAD_JOIN_TIMEOUT_S = 10.0` at :66) skips it exactly as the comment claims.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.trace.json.gz new file mode 100644 index 000000000..a4dd57673 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5864.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.cost.json new file mode 100644 index 000000000..06eb3240f --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 975.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5958/run-20260815-071517-7e0daf", + "recorded_at": "2026-08-15T07:31:31", + "attempts": [ + { + "invocation_id": "inv-a67c5359", + "outcome": "run:run-20260815-071517-7e0daf", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-071517-7e0daf", + "llm_calls": 122, + "input_tokens": 348244, + "output_tokens": 212646, + "usd": 0.455003, + "source": "spans" + } + ], + "attempt_usd_total": 0.455003, + "llm_calls": 122, + "input_tokens": 348244, + "output_tokens": 212646, + "usd": 0.455003, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-071517-7e0daf", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.455, + "minutes": 16.22, + "ci_minutes": 0, + "input_tokens": 348244, + "output_tokens": 212646, + "tool_calls": 327, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6496, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 122, + "by_role": { + "planner": { + "usd": 0.003341, + "input_tokens": 2720, + "output_tokens": 2370, + "calls": 1 + }, + "agent": { + "usd": 0.422316, + "input_tokens": 329743, + "output_tokens": 187509, + "calls": 119 + }, + "reducer": { + "usd": 0.029346, + "input_tokens": 15781, + "output_tokens": 22767, + "calls": 2 + } + }, + "cache_read_tokens": 4706176, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.58, + "gate": 1.52, + "review": 967.83, + "report": 0.06 + }, + "time_to_first_result_s": 972.98 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5958.trace.json.gz", + "trace_bytes": 35073 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.md new file mode 100644 index 000000000..1312681ac --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5958, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4dd07d536f34) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 9 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] input_output_modality_contracts.md promotion gate 'Add explicit CODEOWNERS for the primary I/O paths' is satisfied by new rules 60-63/53-57. +- [claim-verified] diff_stat: 13 files touched (CODEOWNERS + 12 design docs), 0 test files — matches sweep_targets. +- [claim-verified] Coverage-gap list (inputs, outputs, model_extras, plugins, errors.py, request.py, data_entry_keys.py, attention, benchmarks) all have new rules. +- [claim-verified] Non-source sections (*, /.github, /docs, /recipes, /.claude, /tests, /.buildkite, /benchmarks/*, /apps) unchanged vs base. +- [claim-refuted] PR body conflict table says offloader after = 'yuanheng-zhao, Isotr0py, lishunyang12'; actual CODEOWNERS:148 = yuanheng-zhao, david6666666, lishunyang12. +- [claim-refuted] PR body conflict table worker after omits @Gaohan123; actual CODEOWNERS:47 includes Gaohan123. +- [claim-refuted] PR body conflict table says offloader after = 'yuanheng-zhao, Isotr0py, lishunyang12', but CODEOWNERS:148 and offloader.md:6-8 are yuanheng-zhao, david6666666, lishunyang12 (Isotr0py removed, david6666666 added); the 'Design docs updated' section also omits this replacement. +- [claim-refuted] PR body conflict table says worker after = 'tzhouam, yinpeiqi, fake0fan, Sy0307', omitting @Gaohan123 that ar_runtime.md:10 and CODEOWNERS:47 add. +- [claim-refuted] PR body says /vllm_omni/lora/ Not changed, but head:106 narrows it to @SamitHuang @wtomin from base @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao. +- [claim-refuted] PR body offloader 'After' lists Isotr0py; actual head:148 and offloader.md:5-7 both list @david6666666 (body table stale; code and doc agree). +- [claim-refuted] PR body worker 'After' omits @Gaohan123, present at head:47. +- [claim-refuted] PR body 'Design docs updated' names 3 docs, but 12 design docs changed (diff_stat). +- [validated] ar_runtime.md:11-12 core/**+worker/** owners == CODEOWNERS:46-47 (tzhouam, yinpeiqi, fake0fan, Sy0307, Gaohan123). +- [validated] entrypoints.md owners + primary_path_exceptions -> CODEOWNERS:66/68/69 (openai/errors.py->error_contracts, cli/benchmark->benchmarking). + +`.github/CODEOWNERS:137` [major] — `.github/CODEOWNERS:137` adds `/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin` — missing @RuixiangMa — while `docs/design/module/diffusion/diffusion_model_integration.md:5-11` lists @RuixiangMa as an owner of `vllm_omni/diffusion/models/**`. Since CODEOWNERS uses last-match-wins, this more specific rule overrides the umbrella `/vllm_omni/diffusion/` rule and silently drops @RuixiangMa as reviewer for every diffusion/models file. Add @RuixiangMa to line 137 to match the authoritative design doc (line 138 for model_loader already includes him). (evidence: PR-time tree, changed by this diff: `.github/CODEOWNERS:137` `/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin` (no @RuixiangMa); `.github/CODEOWNERS:138` `/vllm_omni/diffusion/model_loader/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa` (includes him); `docs/design/module/diffusion/diffusion_model_integration.md:9` ` - "@RuixiangMa"` and `:11` ` - vllm_omni/diffusion/models/**`; umbrella rule `.github/CODEOWNERS` `/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666 @xuechendi`.) + +`.github/CODEOWNERS:80` [major] — Confirmed. /vllm_omni/model_extras/ adds @princepride and /vllm_omni/plugins/ adds @tjtanaa/@xuechendi, but docs/design/module/model_integration.md (the cited source) lists only @tzhouam and @gcanlin as owners of both vllm_omni/model_extras/** and vllm_omni/plugins/**; its only primary_code_path_owners block covers model_executor/models/**. No design-doc front matter authorizes those handles for those paths, so the rules violate the docs-first contract declared in this same diff. Add scoped primary_code_path_owners entries (or move plugins/** to an authorizing doc) or drop the unsourced handles. (evidence: Changed in this diff, .github/CODEOWNERS:80 `/vllm_omni/model_extras/ @tzhouam @gcanlin @princepride` and .github/CODEOWNERS:81 `/vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi`; cited source docs/design/module/model_integration.md:6-7 ` - "@tzhouam"` ` - "@gcanlin"` with :10 ` - vllm_omni/model_extras/**` and :11 ` - vllm_omni/plugins/**` (only owners), and its primary_code_path_owners block :12-19 covers only `vllm_omni/model_executor/models/**`. Repo-wide grep of docs/design/module: @tjtanaa appears only at execution_platforms.md:7; @xuechendi only in execution_platforms.md:8, omni_connector.md:8 and diffusion/ docs; @princepride only in cache_management.md:7, stage_runtime.md:15, omni_connector.md:6 and diffusion/ docs — none of those docs' primary_code_paths cover model_extras or plugins.) + +`.github/CODEOWNERS:~75` [minor] — CODEOWNERS:75 places the OmniConnector owners on the entire `/vllm_omni/distributed/` tree, but omni_connector.md:12 authorizes only `vllm_omni/distributed/omni_connectors/**` as its primary_code_path. `vllm_omni/distributed/omni_coordinator/**` and `vllm_omni/distributed/ray_utils/**` are stage_runtime.md related paths (lines 30-31), and all three directories exist under `vllm_omni/distributed/`. Either narrow the rule to `/vllm_omni/distributed/omni_connectors/` and add a stage-runtime/coordinator rule, or broaden omni_connector.md's primary_code_paths to authorize the full directory. (evidence: unchanged-by-this-diff? no — all three lines are PR-time tree content. Decisive: `.github/CODEOWNERS:75` `/vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan`; `docs/design/module/omni_connector.md:12` ` - vllm_omni/distributed/omni_connectors/**`; `docs/design/module/stage_runtime.md:30` ` - vllm_omni/distributed/omni_coordinator/**`; `docs/design/module/stage_runtime.md:31` ` - vllm_omni/distributed/ray_utils/**`; `vllm_omni/distributed/` directory listing contains `omni_connectors/`, `omni_coordinator/`, `ray_utils/`.) + +`.github/CODEOWNERS:83` [minor] — .github/CODEOWNERS:83 adds `/vllm_omni/model_executor/stage_configs/ @lishunyang12 @alex-jw-brooks`, but `list_dir vllm_omni/model_executor` shows no `stage_configs/` directory (entries: __init__.py, custom_process_mixin.py, layers/, model_loader/, models/, stage_input_processors/), so the rule matches no files. The source doc still lists the stale path at docs/design/module/vllm_omni_config.md:13. Remove the stale doc path and this rule, or point both at the current location of the stage configs. (evidence: .github/CODEOWNERS:83 `/vllm_omni/model_executor/stage_configs/ @lishunyang12 @alex-jw-brooks` (added by this PR); docs/design/module/vllm_omni_config.md:13 ` - vllm_omni/model_executor/stage_configs/**` (unchanged by this diff, present in PR-time tree); list_dir vllm_omni/model_executor -> `__init__.py, custom_process_mixin.py, layers/, model_loader/, models/, stage_input_processors/` (no stage_configs/).) + +`.github/CODEOWNERS:131` [minor] — The generated rule cites both diffusion_runtime.md and continuous_batching.md, but those docs assign different owners to vllm_omni/diffusion/sched/**: continuous_batching.md (unchanged by this PR) lists only @Isotr0py and @princepride, while diffusion_runtime.md lists @Isotr0py @princepride @SamitHuang @fhfuih. The rule silently adopts the diffusion_runtime list. Reconcile continuous_batching.md's owners with diffusion_runtime.md, or document which doc wins for sched so the authoritative sources stop contradicting the generated rule. (evidence: Changed by this diff: .github/CODEOWNERS:130 `# — docs/design/module/diffusion/diffusion_runtime.md, continuous_batching.md`; .github/CODEOWNERS:131 `/vllm_omni/diffusion/sched/ @Isotr0py @princepride @SamitHuang @fhfuih`. Unchanged by this diff, present in the PR-time tree: docs/design/module/diffusion/continuous_batching.md:6 ` - "@Isotr0py"` :7 ` - "@princepride"` :9 ` - vllm_omni/diffusion/sched/**`. Changed by this diff: docs/design/module/diffusion/diffusion_runtime.md:9 ` - "@fhfuih"` (with :6-8 naming @Isotr0py, @princepride, @SamitHuang for the same path).) + +`.github/CODEOWNERS:6` [minor] — Add an in-repo generator/linter that consumes the design-doc front matter (including the new `primary_code_path_owners` block) and emits `.github/CODEOWNERS`, or remove the header's 'regenerate' instruction and the PR test-plan claim that last-match-wins ordering was verified against '23 representative paths via simulation'. Repo-wide grep confirms no such script exists: `primary_code_path_owners` appears only in docs/design/module/{execution_platforms,model_integration}.md, and no '23 representative' text or simulation artifact is present, so the mapping is hand-maintained and the simulation is not reproducible. (evidence: Unchanged by this diff but present in the PR-time tree: .github/CODEOWNERS:6-7 `# The vllm_omni/** source-tree rules below are derived from the `owners` and` / `# `primary_code_paths` front matter in docs/design/module/*.md. When updating` / `# ownership, update the design doc first, then regenerate the matching rule here.` — instruction to regenerate with no generator in tree. Repo-wide grep: `primary_code_path_owners` matches only docs/design/module/execution_platforms.md:12 and docs/design/module/model_integration.md:12; `CODEOWNERS` matches only .github/CODEOWNERS itself and .claude/skills references; `23 representative` and the PR's simulation artifact are absent from the tree.) + +`tests/entrypoints/test_pd_disaggregation.py:1074` [minor] — Update or remove `tests/entrypoints/test_pd_disaggregation.py:1074`: the referenced `../../vllm_omni/model_executor/stage_configs/qwen3_omni_moe_pd_separation.yaml` no longer exists (the `vllm_omni/model_executor/stage_configs/` directory is gone; stage YAMLs now live under `vllm_omni/deploy/`), so this test silently skips. Note the PD YAML is also absent from `vllm_omni/deploy/` — file a linked follow-up to point the test at the current PD stage config or remove the stale test. Separately, this PR adds a CODEOWNERS rule for `/vllm_omni/model_executor/stage_configs/` that covers a nonexistent path and should be dropped. (evidence: tests/entrypoints/test_pd_disaggregation.py:1074 (unchanged by this diff, present in PR-time tree): `"../../vllm_omni/model_executor/stage_configs/qwen3_omni_moe_pd_separation.yaml",`. PR diff adds: `/vllm_omni/model_executor/stage_configs/ @lishunyang12 @alex-jw-brooks`. Repo-wide grep for `qwen3_omni_moe_pd_separation` returns only that test line; `list_dir vllm_omni/model_executor` shows no `stage_configs/`, and `list_dir vllm_omni/deploy` lists stage YAMLs but no `qwen3_omni_moe_pd_separation.yaml`.) + +`.github/CODEOWNERS:1` [minor] — PR 5958's description section 'Design docs updated (to stay in sync)' lists only 3 of the 12 design docs actually changed. diff_stat shows these 12 modified: docs/design/module/ar_runtime.md, benchmarking.md, cache_management.md, diffusion/diffusion_model_integration.md, diffusion/diffusion_runtime.md, diffusion/offloader.md, diffusion/parallelism.md, entrypoints.md, execution_platforms.md, input_output_modality_contracts.md, model_integration.md, observability.md. Please list all 12 so the summary reconciles with the diff. (evidence: PR 5958 body (via gh_pr_view, not a repo file line) contains the section '## Design docs updated (to stay in sync)' followed by only three bullets: '- `docs/design/module/entrypoints.md` — add @NickCao to `owners`', '- `docs/design/module/diffusion/offloader.md` — add @lishunyang12 to `owners`', '- `docs/design/module/model_integration.md` — add `primary_code_path_owners` block scoping @Sy0307, @amy-why-3459, @linyueqian to `vllm_omni/model_executor/models/**`'. Meanwhile diff_stat (merge-base 636d44e27266) lists 12 changed design docs: 'docs/design/module/ar_runtime.md | 1 +', 'docs/design/module/benchmarking.md | 1 +', 'docs/design/module/cache_management.md | 1 +', '.../diffusion/diffusion_model_integration.md | 5 +-', 'docs/design/module/diffusion/diffusion_runtime.md | 1 +', 'docs/design/module/diffusion/offloader.md | 3 +-', 'docs/design/module/diffusion/parallelism.md | 1 +', 'docs/design/module/entrypoints.md | 1 +', 'docs/design/module/execution_platforms.md | 9 +-', '.../module/input_output_modality_contracts.md | 2 +', 'docs/design/module/model_integration.md | 8 ++', 'docs/design/module/observability.md | 1 +'.) + +**Additional observations (beyond the comment budget):** +- `.github/CODEOWNERS:148` [minor] The PR body's conflict-resolution table disagrees with the committed CODEOWNERS: the worker row omits @Gaohan123 (CODEOWNERS:47 adds them) and the offloader row lists @Isotr0py instead of @david6666666 (CODEOWNERS:148 an +- `.github/CODEOWNERS:106` [minor] The PR description lists `/vllm_omni/lora/` under 'Not changed', but this diff changes `.github/CODEOWNERS:106` from `/vllm_omni/lora/ @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao` (base line 33) +- `.github/CODEOWNERS:6` [minor] Header is incomplete: the vllm_omni CODEOWNERS rules also derive from `primary_code_path_owners` (docs/design/module/execution_platforms.md:12, docs/design/module/model_integration.md:12) and `primary_path_exceptions` (d +- `docs/design/module/execution_platforms.md:12` [minor] The new `primary_code_path_owners` key is introduced at docs/design/module/execution_platforms.md:12 and docs/design/module/model_integration.md:12, but no repo doc defines its semantics: the CODEOWNERS regeneration head + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.trace.json.gz new file mode 100644 index 000000000..e6667435f Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5958.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.cost.json new file mode 100644 index 000000000..0222c5d0b --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1406.9, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r1/runs/pr5978/run-20260815-071824-431e92", + "recorded_at": "2026-08-15T07:41:49", + "attempts": [ + { + "invocation_id": "inv-4c5159d0", + "outcome": "run:run-20260815-071824-431e92", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-071824-431e92", + "llm_calls": 156, + "input_tokens": 1778380, + "output_tokens": 298569, + "usd": 1.230386, + "source": "spans" + } + ], + "attempt_usd_total": 1.230386, + "llm_calls": 156, + "input_tokens": 1778380, + "output_tokens": 298569, + "usd": 1.230386, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-071824-431e92", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2304, + "minutes": 23.41, + "ci_minutes": 0, + "input_tokens": 1778380, + "output_tokens": 298569, + "tool_calls": 276, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0547, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 156, + "by_role": { + "agent": { + "usd": 1.19103, + "input_tokens": 1752502, + "output_tokens": 269180, + "calls": 154 + }, + "reducer": { + "usd": 0.039356, + "input_tokens": 25878, + "output_tokens": 29389, + "calls": 2 + } + }, + "cache_read_tokens": 15622144, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.69, + "gate": 1.98, + "review": 1398.75, + "report": 0.06 + }, + "time_to_first_result_s": 1404.46 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5978.trace.json.gz", + "trace_bytes": 51929 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.md b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.md new file mode 100644 index 000000000..5c79e07fd --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5978, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 86854432db52) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 9 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] audio_duration_seconds has exactly one consumer (_encode_video_audio_conditions_resident:1321-1330); no other prepared-video consumer reads the now-shorter transcode output. +- [resolved] no stale reference to the deleted t2va_similarity path: grep for t2va_similarity returns no matches outside the diff; nightly yml updated in the same change. +- [resolved] Lossless RGB single-decode: Qwen sampling and the video VAE both now use load_video_frames on the same rgb24 stream, so pixel parity holds; residual = full-video uint8 decode (~620MB for 209-frame 768p on rank 0) and decord-vs-av backend are not CPU-tested. +- [resolved] workdir removal from sample_reference_video_frames — repo-wide grep found only pipeline_minimax_h3.py:881 (updated) and the new contract test; no stale caller remains. +- [claim-verified] minimax_h3_align_frame_count(192)==192 and (120)==124 (time_request.py), matching new test I2VA_NUM_FRAMES=192 / REF2VA_NUM_FRAMES=124. +- [claim-verified] probe_video equality compares only width/height/fps/frame_count (helpers.py:89-108), so differing file sizes won't fail `reference_metadata == generated_metadata`. +- [claim-verified] I2VA/Ref2VA multipart fields input_reference/input_references match api_server.py:3190-3191 endpoint signatures. +- [claim-verified] libx264rgb/crf 0/rgb24 prepared stream is decoded by both Qwen sampling and video VAE via load_video_frames, so the lossless-RGB comment in the diff is accurate. +- [claim-verified] new e2e file defines exactly two test functions (i2va + ref2va), matching the reported '2 tests collected'. +- [claim-verified] threshold arithmetic via calc: 0.975190>=0.97, 35.981286>=34, 0.980577>=0.97, 42.650204>=34 -> both reported cases pass the stated gate. +- [claim-verified] frame counts: 8.0*24=192 and align_frame_count(round(5.0*24))==124 per contract test, matching I2VA_NUM_FRAMES=192 / REF2VA_NUM_FRAMES=124. +- [claim-verified] I2VA/Ref2VA frame counts: time_request._align_frame_count(round(8.0*24))=192 and (round(5.0*24))=124 (17n+5 snapping), matching the PR's 192/124. +- [claim-verified] T2VA accuracy coverage removed: diff_stat shows test_minimax_h3_t2va_similarity.py deleted and test-nightly.yml:433 now invokes test_minimax_h3_i2va_ref2va_similarity.py. +- [claim-verified] repo-wide grep for test_minimax_h3_t2va_similarity returned no matches, so the deleted T2VA case has no remaining reference. + +`.buildkite/cuda/test-nightly.yml:430` [major] — This diff deletes the only full-model official-asset T2VA pixel-alignment test (SSIM 0.82 / PSNR 20.0) and replaces the nightly command with the new I2VA/Ref2VA test. t2va is still supported and documented: pipeline_minimax_h3.py:592 includes it in supported_tasks and pipeline_minimax_h3.py:798 still branches `requested_frames = 124 if task == "ref2va" else 209`. Please keep a T2VA-specific gate at the old thresholds (or a separate relaxed lane), or file a tracking issue for the T2VA alignment gap instead of silently dropping the coverage. (evidence: Present in PR-time tree, .buildkite/cuda/test-nightly.yml:430 ` - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"` and :433 ` - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model` (grep of the same file for `test_minimax_h3_t2va_similarity.py` returns no matches). Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:592 `supported_tasks = {"ref2va"} if partition == "ref2va" else {"t2va", "fl2va"}` and :798 `requested_frames = 124 if task == "ref2va" else 209`. Documented in recipes/MiniMaxAI/MiniMax-H3.md:713 `| `task` | `t2va`, `fl2va`, or `ref2va` | Passed in `extra_params`; selects the task-specific DiT |`. The deleted file (not present in PR-time tree, shown in the diff) contained `SSIM_THRESHOLD = 0.82` and `PSNR_THRESHOLD = 20.0`; grep of tests/e2e/accuracy for `t2va` returns no matches.) + +`vllm_omni/diffusion/utils/media_utils.py:194` [minor] — Confirmed. In the shared mux_video_audio_bytes helper, this diff sets `audio_frame.pts = 0` and `audio_frame.time_base = Fraction(1, audio_sample_rate)` for every video-with-audio endpoint. The only added regression test, test_mux_video_audio_marks_aac_priming_timestamp, exercises 32000 Hz (media_utils.py:261) and asserts only `first_packet.pts < 0` (test_video_server.py:269); it does not cover the serving-default 24 kHz path (`_encode_video_bytes` in video_api_utils.py:528 uses `audio_sample_rate=audio_sample_rate or 24000`) nor prove leading silence is no longer exposed. Consider a 24 kHz decoded-waveform/start-time assertion or document the normalization as intended repo-wide. (evidence: vllm_omni/diffusion/utils/media_utils.py:194 `audio_frame.pts = 0` (changed by this diff, present in the PR-time tree) and vllm_omni/diffusion/utils/media_utils.py:195 `audio_frame.time_base = Fraction(1, audio_sample_rate)`. Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/video_api_utils.py:528 `audio_sample_rate=audio_sample_rate or 24000`; tests/entrypoints/openai_api/test_video_server.py:261 `audio_sample_rate=32000`; tests/entrypoints/openai_api/test_video_server.py:269 `assert first_packet.pts < 0`.) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:31` [minor] — This diff adds `tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py`, whose `_model_name()` (lines 106-113) falls back to `snapshot_download(repo_id=MODEL_REPO_ID, revision=MODEL_REVISION)` with `MODEL_REVISION = "main"` (line 31), and the nightly command at `.buildkite/cuda/test-nightly.yml:433` runs the test with no `VLLM_TEST_MINIMAX_H3_*_MODEL` env override. Media assets are SHA-pinned, but the model checkpoint is not, so a push to MiniMaxAI/MiniMax-H3 `main` can move SSIM/PSNR and flake the gate independently of this code. Pin a concrete model revision/SHA, or set the env var in `.buildkite/cuda/test-nightly.yml`, to make the gate reproducible end-to-end. (evidence: tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:31 `MODEL_REVISION = "main"`; tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:113 `revision=MODEL_REVISION,`; tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:107 `configured = os.environ.get(model_env_var)`; .buildkite/cuda/test-nightly.yml:433 `pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model` (unchanged by this diff beyond the file rename; present in the PR-time tree).) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_contract.py:1271` [minor] — In tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1271, test_ref2va_qwen_sampling_decodes_prepared_video_once drives the real sample_reference_video_frames with only a 25-frame fixture (indices 0/12/24), while the PR's new I2VA/Ref2VA e2e cases use 192/124 frames. Parameterize or add cases at 124 and 192 frames so the single-decode and source-index mapping are validated for the production frame counts, not only the small fixture. (evidence: ['tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1258 `decoded = np.arange(25 * 2 * 2 * 3, dtype=np.uint8).reshape(25, 2, 2, 3)` — 25-frame fixture.', 'tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1271 `sampled = reference_video_module.sample_reference_video_frames("prepared.mp4")` — real function called (only _probe_video and load_video_frames are monkeypatched).', 'tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1273 `assert load_calls == ["prepared.mp4"]` — asserts single decode.', 'tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1276 `(decoded[0], decoded[12], decoded[24]),` — sampled indices 0/12/24.', 'tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:93 `I2VA_NUM_FRAMES = 192` — production I2VA frame count (new file in this PR).', 'tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:94 `REF2VA_NUM_FRAMES = 124` — production Ref2VA frame count (new file in this PR).']) + +`vllm_omni/diffusion/models/minimax_h3/reference_video.py:307` [minor] — Contract tests test_r7_r8_ref2va_video_segment_matrix, test_ref2va_two_video_recipe_tolerates_container_rounding, and test_ref2va_two_video_recipe_rejects_real_duration_overflow monkeypatch _transcode_reference_video (test_minimax_h3_contract.py:1237, :1318, :1366), so the newly added ffmpeg `libx264rgb -crf 0 -pix_fmt rgb24` path at reference_video.py:307-313 never runs in unit/PR lanes. Add a fast unit test that invokes the real _transcode_reference_video on a tiny generated video and asserts the output pixel format/codec, so missing ffmpeg/libx264rgb support fails before the nightly full_model H100 gate. (evidence: vllm_omni/diffusion/models/minimax_h3/reference_video.py:307-313 (added by this diff): `"-c:v", "libx264rgb", "-crf", "0", "-preset", "veryfast", "-pix_fmt", "rgb24",`. tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1237 (unchanged by this diff, present in the PR-time tree): `"_transcode_reference_video",` followed at 1238 by `lambda source, **kwargs: transcode_calls.append((source, kwargs)) or "prepared.mp4",` — so the real ffmpeg path does not run; same at line 1366: `monkeypatch.setattr(reference_video_module, "_transcode_reference_video", lambda source, **kwargs: "prepared.mp4")`. Grep across tests/ finds no other `_transcode_reference_video` caller, and the only un-mocked execution is the new e2e whose pytestmark includes `pytest.mark.full_model` under a 4xH100 hardware gate.) + +`vllm_omni/diffusion/models/minimax_h3/reference_video.py:440` [minor] — The new `test_ref2va_qwen_sampling_decodes_prepared_video_once` monkeypatches `load_video_frames`, so neither the decord primary path nor the PyAV fallback in `load_video_frames` (reference_video.py:403-419) is CPU-tested; the RGB parity guarantee is source-inspection-only. Add a CPU test forcing the PyAV fallback (or document decord as the only supported backend). (evidence: Unchanged by this diff, present in the PR-time tree: `vllm_omni/diffusion/models/minimax_h3/reference_video.py:410` `frames = [frame.to_ndarray(format="rgb24") for frame in container.decode(video=0)]` (PyAV fallback) and `reference_video.py:415-419` `reader = decord.VideoReader(path) ... return frames.asnumpy() if hasattr(frames, "asnumpy") else np.asarray(frames)` (decord path, no explicit RGB format). Added by this diff: `tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1267-1270` `monkeypatch.setattr(reference_video_module, "load_video_frames", lambda path: load_calls.append(path) or decoded,)` — the new sampling test stubs the function, so no CPU test exercises either real decode path.) + +`vllm_omni/diffusion/models/minimax_h3/reference_video.py:440` [minor] — `sample_reference_video_frames` now fully decodes the prepared video once (reference_video.py:440) and only then indexes the ~2 fps Qwen sample frames (line 441), while `_encode_video_conditions_resident` independently calls `load_video_frames(item["prepared_path"])` for the video VAE (pipeline_minimax_h3.py:1274), so the same prepared stream is fully decoded twice on rank 0. Since `load_video_frames` already uses `decord.VideoReader`, the Qwen sampling path could decode only the selected indices via `reader.get_batch(indices)` and still see the same lossless RGB pixels while avoiding one full-frame materialization. (evidence: vllm_omni/diffusion/models/minimax_h3/reference_video.py:440 `decoded_frames = load_video_frames(prepared_path)`; vllm_omni/diffusion/models/minimax_h3/reference_video.py:441 `frames = [np.asarray(decoded_frames[source_index]) for source_index in indices]`; unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1274 `self.video_vae.encode_video(load_video_frames(item["prepared_path"])) for item in prepared_videos`.) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:~228` [minor] — tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:228 (and 304) pin both cases to `res={"cuda": "H100"}`, while .buildkite/cuda/test-nightly.yml:433-434 runs this file on `mirror_hardwares: h100_4`; the PR body reports the 0.97 SSIM / 34 dB PSNR gates were calibrated on 4x NVIDIA B300. Re-run at least one case on the H100 mirror or document why the B300 numbers transfer. (evidence: tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:228 `@hardware_test(res={"cuda": "H100"}, num_cards=4)` (also line 304); .buildkite/cuda/test-nightly.yml:433 `pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model` followed by line 434 `mirror_hardwares: h100_4`; PR body (metadata, not tree): `**Hardware:** 4x NVIDIA B300, BF16, Ulysses 4, ring 1, HSDP shard size 4, VAE patch parallel 4, tiled VAE, FlashAttention, 50 steps, seed 0.`) + +**Additional observations (beyond the comment budget):** +- `tests/entrypoints/openai_api/test_video_server.py:269` [minor] Confirmed: this diff adds `test_mux_video_audio_marks_aac_priming_timestamp` to `tests/entrypoints/openai_api/test_video_server.py` (a file marked `core_model` + `cpu` at line 41), and `gh_ci_read(5978)` shows `buildkite +- `tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1525` [minor] This test monkeypatches `_dit_rank_world` to a benign lambda before asserting the ValueError, so it would still pass if the `max_duration_seconds <= 0` check were moved after the `_dit_rank_world()` call — it proves only + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.trace.json.gz new file mode 100644 index 000000000..bc6cc1f2e Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r1/pr5978.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/manifest.json b/eval/dataset/arms/copilot_v15_holdout4_r2/manifest.json new file mode 100644 index 000000000..cd4cfa454 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v15_holdout4_r2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout4" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.cost.json new file mode 100644 index 000000000..1e86a443f --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1366.9, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5608/run-20260815-135724-2c0b95", + "recorded_at": "2026-08-15T14:20:09", + "attempts": [ + { + "invocation_id": "inv-340969ab", + "outcome": "run:run-20260815-135724-2c0b95", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-135724-2c0b95", + "llm_calls": 133, + "input_tokens": 656163, + "output_tokens": 273022, + "usd": 0.715099, + "source": "spans" + } + ], + "attempt_usd_total": 0.715099, + "llm_calls": 133, + "input_tokens": 656163, + "output_tokens": 273022, + "usd": 0.715099, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-135724-2c0b95", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7151, + "minutes": 22.74, + "ci_minutes": 0, + "input_tokens": 656163, + "output_tokens": 273022, + "tool_calls": 239, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.87, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 133, + "by_role": { + "agent": { + "usd": 0.682251, + "input_tokens": 638928, + "output_tokens": 247419, + "calls": 131 + }, + "reducer": { + "usd": 0.032848, + "input_tokens": 17235, + "output_tokens": 25603, + "calls": 2 + } + }, + "cache_read_tokens": 8800384, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.47, + "gate": 1.98, + "review": 1358.81, + "report": 0.06 + }, + "time_to_first_result_s": 1364.31 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5608.trace.json.gz", + "trace_bytes": 37710 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.md new file mode 100644 index 000000000..0581bbb08 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.md @@ -0,0 +1,62 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5608, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head de8517f6f2bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Fast path is byte-identical to base (old code did cos/sin.unsqueeze(unsqueeze_dim) then npu_rotary_mul); file_at_base confirms the only consumer change is the helper call, so supported shapes regress nothing. +- [resolved] Test drives the real entry point: _load_source_module executes the actual qwen3_tts_tokenizer_v2.py and the real rotary_embedding.py with fake torch_npu, not a mocked helper. +- [resolved] concern 'does the test drive the real entry point': it calls tokenizer.apply_rotary_pos_emb after apply_qwen3_tts_tokenizer_v2_patch() has replaced it with _apply_rotary_pos_emb_npu, and the fake npu_rotary_mul records the transposed cos — genuine dispatch coverage (not asserting the injected value). +- [resolved] 3D-input concern: no in-repo caller passes non-4D with unsqueeze_dim=1 (only tokenizer_v2 4D callers); residual future-caller contract issue filed as a nit. +- [claim-verified] Reported stage-1 threshold: decoder hidden_size=1024/heads=16 => head_dim=64; first ramp chunk seq=4 => B*16<=32 => B<=2, so concurrency>=4 falls back, matching the PR body. +- [claim-verified] PR body's constraint `batch*heads <= seq_len*8` matches rotary_embedding.py:26; the CANN error 'B*N is large or D is not aligned' matches the guard's two rejection branches. +- [claim-verified] diff_stat: exactly 3 files changed (test, new helper, tokenizer patch) — matches the stated scope. +- [claim-verified] No public default/API/protocol changed: _apply_rotary_pos_emb_npu signature and the module-global patch target are unchanged; the new helper is internal. +- [claim-verified] PR's 'concurrency >= 4' trigger: decoder batch=4, heads=16, seq=4 -> 4*16=64 > 4*8=32 -> fallback path (calc). +- [claim-verified] PR perf table not self-refuting: c=1/c=4 throughput 1.92/6.62 vs audio/(prompts*E2EL/concurrency) ~1.93/~6.65; c=8/c=16 overlap makes closed-form check inconclusive, not contradictory. +- [claim-refuted] PR-body 'concurrency >= 4' completeness: its own c=16 row shows Streaming continuity OK rate 8.33% and Mean AUDIO_RTF 1.74 — the fix stops the crash but c=16 does not meet real-time; the body does not call this out. +- [claim-refuted] module docstring 'Shared NPU rotary embedding helpers' — grep shows only one consumer (tokenizer_v2); the other two BNSD npu_rotary_mul sites are left unguarded. +- [claim-refuted] PR claims the short-seq B*N>seq*8 crash; the changed test q=(1,2,3,2) (line 538) is rejected only because head_dim=2 is D-misaligned (float32: 1%8=1), so the `batch_size * num_heads <= seq_len * 8` term is never the deciding condition in any test. +- [validated] Fallback transpose math: q [B,N,S,D] -> transpose(1,2).contiguous() [B,S,N,D]; cos.unsqueeze(2) [B,S,1,D]; return transpose(1,2) restores BNSD; test asserts q_out==q+1 and k_out==k+1 round-trip. + +`tests/platforms/npu/test_310p_patches.py:547` [major] — tests/platforms/npu/test_310p_patches.py:547 now asserts (1,3,1,2), which only proves the BSND fallback executed. The input q=torch.zeros(1,2,3,2) is rejected by the D-alignment conjunct (head_dim//2 % 8 == 1), so the new short-seq predicate batch_size*num_heads <= seq_len*8 (2<=24) is short-circuited and never decides; a reversed or broken predicate would still pass CI. Add cases for the supported fast path (e.g. fp32 q=(1,2,3,16)) and the true short-seq fallback (e.g. q=(2,8,1,16)), asserting the transposed rotary_calls[i][0] layout. (evidence: tests/platforms/npu/test_310p_patches.py:538 `q = torch.zeros(1, 2, 3, 2)` +tests/platforms/npu/test_310p_patches.py:547 `assert rotary_calls[0][1].shape == (1, 3, 1, 2)` +vllm_omni/platforms/npu/layers/rotary_embedding.py:27 `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8` +With fp32 head_dim=2: half_dim_alignment = 32//4 = 8, head_dim//2 = 1, 1%8 == 1, so the first conjunct is False and `batch_size * num_heads <= seq_len * 8` is short-circuited; the (1,3,1,2) cos shape is only reachable via the BSND fallback (cos.unsqueeze(2)), not the fast path (cos.unsqueeze(1) -> (1,1,3,2)).) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:31` [major] — PR adds npu_rotary_mul_with_bsnd_fallback but migrates only qwen3_tts_tokenizer_v2.py:31-32. The same BNSD unguarded calls remain at vllm_omni/model_executor/models/common/qwen3_code_predictor.py:260-261 and vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184-185. With CodePredictor defaults (num_attention_heads=16, num_code_groups=32 ⇒ seq_len=33), the CANN limit batch*num_heads <= seq_len*8 permits batch ≤ 16; concurrency 17 already exceeds it and deploy max_num_seqs=64 is far beyond. Migrate these two sites to the new helper or prove their seq_len is always large enough; otherwise Stage-0 remains exposed to the same BNSD tiler crash this PR fixes for Stage-1. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/common/qwen3_code_predictor.py:260 `q = torch_npu.npu_rotary_mul(q, cos, sin)` and :261 `k = torch_npu.npu_rotary_mul(k, cos, sin)`. unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184 `q = torch_npu.npu_rotary_mul(q, cos, sin)` and :185 `k = torch_npu.npu_rotary_mul(k, cos, sin)`. added by this diff: vllm_omni/platforms/npu/layers/rotary_embedding.py:31 `def npu_rotary_mul_with_bsnd_fallback(` and :44 `if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):` with :25 `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8`. config defaults: vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py:198 `num_attention_heads=16,` and :215 `num_code_groups=32,`; vllm_omni/deploy/qwen3_tts.yaml:44 `max_num_seqs: 64`.) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:57` [minor] — The BSND fallback explicitly makes the transposed input contiguous (`hidden_states_bsnd = hidden_states.transpose(1, 2).contiguous()` at rotary_embedding.py:51) but returns the fused op's transposed result as a non-contiguous view (`return output_bsnd.transpose(1, 2)` at rotary_embedding.py:57), unlike the fast path which returns the fused op output directly. Return `output_bsnd.transpose(1, 2).contiguous()` for branch parity. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:51 ` hidden_states_bsnd = hidden_states.transpose(1, 2).contiguous()` and vllm_omni/platforms/npu/layers/rotary_embedding.py:57 ` return output_bsnd.transpose(1, 2)` — both lines are in the PR diff (new file); the input side materializes the transposed tensor as contiguous, while the output side returns the transpose as a non-contiguous view.) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:51` [minor] — The BSND fallback adds a transpose + contiguous copy (`rotary_embedding.py:51`) for every rejected BNSD tensor, and the qwen3_tts_tokenizer_v2 consumer invokes it for both q and k. The PR body's own benchmark shows c=16 is not streaming-stable: Streaming continuity OK rate 8.33%, Mean AUDIO_RTF 1.74, Mean AUDIO_UNDERRUN 0.48s. Either scope the validation claim (streaming-stable up to c=8, crash-fixed only at c=16) or investigate the fallback/perf before advertising c=16. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:51 ` hidden_states_bsnd = hidden_states.transpose(1, 2).contiguous()` — present in the diff, decisive line for the transpose/copy cost. PR #5608 body benchmark table (not a repo file, quoted from gh_pr_view): `| Streaming continuity OK rate | 100.00% | 100.00% | 100.00% | 8.33% |`; `| Mean AUDIO_RTF | 0.52 | 0.60 | 0.72 | 1.74 |`; `| Mean AUDIO_UNDERRUN (s) | 0.00 | 0.00 | 0.01 | 0.48 |`.) + +`PR description:0` [minor] — Substantiate the WER 0.982% claim: the PR body provides a command/config only for the E2EL/RTF benchmark (bench_tts.py, --concurrency 1 4 8 16, codec_chunk_ramp enabled) and no command/config for the WER evaluation, so the WER cannot be tied to that ramp/concurrency scenario. Add the exact WER benchmark invocation and config, or state explicitly that WER was measured outside this PR's concurrency scenario. (evidence: PR #5608 body (via gh_pr_view), Accuracy section: '### Accuracy \nThe WER evaluation results on the seed-tts-eval dataset are as follows:\n\n| Metric | Results |\n|---|---:|\n| WER | 0.982% |' — no command or config accompanies the WER number. The only command in the body is in the Performance section: 'nohup python /vllm-workspace/vllm-omni/benchmarks/tts/bench_tts.py \\n --model /nas/disk1/Qwen3-TTS-12Hz-1.7B-CustomVoice \\n --task default_voice \\n --dataset-path ./seedtts_testset \\n --locale zh \\n --concurrency 1 4 8 16\\n --num-prompts 48 \\n --host 55.122.12.76 \\n --port 30488 \\n --output-dir ./results > test.log 2>&1 &' — that command's results table lists E2EL, AUDIO_RTF, AUDIO_TTFP, underrun etc. and contains no WER metric, so the WER claim is not tied to the ramp/concurrency configuration by the PR body.) + +`vllm_omni/platforms/npu/platform.py:~48` [minor] — The diff routes tokenizer RoPE through `npu_rotary_mul_with_bsnd_fallback` (`qwen3_tts_tokenizer_v2.py:31-32`), and `NPUOmniPlatform.__init__` installs that patch unconditionally before applying 310P patches (`platform.py:48-49`). PR validation is only 910B3/CANN 9.0.0, while 310P runs the shared 12Hz decoder in fp16 via `_Qwen3TTSCode2Wav310P._npu_decoder_runtime_dtype` (`_310p/patch/qwen3_tts.py:22,127`). Confirm the BSND fallback on 310P or gate the helper to the validated arch. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/platform.py:48 ` apply_qwen3_tts_tokenizer_v2_patch()` and :49 ` apply_310p_patches()` — the tokenizer RoPE patch is installed for every NPU platform with no arch gate before the 310P patch. In the diff: vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:31-32 ` npu_rotary_mul_with_bsnd_fallback(q, cos, sin, unsqueeze_dim),` / ` npu_rotary_mul_with_bsnd_fallback(k, cos, sin, unsqueeze_dim),` — the new fallback path. Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:22 `_RUNTIME_DTYPE = torch.float16` and :127 ` def _npu_decoder_runtime_dtype(self, device: torch.device) -> torch.dtype:` (body returns `_RUNTIME_DTYPE`) — 310P runs the shared decoder in fp16. PR body lists only `Hardware: 910B3 * 1` / `CANN: 9.0.0`.) + +`tests/platforms/npu/test_310p_patches.py:492` [nit] — Diff only modifies NPU files and adds a CPU-marked test under tests/platforms/npu (pytestmark = [core_model, cpu]). The only red gate is buildkite/vllm-omni-amd-ci, and no .buildkite file collects tests/platforms/npu (AMD jobs collect tests/diffusion, tests/model_executor, tests/e2e, tests/engine). This test is unlikely to be the cause of the AMD failure — please confirm the failure is pre-existing on main or link the job log if it did collect this test. (evidence: ['tests/platforms/npu/test_310p_patches.py:21 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — whole file is CPU/core_model-marked.', 'tests/platforms/npu/test_310p_patches.py:492 `rotary_path = _repo_root() / "vllm_omni" / "platforms" / "npu" / "layers" / "rotary_embedding.py"` — anchor line present verbatim.', '.buildkite/amd/test-amd-merge.yml:21 `"timeout 60m pytest -sv tests/diffusion -m \'core_model and cpu\' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — AMD CPU lane collects tests/diffusion, not tests/platforms/npu.', '.buildkite/amd/test-amd-merge.yml:32 `"timeout 60m pytest -sv tests/model_executor -m \'core_model and cpu\' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — AMD CPU lane collects tests/model_executor, not tests/platforms/npu.', 'grep across .buildkite for `platforms/npu` and `tests/platforms` returned no matches — no .buildkite file collects tests/platforms/npu (unchanged by this diff, present in the PR-time tree).', 'gh_ci_read(5608): buildkite/vllm-omni-amd-ci = FAILURE, buildkite/vllm-omni-npu-ci = SUCCESS — red gate is AMD while the NPU lane passes.']) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:19` [nit] — In `_bnsd_rotary_shape_is_supported`, non-4D input returns False, and with `unsqueeze_dim=1` the fallback at line 51 silently applies `transpose(1,2)` — for a 3D (B,S,D) tensor that yields (B,D,S) instead of the pre-PR pass-through. The current Qwen3-TTS caller supplies 4D q/k, so this is future-proofing: assert `hidden_states.ndim == 4` (or otherwise reject non-4D) before the BSND fallback. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:19 ` if hidden_states.ndim != 4:` / :20 ` return False` / :44 ` if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):` / :51 ` hidden_states_bsnd = hidden_states.transpose(1, 2).contiguous()`. Consumer unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py:362 ` query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)` with query_states/key_states produced by `.transpose(1, 2)` at lines 355-356, i.e. always 4D.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.trace.json.gz new file mode 100644 index 000000000..11bb77930 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5608.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.cost.json new file mode 100644 index 000000000..87336d8cd --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1663.8, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5720/run-20260815-135726-1a3997", + "recorded_at": "2026-08-15T14:25:06", + "attempts": [ + { + "invocation_id": "inv-74d79d15", + "outcome": "run:run-20260815-135726-1a3997", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-135726-1a3997", + "llm_calls": 184, + "input_tokens": 1869171, + "output_tokens": 375209, + "usd": 1.45497, + "source": "spans" + } + ], + "attempt_usd_total": 1.45497, + "llm_calls": 184, + "input_tokens": 1869171, + "output_tokens": 375209, + "usd": 1.45497, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-135726-1a3997", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.455, + "minutes": 27.67, + "ci_minutes": 0, + "input_tokens": 1869171, + "output_tokens": 375209, + "tool_calls": 306, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1907, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 184, + "by_role": { + "agent": { + "usd": 1.417382, + "input_tokens": 1831435, + "output_tokens": 350342, + "calls": 182 + }, + "reducer": { + "usd": 0.037587, + "input_tokens": 37736, + "output_tokens": 24867, + "calls": 2 + } + }, + "cache_read_tokens": 19909760, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.11, + "gate": 1.92, + "review": 1653.95, + "report": 0.06 + }, + "time_to_first_result_s": 1660.03 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5720.trace.json.gz", + "trace_bytes": 65082 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.md new file mode 100644 index 000000000..0b7bae090 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5720, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head b4733f6bfd98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] Ref2VA-in-combined-mode resident-block skip: `diffuse` passes `enabled=transformer is self.transformer` and new test `test_distributed_layerwise_resident_blocks_can_be_skipped` covers the disabled path. +- [claim-verified] Default combined mode loads both DiTs: `_minimax_h3_partition_for_task(None/"auto") == "combined"` and `__init__` creates `transformers_ref`; parametrized test confirms. +- [claim-verified] `--task-type ref2va` downloads only the selected partition: `MINIMAX_H3_TASK_DOWNLOAD_PATTERNS["ref2va"] == ["Ref2VA/**"]`. +- [claim-verified] PR 'By default initialize both DiTs' holds: `_minimax_h3_partition_for_task(None, repo_id)` returns `combined` and `__init__` creates `transformers_ref` (pipeline_minimax_h3.py:573-618). +- [claim-verified] 'task-type fl2va/ref2va loads only one DiT' holds: non-combined branch leaves `ref2va_model_path=None` and `self._dit_modules=["transformer"]`; `test_pipeline_loads_task_selected_dits_and_shared_components_once` asserts `len(created["dit"])==expected_dits`. +- [claim-verified] 'shared components loaded once' holds: combined mode builds tokenizer/processor/text_encoder/video_vae/audio_vae from the FL2VA path only; test asserts each created list has length 1 and tokenizer/processor calls == 1. +- [claim-verified] docs 'single-task downloads only selected partition' holds: `MINIMAX_H3_TASK_DOWNLOAD_PATTERNS` and `_resolve_minimax_h3_model_root` select it (pipeline_minimax_h3.py:115-161). +- [claim-verified] PR 'share text_encoder and vae' holds at code level. +- [claim-verified] test_pipeline_loads_task_selected_dits_and_shared_components_once asserts download allow_patterns against the constants, but download is monkeypatched so the real HF repo layout is never validated. +- [claim-refuted] PR/test claim that Qwen3-TTS --task-type is validated: guard in vllm_omni/config/model.py:215 checks model_arch == 'Qwen3TTSTalkerForConditionalGenerationARVLLM', but production qwen3_tts/pipeline.py:17 and registry keys use 'Qwen3TTSTalkerForConditionalGeneration' (no ARVLLM); grep shows ARVLLM only in model.py and tests. +- [claim-refuted] tests/entrypoints/test_resolve_dreamzero_config.py:51 2-arg get_hf_file_to_dict monkeypatch now receives revision= from get_diffusion_model_index -> TypeError; test not updated in this diff. +- [claim-refuted] tests/entrypoints/test_utils.py:296 test_glm_image_diffusers_format_resolution patches file_or_path_exists but resolve_model_config_path now calls get_diffusion_model_index -> branch no longer hermetic; test not updated. +- [claim-refuted] recipes/MiniMaxAI/MiniMax-H3-NPU.md:109 'Layerwise offload applies to both DiTs' is overstated - resident_dit_paths={transformer} so only transformer gets resident layers; the statement is true only for the streaming hook portion, not resident-layer offload +- [claim-refuted] recipes/MiniMaxAI/MiniMax-H3.md:200 run_h3_2gpu_all_tasks.sh claim: examples/offline_inference has no minimax_h3 subdirectory (kept comments item already flags stale path) + +`vllm_omni/diffusion/utils/hf_utils.py:~23` [major] — Confirmed: `get_diffusion_model_index` (hf_utils.py:22-26) calls `get_hf_file_to_dict` for both `model_index.json` and `modular_model_index.json` with no per-file guard, and both consumers — `is_diffusion_model` strategy 2 (hf_utils.py:101) and `resolve_model_class_name` (data.py:557) — now invoke the helper without the try/except the pre-PR code had. A gated-repo/network/offline error from the first lookup escapes model detection instead of falling through to strategy 3 / bagel / dreamzero, and the second index is skipped. Wrap each per-file lookup (or the loop) so failures degrade to the next index and preserve the old fall-through. (evidence: hf_utils.py:22-23 `for filename in DIFFUSION_MODEL_INDEX_FILES:` / `config = get_hf_file_to_dict(filename, model_name, revision=revision)` — the new helper's per-file lookup has no try/except; hf_utils.py:101 `config_dict = get_diffusion_model_index(model_name)` — strategy 2 of `is_diffusion_model` calls the unguarded helper (the diff removed the old try/except around `get_hf_file_to_dict`); data.py:557 `model_index = get_diffusion_model_index(model)` — `resolve_model_class_name` likewise dropped its guard; unchanged by this diff, present in the PR-time tree: config_factory.py:168-169 `try:` / `config_dict = get_hf_file_to_dict("config.json", model, revision=None)` — the repo's own pattern wraps the same vLLM utility in try/except, corroborating that it can raise.) + +`buildkite/vllm-omni-amd-ci:0` [major] — buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are both FAILURE on PR 5720 at head. The PR modifies core diffusion modules these lanes exercise (vllm_omni/diffusion/data.py, registry.py, worker/diffusion_model_runner.py, cache/cachedit/backend.py). Pull the lane logs and diff-attribute pre-existing vs introduced before treating this as resolved; PR 5720 is already merged, so this is now a post-merge follow-up. (evidence: ['gh_ci_read(5720) (CI check state, not a repo file line): `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`', 'gh_ci_read(5720) (CI check state, not a repo file line): `{"bucket":"fail","name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}`', 'gh_pr_view(5720).files includes `vllm_omni/diffusion/data.py`, `vllm_omni/diffusion/registry.py`, `vllm_omni/diffusion/worker/diffusion_model_runner.py`, `vllm_omni/diffusion/cache/cachedit/backend.py`']) + +`vllm_omni/config/model.py:~20` [major] — The new `_validate_startup_task_type()` (vllm_omni/config/model.py:215) gates on `model_arch == "Qwen3TTSTalkerForConditionalGenerationARVLLM"`, but the production Qwen3-TTS pipeline is registered as `Qwen3TTSTalkerForConditionalGeneration` (vllm_omni/model_executor/models/qwen3_tts/pipeline.py:20, not :17; model_executor/models/registry.py:157). `ARVLLM` appears only in this guard, the adjacent `_patch_qwen3_tts` guard, and tests/engine/test_arg_utils.py — never as a production assignment. Because the PR removed the argparse `choices` from `--task-type`, invalid values such as `--task-type fl2va` are no longer rejected at startup for real Qwen3-TTS. Compare against `Qwen3TTSTalkerForConditionalGeneration` (or derive from the stage model type) and drive the test with the production arch. (evidence: vllm_omni/config/model.py:215 (added by this diff): ` if self.model_arch != "Qwen3TTSTalkerForConditionalGenerationARVLLM" or self.task_type is None:` — startup validation only runs for the ARVLLM string. tests/engine/test_arg_utils.py:136 (added by this diff): ` model_arch="Qwen3TTSTalkerForConditionalGenerationARVLLM",` — the new test uses the same non-production string. Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/qwen3_tts/pipeline.py:20 ` model_arch="Qwen3TTSTalkerForConditionalGeneration",` and vllm_omni/model_executor/models/registry.py:157 ` "Qwen3TTSTalkerForConditionalGeneration": (` — production uses the non-ARVLLM arch. Repo-wide grep for `Qwen3TTSTalkerForConditionalGenerationARVLLM` matches only config/model.py:215, config/model.py:283, and tests/engine/test_arg_utils.py.) + +`vllm_omni/config/config_factory.py:~186` [major] — config_factory.py:186 still calls get_hf_file_to_dict("model_index.json", model, revision=None) directly, so _try_infer_model_type cannot resolve a modular Diffusers repo that exposes only modular_model_index.json. This PR makes modular_model_index.json a first-class index via get_diffusion_model_index and migrates data.py/entrypoints/utils.py, but leaves this engine-side pipeline-inference consumer on the old path. Migrate this site to get_diffusion_model_index or document why it intentionally excludes modular repos. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/config/config_factory.py:186 ` model_index = get_hf_file_to_dict("model_index.json", model, revision=None)` — reads only model_index.json, so it cannot see a repo that exposes only modular_model_index.json. Sibling consumers were migrated in this diff and I read them: vllm_omni/entrypoints/utils.py `_try_get_class_name_from_diffusers_config` now does `model_index = get_diffusion_model_index(model)`, and `resolve_model_config_path` does `if get_diffusion_model_index(model) is not None:`.) + +`vllm_omni/entrypoints/utils.py:~284` [major] — `resolve_model_config_path` at vllm_omni/entrypoints/utils.py:284 now gates the Diffusers branch on `get_diffusion_model_index(model) is not None`, which performs real `get_hf_file_to_dict` lookups for `model_index.json`/`modular_model_index.json`. `test_glm_image_diffusers_format_resolution` still stubs only `vllm_omni.entrypoints.utils.file_or_path_exists` with return_value=True (tests/entrypoints/test_utils.py:296-297), so that stub no longer controls this branch and the test becomes network-dependent/offline-failing for `zai-org/GLM-Image`. Patch `get_diffusion_model_index` (or `get_hf_file_to_dict`) in this test, or keep a stubable seam. (evidence: vllm_omni/entrypoints/utils.py:284 ` if get_diffusion_model_index(model) is not None:` — changed branch gate in the PR-time tree. tests/entrypoints/test_utils.py:296-297 `"vllm_omni.entrypoints.utils.file_or_path_exists",` / `return_value=True,` — unchanged by this diff (not present in the PR patch) and no longer controls that branch; the test still stubs only `file_or_path_exists`, so the new `get_diffusion_model_index` path does a real HF lookup.) + +`recipes/MiniMaxAI/MiniMax-H3-NPU.md:112` [major] — The diff adds the sentence 'Layerwise offload applies to both DiTs.' for the NPU combined-service profile. Resident-layer offload only applies to the primary `transformer` (`_offload_plan.resident_dit_paths=frozenset({"transformer"})`), and `diffuse()` skips resident staging for `transformers_ref` via `_resident_dit_layers_on_device(enabled=transformer is self.transformer)`. Streaming hooks do apply to both DiTs, but the unqualified doc claim is false for the resident-block portion. Rephrase to 'Streaming layerwise offload applies to both DiTs; only the FL2VA transformer keeps its leading blocks resident'. (evidence: recipes/MiniMaxAI/MiniMax-H3-NPU.md:110 `partition restart is required. Layerwise offload applies to both DiTs.` (added by this diff); vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:518 `resident_dit_paths=frozenset({"transformer"}),` (unchanged context in this diff, present in PR-time tree); vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1477 `with self._resident_dit_layers_on_device(enabled=transformer is self.transformer):` (added by this diff).) + +`vllm_omni/entrypoints/cli/serve.py:~216` [minor] — `serve.py:216` help says 'The selected model validates supported values', but after this diff only two paths validate: `model.py:215-219` validates only Qwen3-TTS AR models, and `pipeline_minimax_h3.py:125-139` is the only diffusion pipeline that rejects unknown values (other diffusion pipelines never read `OmniDiffusionConfig.task_type`). Every other model silently accepts and ignores an arbitrary `--task-type`, so the help text overstates the guarantee. Either scope the help text to validating models or reject unknown task types centrally. (evidence: ['vllm_omni/entrypoints/cli/serve.py:216 `help="Model-defined startup task type. The selected model validates "` — help text claims model-defined validation', 'vllm_omni/config/model.py:215 `if self.model_arch != "Qwen3TTSTalkerForConditionalGenerationARVLLM" or self.task_type is None:` — early return means only Qwen3-TTS AR models reach the validation below', 'vllm_omni/config/model.py:217 `if self.task_type not in _QWEN3_TTS_TASK_TYPES:` — the only AR validation path', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:139 `raise ValueError(f"MiniMax-H3 task_type must be one of auto, t2va, fl2va, or ref2va; got {task_type!r}")` — the only diffusion pipeline that rejects unknown task_type', 'grep across vllm_omni/diffusion for `task_type` found no other pipeline reading `OmniDiffusionConfig.task_type` (only data.py field definition and MiniMax-H3 pipeline), so other diffusion models accept arbitrary values without validation.']) + +`recipes/MiniMaxAI/MiniMax-H3.md:197` [minor] — recipes/MiniMaxAI/MiniMax-H3.md:197 still references examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh, but examples/offline_inference has no minimax_h3 subdirectory. Point to the actual example path or remove the stale reference. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:197 `bash examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh` — unchanged by this diff, present in the PR-time tree; list_dir(examples/offline_inference) returns only audex/, aura_omni/, covo_audio/, custom_pipeline/, dreamzero/, dynin_omni/, hunyuan_image3/, image_to_image/, image_to_video/, internvla_a1/, lance/, mimo_audio/, ming_flash_omni/, minicpmo/, personaplex/, qwen2_5_omni/, qwen3_omni/, speech_to_video/, step_audio2/, text_to_audio/, text_to_image/, text_to_speech/, text_to_video/, x_to_text/, x_to_video_audio/ (no minimax_h3), and grep for run_h3_2gpu_all_tasks.sh under examples/ returns no matches.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/entrypoints/omni_base.py:81` [minor] omni_base.py:81 hardcodes `revision=None` in the modular-deferral probe while `omni_snapshot_download(model_id)` has no revision parameter, so a `--revision ` serve checks the repository default branch for `modular_ +- `tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:307` [minor] The diff adds a contract test that monkeypatches `download_weights_from_hf_specific` with `fake_download` returning the test-local `tmp_path` and asserts only the `allow_patterns` handed to it (test_minimax_h3_contract.p +- `vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:518` [minor] The diff changes `_dit_modules` to `["transformer", "transformers_ref"]` while `_offload_plan` keeps `resident_dit_paths=frozenset({"transformer"})`. In combined mode with distributed layerwise offload, ModuleDiscovery d +- `recipes/MiniMaxAI/MiniMax-H3-NPU.md:175` [minor] Confirmed. Line 112 (`The same endpoint accepts task=t2va, task=fl2va, and task=ref2va; no partition restart is required.`) invites all tasks on one server, while line 118 (`above. Both are validated for T2VA only.`) and + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.trace.json.gz new file mode 100644 index 000000000..8adaaf4f4 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5720.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.cost.json new file mode 100644 index 000000000..4026ff922 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1116.7, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5723/run-20260815-135726-8ea718", + "recorded_at": "2026-08-15T14:15:59", + "attempts": [ + { + "invocation_id": "inv-8275c864", + "outcome": "run:run-20260815-135726-8ea718", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-135726-8ea718", + "llm_calls": 106, + "input_tokens": 797758, + "output_tokens": 162397, + "usd": 0.577946, + "source": "spans" + } + ], + "attempt_usd_total": 0.577946, + "llm_calls": 106, + "input_tokens": 797758, + "output_tokens": 162397, + "usd": 0.577946, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-135726-8ea718", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5779, + "minutes": 18.54, + "ci_minutes": 0, + "input_tokens": 797758, + "output_tokens": 162397, + "tool_calls": 172, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7438, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 106, + "by_role": { + "agent": { + "usd": 0.551529, + "input_tokens": 785492, + "output_tokens": 141421, + "calls": 104 + }, + "reducer": { + "usd": 0.026417, + "input_tokens": 12266, + "output_tokens": 20976, + "calls": 2 + } + }, + "cache_read_tokens": 6811648, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.34, + "gate": 1.95, + "review": 1107.24, + "report": 0.06 + }, + "time_to_first_result_s": 1112.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5723.trace.json.gz", + "trace_bytes": 29492 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.md new file mode 100644 index 000000000..38cca3224 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.md @@ -0,0 +1,57 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5723, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 93c0112c114e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [claim-verified] FLASH_ATTN resolves to AITER on ROCm: vllm_omni/platforms/rocm/platform.py:101 `aiter_supported = is_aiter_found_and_supported() and 90 < capability < 100` gates FLASH_ATTN; vllm_omni/diffusion/attention/backends/utils/fa.py:40-44 imports `from aiter import flash_attn_func, flash_attn_varlen_func`. +- [claim-verified] 'The VAE uses AITER GroupNorm on ROCm': vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py:16 `from aiter.ops.groupnorm import GroupNorm as AiterGroupNorm` and line 48 replaces VAE GroupNorm when aiter is found. +- [claim-verified] 'FA4 is CUDA-only': pyproject.toml:100-102 `fa4 = ["flash-attn-4[cu13]==4.0.0b18"]` — CUDA 13/Blackwell-specific extra. +- [claim-verified] gfx942/gfx950 capability gate: rocm/platform.py computes capability=major*10+minor and `90 < capability < 100`, so gfx942 (9,4) and gfx950 (9,5) pass; MI325X would also pass (same gfx942) — the doc's 'not listed' is a validation-scoping choice, not a code gate. +- [claim-verified] PR numbers are internally consistent: 1/0.73 = 1.37 it/s; gfx942 T2VA components 0.09+244.04+4.15=248.28 vs E2E 267.42 leaves 19.14 s overhead; FL2VA 13.98+257.58+4.11=275.67 vs E2E 287.07 leaves 11.40 s overhead. +- [claim-verified] `--diffusion-attention-backend FLASH_ATTN` is a valid flag value — tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json:33 and docs/user_guide/diffusion/attention_backends.md use it. +- [claim-verified] FLASH_ATTN→AITER on ROCm — docs/design/module/dit_module.md:505 'dispatches to FA2/FA3 on CUDA, AITER on ROCm, and mindiesd on Ascend NPU'. +- [claim-verified] VAE AITER GroupNorm on ROCm — vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py:3 'Patch initialize_model to replace VAE GroupNorm with AITER GroupNorm on ROCm.' +- [claim-verified] vLLM pin `vllm==0.26.0+rocm723` + `https://wheels.vllm.ai/rocm/0.26.0/rocm723` matches docs/getting_started/installation/gpu/rocm.inc.md. +- [claim-refuted] supported_models.md says 'MiniMax H3 AMD GPU support is validated' (new line 101) but the MiniMax H3 row at line 47 is `| `MiniMaxH3Pipeline` | ... | ✅︎ | | | |` — the AMD GPU column is empty and no H3 marker exists; table legend 'Empty cells mean not listed as supported on that backend' directly contradicts the footnote. +- [claim-refuted] Recipe Known-limitations (line 724) says 'the current vLLM-Omni path takes exactly one image plus one audio reference, or one or more videos with no separate `audio_reference`' — refuted by model_metadata.py (`max_multimodal_image_inputs=9, supports_mixed_reference_inputs=True`), pipeline `_validate_ref2va_reference_counts` (≤9 images/≤3 videos/≤3 audios/≤12 total), and forward() `audio_condition = torch.cat([embedded_audio_condition, external_audio_condition])`. +- [claim-refuted] PR description claims gfx950 'single + 4-GPU' validation, but the evidence table has only a 1x gfx950 row (line 481) and recipes line 469 says text-encoder TP was 'not exercised' on gfx950. +- [claim-refuted] PR body says 'mark MiniMax H3 AMD GPU supported' but the `MiniMaxH3Pipeline` row AMD cell is still empty (`... | ✅︎ | | | |`) and the legend says empty cells mean not listed as supported. +- [claim-refuted] gfx942 E2E does not equal the listed component sum: T2VA 0.09+244.04+4.15=248.28 vs 267.42 (Δ19.14 s); FL2VA 13.98+257.58+4.11=275.67 vs 287.07 (Δ11.40 s); residual is unlabeled. + +`docs/models/supported_models.md:101` [major] — docs/models/supported_models.md: the new footnote at line 101 asserts MiniMax H3 AMD support is validated, but the MiniMaxH3Pipeline row (line 47) still has an empty AMD GPU cell and no row carries `H3`. The legend at line 99 says empty cells mean 'not listed as supported on that backend', so the table contradicts the footnote and the PR's goal to mark AMD support is not delivered. Add `✅︎H3` to the row's AMD GPU cell, or remove the footnote. (evidence: docs/models/supported_models.md:47 (pre-existing, present in PR-time tree; not changed by this diff): `| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |` — AMD GPU cell empty. docs/models/supported_models.md:99 (pre-existing, present in PR-time tree; not changed by this diff): `✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.` docs/models/supported_models.md:101 (added by this diff): `H3 MiniMax H3 AMD GPU support is validated on gfx942 (MI300X) and`) + +`recipes/MiniMaxAI/MiniMax-H3.md:~729` [major] — At recipes/MiniMaxAI/MiniMax-H3.md:729 the new bullet says the serving path 'takes exactly one image plus one audio reference, or one or more videos with no separate audio_reference', but this contradicts the PR-time admission contract: MiniMaxH3Pipeline metadata declares max_multimodal_image_inputs=9 and supports_mixed_reference_inputs=True, _validate_ref2va_reference_counts only rejects >9 images / >3 videos / >3 audios / >12 total, and the recipe's own 'Official input matrix and limits' table lists video+audio and mixed image/video/audio with 'audio requires a visual reference'. Replace the bullet with the actual limits (images ≤9, videos ≤3, audios ≤3, total ≤12; video+standalone audio allowed) or delete it. (evidence: Diff-added text, recipes/MiniMaxAI/MiniMax-H3.md:729: ` request; the current vLLM-Omni path takes exactly one image plus one audio` (continuing 'reference, or one or more videos with no separate `audio_reference`'). Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/model_metadata.py:42-43 ` max_multimodal_image_inputs=9,` / ` supports_mixed_reference_inputs=True,`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:309-325 `_validate_ref2va_reference_counts` body includes `if image_count > 9:`, `if video_count > 3:`, `if audio_count > 3:`, and `if image_count + video_count + audio_count > 12:` with raises 'ref2va accepts at most 9 image references', 'at most 3 video references', 'at most 3 standalone audio references', 'at most 12 total references'. Same recipe, 'Official input matrix and limits' table: `images ≤9, videos ≤3, audios ≤3, total references ≤12; audio requires a visual reference` with Ref2VA supporting `video+audio` and `mixed image/video/audio`.) + +`recipes/MiniMaxAI/MiniMax-H3.md:~426` [major] — The new ROCm single- and four-GPU commands at lines 426 and 446 use `export MODEL="${MODEL_ROOT}/FL2VA"`, but the AMD ROCm section never defines MODEL_ROOT. The only assignment in the file is `MODEL_ROOT=/path/to/MiniMax-H3 \` at line 205, a command-local env prefix for the CUDA run-script, not an exported variable available to a user following only the AMD section; such a user gets an empty variable and serves `/FL2VA`. Add `export MODEL_ROOT=/path/to/MiniMax-H3` (with a note that the checkpoint must be pre-downloaded) at the top of the ROCm section, or use a repo ID. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:426 (added by this diff) `export MODEL="${MODEL_ROOT}/FL2VA"`; recipes/MiniMaxAI/MiniMax-H3.md:446 (added by this diff) `export MODEL="${MODEL_ROOT}/FL2VA"`; recipes/MiniMaxAI/MiniMax-H3.md:205 (pre-existing, not in this diff) `MODEL_ROOT=/path/to/MiniMax-H3 \` — the only `MODEL_ROOT=` assignment in the file, and it is a command-local env prefix for the CUDA run-script, so it is not exported to the AMD section.) + +`recipes/MiniMaxAI/MiniMax-H3.md:405` [minor] — Line 405 tells ROCm users to 'drop the CUDA-only FLASHINFER_DISABLE_VERSION_CHECK', but no command in the recipe sets that variable (grep across recipes/ finds only this line), and the engine injects it internally via os.environ.setdefault in stage_engine_core_proc.py:122. Reword to something like 'no FlashInfer version-check env var is needed on ROCm' so it does not tell users to remove a setting they never added. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:405 `CUDA-only `FLASHINFER_DISABLE_VERSION_CHECK`, and install without the `[fa4]` extra` (added by this diff; grep across recipes/ returns only this occurrence) — no CUDA command in the recipe sets the variable. vllm_omni/engine/stage_engine_core_proc.py:122 ` os.environ.setdefault("FLASHINFER_DISABLE_VERSION_CHECK", "1")` (unchanged by this diff, present in the PR-time tree; the engine injects it internally, so a user-facing 'drop' instruction is misleading).) + +`recipes/MiniMaxAI/MiniMax-H3.md:479` [minor] — At recipes/MiniMaxAI/MiniMax-H3.md:479-480, label the residual overhead in the ROCm evidence table so the E2E figures reconcile with the component sums: T2VA 0.09+244.04+4.15=248.28 s vs 267.42 s client E2E (Δ19.14 s); FL2VA 13.98+257.58+4.11=275.67 s vs 287.07 s client E2E (Δ11.40 s). Currently the residual is unlabeled. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:479 `| T2VA, 1344x768, 209 frames, 50 steps | 4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile | encode 0.09 s, denoise 244.04 s, decode 4.15 s, 267.42 s client E2E; H.264 24 FPS + 32 kHz stereo AAC |` — 0.09+244.04+4.15=248.28, not 267.42. recipes/MiniMaxAI/MiniMax-H3.md:480 `| FL2VA, 1344x768, 209 frames, 50 steps | 4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile | encode 13.98 s, denoise 257.58 s, decode 4.11 s, 287.07 s client E2E; H.264 24 FPS + 32 kHz stereo AAC |` — 13.98+257.58+4.11=275.67, not 287.07. No table cell labels or footnotes explain the residual between the component sum and the E2E figure.) + +`docs/models/supported_models.md:102` [minor] — New footnote introduces a third gfx950 SKU label: docs/models/supported_models.md:102 says "gfx950 (MI350)", while recipes/cosmos3/Cosmos3-Nano.md:434 says "gfx950 / MI350X" and docs/user_guide/quantization/mxfp4.md:42 says "gfx950 / MI355X". Align on one convention (or name the exact validated SKU) so the docs do not disagree. (evidence: docs/models/supported_models.md:102 `gfx950 (MI350) in BF16, using the AITER `FLASH_ATTN` diffusion attention backend.` — added by this PR's diff. Unchanged by this diff, present in the PR-time tree: recipes/cosmos3/Cosmos3-Nano.md:434 `- Driver / runtime: ROCm, HIP 7.2, **gfx942 or gfx950** (validated on gfx950 / MI350X,` and docs/user_guide/quantization/mxfp4.md:42 `| AMD ROCm (gfx950 / MI355X) | ✅ |`.) + +`recipes/MiniMaxAI/MiniMax-H3.md:469` [minor] — The four-GPU ROCm command at recipes/MiniMaxAI/MiniMax-H3.md:460 uses `--text-encoder-tp-size 4`, but lines 468–469 state that `--text-encoder-tp-size` was not exercised on gfx950, and the Validated ROCm evidence table lists only a 1x gfx950 row (line 481) while the PR body claims 4-GPU gfx950 T2VA validation. Add the 4-GPU gfx950 evidence row/config or narrow the PR description to match the evidence table. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:468-469 "supports only its native `tile` mode. `--text-encoder-tp-size` is validated on / gfx942; on gfx950 it was not exercised."; recipes/MiniMaxAI/MiniMax-H3.md:460 " --text-encoder-tp-size 4 \\" (documented four-GPU ROCm command); recipes/MiniMaxAI/MiniMax-H3.md:481 "| T2VA, 832x480, ~4 s, 40 steps | 1x gfx950 (MI350), FLASH_ATTN, CPU offload | valid MP4 (H.264 + synced audio); ~0.73 s/denoise-step (~1.37 it/s), ~55 s client E2E incl. warmup |"; PR 5723 body: "MiniMax H3 T2VA generates valid H.264 + 32 kHz audio on gfx950 (single + 4-GPU; FLASH_ATTN ~2× faster than SDPA)") + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.trace.json.gz new file mode 100644 index 000000000..da9b9f654 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5723.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.cost.json new file mode 100644 index 000000000..f89ddebf4 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1614.2, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5756/run-20260815-135724-64af42", + "recorded_at": "2026-08-15T14:24:16", + "attempts": [ + { + "invocation_id": "inv-9a1ef195", + "outcome": "run:run-20260815-135724-64af42", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-135724-64af42", + "llm_calls": 140, + "input_tokens": 1101775, + "output_tokens": 231383, + "usd": 0.900251, + "source": "spans" + } + ], + "attempt_usd_total": 0.900251, + "llm_calls": 140, + "input_tokens": 1101775, + "output_tokens": 231383, + "usd": 0.900251, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-135724-64af42", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9003, + "minutes": 26.87, + "ci_minutes": 0, + "input_tokens": 1101775, + "output_tokens": 231383, + "tool_calls": 230, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0035, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 140, + "by_role": { + "planner": { + "usd": 0.001835, + "input_tokens": 2429, + "output_tokens": 1072, + "calls": 1 + }, + "agent": { + "usd": 0.847642, + "input_tokens": 1070577, + "output_tokens": 191277, + "calls": 136 + }, + "reducer": { + "usd": 0.050774, + "input_tokens": 28769, + "output_tokens": 39034, + "calls": 3 + } + }, + "cache_read_tokens": 12898176, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.05, + "gate": 1.67, + "review": 1606.37, + "report": 0.06 + }, + "time_to_first_result_s": 1612.13 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5756.trace.json.gz", + "trace_bytes": 42475 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.md new file mode 100644 index 000000000..467950927 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5756, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 43592108ac79) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 4 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] `images[0]`/`audios[0]` are guarded by `len(images)==1 and len(audios)==1` (api_client.py:320), so no empty-index risk. +- [resolved] `extra_body`→`extra_params` multipart field rename: the old `extra_body` field was not accepted by the /videos endpoint (server only reads `extra_params`, api_server.py:3009/3073); no caller passes extra kwargs today, so the no-builder branch is inert — rename is a fix, not a regression. +- [resolved] model_params['type'] pop: nodes.py:221-223 now leaves `type` and api_client.py:357-359 strips it for all models before dispatch; Wan never sets `type` (nodes.py:750 returns WanModelSpecificParams(kwargs) without it), so no stale consumer. +- [resolved] lookup_model_spec lstrip fix (models.py:128-129, commit 43592108) resolves the leading-slash path case; `split('/', 1)[-1]` keeps 'MiniMax-H3' in the lookup key for the shipped snapshot path. +- [claim-verified] '20 passed' matches parametrization arithmetic: 8 image + 2 understanding + 4 tts + 4 video + 2 minimax_h3 = 20 (calc). +- [claim-verified] No new external dependency calls; video_to_bytes/audio_to_base64 are pre-existing internal helpers. +- [claim-verified] PR test table '20 passed': matrix = 8 image + 2 understanding + 4 TTS + 4 video + 2 H3 Ref2VA = 20 (calc). +- [claim-verified] gate_report says no failing checks; no CI attribution work needed. +- [validated] Only in-repo caller of api_client.generate_video is nodes.py:228; no stale `image=` or `extra_body=` video callers remain (grep apps/ComfyUI-vLLM-Omni). +- [validated] Server /v1/videos `_parse_video_form` accepts `input_references`, `input_reference`, `audio_reference`, `extra_params`, `flow_shift` (api_server.py:2981-2992, 3033-3054) — client field names match. +- [validated] `flow_shift` top-level form maps to `extra_args["flow_shift"]` and `extra_params` merges into `extra_args` (serving_video.py:196-207); MiniMax-H3 builder output reaches the pipeline. +- [validated] Multi-video ref2va temp files are cleaned in `_run_video_generation_job` finally and the sync endpoint finally (api_server.py); no new leak on the new upload path. +- [validated] lookup_model_spec's new split keeps the full path remainder, so `/path/to/MiniMax-H3/FL2VA` matches; no regression for existing single-slash patterns. +- [validated] server multipart contract: client fields match api_server.py:2981/2982/2985/2998/3009 (`input_reference`, `input_references`, `audio_reference`, `flow_shift`, `extra_params`). + +`apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:~1` [minor] — The shipped workflow hardcodes `/workspace/models/hub/models--MiniMaxAI--MiniMax-H3/snapshots/428986c4aa5280855b9ac26eff8d1a46b5999cd0/FL2VA` as the `model` widget value. `_parse_video_form` rejects any `request.model` that differs from `effective_model_name`, and the MiniMax-H3 recipe serves `"${MODEL_ROOT}/FL2VA"`, so following the recipe as-is yields a Model mismatch 400 until the field is edited. Make the model value a portable placeholder or note that it must equal the served model name (or use `--served-model-name`). (evidence: unchanged by this diff, present in the PR-time tree: apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:1 `"widgets_values":["http://localhost:8000/v1","/workspace/models/hub/models--MiniMaxAI--MiniMax-H3/snapshots/428986c4aa5280855b9ac26eff8d1a46b5999cd0/FL2VA",...` — machine-specific snapshot path in the model widget. vllm_omni/entrypoints/openai/api_server.py:3087-3088 `effective_model_name = handler.model_name or app_model_name or request.model or "unknown"` and `if request.model is not None and effective_model_name is not None and request.model != effective_model_name:` raise the mismatch 400. recipes/MiniMaxAI/MiniMax-H3.md:60 `export MODEL="${MODEL_ROOT}/FL2VA"` — recipe serves a path different from the workflow's hardcoded snapshot path.) + +`apps/ComfyUI-vLLM-Omni/README.md:108` [minor] — Confirmed. apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:164 sets the shared Generate Video node's default to `"fps": ("INT", {"default": 16, "min": 1})`, and api_client.py passes that value straight through as the multipart `fps` field. vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:399-401 raises `ValueError("MiniMax H3 output fps is fixed at 24")` whenever `fps != MINIMAX_H3_FPS` (24). The README at line 105 now advertises MiniMax-H3 on this shared node without documenting the fps=24 requirement, so keeping the default while only changing the model string yields a server-side error. Add a MiniMax-H3-specific default/validation or document fps=24 in the README tip. (evidence: ['apps/ComfyUI-vLLM-Omni/README.md:105 `### Text-to-video and image-to-video generation (e.g., Wan, MiniMax-H3)` (in the PR diff; advertises MiniMax-H3 on the shared node)', 'apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:164 `"fps": ("INT", {"default": 16, "min": 1}),`', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:83 `MINIMAX_H3_FPS = 24` (unchanged by this diff, present in the PR-time tree)', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:399-401 `fps = int(sampling.fps or MINIMAX_H3_FPS)` / `if fps != MINIMAX_H3_FPS:` / `raise ValueError(f"MiniMax H3 output fps is fixed at {MINIMAX_H3_FPS}")` (unchanged by this diff, present in the PR-time tree)']) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:~57` [minor] — The MiniMax-H3 params builder only emits `flow_shift` as a top-level field and merges `task` and `audio_flow_shift` into `extra_params`; no `duration` is ever produced. The shared video node defaults `num_frames` to 41 (≈1.7s at 24fps) and forwards it verbatim, while recipes/MiniMaxAI/MiniMax-H3.md requires `duration` in `extra_params` to cover the complete reference audio, otherwise the reference soundtrack is truncated. Add a `duration` input to VLLMOmniMiniMaxH3Params (or derive it from the reference audio length) so the ComfyUI Ref2VA image+audio path can request a clip long enough for the reference audio. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:47 `if "flow_shift" in params:` models.py:48 ` form_fields["flow_shift"] = params.pop("flow_shift")` models.py:49-52 `for key in ("audio_flow_shift",):` / `if key in params:` / ` merged_extra_params[key] = params.pop(key)` models.py:56-57 `if merged_extra_params:` / ` form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)` — no `duration` key is ever added. apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:165 `"num_frames": ("INT", {"default": 41, "min": 1}),` — default 41 frames ≈ 1.7s at 24fps. apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:752-773 `VLLMOmniMiniMaxH3Params.INPUT_TYPES` exposes only `audio_flow_shift` and `flow_shift`. recipes/MiniMaxAI/MiniMax-H3.md:252-253 `The requested duration should cover the complete audio. If `duration` is` / `shorter, the reference soundtrack is truncated to the generated clip.`) + +`tests/e2e/features/comfyui/test_comfyui_integration.py:891` [minor] — Confirmed: the new MiniMax-H3 tests mock AsyncOmni.generate (module docstring and the mock_async_omni fixture), so MiniMaxH3Pipeline._resolve_shape never runs. They pass fps=VIDEO_FPS (8) and num_frames=VIDEO_NUM_FRAMES (5), while the real pipeline rejects fps != 24 at pipeline_minimax_h3.py:399-401. The two new cases are both ref2va (ref2va-image-audio, ref2va-multi-video); the shipped example workflow's frame->/FL2VA route (client sets task='fl2va') has no MiniMax-H3 test. Please add a frame->fl2va case and use runnable H3 values (fps=24, recipe-aligned num_frames). Note: since the fixture still mocks generate, the fl2va case will only cover node/client task routing; replacing the mock or adding a pipeline-level test is needed to actually reach _resolve_shape. (evidence: ['tests/e2e/features/comfyui/test_comfyui_integration.py:2 `Integration tests for ComfyUI nodes that use the Omni API client, with a mocked AsyncOmni and a real API server running in a background process.` (unchanged by this diff, present in PR-time tree)', 'tests/e2e/features/comfyui/test_comfyui_integration.py:86 `VIDEO_FPS = 8` (unchanged by this diff, present in PR-time tree)', 'tests/e2e/features/comfyui/test_comfyui_integration.py:87 `VIDEO_NUM_FRAMES = 5` (unchanged by this diff, present in PR-time tree)', 'tests/e2e/features/comfyui/test_comfyui_integration.py:891 `async def test_video_generation_node_minimax_h3_ref2va(` (added in this diff)', 'tests/e2e/features/comfyui/test_comfyui_integration.py:927 ` fps=VIDEO_FPS,` (added in this diff)', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:399 ` fps = int(sampling.fps or MINIMAX_H3_FPS)` / :400 ` if fps != MINIMAX_H3_FPS:` / :401 ` raise ValueError(f"MiniMax H3 output fps is fixed at {MINIMAX_H3_FPS}")` — unchanged by this diff, present in PR-time tree; MINIMAX_H3_FPS = 24 at :83']) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:168` [minor] — VLLMOmniGenerateVideo's IMAGE socket is renamed from `image` to `frame` (nodes.py:168) with no legacy `image` input kept. ComfyUI matches saved workflow links by input name, so existing user workflows connecting IMAGE -> `image` will silently lose that link after the extension updates. The README documents the new `frame` contract but does not warn about the rename. Please either keep a hidden legacy `image` input (alias mapped to `frame`) or add an explicit upgrade/breaking-change note in the README. (evidence: PR-time tree apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:168 `"frame": ("IMAGE",),` is now the only IMAGE socket in VLLMOmniGenerateVideo.INPUT_TYPES. Pre-PR file (file_at_base, same region) had `"image": ("IMAGE",),` in that slot, and the PR diff shows the exact replacement (`- "image": ("IMAGE",)` / `+ "frame": ("IMAGE",)`). A PR-time grep for `"image": ("IMAGE",)` returns only nodes.py:57 (VLLMOmniGenerateImage) and nodes.py:258 (VLLMOmniUnderstanding), so no hidden legacy `image` alias exists in the video node. README.md:120 now says `Connect a **frame** image for first-frame / image-to-video (e.g., Wan I2V, MiniMax-H3 FL2VA).` with no mention that the previous `image` socket was removed.) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:71` [minor] — This diff raises the shared VLLMOmniClient default max_poll_duration from 5 to 30 minutes (60*5 -> 60*30), so every video model routed through the Generate Video node (Wan, unregistered models, and MiniMax-H3) now inherits the longer polling timeout, not just MiniMax-H3 Ref2VA. All five VLLMOmniClient(url) instantiations in nodes.py (93/227/297/408/477) use the default and only line 227 calls generate_video, the sole consumer of max_poll_duration. Please confirm this global 6x timeout increase is intentional, or scope it to the H3 params-builder path. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:71 `max_poll_duration: float = 60 * 30,` (changed by this diff; pre-PR base read `max_poll_duration: float = 60 * 5`). apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:387 `deadline = asyncio.get_running_loop().time() + self.max_poll_duration`. apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:93,227,297,408,477 each contain ` client = VLLMOmniClient(url)` with no timeout/max_poll_duration argument; nodes.py:228 ` output = await client.generate_video(` is the only generate_video call among those sites.) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:~355` [minor] — This new multi-video upload loop hardcodes `reference_{idx}.mp4` and `video/mp4` for every reference video (api_client.py:355-361). The recipe this node implements sends the second reference as quicktime (recipes/MiniMaxAI/MiniMax-H3.md:264 `BACKGROUND_VIDEO=...mov`; :276 `type=video/quicktime`). `video_to_bytes` only sets `output_buffer.name = filename` and delegates to `video.save_to` (format.py:129-134), so container/MIME handling depends on the external comfy_api `VideoInput.save_to`; a .mov source that is not transcoded to MP4 would be mislabeled to the server. Verify with a real .mov second-reference Ref2VA request and pass through each video's original extension/MIME (or explicitly transcode/override) instead of hardcoding MP4. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:355 ` for idx, video in enumerate(reference_videos, start=1):` ; api_client.py:356 ` video_filename = f"reference_{idx}.mp4"` ; api_client.py:360 ` filename=video_filename,` ; api_client.py:361 ` content_type="video/mp4",` ; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py:129 `def video_to_bytes(video: VideoInput, filename: str = "video.mp4") -> BytesIO:` ; format.py:131-132 ` output_buffer.name = filename` / ` video.save_to(output_buffer)` ; recipes/MiniMaxAI/MiniMax-H3.md:264 `export BACKGROUND_VIDEO=/path/to/fairytale_background.mov` ; recipes/MiniMaxAI/MiniMax-H3.md:276 ` -F "input_references=@${BACKGROUND_VIDEO};type=video/quicktime" \`) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:~223` [nit] — nodes.py:223 says `model_params["type"]` is used in api_client, but api_client.py:359 pops it (`model_params.pop("type", None)`) and models.py:44 pops it again defensively. Reword the comment to say it is an internal ComfyUI-only tag that the client strips, or delete the commented-out block. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:223 ` # model_params["type"] is also an internal field, but is used in api_client`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:359 ` model_params.pop("type", None)`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:44 ` params.pop("type", None)`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.trace.json.gz new file mode 100644 index 000000000..2a75fad2f Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5756.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.cost.json new file mode 100644 index 000000000..4bc726426 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1541.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5779/run-20260815-135725-148d95", + "recorded_at": "2026-08-15T14:23:04", + "attempts": [ + { + "invocation_id": "inv-c126d53d", + "outcome": "run:run-20260815-135725-148d95", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-135725-148d95", + "llm_calls": 186, + "input_tokens": 1509229, + "output_tokens": 319239, + "usd": 1.250654, + "source": "spans" + } + ], + "attempt_usd_total": 1.250654, + "llm_calls": 186, + "input_tokens": 1509229, + "output_tokens": 319239, + "usd": 1.250654, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-135725-148d95", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2507, + "minutes": 25.64, + "ci_minutes": 0, + "input_tokens": 1509229, + "output_tokens": 319239, + "tool_calls": 334, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0986, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 186, + "by_role": { + "agent": { + "usd": 1.20884, + "input_tokens": 1483712, + "output_tokens": 287555, + "calls": 183 + }, + "reducer": { + "usd": 0.041815, + "input_tokens": 25517, + "output_tokens": 31684, + "calls": 3 + } + }, + "cache_read_tokens": 18222208, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.35, + "gate": 1.62, + "review": 1531.09, + "report": 0.06 + }, + "time_to_first_result_s": 1538.11 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5779.trace.json.gz", + "trace_bytes": 47229 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.md new file mode 100644 index 000000000..f03dd852f --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.md @@ -0,0 +1,68 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5779, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f092a92eb579) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] H3 default TRTLLM + allgather_degree>1 raises at init instead of silently producing wrong attention (layer.py:96-100); residual: explicit TRTLLM for non-H3 models with allgather>1 also now raises — intended per commit 'reject TRTLLM AllGather-KV'. +- [resolved] TRTLLM leaves supports_prefix_kv_slicing at its False default (abstract.py:21), so H3 always materializes the prefix mask and reaches the new trim path (minimax_h3_transformer.py:407-411). +- [resolved] Issue #5771 root causes fixed: prefix-valid packed masks now accepted (trtllm_attn.py:316-338) and denoise timestep published (pipeline_minimax_h3.py:1376-1378); residual = per-layer .item() syncs noted in review comments. +- [claim-verified] 58816-58758 = 58, matching the PR body's padding token count. +- [claim-verified] 'make TRTLLM the H3 default' is delivered: model_metadata.py:40-46 sets attention_mask_free=True and cuda/platform.py:211-218 returns TRTLLM_ATTN when allow_trtllm_default and compute_capability.major==10 and head_size==128 and trtllm_gen_available. +- [claim-verified] packed cu_seqlens reuse/prefix-mask trim/zero-pad implemented at trtllm_attn.py:257-404; tests cover trim, non-prefix rejection, incomplete-metadata rejection, and coverage rejection. +- [claim-verified] SAGE short-sequence dense fallback implemented via use_sage = quant.enabled and torch.all(seq_lens >= k_block_size); test_sage_short_sequence_uses_dense_kernel drives it. +- [claim-verified] recipe 'B200 also supports int8 Q/K' is consistent with docs ('int8 QK kernels are compiled for SM100 only') but unvalidated here (INT8 skipped on SM103). +- [claim-verified] new tests prove route plumbing (mocked kernel asserts trimmed cu_seqlens/seq_lens) and numerics (real-kernel tests vs SDPA); no authored-value-only assertion found. +- [claim-verified] PR body's changed-file list matches diff_stat (11 files, 371 insertions / 34 deletions). +- [validated] test_packed_metadata_trims_padding_mask drives real forward_cuda with a mocked kernel and asserts query/key/value=6, cu=[0,6], and zero tail (test_trtllm_attn.py:212-247). +- [validated] test_sage_short_sequence_uses_dense_kernel replaces _sage_quantize_fn with pytest.fail and asserts the dense fallback runs (test_trtllm_attn.py:464-475). +- [validated] test_mask_free_allowlist_gates_trtllm_default asserts MiniMaxH3Pipeline.attention_mask_free is True (test_trtllm_attn.py:189-196). +- [validated] minimax_h3_denoise_loop `on_step`→`on_step_end` rename has exactly one in-repo caller (pipeline_minimax_h3.py:1364) and it is updated; grep for `on_step=` returns no matches, so no stale consumer. + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:~320` [major] — Confirmed: in the packed branch, forward_cuda synchronizes host/device on every call — `int(cu_seq_lens_q[-1].item())`/kv (line 300), the trailing-empty loop reads (307-308), `valid_tokens = int(flat_mask.sum().item())` (320), `torch.equal(flat_mask, expected_mask)` (322), and the two boundary `.item()` reads (330-331); with SAGE, `torch.all(...).item()` (382) adds another. These run inside the `@torch.compiler.disable` eager island `_run_packed_attention`, i.e. once per attention site per denoise step (~50 DiT blocks, so hundreds of syncs per step, thousands over a 50-step loop). The producer already supplies host-int `extra['valid_kv_length'] = used` (minimax_h3_transformer.py:417) and warns against per-layer `.item()` reads (lines 404-406), but this backend never reads it. Use `extra.get('valid_kv_length')` as the trim length and avoid re-deriving/validating it from the on-device mask on the hot path. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:300 `if int(cu_seq_lens_q[-1].item()) != q.shape[0] or int(cu_seq_lens_kv[-1].item()) != k.shape[0]:`; :307-308 `and int(cu_seq_lens_q[-1].item()) == int(cu_seq_lens_q[-2].item())` / `and int(cu_seq_lens_kv[-1].item()) == int(cu_seq_lens_kv[-2].item())`; :320 `valid_tokens = int(flat_mask.sum().item())`; :322 `if not torch.equal(flat_mask, expected_mask):`; :330-331 `q_end = int(q_boundary.item()) + 1` / `kv_end = int(kv_boundary.item()) + 1`. Producer (unchanged by this diff, present in PR-time tree): vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:417 `"valid_kv_length": used,`; :404-406 `# max_seqlen is already the first (real) packed document length. Do\n # not read the CUDA cu_seqlens scalars here: this function runs once\n # per layer and .item() would serialize every attention launch.`; vllm_omni/diffusion/attention/backends/abstract.py:21 `supports_prefix_kv_slicing: bool = False` (TRTLLM backend does not override it, so padded H3 rows construct an attn_mask and execute the mask-related syncs).) + +`tests/diffusion/attention/test_trtllm_attn.py:212` [major] — tests/diffusion/attention/test_trtllm_attn.py is not exercised by any CI lane. The module-level pytestmark (line 14) is `[core_model, diffusion, cuda]` with no `cpu`, so the merge/ready lane `pytest -sv tests/diffusion -m 'core_model and cpu'` collects but deselects every test; the ready-gate's other top-level diffusion lane `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` (test-ready.yml:68) does not recurse into tests/diffusion/attention/. No Buildkite file references tests/diffusion/attention. The CPU-runnable regressions (packed metadata trim/reject/complete/cover, allowlist gate, SAGE config validation, denoise-progress mixin) therefore never run automatically, and the PR's '37 passed, 1 skipped' is a manual B300 run only. Add a tests/diffusion/attention lane or mark/move the CPU-runnable tests into a `core_model and cpu`-selected file. (evidence: PR-time tree, unchanged by this diff: tests/diffusion/attention/test_trtllm_attn.py:14 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]`; .buildkite/cuda/test-merge.yml:18 `"timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/cuda/test-ready.yml:68 `- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"`. Repo-wide grep of the PR-time tree finds no Buildkite/GitHub Actions command referencing `tests/diffusion/attention` or `test_trtllm_attn`.) + +`vllm_omni/diffusion/attention/layer.py:96` [minor] — Add a regression test for the new TRTLLM_ATTN + allgather_degree>1 ValueError at layer.py:96-100. No test covers this branch, and the guard is broader than the H3 default: explicit TRTLLM on non-H3 models with allgather_degree>1 now also raises (token refiner bypasses it via skip_sequence_parallel=True), so the rejection behavior should be pinned down. (evidence: vllm_omni/diffusion/attention/layer.py:96-100 `if not skip_sequence_parallel and allgather_degree > 1 and attn_backend_cls.get_name() == "TRTLLM_ATTN": + raise ValueError( + "TRTLLM_ATTN does not support AllGather-KV sequence parallelism. " + "Set --allgather-degree 1 or select another diffusion attention backend." + )` — new guard present in the diff. No test references this rejection (grep for the message across tests/ returned no matches). The model-agnostic guard is only gated on backend name + allgather_degree, so non-H3 models with explicit TRTLLM_ATTN also raise. Unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:589 `skip_sequence_parallel=True` — token refiner bypasses the guard.) + +`recipes/MiniMaxAI/MiniMax-H3.md:248` [minor] — recipes/MiniMaxAI/MiniMax-H3.md:248 — Remove or substantiate the 'within 2%' dense-TRTLLM-vs-FA4 latency claim. PR #5779's body reports only average PSNR 27.10 dB / SSIM 0.8880 for the cold dense-TRTLLM run and explicitly states its end-to-end latency 'is not reported as steady-state backend performance'; no latency source backs the equivalence claim. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:247-248 `Stable measurements with the four-GPU profile above put dense `TRTLLM_ATTN` +and FA4 within 2% of each other.` — the claim added by this diff (grep confirms line 248 contains 'within 2%'). PR #5779 body (no repo file): `Encoded-video comparison against FA4: average PSNR 27.10 dB and SSIM 0.8880.` and `This was a cold TRTLLM service run and included regional/VAE compilation plus slow CPU MP4 encoding, so its end-to-end latency is not reported as steady-state backend performance.`) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:~342` [minor] — trtllm_attn.py:342 — after the prefix trim, max_q_len/max_kv_len are still read from extra unchanged and passed to trtllm_ragged_attention_deepseek (line 383). The H3 producer sets max_seqlen_* to the first document length and all new tests use max_seqlen_* == used, so this is currently consistent, but the backend now accepts arbitrary packed producers. Recompute max from the trimmed seq_lens, or assert max_q_len <= q.shape[0] and max_kv_len <= k.shape[0] after trimming. (evidence: ['vllm_omni/diffusion/attention/backends/trtllm_attn.py:334 ` q = q[:valid_tokens]` (followed by lines 335-338 trimming k, v, cu_seq_lens_q and cu_seq_lens_kv) — the packed prefix trim happens before the max values are read.', 'vllm_omni/diffusion/attention/backends/trtllm_attn.py:342 ` max_q_len = int(extra["max_seqlen_q"])` — max is read unchanged from extra after the trim (line 343 reads max_kv_len the same way).', 'vllm_omni/diffusion/attention/backends/trtllm_attn.py:383 ` max_q_len=max_q_len,` — the unchanged value is passed into trtllm_ragged_attention_deepseek.', 'vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:415 ` "max_seqlen_q": max_seqlen,` with the preceding comment \'max_seqlen is already the first (real) packed document length\' — present in the PR-time tree, showing the current producer is consistent; no assertion protects against a producer that supplies the padded total.']) + +`vllm_omni/diffusion/model_metadata.py:44` [minor] — This diff sets attention_mask_free=True for MiniMaxH3Pipeline (model_metadata.py:46); layer.py:83 feeds that into allow_trtllm_default, and platform.py defaults to TRTLLM_ATTN for any compute_capability.major == 10 (both sm_100 and sm_103) when flashinfer and head_dim 128 are present. Since FL2VA and Ref2VA share MiniMaxH3Pipeline, the default applies to both partitions, but the PR body's Testing section reports only B300 (sm_103) FL2VA runs. The recipe (recipes/MiniMaxAI/MiniMax-H3.md:265) advertises 'B200 also supports int8 Q/K' while the PR body says the INT8 SAGE kernel was skipped/unavailable on SM103. Please either add a B200/sm_100 and Ref2VA validation row (or scope the advertised default), and hold the B200 int8 SAGE claim until it has a run. (evidence: ['vllm_omni/diffusion/model_metadata.py:46 (this diff) ` attention_mask_free=True,` inside the `"MiniMaxH3Pipeline": DiffusionModelMetadata(` block.', 'vllm_omni/diffusion/attention/layer.py:83 (unchanged by this diff, present in PR-time tree) ` allow_trtllm_default = get_diffusion_model_metadata(model_class_name).attention_mask_free` — the metadata flag is the only gate for the TRTLLM default.', 'vllm_omni/platforms/cuda/platform.py:208 (unchanged by this diff) ` and compute_capability.major == 10` inside `trtllm_attn_default_ok = (allow_trtllm_default and compute_capability is not None ...)` — gates the default for both sm_100 and sm_103.', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:2 `"""vLLM-Omni pipeline for MiniMax H3 FL2VA and Ref2VA partitions."""` — both partitions use the same pipeline class, so the class-name-keyed metadata applies to both.', 'recipes/MiniMaxAI/MiniMax-H3.md:265 (this diff) `fp8_e4m3` for Q/K; B200 also supports `int8` Q/K. The TRTLLM SAGE path fixes V` — advertises B200 int8 SAGE with no B200 run.', 'PR body (PR 5779): `the skipped INT8 SAGE kernel is unavailable on SM103` and the Testing section lists only `B300 SXM6 AC ... 209 frames` FL2VA runs — no B200/sm_100 or Ref2VA TRTLLM evidence.']) + +`recipes/MiniMaxAI/MiniMax-H3.md:~38` [minor] — Reword recipes/MiniMaxAI/MiniMax-H3.md:38-39 from 'Install the optional dependency only for the four-GPU B300/GB200 `FLASH_ATTN` profile:' to 'Install the optional dependency only if you explicitly select `--diffusion-attention-backend FLASH_ATTN` on Blackwell; the default four-GPU TRTLLM_ATTN profile does not need it.' (evidence: recipes/MiniMaxAI/MiniMax-H3.md:38-39 `Install the optional dependency only for the four-GPU B300/GB200 `FLASH_ATTN` profile:` — stale sentence. recipes/MiniMaxAI/MiniMax-H3.md:209 `- dense BF16 `TRTLLM_ATTN`, with Ring and TP left at 1.` — the four-GPU profile now uses TRTLLM_ATTN. recipes/MiniMaxAI/MiniMax-H3.md:244 `--diffusion-attention-backend TRTLLM_ATTN` and recipes/MiniMaxAI/MiniMax-H3.md:256 `--diffusion-attention-backend FLASH_ATTN` — FA4/FLASH_ATTN is now documented only as the explicit opt-in; a grep of the file shows the four-GPU command blocks contain no `--diffusion-attention-backend` line (backend flags remain only at lines 102, 142, 244, 256).) + +`recipes/MiniMaxAI/MiniMax-H3.md:~595` [minor] — At recipes/MiniMaxAI/MiniMax-H3.md:595, the 'Validated four-GPU evidence' table has no attention-backend column, and the four-GPU profile directly above it was changed in this PR from explicit `--diffusion-attention-backend FLASH_ATTN` to the TRTLLM default. The PR body also states the dense TRTLLM 50-step end-to-end latency was not reported as steady-state. Annotate the backend that produced each row, or re-measure under the TRTLLM default so the recommended route has a matching validation row. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:590 `| Workload | Configuration | Observed result |`; recipes/MiniMaxAI/MiniMax-H3.md:595 `| FL2VA, 209 frames, 1248x768 | no offload, U4, VPP4 tile, regional compile | 86.964 s mean HTTP client latency |` — the table is unchanged by this diff (present at the merge-base) and contains no attention-backend column. recipes/MiniMaxAI/MiniMax-H3.md:209 `- dense BF16 `TRTLLM_ATTN`, with Ring and TP left at 1.` and recipes/MiniMaxAI/MiniMax-H3.md:240 `` `TRTLLM_ATTN`; no attention backend flag is required.`` — the four-GPU profile now defaults to TRTLLM without a backend flag, whereas the diff removed the old `--diffusion-attention-backend FLASH_ATTN` from that command.) + +**Additional observations (beyond the comment budget):** +- `recipes/MiniMaxAI/MiniMax-H3.md:209` [minor] recipes/MiniMaxAI/MiniMax-H3.md:209 and the text-encoder-TP four-GPU command drop `--diffusion-attention-backend FLASH_ATTN` and therefore depend on the TRTLLM platform default, but the Prerequisites install only `uv pip +- `docs/user_guide/diffusion/attention_backends.md:17` [minor] docs/user_guide/diffusion/attention_backends.md:17 advertises a packed-metadata entry point without stating the contract enforced in vllm_omni/diffusion/attention/backends/trtllm_attn.py:261-406. Document the required ke + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.trace.json.gz new file mode 100644 index 000000000..7e3146170 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5779.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.cost.json new file mode 100644 index 000000000..6fde6ed88 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1635.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5801/run-20260815-141601-393b5e", + "recorded_at": "2026-08-15T14:43:14", + "attempts": [ + { + "invocation_id": "inv-e2434fdf", + "outcome": "run:run-20260815-141601-393b5e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-141601-393b5e", + "llm_calls": 161, + "input_tokens": 1023186, + "output_tokens": 334167, + "usd": 0.929344, + "source": "spans" + } + ], + "attempt_usd_total": 0.929344, + "llm_calls": 161, + "input_tokens": 1023186, + "output_tokens": 334167, + "usd": 0.929344, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-141601-393b5e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9293, + "minutes": 27.22, + "ci_minutes": 0, + "input_tokens": 1023186, + "output_tokens": 334167, + "tool_calls": 326, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.019, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 161, + "by_role": { + "agent": { + "usd": 0.876918, + "input_tokens": 995799, + "output_tokens": 293295, + "calls": 158 + }, + "reducer": { + "usd": 0.052426, + "input_tokens": 27387, + "output_tokens": 40872, + "calls": 3 + } + }, + "cache_read_tokens": 10574080, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.52, + "gate": 1.55, + "review": 1627.85, + "report": 0.06 + }, + "time_to_first_result_s": 1632.96 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5801.trace.json.gz", + "trace_bytes": 42177 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.md new file mode 100644 index 000000000..17110eba7 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5801, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1ef0cb1562a6) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 12 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] base rope.py RotaryEmbeddingWan.forward_npu passed 5 args to the 4-param apply_rotary_emb_mindiesd (would TypeError); this PR adds half_head_dim and fixes the call — residual: the real mindiesd rank contract is still unverified. +- [resolved] Earlier branch revision f43158e9 had forward_native compute in self.dtype (would drop fp32 accumulation); final head restored x.to(torch.float32) at norm.py:131. Residual: eager CUDA fused path still computes bf16 internally (test_norm.py:625) and is not parity-checked for H3. +- [claim-verified] PR body claims MiniMax-H3 starts using Omni RMSNorm + RoPE; confirmed: minimax_h3_transformer.py:36 imports RMSNorm, :335 constructs `RotaryEmbedding(is_neox_style=True, half_head_dim=False)`, and `_apply_rope` calls `self.rope(...)` at :361. +- [claim-refuted] PR body performance claim '-3.35%': arithmetic (491014-475070)/491014 = 3.2472%, not 3.35%; the quoted latency delta is -15944 ms, or -3.25%. +- [claim-refuted] PR body says -3.35%, but its own numbers give 3.247% (15944/491014), i.e. -3.25%. +- [claim-refuted] PR body '-3.35%' recomputes to -3.247% ((491014-475070)/491014); minor rounding/arithmetic drift in the stated latency gain. +- [claim-refuted] PR body E2E numbers: (491014-475070)/491014 = 3.247% via calc, not the claimed -3.35%; the stated numbers self-contradict. +- [validated] RMSNorm native-fp32 contract: `forward_native` at norm.py:126-132 still does `x = x.to(torch.float32)` and `self.weight.to(torch.float32)`; `_norm` now passes `dtype=_BF16_DTYPE` so the checkpoint gamma stays bf16 — matches the stated purpose. +- [validated] Native RoPE equivalence: pre-PR `_apply_rope` computed `(x_rot*cos)+(_rotate_half(x_rot)*sin)`; `apply_rotary_emb_torch` (rope.py:22-37) computes the same `x*cos + rotate_half(x)*sin` for the non-interleaved path, and `_prepare_half_head_dim_cos_sin` slices 96 full-dims down to 48 half-dims — equal to the removed manual half-slicing for the duplicated H3 frequencies. +- [validated] PR state: gh_pr_view shows state MERGED and mergeable UNKNOWN at head 1ef0cb15; merge commit 1ef0cb15 is a plain `Merge branch 'main'` with no semantic conflict in the three touched files. +- [validated] No missing pytestmark: test_norm.py:12, test_rope_broadcast.py:21, test_minimax_h3_parallel.py:8 all carry core_model+cpu markers, so the new tests are collected/selected by the AMD CPU lane. +- [validated] RMSNorm dtype default stays torch.float32 (norm.py:68), so existing callers keep fp32 weight; only minimax_h3_transformer.py:177 passes dtype=bf16. +- [validated] RMSNorm.forward_native casts x and weight to float32 (norm.py:130-134), preserving the fp32 accumulation contract. +- [validated] H3 RoPE native parity holds: test computes expected from duplicated freqs with atol=0 and matches the first-half slice at rope.py:125. + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177` [minor] — This diff moves H3 `_norm` from `nn.RMSNorm` to the custom `RMSNorm` (minimax_h3_transformer.py:177). In the Test Plan's serve command `--enforce-eager` is absent and `enforce_eager` defaults to False (omni_config.py:269); H3 declares `_repeated_blocks = ["MiniMaxH3DiTBlock"]` (minimax_h3_transformer.py:798) so its blocks are regionally compiled, and `RMSNorm.forward_cuda` returns `forward_native` whenever `torch.compiler.is_compiling()` (norm.py:92-93). The fused `vllm._custom_ops.rms_norm` path therefore cannot contribute to the reported E2E gain — only RoPE (and the native RMSNorm fallback) can. Scope the claim to RoPE and/or add an `--enforce-eager` H3 benchmark plus eager q/k RMSNorm parity, because the fused bf16 kernel computes in bf16 internally (test_norm.py:625) and the current visual precision comparison runs the compile/native path, not the fused one. (evidence: ['Added by this diff: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177 ` return RMSNorm(size, eps=eps, dtype=dtype)` — `_norm` now builds the custom op instead of `nn.RMSNorm`.', 'Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/layers/norm.py:92-93 ` if torch.compiler.is_compiling():` / ` return self.forward_native(x)` — the fused op is bypassed during compile tracing.', 'Unchanged by this diff, present in the PR-time tree: vllm_omni/config/omni_config.py:269 ` enforce_eager: bool = False` — default compile path, matching the Test Plan serve command without `--enforce-eager`.', 'Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:798 ` _repeated_blocks = ["MiniMaxH3DiTBlock"]` — regional compile target.', 'Unchanged by this diff (pre-existing line in the modified test file), present in the PR-time tree: tests/diffusion/layers/test_norm.py:625 ` (vllm._custom_ops.rms_norm) computes in bf16 precision internally,` — documents the eager-only fused precision loss the visual comparison does not cover.']) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177` [minor] — The comment above `_norm` still describes `torch.nn.RMSNorm` and claims fp32 accumulation, but the function now returns the custom `RMSNorm`. On CUDA the dispatch path is `forward_cuda` -> `_forward_fused` -> `vllm._custom_ops.rms_norm`, which tests/diffusion/layers/test_norm.py documents as computing in bf16 internally; fp32 accumulation holds only for `forward_native` (and the torch.compile fallback). Please update the comment to the real contract and confirm reduced-precision fused norm is intended for H3. (evidence: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:175 ` # torch.nn.RMSNorm upcasts reduced-precision inputs for the variance` ; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177 ` return RMSNorm(size, eps=eps, dtype=dtype)` ; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/layers/norm.py:95 ` return self._forward_fused(x)` ; vllm_omni/diffusion/layers/norm.py:80 ` fused_rms_norm(out, x_2d, self.weight.data, self.variance_epsilon)` ; tests/diffusion/layers/test_norm.py:625 ` (vllm._custom_ops.rms_norm) computes in bf16 precision internally,`) + +`tests/diffusion/layers/test_norm.py:322` [minor] — test_norm.py:328-337 — the mocked `npu_rms_norm` returns `(x,)` unchanged and the test only asserts `norm.forward_npu(x) is x`, `captured["gamma"] is norm.weight`, `captured["gamma"].dtype == torch.bfloat16`, and epsilon. It never asserts output dtype or values under the real torch_npu contract, so a bf16 gamma/input path that misbehaves in the real kernel would not be caught. Consider asserting output dtype/value contract or naming the platform test covering the real NPU kernel. (evidence: tests/diffusion/layers/test_norm.py:331 ` return (x,)`; tests/diffusion/layers/test_norm.py:337 ` assert norm.forward_npu(x) is x`; tests/diffusion/layers/test_norm.py:339 ` assert captured["gamma"].dtype == torch.bfloat16` — mocked op returns input unchanged, so no output dtype/value contract of the real `torch_npu.npu_rms_norm` is verified.) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:~335` [minor] — This line introduces a new per-attention RotaryEmbedding that runs through CustomOp.forward dispatch (the default _forward_method set by CustomOp.__init__, custom_op.py:17; CUDA -> forward_cuda at custom_op.py:22-23, ROCm -> forward_hip). The only H3 RoPE correctness test overrides `attention.rope._forward_method = attention.rope.forward_native` (test_minimax_h3_parallel.py:73), so the fused CUDA/HIP dispatch path the PR title claims is never exercised. With default `diffusion_compile_granularity='regional'` (data.py:716) and `_run_packed_attention` marked `@torch.compiler.disable` (minimax_h3_transformer.py:367), `_apply_rope` runs inside the compiled region and hits forward_cuda/forward_hip, which feed half-dim cos/sin from _prepare_half_head_dim_cos_sin into vllm's apply_rotary_emb. Remove the native override and drive _apply_rope through CustomOp.forward (or add a CUDA/HIP dispatch test) to prove the fused path. (evidence: Diff lines: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335 ` self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)`; tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:73 ` attention.rope._forward_method = attention.rope.forward_native`. Unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/layers/custom_op.py:17 ` self._forward_method = self.dispatch_forward()` and :22-23 ` elif current_omni_platform.is_cuda():` / ` return self.forward_cuda`; vllm_omni/diffusion/data.py:716 ` diffusion_compile_granularity: str = "regional"`; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:367 ` @torch.compiler.disable`.) + +`vllm_omni/diffusion/layers/rope.py:~122` [minor] — `_prepare_half_head_dim_cos_sin` (vllm_omni/diffusion/layers/rope.py:114-127) now centralizes the 3D→2D batch-dim strip for the CUDA/HIP/native paths, but it still silently keeps only `cos[0]`/`sin[0]` and has no assertion that all batch elements share the same rotary encoding. Any 3D per-batch cos/sin reaching this helper loses batch elements 1..B-1. Add `assert cos.shape[0] == 1 or torch.equal(cos[0].expand_as(cos), cos)` (same for sin), or restore backend-local handling with a documented caller contract. Note: current Flux2 `apply_rope_to_qk` callers build 2D `[S, D]` cos/sin, so this is a defensive gap rather than an active failure today. (evidence: Present in the PR diff: vllm_omni/diffusion/layers/rope.py:120 `if cos.dim() == 3:`; :121 `# All batch elements share the same rotary position encoding.`; :122 `cos = cos[0]`; :123 `sin = sin[0]`. The helper `_prepare_half_head_dim_cos_sin` (line 114) contains no assertion before this strip (grep for `assert cos` returns no matches), so 3D per-batch cos/sin are silently reduced to the first batch element.) + +`vllm_omni/diffusion/layers/rope.py:125` [minor] — For `half_head_dim=False`, `_prepare_half_head_dim_cos_sin` silently keeps only the first half of full-dim cos/sin (`cos[..., : cos.shape[-1] // 2]` for neox style). This is only valid when full-dim frequencies are two identical halves — true for H3 (`MiniMaxH3Rope.forward` returns `torch.cat((half, half), dim=-1)`), but the invariant is neither asserted nor documented. Add an assert (e.g. `cos[..., :d//2] == cos[..., d//2:]`) or a docstring so a future `half_head_dim=False` caller with distinct full frequencies fails loudly instead of silently mis-rotating. (evidence: vllm_omni/diffusion/layers/rope.py:125 ` cos = cos[..., ::2] if self.interleaved else cos[..., : cos.shape[-1] // 2]` — neox-style branch discards the second half of full-dim cos/sin with no assert. vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:227 ` return torch.cat((half, half), dim=-1) # [S, 96]` — H3's only current full-dim producer duplicates halves, unchanged by this diff, present in the PR-time tree. vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335 ` self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)` — the sole half_head_dim=False consumer.) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335` [minor] — This PR bundles two independent kernel swaps — self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False) at minimax_h3_transformer.py:335 and the _norm swap to RMSNorm at line 177 — but the Test Result reports one combined E2E latency (491014ms -> 475070ms, -3.35%) and one visual precision video. That cannot attribute the gain to either kernel or show whether either changes output numerics. Split into (a) RoPE-only and (b) RMSNorm-only runs with the same seed/config, and report per-cell latency plus a quantitative accuracy metric (e.g. max abs diff or PSNR/SSIM on decoded frames). (evidence: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335 ` self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)`; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177 ` return RMSNorm(size, eps=eps, dtype=dtype)`; PR 5801 description (untrusted data, not in diff): `E2E Latency: 491014ms -> 475070ms. **-3.35%**` and `**Precision:** ori (left) vs opt (right)` with a single comparison video — no per-kernel split.) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335` [minor] — Anchor the E2E claim to the head being merged: the PR body reports commit 5215e03a91adecbb5ffece29aa74360a7569d0c5, but head 1ef0cb1562a6 is a merge that pulled main's H3 transformer/pipeline/offloader changes, so the 491014→475070 latency delta may not describe the reviewed tree. Re-run the serve benchmark at head, or explicitly label the body numbers as pre-merge and add a head measurement. (evidence: PR body (GitHub metadata, not a file): '**vLLM-Omni Commit:** 5215e03a91adecbb5ffece29aa74360a7569d0c5'. git head 1ef0cb1562a67a0ecb1d773fb582865f386fd8c5: 'Merge: aaeaa000 9235b0ae' and 'Merge branch 'main' into h3_opt'; its stat lists main-pulled H3 files such as 'vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py | 35 +-', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | 135 ++++--', and 'vllm_omni/diffusion/offloader/distributed_layerwise_backend.py | 465 ++++++++++++++++-----'. Cited line exists in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335 `self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177` [minor] Correct the PR body's performance percentage: 'E2E Latency: 491014ms -> 475070ms' is a 3.25% reduction ((491014-475070)/491014 = 3.2472%, delta -15944 ms), but the body states -3.35%. Update the claim to -3.25%. +- `tests/diffusion/layers/test_norm.py:317` [minor] This test only proves `forward_native` is self-consistent with the formula written in the test itself: `expected` multiplies by `norm.weight.float()` exactly like the implementation at vllm_omni/diffusion/layers/norm.py: +- `tests/diffusion/layers/test_rope_broadcast.py:242` [minor] This test monkeypatches `sys.modules['mindiesd']` after module import and then calls the free function `apply_rotary_emb_mindiesd` directly, so it never exercises `RotaryEmbedding.forward_npu` or the `self.has_mindie` fl +- `.buildkite/amd/test-amd-ready.yml:19` [minor] `buildkite/vllm-omni-amd-ci` is FAILURE on this PR. The changed test files all carry `core_model`+`cpu` markers (tests/diffusion/layers/test_norm.py:12, tests/diffusion/layers/test_rope_broadcast.py:21, tests/diffusion/m + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.trace.json.gz new file mode 100644 index 000000000..2efc22da4 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5801.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.cost.json new file mode 100644 index 000000000..ba921255d --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 535.5, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5833/run-20260815-142011-000b01", + "recorded_at": "2026-08-15T14:29:05", + "attempts": [ + { + "invocation_id": "inv-f53a8978", + "outcome": "run:run-20260815-142011-000b01", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-142011-000b01", + "llm_calls": 52, + "input_tokens": 137310, + "output_tokens": 90311, + "usd": 0.184368, + "source": "spans" + } + ], + "attempt_usd_total": 0.184368, + "llm_calls": 52, + "input_tokens": 137310, + "output_tokens": 90311, + "usd": 0.184368, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-142011-000b01", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1844, + "minutes": 8.89, + "ci_minutes": 0, + "input_tokens": 137310, + "output_tokens": 90311, + "tool_calls": 108, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3699, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 52, + "by_role": { + "agent": { + "usd": 0.166225, + "input_tokens": 130390, + "output_tokens": 75554, + "calls": 50 + }, + "reducer": { + "usd": 0.018143, + "input_tokens": 6920, + "output_tokens": 14757, + "calls": 2 + } + }, + "cache_read_tokens": 1776000, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.79, + "gate": 1.68, + "review": 527.62, + "report": 0.07 + }, + "time_to_first_result_s": 533.13 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5833.trace.json.gz", + "trace_bytes": 19548 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.md new file mode 100644 index 000000000..236c29031 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5833, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac67e8874d00) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] ar_diffusion_pipeline_capability.md is absent from nav/index, but its own header marks it 'experimental and non-normative... not yet part of the stable module-design hierarchy' — deliberate exclusion, not an omission. +- [claim-verified] diff_stat shows exactly 2 files changed: docs/.nav.yml and docs/design/index.md — matches PR body claim 'final PR diff contains only docs/.nav.yml and docs/design/index.md'. +- [claim-verified] No design-document content modified at this head: both touched files are navigation/landing only, and every linked document pre-exists (read_file + list_dir of docs/design). +- [claim-verified] diff_stat shows only docs/.nav.yml and docs/design/index.md changed (91 insertions / 24 deletions), matching the PR's 'only these two files' claim. +- [claim-verified] 'does not add or modify design documents' holds at this head: no .md under docs/design/feature|module changed; only nav + index touched. +- [claim-verified] 'final PR diff contains only docs/.nav.yml and docs/design/index.md' — diff_stat output: 2 files changed. +- [claim-verified] 'does not add or modify design documents' — the diff contains no docs/design/feature/*.md changes; async_diffusion_output.md, pipeline_parallel.md and qwen3_omni_tts_performance_optimization.md pre-exist and are only newly linked. +- [claim-verified] 'existing document URLs stable' — no file renamed or moved in the diff; connector pages keep their individual URLs. +- [validated] Every old nav feature/module entry is retained in the new nav: disaggregated, ray_based, prefix_caching, cfg/expert/sequence/tensor/vae/hsdp, cache_dit, teacache, skip_softmax, async_chunk, omni_async_output_materialization, diffusion_continuous_batching, ar/dit/entrypoint/async_omni modules (file_at_base docs/.nav.yml vs head read). +- [validated] Duplicate vae_parallel.md entry removed: old nav listed design/feature/vae_parallel.md twice; new nav lists it once. +- [validated] omni_connectors/ directory entry expanded to exactly the 6 existing connector .md files (list_dir shows 6; all 6 appear in new nav and index.md). +- [validated] Newly linked targets exist: docs/design/feature/pipeline_parallel.md, docs/design/feature/async_diffusion_output.md, docs/design/metrics.md, docs/design/qwen3_omni_tts_performance_optimization.md. +- [validated] docs/user_guide/quantization/overview.md exists, so both nav leaf and index.md ../ link resolve (list_dir docs/user_guide/quantization). +- [validated] index.md relative link `../user_guide/quantization/overview.md` resolves from docs/design/ to docs/user_guide/quantization/overview.md (correct parent hop). + +`docs/design/index.md:53` [minor] — docs/design/index.md adds `#### Quantization` (line 49) and leaves the unchanged `Cache-DiT`, `TeaCache`, and `Diffusion Continuous Batching` bullets (lines 53-55) directly beneath it, presenting cache/batching docs as quantization docs. In docs/.nav.yml those three are siblings of `Quantization` under `Diffusion Acceleration` (lines 131-134). Add a `#### Cache Acceleration` heading before Cache-DiT/TeaCache and move `Diffusion Continuous Batching` out as a direct sibling under `### Diffusion acceleration` so the landing page matches the sidebar. (evidence: docs/design/index.md:49 `#### Quantization`; docs/design/index.md:53 `- [Cache-DiT](feature/cache_dit.md)`; docs/design/index.md:54 `- [TeaCache](feature/teacache.md)`; docs/design/index.md:55 `- [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)` — all nested under Quantization on the landing page. Contrast docs/.nav.yml:131 ` - Quantization: user_guide/quantization/overview.md`; docs/.nav.yml:132 ` - Cache-DiT: design/feature/cache_dit.md`; docs/.nav.yml:133 ` - TeaCache: design/feature/teacache.md`; docs/.nav.yml:134 ` - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md` — siblings under `Diffusion Acceleration`, not under Quantization.) + +`docs/.nav.yml:133` [minor] — This diff adds `- Quantization: user_guide/quantization/overview.md` at docs/.nav.yml:133 under `Diffusion Acceleration`, but the linked overview (docs/user_guide/quantization/overview.md:4-5) states the `quantization_config` entrypoint is shared by diffusion-only, multi-stage omni/TTS, and multi-stage diffusion models, so classifying it as diffusion acceleration is misleading and duplicates the existing link at docs/.nav.yml:76. Move the link out of `Diffusion Acceleration` (e.g. a general quantization group in Design Documents) or relabel it to reflect the general scope. (evidence: docs/.nav.yml:133 ` - Quantization: user_guide/quantization/overview.md` (added by this diff, nested under `Diffusion Acceleration`); docs/.nav.yml:76 ` - Overview: user_guide/quantization/overview.md` (unchanged by this diff — pre-existing link under User Guide > Features > Quantization); docs/user_guide/quantization/overview.md:4-5 `path. The same configuration entrypoint is used across diffusion-only models,` / `multi-stage omni/TTS models, and multi-stage diffusion models, but each model` (unchanged by this diff — the linked page states the entrypoint spans all model types, not just diffusion acceleration).) + +`docs/design/index.md:4` [nit] — docs/design/index.md:4 now states the sidebar groups documents by system concern, but repo-wide grep for `ar_diffusion_pipeline_capability` returns no matches in any file content; the file exists at docs/design/ar_diffusion_pipeline_capability.md and is referenced by neither docs/design/index.md nor docs/.nav.yml, while its own status note marks it experimental/non-normative and outside the stable hierarchy. Consider adding one sentence acknowledging the note is intentionally unlisted (or list it under an Experimental subsection). (evidence: docs/design/index.md:4 `vLLM-Omni. The sidebar groups documents by the system concern they describe,`; docs/design/ar_diffusion_pipeline_capability.md:3 `> **Status:** experimental and non-normative. This note documents the current`; repo-wide grep for `ar_diffusion_pipeline_capability` over the PR-time tree returned no matches, while the directory listing shows `docs/design/ar_diffusion_pipeline_capability.md` exists — so neither the design index nor the nav references the unlisted note.) + +`docs/.nav.yml:109` [nit] — docs/.nav.yml:109 labels the link 'Ray-based Execution', and docs/design/index.md:16 uses the same label, but docs/design/feature/ray_based_execution.md:1 still has H1 '# Distributed utils'. Align the page H1 with the nav/index label (preferred) or relabel nav/index. (evidence: changed by this diff: docs/.nav.yml:109 `- Ray-based Execution: design/feature/ray_based_execution.md`; changed by this diff: docs/design/index.md:16 `- [Ray-based Execution](feature/ray_based_execution.md)`; unchanged by this diff, present in the PR-time tree: docs/design/feature/ray_based_execution.md:1 `# Distributed utils`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.trace.json.gz new file mode 100644 index 000000000..c37ca438a Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5833.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.cost.json new file mode 100644 index 000000000..1985a05c7 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1534.0, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5864/run-20260815-142306-3d9e37", + "recorded_at": "2026-08-15T14:48:38", + "attempts": [ + { + "invocation_id": "inv-e50cf9da", + "outcome": "run:run-20260815-142306-3d9e37", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-142306-3d9e37", + "llm_calls": 167, + "input_tokens": 1382438, + "output_tokens": 325468, + "usd": 1.205381, + "source": "spans" + } + ], + "attempt_usd_total": 1.205381, + "llm_calls": 167, + "input_tokens": 1382438, + "output_tokens": 325468, + "usd": 1.205381, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-142306-3d9e37", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2054, + "minutes": 25.53, + "ci_minutes": 0, + "input_tokens": 1382438, + "output_tokens": 325468, + "tool_calls": 283, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0832, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 167, + "by_role": { + "agent": { + "usd": 1.16775, + "input_tokens": 1346921, + "output_tokens": 300017, + "calls": 165 + }, + "reducer": { + "usd": 0.037631, + "input_tokens": 35517, + "output_tokens": 25451, + "calls": 2 + } + }, + "cache_read_tokens": 17559552, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.09, + "gate": 1.56, + "review": 1525.98, + "report": 0.06 + }, + "time_to_first_result_s": 1531.68 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5864.trace.json.gz", + "trace_bytes": 59274 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.md new file mode 100644 index 000000000..17cb2efa6 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.md @@ -0,0 +1,69 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5864, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 07fc831e7a43) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Concern: does output_rank=None make all ranks reply on control RPCs and leave stale replies? At head `output_rank = None if multi_rank_reply else (... else 0)`, so control RPCs (exec_all_ranks=False) keep output_rank=0 and only rank 0 replies — residual: DP multi-rank replies are the only path emitting multiple replies. +- [resolved] Concern: hardcoded primary rank = dp_rank*replica_size could mismatch real topology? Verified initialize_model_parallel uses RankGenerator(order='tp-sp-pp-cfg-dp') so primary ranks are exactly dp_rank*replica_size — residual: contract is duplicated and only asserted against mocked queues, not the real group construction (comment 3). +- [resolved] collect_rank_status flip is safe: only the DP multi call uses unique_reply_rank=None+exec_all_ranks=True; control-plane RPCs still use exec_all_ranks=False so collect_rank_status stays True. +- [claim-verified] multiproc result path now uses per-worker queues: multiproc_executor.py:123 `self._result_mqs = [self._init_result_queue(handle) for handle in result_handles]` and _launch_workers collects every rank's result_handle (line 347). +- [claim-verified] DP wave preflight rejects before dispatch: execute_request raises ValueError for incompatible keys/extra_args/empty prompt before collective_rpc; tests assert `executor.collective_rpc.assert_not_called()`. +- [claim-verified] fail-closed on partial wave: `_fail_closed_on_dp_wave_timeout` sets `_is_failed=True` then `self.shutdown()` (multiproc_executor.py:414-431); test_dlo_dp_partial_reply_times_out_and_fails_closed covers it. +- [claim-verified] single shutdown owner with global 15s/5s budget implemented in `_ExecutorShutdownCleaner.__call__` and asserted by TestExecutorShutdownCleaner.test_worker_joins_share_one_global_deadline. +- [claim-verified] test_per_worker_result_queues_release_nested_numpy_shm_and_processes actually spawns 2 workers, pumps both queues, unpacks nested numpy SHM, and asserts exitcode 0 + no leaked segment. +- [claim-verified] PR description claim 'request wave hung' is addressed by `_DLO_DP_WAVE_TIMEOUT_S` + `_fail_closed_on_dp_wave_timeout` in multiproc_executor.py:414-431; partial-reply timeout is exercised by test_dlo_dp_partial_reply_times_out_and_fails_closed. +- [claim-refuted] The PR removes the only mechanism that made DLO DP waves form by default: engine no longer sets request_batch_max_wait_ms=500 when 0, OmniDiffusionConfig default is 0.0 (data.py:845), async serving passes default 0.0 (async_omni_engine.py:1026), and _wait_for_request_batch_admission_locked returns immediately at `if max_wait_s == 0: return` — concurrency silently serializes under default config. +- [claim-refuted] architecture 'RequestScheduler collect up to DP-size requests' is not delivered by default: config default request_batch_max_wait_ms=0.0 makes the admission wait return immediately (see comment 1). +- [validated] sampling_params.clone() is copy.deepcopy (inputs/data.py:372-373), so inline add_request_async cloning actually isolates guidance_scale/extra_args between concurrent requests. +- [validated] WorkerProc.shutdown joins the async-output thread before releasing result_mq, and finally-block releases mq/result_mq even when worker.shutdown raises (tests confirm). +- [validated] request-mode DP collection reads num_responses from the shared _sync_result_buffer (pumps are sole readers of worker queues) and _validate_wave_id discards stale wave_ids; worker tags reply dicts with wave_id (_execute_rpc). + +`vllm_omni/diffusion/diffusion_engine.py:251` [major] — Confirmed. In `_init_runtime_state` the diff removes `self.od_config.request_batch_max_wait_ms = 500.0`, and with the default 0.0 (vllm_omni/diffusion/data.py:845) `_wait_for_request_batch_admission_locked` exits at diffusion_engine.py:479-480 (`if max_wait_s == 0: return`), so two sequential DLO DP requests are scheduled one-by-one instead of accumulating into a `dp_size` wave. The example only works because it passes `--batch-wait-ms 500`. Restore a local effective wait for `dp_concurrent` without mutating `od_config`, or require/document `request_batch_max_wait_ms > 0` for DLO DP, and add an integration test that two sequential requests under the default form one wave. (evidence: vllm_omni/diffusion/diffusion_engine.py:478-480 `max_wait_s = self.od_config.request_batch_max_wait_ms / 1000.0` / `if max_wait_s == 0:` / `return` — the admission gate exits immediately when the wait is zero. vllm_omni/diffusion/data.py:845 `request_batch_max_wait_ms: float = 0.0` — unchanged by this diff, present in the PR-time tree, so the default is zero. vllm_omni/diffusion/diffusion_engine.py:250-253 `logger.info(... "dp_concurrent: max_num_running_reqs=%d, batch_wait=%sms", dp_size, self.od_config.request_batch_max_wait_ms ...)` — `_init_runtime_state` now only reads the config value and never assigns 500.0 (the assignment is removed in the diff).) + +`vllm_omni/diffusion/executor/multiproc_executor.py:734` [minor] — Confirmed with corrections. multiproc_executor.py:734 computes DP primary queues as `primary_ranks = [dp_rank * replica_size for dp_rank in range(num_responses)]` (replica_size = tp*sp*pp*cfg), and the worker's reply gate at diffusion_worker.py:1011 sends only from SP/CFG/TP/PP rank==0 primaries. The two agree because initialize_model_parallel hard-codes RankGenerator `order="tp-sp-pp-cfg-dp"`; allgather/ulysses/ring only alter sequence_parallel_size (already inside replica_size) and HSDP is a separate fs dimension not combined with this DLO DP path, so those degrees do not change this rank order today. The actual gap is that this layout assumption is duplicated implicitly in the executor and in test_step_execution_reads_each_dp_primary_result_queue, whose mocked queues are keyed by global-rank index with the same hard-coded expected_primary_ranks, so a future RankGenerator order change would not be caught. Keep the minor severity; consider deriving primary ranks from get_dp_group()/the initialized TP/SP/PP/CFG groups or adding an assertion against the RankGenerator order, and document the supported axes. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:734 `primary_ranks = [dp_rank * replica_size for dp_rank in range(num_responses)]` — added by this diff; vllm_omni/diffusion/worker/diffusion_worker.py:1011 `should_reply = is_primary_in_replica and self.result_mq is not None` — unchanged by this diff, present in the PR-time tree, where the `is_primary_in_replica` predicate immediately above requires `get_sequence_parallel_rank() == 0 and get_classifier_free_guidance_rank() == 0 and get_tensor_model_parallel_rank() == 0 and get_pipeline_parallel_rank() == 0`; vllm_omni/diffusion/distributed/parallel_state.py (inside `initialize_model_parallel`) `RankGenerator(..., order="tp-sp-pp-cfg-dp")` — unchanged by this diff, the order that makes `dp_rank * tp*sp*pp*cfg` equal the primary rank.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:930` [minor] — WorkerProc.shutdown only releases result_mq when the async-output thread is absent or has exited. If that thread remains alive after the 10s join, the SHM-backed result queue is skipped and left for process-exit cleanup, weakening the full-IPC-release guarantee this diff adds in worker_main's finally block. Consider explicitly disposing result_mq on the timeout path as well (once the thread can no longer enqueue), or prove process-exit finalization reliably unlinks the segment. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:930 `def shutdown(self) -> None:` ... and inside it: `if self.result_mq is not None and (self._async_output_thread is None or not self._async_output_thread.is_alive()):` followed by `result_mq = self.result_mq`, `self.result_mq = None`, `result_mq.shutdown()`, `del result_mq`, `gc.collect()` — so when `self._async_output_thread.is_alive()` is True after the 10s join, the shutdown call is skipped. The caller path is unchanged by this diff: worker_main's finally block at vllm_omni/diffusion/worker/diffusion_worker.py:1238 invokes `worker_proc.shutdown()` then `worker_proc.context.term()` before the worker process returns.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:47` [minor] — `_is_empty_dp_prompt` (multiproc_executor.py:47) only treats None, empty str, and dicts with a truthy 'prompt' key as empty. OmniPromptType also admits token prompts, embeds prompts, OmniCustomPrompt (`prompt_ids`), and list forms; an empty prompt in those shapes returns False and is still dispatched. The new wave timeout and fail-closed shutdown bound the resulting hang, so this is a robustness gap rather than a permanent hang, but the preflight does not cover all valid prompt shapes — mirror the worker's prompt normalization or reject unknown shapes, and add list/tokenized-empty cases. (evidence: In the PR diff, vllm_omni/diffusion/executor/multiproc_executor.py:41-49: `def _is_empty_dp_prompt(prompt: object) -> bool:` / ` """Return whether a DP request has no usable text prompt."""` / ` if prompt is None:` / ` return True` / ` if isinstance(prompt, str):` / ` return not prompt` / ` if isinstance(prompt, dict) and "prompt" in prompt:` / ` return not prompt["prompt"]` / ` return False`. Unchanged by this diff but present in the PR-time tree, vllm_omni/inputs/data.py:151: `OmniPromptType: TypeAlias = PromptType | OmniTextPrompt | OmniTokensPrompt | OmniEmbedsPrompt | OmniCustomPrompt` — OmniTokensPrompt is a TokensPrompt (`prompt_token_ids`), OmniCustomPrompt carries `prompt_ids`, and PromptType permits list forms, none of which the helper inspects.) + +`tests/diffusion/test_inline_stage_diffusion_client.py:166` [minor] — In tests/diffusion/test_inline_stage_diffusion_client.py, test_inline_shutdown_runs_after_orchestrator_premark (line 166) only pre-marks _shutting_down=True and asserts mock_engine.close idempotency; it never enqueues a collective_rpc_async task into the single-worker ThreadPoolExecutor, so it cannot show that the shutdown() reorder (engine.close before executor.shutdown(wait=True, cancel_futures=True)) avoids teardown blocking behind a pending control RPC. Add a regression test that submits a blocking control RPC via collective_rpc_async and asserts shutdown() completes, or assert the relative call order of engine.close and executor.shutdown. (evidence: tests/diffusion/test_inline_stage_diffusion_client.py:166 `def test_inline_shutdown_runs_after_orchestrator_premark(client, mock_engine):` +tests/diffusion/test_inline_stage_diffusion_client.py:169 ` client._shutting_down = True` +tests/diffusion/test_inline_stage_diffusion_client.py:171 ` client.shutdown()` +tests/diffusion/test_inline_stage_diffusion_client.py:173 ` mock_engine.close.assert_called_once()` +tests/diffusion/test_inline_stage_diffusion_client.py:174 ` assert client._shutdown_complete` +tests/diffusion/test_inline_stage_diffusion_client.py:176 ` client.shutdown()` +tests/diffusion/test_inline_stage_diffusion_client.py:177 ` mock_engine.close.assert_called_once()` +The test body contains no collective_rpc_async/run_in_executor call and no assertion on the order of _engine.close vs _executor.shutdown. The reorder it is supposed to protect is in the changed source hunk (vllm_omni/diffusion/inline_stage_diffusion_client.py): `self._engine.close()` now runs before `self._executor.shutdown(wait=True, cancel_futures=True)`.) + +`vllm_omni/diffusion/inline_stage_diffusion_client.py:321` [minor] — The reorder to close() before executor.shutdown only unblocks RPCs still queued. Once a control RPC is dequeued, the busy-loop thread is stuck in executor.collective_rpc and the inline thread-pool worker is stuck in DiffusionEngine.collective_rpc's task.future.result(); DiffusionEngine.close() joins the busy loop for 10s, logs 'Worker thread did not terminate within 10s; scheduler and executor shutdown will be deferred.', and returns without failing that in-flight future or shutting the executor, so the subsequent _executor.shutdown(wait=True) can still block indefinitely on a hung worker. Make close() fail in-flight RPC futures or mark the executor failed on this early return (or bound control-RPC timeouts). (evidence: inline_stage_diffusion_client.py:321 `self._engine.close()` and inline_stage_diffusion_client.py:326 `self._executor.shutdown(wait=True, cancel_futures=True)` — changed by this diff. Unchanged by this diff, present in PR-time tree: diffusion_engine.py:964 `return task.future.result(timeout=timeout)` (the inline thread-pool worker blocks here); diffusion_engine.py:553 `result = self.executor.collective_rpc(` (the busy-loop thread blocks here); diffusion_engine.py:1033-1036 `logger.warning( + "Worker thread did not terminate within 10s; scheduler and executor shutdown will be deferred." + ) + return` — close() returns early without failing the in-flight future or calling self.executor.shutdown().) + +`vllm_omni/diffusion/executor/multiproc_executor.py:486` [minor] — The request-batch compatibility key (RequestBatchSamplingParamsKey) deliberately excludes `extra_args` and has no `prompt` field, and `_can_schedule_waiting` compares only that key. In DLO DP mode `max_num_running_reqs` is `dp_size`, so two requests differing only in `extra_args` — or a valid request paired with an empty prompt — are co-scheduled and reach `execute_request`, where the pre-dispatch preflight at multiproc_executor.py:486 raises for the whole wave. The valid request is collateral damage instead of being served serially. Add DLO-specific wave compatibility (extra_args/prompt) at schedule time or a serial fallback for incompatible waves. (evidence: vllm_omni/diffusion/sched/interface.py:69 `class RequestBatchSamplingParamsKey:` with docstring: "Only request-batch-wide fields belong here. Request-local values such as seeds, generators, latent tensors, timesteps, and pipeline-specific ``extra_args`` are read per request from"; vllm_omni/diffusion/sched/request_scheduler.py:22 `_REQUEST_BATCH_SAMPLING_PARAMS_KEY_FIELD_NAMES = frozenset(field.name for field in fields(RequestBatchSamplingParamsKey)) - {"lora_int_id"}`; vllm_omni/diffusion/sched/base_scheduler.py:260 `def _can_schedule_waiting(self, state: SchedulerRequestState) -> bool:` ... :265 `return current_key is not None and current_key == state.sampling_params_key`; vllm_omni/diffusion/executor/multiproc_executor.py:486 `empty_prompt_ids = [nr.request_id for nr in new_reqs if _is_empty_dp_prompt(nr.req.prompt)]` (preceded by the extra_args-signature ValueError and followed by `raise ValueError("DP multi-concurrency requires a non-empty prompt for every request; ...")` before the RPC try block).) + +`docs/user_guide/diffusion/cpu_offload_diffusion.md:~299,309` [minor] — This diff changes DLO DP preflight so that a wave where every request has num_inference_steps=None is accepted as compatible (multiproc_executor.py:467-469; test_dlo_dp_allows_shared_default_denoise_steps). The user guide and recipe still say None is not allowed (docs/user_guide/diffusion/cpu_offload_diffusion.md:299,309; recipes/cosmos3/Cosmos3-DistOffload.md:124-125). Update both docs to match the new allowed case. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:467-469 (in this diff) ` # and LoRA settings. Two default (None) step counts are valid and` / ` # resolve identically inside the same pipeline.` / ` compatibility_keys = [build_request_batch_sampling_params_key(nr.req) for nr in new_reqs]` — None==None passes this compatibility check. tests/diffusion/test_multiproc_engine_concurrency.py:397 (in this diff) ` def test_dlo_dp_allows_shared_default_denoise_steps(self):` with both `num_inference_steps = None` asserting `executor.collective_rpc.assert_called_once()`. Unchanged by this diff, present in the PR-time tree: docs/user_guide/diffusion/cpu_offload_diffusion.md:299 `- `num_inference_steps` must be specified explicitly (None is not allowed)`; docs/user_guide/diffusion/cpu_offload_diffusion.md:309 `- `num_inference_steps=None` is not allowed in DP multi-concurrency mode`; recipes/cosmos3/Cosmos3-DistOffload.md:125 `> `num_inference_steps=None` is not allowed because it may resolve differently`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.trace.json.gz new file mode 100644 index 000000000..a20fe1373 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5864.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.cost.json new file mode 100644 index 000000000..bc097eccd --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1466.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5958/run-20260815-142419-66a8da", + "recorded_at": "2026-08-15T14:48:43", + "attempts": [ + { + "invocation_id": "inv-cc982826", + "outcome": "run:run-20260815-142419-66a8da", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-142419-66a8da", + "llm_calls": 112, + "input_tokens": 260390, + "output_tokens": 260866, + "usd": 0.448479, + "source": "spans" + } + ], + "attempt_usd_total": 0.448479, + "llm_calls": 112, + "input_tokens": 260390, + "output_tokens": 260866, + "usd": 0.448479, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-142419-66a8da", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4485, + "minutes": 24.4, + "ci_minutes": 0, + "input_tokens": 260390, + "output_tokens": 260866, + "tool_calls": 233, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7838, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 112, + "by_role": { + "planner": { + "usd": 0.002576, + "input_tokens": 2720, + "output_tokens": 1674, + "calls": 1 + }, + "agent": { + "usd": 0.393031, + "input_tokens": 235488, + "output_tokens": 216634, + "calls": 108 + }, + "reducer": { + "usd": 0.052872, + "input_tokens": 22182, + "output_tokens": 42558, + "calls": 3 + } + }, + "cache_read_tokens": 3378560, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.14, + "gate": 2.11, + "review": 1457.64, + "report": 0.06 + }, + "time_to_first_result_s": 1463.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5958.trace.json.gz", + "trace_bytes": 33688 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.md new file mode 100644 index 000000000..ba129083d --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5958, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4dd07d536f34) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 9 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] continuous_batching.md and diffusion_runtime.md both own diffusion/sched/** (2 vs 4 owners); CODEOWNERS uses the 4-owner superset, so no reviewer is dropped. +- [claim-verified] Coverage-gap list (inputs/outputs/model_extras/plugins/errors.py/request.py/data_entry_keys.py/attention/benchmarks) had no rules at base; all are now covered. +- [claim-verified] 'Coverage gaps filled' list (inputs/, outputs/, errors.py, request.py, data_entry_keys.py, attention/, benchmarks/) all now have rules and were absent from base CODEOWNERS (diff). +- [claim-verified] `*` default owners unchanged: `* @Gaohan123 @hsliuustc0106 @ywang96` (CODEOWNERS:12). +- [claim-verified] CI green: pre-commit SUCCESS, DCO SUCCESS, readthedocs SUCCESS (gh_ci_read). +- [claim-verified] PR body lists `/vllm_omni/attention/` under 'Coverage gaps filled (previously fell to the * catch-all)'; file_at_base CODEOWNERS has no attention rule, so the claim is true. +- [claim-refuted] PR body 'Not changed: /vllm_omni/lora/' — diff narrows lora from 6 to 2 owners. +- [claim-refuted] PR body offloader row 'After = yuanheng-zhao, Isotr0py, lishunyang12' — diff and offloader.md say @david6666666, not @Isotr0py. +- [claim-refuted] PR body worker row 'After' omits @Gaohan123 — ar_runtime.md and CODEOWNERS include @Gaohan123. +- [claim-refuted] PR body 'Not changed' lists /vllm_omni/lora/, but the diff narrows lora 6→2 owners (princepride,ZJY0516,RuixiangMa,yuanheng-zhao dropped) → review comment 3. +- [claim-refuted] PR body conflicts table omits the model_executor/models 9→5 shift (drops ZeldaHuang,princepride,yuanheng-zhao,Bounty-hunter,NickCao; adds tzhouam) → review comment 6. +- [claim-refuted] PR body worker after-state lists 4 owners but the new rule has 5 (omits @Gaohan123). +- [claim-refuted] PR body 'Design docs updated' section lists only 3 docs; the diff updates 12 docs (ar_runtime, benchmarking, cache_management, diffusion_runtime, diffusion_model_integration, parallelism, execution_platforms, input_output_modality_contracts, observability omitted). +- [claim-refuted] PR body lists /vllm_omni/lora/ under 'Not changed', but the diff narrows it from 6 owners to @SamitHuang @wtomin; CODEOWNERS comment itself says 'ownership intentionally narrowed'. + +`.github/CODEOWNERS:81` [major] — CODEOWNERS:81 `/vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi` contradicts its cited source docs/design/module/model_integration.md, whose front matter owns `vllm_omni/plugins/**` with owners @tzhouam @gcanlin (no primary_code_path_owners entry for plugins); the rule drops @tzhouam and adds @tjtanaa/@xuechendi, who are execution_platforms.md owners. CODEOWNERS:80 `/vllm_omni/model_extras/ @tzhouam @gcanlin @princepride` likewise adds unsourced @princepride. Either set both rules to @tzhouam @gcanlin or add primary_code_path_owners entries in model_integration.md justifying the extra owners. (evidence: ['.github/CODEOWNERS:81 `/vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi`', '.github/CODEOWNERS:80 `/vllm_omni/model_extras/ @tzhouam @gcanlin @princepride`', 'docs/design/module/model_integration.md:6 ` - "@tzhouam"`', 'docs/design/module/model_integration.md:7 ` - "@gcanlin"`', 'docs/design/module/model_integration.md:11 ` - vllm_omni/plugins/**`', 'docs/design/module/execution_platforms.md:7 ` - "@tjtanaa"` (unchanged by this diff; present in PR-time tree — corroborates that @tjtanaa is an execution_platforms owner, not a model_integration owner)']) + +`.github/CODEOWNERS:137` [major] — .github/CODEOWNERS:137 assigns only three owners to /vllm_omni/diffusion/models/, but the cited design doc lists four owners (@fhfuih, @Bounty-hunter, @wtomin, @RuixiangMa) for primary_code_paths that include vllm_omni/diffusion/models/**. Since the doc has a flat owners list (no primary_code_path_owners block) and the CODEOWNERS header says rules derive from that front matter, @RuixiangMa is silently dropped from the models path. Add @RuixiangMa here, or document the split with a primary_code_path_owners block in the design doc. (evidence: .github/CODEOWNERS:137 `/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin`; .github/CODEOWNERS:138 `/vllm_omni/diffusion/model_loader/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa`; docs/design/module/diffusion/diffusion_model_integration.md:9 ` - "@RuixiangMa"`; docs/design/module/diffusion/diffusion_model_integration.md:11 ` - vllm_omni/diffusion/models/**`) + +`.github/CODEOWNERS:79` [minor] — Head .github/CODEOWNERS:79 narrows /vllm_omni/model_executor/models/ from 9 reviewers (@linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin @NickCao) to 5 (@tzhouam @gcanlin @Sy0307 @amy-why-3459 @linyueqian), dropping @ZeldaHuang, @princepride, @yuanheng-zhao, @Bounty-hunter, and @NickCao. This exactly matches docs/design/module/model_integration.md owners (@tzhouam, @gcanlin) plus its models/** primary_code_path_owners (@Sy0307, @amy-why-3459, @linyueqian). Since these now become the standing auto-reviewers for the whole model tree, please confirm the design doc's owner list is current before this replaces prior auto-assignment. (Note: I could not find in-tree evidence that the dropped names own specific model directories under this path.) (evidence: ['.github/CODEOWNERS:79 `/vllm_omni/model_executor/models/ @tzhouam @gcanlin @Sy0307 @amy-why-3459 @linyueqian`', '.github/CODEOWNERS (merge-base, removed by this diff): `/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin @NickCao`', 'docs/design/module/model_integration.md:6-7 `owners:` / ` - "@tzhouam"` / ` - "@gcanlin"`', 'docs/design/module/model_integration.md:14-19 `primary_code_path_owners:` ... `- paths:` / ` - vllm_omni/model_executor/models/**` / ` owners:` / ` - "@Sy0307"` / ` - "@amy-why-3459"` / ` - "@linyueqian"`']) + +`.github/CODEOWNERS:~75` [minor] — Confirmed. The CODEOWNERS rule at line 75 (`/vllm_omni/distributed/ ...`) covers the entire distributed tree, but the cited source doc omni_connector.md only owns `vllm_omni/distributed/omni_connectors/**` as primary. In-tree subdirectories `vllm_omni/distributed/omni_coordinator/**` and `vllm_omni/distributed/ray_utils/**` are only related paths of stage_runtime.md (omni_coordinator also in engine_orchestration.md), so the generated rule is broader than the docs it claims to derive from. Recommend narrowing to `/vllm_omni/distributed/omni_connectors/**` or adding explicit rules for the other sub-trees. (evidence: .github/CODEOWNERS:75 `/vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan`; docs/design/module/omni_connector.md:11 ` - vllm_omni/distributed/omni_connectors/**` (primary_code_paths); docs/design/module/omni_connector.md:13 ` - vllm_omni/platforms/*/omni_connectors/**` (related_code_paths); docs/design/module/stage_runtime.md:30 ` - vllm_omni/distributed/omni_coordinator/**` and stage_runtime.md:31 ` - vllm_omni/distributed/ray_utils/**` (both under related_code_paths, unchanged by this diff, present in PR-time tree); engine_orchestration.md:30 ` - vllm_omni/distributed/omni_coordinator/**` (related_code_paths).) + +`.github/CODEOWNERS:106` [minor] — PR body's 'Not changed' bullet (`/vllm_omni/lora/` and `/vllm_omni/transformers_utils/`) is inaccurate for LoRA: the diff narrows `.github/CODEOWNERS:106` to `/vllm_omni/lora/ @SamitHuang @wtomin`, dropping @princepride, @ZJY0516, @RuixiangMa, and @yuanheng-zhao from review assignment. The CODEOWNERS comment at line 105 already records 'no dedicated design doc; ownership intentionally narrowed'. Move this narrowing to 'What changed' or restore the prior owners so the body matches the diff. (evidence: PR-time tree .github/CODEOWNERS:106 `/vllm_omni/lora/ @SamitHuang @wtomin`; pre-PR .github/CODEOWNERS:33 `/vllm_omni/lora/ @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao` (removed by this diff, verified via file_at_base); PR body 'Not changed' bullet: `/vllm_omni/lora/` and `/vllm_omni/transformers_utils/` (no dedicated design doc defining ownership).) + +`.github/CODEOWNERS:47` [minor] — Correct the PR body's worker conflict-table row: 'After' lists tzhouam, yinpeiqi, fake0fan, Sy0307, but the merged /vllm_omni/worker/ rule in .github/CODEOWNERS:47 and docs/design/module/ar_runtime.md both include @Gaohan123 as a fifth owner. Add @Gaohan123 to that row so the body matches the merged rule. (evidence: Unchanged outside diff: PR body conflict table row `/vllm_omni/worker/` | `tzhouam, gcanlin` | `tzhouam, yinpeiqi, fake0fan, Sy0307` — omits @Gaohan123. .github/CODEOWNERS:47 `/vllm_omni/worker/ @tzhouam @yinpeiqi @fake0fan @Sy0307 @Gaohan123` (present in the diff). docs/design/module/ar_runtime.md:10 ` - "@Gaohan123"` (present in the diff).) + +`.github/CODEOWNERS:1` [minor] — The PR body's 'Design docs updated (to stay in sync)' section names only entrypoints.md, diffusion/offloader.md, and model_integration.md, but this diff modifies 12 docs/design/module/*.md files; add ar_runtime.md, benchmarking.md, cache_management.md, diffusion/diffusion_model_integration.md, diffusion/diffusion_runtime.md, diffusion/parallelism.md, execution_platforms.md, input_output_modality_contracts.md, and observability.md to the section. (evidence: gh_pr_view PR 5958 body: `## Design docs updated (to stay in sync)` followed by exactly three bullets — `- \`docs/design/module/entrypoints.md\` — add @NickCao to \`owners\``, `- \`docs/design/module/diffusion/offloader.md\` — add @lishunyang12 to \`owners\``, `- \`docs/design/module/model_integration.md\` — add \`primary_code_path_owners\` block scoping @Sy0307, @amy-why-3459, @linyueqian to \`vllm_omni/model_executor/models/**\``. diff_stat for the same PR lists 12 changed docs/design/module files: ar_runtime.md, benchmarking.md, cache_management.md, diffusion/diffusion_model_integration.md, diffusion/diffusion_runtime.md, diffusion/offloader.md, diffusion/parallelism.md, entrypoints.md, execution_platforms.md, input_output_modality_contracts.md, model_integration.md, observability.md — proving 9 listed design docs are omitted from the PR body's section.) + +`.github/CODEOWNERS:8` [minor] — The header promises these rules are regenerated from the design docs, but the file has already drifted: docs/design/module/diffusion/diffusion_model_integration.md lists @RuixiangMa among owners for vllm_omni/diffusion/models/**, yet .github/CODEOWNERS:137 omits them (line 138 model_loader/ from the same doc includes @RuixiangMa). The test-plan '23 representative paths via simulation' is a manual checkbox, not a committed check. Commit the generator/check (parse docs/design/module/**/*.md front matter and assert it matches these rules) so the file cannot silently drift again. (evidence: .github/CODEOWNERS:137 `/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin` — omits @RuixiangMa; .github/CODEOWNERS:138 `/vllm_omni/diffusion/model_loader/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa` — the same cited doc's other primary path retains @RuixiangMa; docs/design/module/diffusion/diffusion_model_integration.md (this PR's diff) sets owners `@fhfuih @Bounty-hunter @wtomin @RuixiangMa` for primary_code_paths including `vllm_omni/diffusion/models/**`. Both files are in this PR's changed set.) + +**Additional observations (beyond the comment budget):** +- `.github/CODEOWNERS:148` [minor] Confirmed: .github/CODEOWNERS:148 is `/vllm_omni/diffusion/offloader/ @yuanheng-zhao @david6666666 @lishunyang12`, and docs/design/module/diffusion/offloader.md owners were updated to match. The PR body is stale on this +- `docs/design/module/diffusion/diffusion_model_integration.md:6` [minor] diffusion_model_integration.md:5-9 removes @Isotr0py and @ZJY0516 from the owners and adds @fhfuih, @Bounty-hunter, @RuixiangMa (keeping @wtomin). This cascades into the new CODEOWNERS rules at lines 137-139, which via l +- `docs/design/module/input_output_modality_contracts.md:126` [minor] This PR adds explicit CODEOWNERS rules for vllm_omni/inputs/, outputs/, request.py, data_entry_keys.py, and the five engine I/O files, satisfying the first bullet of the I/O page's Promotion gate — but the gate still lis +- `docs/design/module/execution_platforms.md:12` [minor] Confirmed. The new CODEOWNERS header claims rules are derived only from `owners` and `primary_code_paths`, but this PR introduces `primary_code_path_owners` in execution_platforms.md:12 and model_integration.md:12 and ac + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.trace.json.gz new file mode 100644 index 000000000..86cce74b5 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5958.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.cost.json b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.cost.json new file mode 100644 index 000000000..9ff95b5ef --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1550.8, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_holdout4_r2/runs/pr5978/run-20260815-142508-c5c3dc", + "recorded_at": "2026-08-15T14:50:57", + "attempts": [ + { + "invocation_id": "inv-abde8667", + "outcome": "run:run-20260815-142508-c5c3dc", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-142508-c5c3dc", + "llm_calls": 151, + "input_tokens": 1490991, + "output_tokens": 300636, + "usd": 1.098003, + "source": "spans" + } + ], + "attempt_usd_total": 1.098003, + "llm_calls": 151, + "input_tokens": 1490991, + "output_tokens": 300636, + "usd": 1.098003, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-142508-c5c3dc", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.098, + "minutes": 25.81, + "ci_minutes": 0, + "input_tokens": 1490991, + "output_tokens": 300636, + "tool_calls": 273, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0542, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 151, + "by_role": { + "agent": { + "usd": 1.046944, + "input_tokens": 1458504, + "output_tokens": 262256, + "calls": 148 + }, + "reducer": { + "usd": 0.051059, + "input_tokens": 32487, + "output_tokens": 38380, + "calls": 3 + } + }, + "cache_read_tokens": 13508736, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.22, + "gate": 1.57, + "review": 1542.95, + "report": 0.06 + }, + "time_to_first_result_s": 1548.79 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5978.trace.json.gz", + "trace_bytes": 49313 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.md b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.md new file mode 100644 index 000000000..d04ab78d9 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5978, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 86854432db52) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 11 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] PR numbers self-consistent: calc(0.975190>=0.97, 35.981286>=34, 0.980577>=0.97, 42.650204>=34) all True. +- [claim-verified] Frame-count contract: I2VA 8.0s -> align(round(8*24)=192)=192; Ref2VA 5.0s -> align(120)=124 (time_request._align_frame_count 17n+5), matching I2VA_NUM_FRAMES=192 / REF2VA_NUM_FRAMES=124. +- [claim-verified] Common gate SSIM_THRESHOLD=0.97 / PSNR_THRESHOLD=34.0 and 1344x768/24fps/AAC 32k stereo are asserted in the new e2e _assert_official_video/_probe_audio. +- [claim-verified] New e2e file contains exactly two test functions, matching '2 tests collected'. +- [claim-verified] .buildkite/cuda/test-nightly.yml now invokes test_minimax_h3_i2va_ref2va_similarity.py under the I2VA/Ref2VA label. +- [claim-verified] profiler exposure: _encode_audio_conditions added to _PROFILER_TARGETS and the method exists on MiniMaxH3Pipeline. +- [claim-verified] T2VA accuracy test removed and nightly path swapped: diff_stat shows test_minimax_h3_t2va_similarity.py -216 lines deleted; test-nightly.yml:430-433 now runs test_minimax_h3_i2va_ref2va_similarity.py. +- [claim-verified] common gate thresholds: assert_video_similarity_metrics enforces ssim>=threshold and psnr>=threshold (helpers.py:289-340); the new e2e uses SSIM_THRESHOLD=0.97 / PSNR_THRESHOLD=34.0. +- [claim-verified] 'Expose standalone audio encoding in profiler': _PROFILER_TARGETS adds _encode_audio_conditions (pipeline:541); profiler wrapper uses *args/**kwargs so the kw-only arg is safe. +- [claim-verified] 'one accuracy gate SSIM>=0.97 PSNR>=34 dB': e2e defaults SSIM_THRESHOLD=0.97, PSNR_THRESHOLD=34.0 and both tests call _assert_official_video without overriding. +- [claim-verified] 'frame counts 192/124': I2VA_NUM_FRAMES=192, REF2VA_NUM_FRAMES=124 asserted via assert_video_metadata. +- [claim-verified] 'remove obsolete T2VA and update nightly path': diff deletes test_minimax_h3_t2va_similarity.py and yaml now runs test_minimax_h3_i2va_ref2va_similarity.py. +- [claim-verified] reported e2e metrics (I2VA 0.975190/35.981286, Ref2VA 0.980577/42.650204) are consistent with the gates; not independently reproduced. +- [claim-verified] diff_stat confirms the T2VA accuracy file is deleted and the I2VA/Ref2VA file added, with the nightly label/command updated accordingly (the T2VA-gate-removal concern is already covered by a prior comment at .buildkite/cuda/test-nightly.yml:433). + +`tests/entrypoints/openai_api/test_video_server.py:253` [minor] — This diff adds `test_mux_video_audio_marks_aac_priming_timestamp` to the `core_model and cpu` entrypoints suite, and both red Buildkite lanes exercise this diff surface: `buildkite/vllm-omni` runs `tests/entrypoints tests/engine -m 'core_model and cpu'` and `buildkite/vllm-omni-amd-ci` runs `tests/diffusion -m 'core_model and cpu'` (which includes the new contract tests). The `assert first_packet.pts < 0` assertion is the most plausible in-diff mechanism for the vllm-omni lane because it depends on PyAV/FFmpeg AAC muxer edit-list behavior; please confirm whether both failures are pre-existing on main or introduced here before merging. (evidence: .buildkite/cuda/test-merge.yml: 'timeout 20m pytest -sv tests/entrypoints tests/engine -m core_model and cpu'; .buildkite/amd/test-amd-merge.yml: 'timeout 60m pytest -sv tests/diffusion -m core_model and cpu'; tests/entrypoints/openai_api/test_video_server.py:266: 'assert first_packet.pts < 0') + +`.buildkite/cuda/test-nightly.yml:~433` [minor] — This diff replaces the MiniMax H3 T2VA nightly accuracy command with test_minimax_h3_i2va_ref2va_similarity.py and deletes test_minimax_h3_t2va_similarity.py, so T2VA — still a supported task — loses its only full-model accuracy gate. Keep the previous T2VA case at its own thresholds (SSIM 0.82 / PSNR 20.0) or file a tracking issue; otherwise a T2VA pixel/audio regression ships with no nightly signal (remaining T2VA tests are local_model smokes only). (evidence: ['.buildkite/cuda/test-nightly.yml:433 ` - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model` — the only MiniMax H3 X2V accuracy command in the PR-time nightly file; the old T2VA command is absent.', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:592 ` supported_tasks = {"ref2va"} if partition == "ref2va" else {"t2va", "fl2va"}` — T2VA remains a supported task in the non-ref2va partition.', 'tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:14 ` pytest.mark.local_model,` and tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:34 `def test_minimax_h3_t2va_ulysses8_smoke():` — remaining T2VA execution tests are local_model smokes, not full_model accuracy tests.', 'Deleted by this diff (removed lines in pr_diff, present pre-PR): `SSIM_THRESHOLD = 0.82` and `PSNR_THRESHOLD = 20.0` from tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py — the only prior T2VA full-model accuracy thresholds, confirming the gate was dropped rather than moved.']) + +`vllm_omni/diffusion/utils/media_utils.py:~194` [minor] — `mux_video_audio_bytes` is the shared muxer behind `_encode_video_bytes` (video_api_utils.py:524) for every video-with-audio endpoint, and this diff changes its AAC priming timing for all callers by setting `audio_frame.pts = 0` with a sample-rate time_base. The PR does add a focused unit assertion (`test_mux_video_audio_marks_aac_priming_timestamp`, which only checks `first_packet.pts < 0`), but there is no non-H3 end-to-end audio-path validation. Confirm this global normalization is intended for all consumers or make it opt-in/parameterized, and validate at least one non-H3 audio path against the new output. (evidence: vllm_omni/diffusion/utils/media_utils.py:194 `audio_frame.pts = 0` and :195 `audio_frame.time_base = Fraction(1, audio_sample_rate)` — added in this diff inside `mux_video_audio_bytes` (line 127), applied whenever `audio_waveform is not None`. Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/video_api_utils.py:524 `return mux_video_audio_bytes(` — the shared path used by `_encode_video_bytes` for all video endpoints. Also in the diff: tests/entrypoints/openai_api/test_video_server.py `def test_mux_video_audio_marks_aac_priming_timestamp()` asserts only `assert first_packet.pts < 0`.) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:31` [minor] — tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:31 sets MODEL_REVISION = "main", and _model_name() at lines 112-115 passes revision=MODEL_REVISION to snapshot_download for the FL2VA/Ref2VA weights. Only the media assets are SHA-pinned via _download_reference_asset, so the default model snapshot remains a moving target; pin a concrete revision/commit for the model download as well. (evidence: tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:31 `MODEL_REVISION = "main"`; tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:112-115 `snapshot_root = snapshot_download(` / `repo_id=MODEL_REPO_ID,` / `revision=MODEL_REVISION,` / `allow_patterns=[f"{component}/**"],`) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~1728` [minor] — pipeline_minimax_h3.py:1728 bounds every standalone Ref2VA audio reference to the generated video duration (`max_duration_seconds=float(num_frames) / float(sampling.fps or MINIMAX_H3_FPS)`) before `audio_vae.encode_waveform`. The new e2e Ref2VA case only gates video SSIM/PSNR and audio container metadata (codec/sample_rate/channels at lines 213-217), so an audio-content or voice-timbre regression from this truncation would pass unnoticed. Add an audio-content check for the Ref2VA path (or name the existing one) and verify it still passes with a reference longer than the output, or justify why a timbre reference may be truncated to the output window. (evidence: pipeline_minimax_h3.py:1728 `max_duration_seconds=float(num_frames) / float(sampling.fps or MINIMAX_H3_FPS),`; pipeline_minimax_h3.py:1221 `max_samples = int(round(max_duration_seconds * int(sample_rate)))`; pipeline_minimax_h3.py:1222 `waveform = waveform[..., :max_samples]`; tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:213 `expected_audio = {"codec_name": "aac", "sample_rate": 32000, "channels": 2}`; tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:216-217 `assert _probe_audio(reference_path) == expected_audio` / `assert _probe_audio(generated_path) == expected_audio` — the test probes only container metadata, no audio-content metric.) + +`tests/entrypoints/openai_api/test_video_server.py:269` [minor] — The added test `test_mux_video_audio_marks_aac_priming_timestamp` asserts `first_packet.pts < 0`, relying on the AAC negative priming timestamp produced by `audio_frame.pts = 0` / `time_base = Fraction(1, audio_sample_rate)` in media_utils.py. The production floor is `av>=14.0.0` (requirements/common.txt:13) while the dev/test pin is `av==18.0.0` (pyproject.toml:54). Run this test under the CI's actual av/FFmpeg and ideally against the oldest allowed av (14.x), or raise the floor to the first version that guarantees the negative PTS; otherwise this assertion can be red in one lane and green in another. (evidence: tests/entrypoints/openai_api/test_video_server.py:269 ` assert first_packet.pts < 0` — added by this diff; vllm_omni/diffusion/utils/media_utils.py:194 ` audio_frame.pts = 0` and vllm_omni/diffusion/utils/media_utils.py:195 ` audio_frame.time_base = Fraction(1, audio_sample_rate)` — added by this diff; requirements/common.txt:13 `av>=14.0.0` — unchanged by this diff, present in the PR-time tree; pyproject.toml:54 ` "av==18.0.0", # for ComfyUI tests` — unchanged by this diff, present in the PR-time tree.) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:101` [minor] — The PR body reports the passing I2VA number (SSIM 0.975190) from 4x B300, but the nightly step that actually runs this file is pinned to `mirror_hardwares: h100_4`. 0.975190 has only ~0.5% headroom over the 0.97 gate, so a B300→H100 numerics shift can flip the gate. Confirm the accuracy case passes on H100 (the gate hardware) rather than only on the B300 hardware quoted in the description. (evidence: tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:101 `SSIM_THRESHOLD = 0.97`; .buildkite/cuda/test-nightly.yml:430 `- label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"` and .buildkite/cuda/test-nightly.yml:434 `mirror_hardwares: h100_4` (unchanged by this diff's label/command edit, present in PR-time tree); PR body: '**Hardware:** 4x NVIDIA B300 ... I2VA: SSIM `0.975190`, PSNR `35.981286 dB`, passing the common gate.') + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1528` [minor] — The test `test_ref2va_audio_duration_validation_precedes_rank_branch` only asserts the ValueError. Since `_dit_rank_world` is monkeypatched to `lambda: (None, 1, 2)` and never raises, an implementation that called `_dit_rank_world` first would still pass. Wrap `_dit_rank_world` in a Mock/spy and assert it was not called before the ValueError so a regression moving validation after the rank branch fails this test. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1525 `monkeypatch.setattr(pipeline_module, "_dit_rank_world", lambda: (None, 1, 2))`; tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1528-1532 `with pytest.raises(ValueError, match="max_duration_seconds must be positive"):` ... `pipeline._encode_audio_conditions([(waveform, 10)], max_duration_seconds=0)` — no assertion records whether `_dit_rank_world` ran; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1213 `raise ValueError("max_duration_seconds must be positive")` precedes vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1214 `_, rank, _ = _dit_rank_world()` — the implementation is correctly ordered, but the test would not catch a reordering.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/models/minimax_h3/reference_video.py:307` [minor] This diff makes `_transcode_reference_video` (reference_video.py:307) encode every Ref2VA reference video with `libx264rgb -crf 0 -pix_fmt rgb24` (lossless RGB), and that transcode runs on the production serving path via +- `tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1545` [minor] This diff adds test_ref2va_standalone_audio_condition_is_bounded_to_output_duration, but its FakeAudioVAE.encode_waveform returns the truncated sample count (`waveform.shape[-1]`, line 1545), so the assertion at line 155 +- `vllm_omni/diffusion/models/minimax_h3/reference_video.py:440` [minor] This diff changes Qwen sampling to decode the entire prepared video (load_video_frames(prepared_path)) on rank 0 just to sample ~2fps frames; load_video_frames materializes every frame, so a max-area 768x1344, 15s/24fps +- `vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1324` [minor] This diff changes embedded reference-video audio extraction to pass `audio_duration_seconds` (bounded to `min(duration, target_frame_count/MINIMAX_H3_FPS)`) into `load_video_audio` instead of the raw `duration_seconds`, + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.trace.json.gz b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.trace.json.gz new file mode 100644 index 000000000..32b3a5be6 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_holdout4_r2/pr5978.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15_smoke/manifest.json b/eval/dataset/arms/copilot_v15_smoke/manifest.json new file mode 100644 index 000000000..faac5b4a6 --- /dev/null +++ b/eval/dataset/arms/copilot_v15_smoke/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v15_smoke", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout3" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_smoke/pr5713.cost.json b/eval/dataset/arms/copilot_v15_smoke/pr5713.cost.json new file mode 100644 index 000000000..dc329596c --- /dev/null +++ b/eval/dataset/arms/copilot_v15_smoke/pr5713.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1260.0, + "split": "holdout3", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15_smoke/runs/pr5713/run-20260815-062547-42142b", + "recorded_at": "2026-08-15T06:46:45", + "attempts": [ + { + "invocation_id": "inv-4459852f", + "outcome": "run:run-20260815-062547-42142b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-062547-42142b", + "llm_calls": 128, + "input_tokens": 1041139, + "output_tokens": 249615, + "usd": 0.87769, + "source": "spans" + } + ], + "attempt_usd_total": 0.87769, + "llm_calls": 128, + "input_tokens": 1041139, + "output_tokens": 249615, + "usd": 0.87769, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-062547-42142b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8777, + "minutes": 20.96, + "ci_minutes": 0, + "input_tokens": 1041139, + "output_tokens": 249615, + "tool_calls": 238, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8988, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 128, + "by_role": { + "planner": { + "usd": 0.002763, + "input_tokens": 15, + "output_tokens": 2452, + "calls": 1 + }, + "agent": { + "usd": 0.838524, + "input_tokens": 1021257, + "output_tokens": 219003, + "calls": 124 + }, + "reducer": { + "usd": 0.036402, + "input_tokens": 19867, + "output_tokens": 28160, + "calls": 3 + } + }, + "cache_read_tokens": 11926144, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.91, + "gate": 1.55, + "review": 1252.16, + "report": 0.06 + }, + "time_to_first_result_s": 1257.66 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5713.trace.json.gz", + "trace_bytes": 37312 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15_smoke/pr5713.md b/eval/dataset/arms/copilot_v15_smoke/pr5713.md new file mode 100644 index 000000000..ba3fae8be --- /dev/null +++ b/eval/dataset/arms/copilot_v15_smoke/pr5713.md @@ -0,0 +1,73 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5713, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6a464218a760) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Worker-process VRAM is measured correctly elsewhere: test_cross_device_cleanup and test_llm_sleep_ack use get_device_global_memory_used_gib (driver-wide); only the merged test_diffusion_sleep_wake_generate uses the per-process get_vram_info for its restore check (see review comment). +- [claim-verified] diff_stat shows only tests/dfx/.../test_invalid_server_control.py, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py, tests/entrypoints/test_omni_sleep_mode.py changed; PR claim 'No production vllm_omni code changes' holds. +- [claim-verified] adding server_args=['--enable-sleep-mode'] to _QWEN_IMAGE does not alter the invalid-JSON cases: FastAPI body validation raises before the route body executes (vllm_omni/entrypoints/openai/api_server.py:3485-3493), and the un-skipped cases are all validation-level. +- [claim-verified] wake route has no try/except around `await engine_client.wake_up(...)` (api_server.py:3497-3511), so the new CPU route-propagation test correctly verifies NotImplementedError is not swallowed. +- [claim-verified] No production `vllm_omni/` changes at this head: the diff touches only the three test files listed in sweep_targets. +- [claim-verified] `send_omni_sleep_http_request` always returns `[resp]`, so `sleep_resps[0]` at test_invalid_server_control.py:125 is safe; the subsequent `assert sleep_resp.status_code == 200` guards the non-2xx case. +- [claim-refuted] PR body '2100.9s -> 901.24s, 49min -> 35min': seconds convert to 35.0min -> 15.0min; minutes imply a 14min delta while seconds imply 20.0min — internally inconsistent. +- [validated] test_wakeup_route_propagates_not_implemented_error exercises the real route: api_server.py omni_wakeup (3497-3511) does not catch NotImplementedError and sleeping_set.remove runs after the await, so raise_server_exceptions=True re-raises and sleeping_stages=={0} holds. +- [validated] e2e 501 message: AsyncOmni.wake_up (async_omni.py:1203-1209) raises NotImplementedError containing both 'sleep(level=2)' and 'not yet implemented', matching the err_message tuple in the new e2e test. +- [validated] omni_server fixture is module-scoped (tests/helpers/fixtures/runtime.py:34-43), so the new level-2 test shares the server with invalid-JSON cases as the comment states. +- [validated] tests/helpers/runtime.py send_omni_sleep_http_request always returns `[resp]` (single element), so `sleep_resps[0]` at test_invalid_server_control.py:125 cannot IndexError. +- [validated] pyproject.toml pins pytest-asyncio==1.4.0 and asyncio_mode='auto', so `pytest_asyncio.fixture(scope='class', loop_scope='class')` and `@pytest.mark.asyncio(loop_scope='class')` are supported. +- [validated] assert_http_error treats tuple err_message with sequence_match='all', so the new level-2 test requires BOTH 'sleep(level=2)' AND 'not yet implemented' in the 501 body. +- [validated] tests/entrypoints/test_omni_sleep_mode.py:194-197 — diffusion fixture: TP=2 via `parallel_config.tensor_parallel_size=2` with `runtime.devices="0,1"`; enable_sleep_mode set both at stage and AsyncOmni level; no new defect. + +`tests/entrypoints/test_omni_sleep_mode.py:~360` [major] — The merged `test_diffusion_sleep_wake_generate` re-adds the lifecycle reclaim assertion as an OR between the worker ACK and a per-process VRAM reading, but the fallback arm can never fail: `vram_sleeping` comes from `get_vram_info` (line 46: per-process CUDA allocator stats; does not see other processes), while the diffusion stage runs in a child process (line 181: `"runtime": {"process": True, "devices": "0,1", ...}`), so the test process never holds the worker weights and `vram_sleeping < 5.0` is effectively always true. That makes the sleep-reclaim assertion unfalsifiable, and the restore assertion `abs(vram_restored - vram_initial) < 3.0` (line 373) is near-vacuous for the same reason. Use the driver-view helper `get_device_global_memory_used_gib` for both fallbacks (as `test_cross_device_cleanup` does) or assert `reported_freed_gib > 14.0` alone, keeping post-wake generate as the real restore check. (evidence: tests/entrypoints/test_omni_sleep_mode.py:360 `assert reported_freed_gib > 14.0 or vram_sleeping < 5.0, (`; tests/entrypoints/test_omni_sleep_mode.py:46 `"""Per-**process** CUDA allocator stats (GiB). Does not see other processes' usage."""`; tests/entrypoints/test_omni_sleep_mode.py:181 `"runtime": {"process": True, "devices": "0,1", "max_batch_size": 1},`; tests/entrypoints/test_omni_sleep_mode.py:373 `assert abs(vram_restored - vram_initial) < 3.0, "VRAM failed to restore to initial levels"`) + +`PR description:54` [minor] — PR description:54 Merge CI timing is self-contradictory: 2100.9s -> 901.24s equals 35.0min -> 15.0min (Δ 20.0min), not 49min -> 35min (Δ 14min). Present one consistent before/after pair and link the source Buildkite run. (evidence: PR description:54 (PR body, not code diff) — `### Merge CI (**2100.9s -> 901.24s**, **49min -> 35min**)`. Conversion check: 2100.9/60=35.015 min -> 35.0min, 901.24/60=15.021 min -> 15.0min, Δ=1199.66s≈20.0min; the adjacent '49min -> 35min' implies Δ=14min — the two representations contradict each other.) + +`tests/entrypoints/openai_api/test_omni_sleep_wakeup.py:178` [minor] — The new route test at 178-196 proves exception propagation and that a failed wake does not clear sleeping_stages, but it uses raise_server_exceptions=True and therefore does not verify the user-visible NotImplementedError→HTTP 501 mapping. The docstring already defers that mapping to test_wakeup_after_level2_sleep_fails; add an explicit note that this live-H100 mapping is weekly-CI-only, or extend the route test to assert the 501 response body contains both 'sleep(level=2)' and 'not yet implemented'. (evidence: tests/entrypoints/openai_api/test_omni_sleep_wakeup.py:178 `def test_wakeup_route_propagates_not_implemented_error(sleep_capable_engine, mocker):`; :185 `sleep_capable_engine.wake_up = mocker.AsyncMock(side_effect=NotImplementedError("engine wake_up refused"))`; :189 `client = TestClient(app, raise_server_exceptions=True)`; :191-192 `with pytest.raises(NotImplementedError, match="engine wake_up refused"):` / `client.post("/v1/omni/wakeup", json={"stage_ids": [0]})`; :195 `assert app.state.sleeping_stages == {0}` — these changed lines prove the test only exercises in-process propagation, not an HTTP 501 response.) + +`tests/entrypoints/test_omni_sleep_mode.py:1` [minor] — Restore an explicit inter-class cleanup between the LLM and diffusion classes: removing clean_gpu_memory_between_tests leaves engine.shutdown()+1.5s sleep as the only gap, while both classes reallocate on the same GPUs 0,1. That is weaker than the previous per-test memory reset and risks stale GPU memory/OOM when the classes run back-to-back. (evidence: [residual] removing `clean_gpu_memory_between_tests` leaves engine.shutdown()+1.5s sleep as the only gap between the LLM class and the diffusion class, which reallocate on the same GPUs 0,1.) + +`tests/entrypoints/test_omni_sleep_mode.py:32` [minor] — The gate buildkite/vllm-omni-amd-ci is red at head while buildkite/vllm-omni passes. This diff touches only tests/entrypoints and tests/dfx/reliability, and neither tests/entrypoints nor tests/dfx appears in any .buildkite/amd pipeline (grep for 'tests/entrypoints' matches only .buildkite/cuda/*.yml). The AMD lanes collect only tests/diffusion, tests/model_executor, tests/e2e and tests/engine, so the red check does not exercise this PR's surface. Please confirm the failure is pre-existing on main rather than introduced here before treating it as a merge blocker. (evidence: tests/entrypoints/test_omni_sleep_mode.py:32 `def clean_device_envs():` (anchor exists verbatim). .buildkite/amd/test-amd-merge.yml:21 `"timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/amd/test-amd-merge.yml:32 `"timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/amd/test-amd-merge.yml:100 `- timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py`; .buildkite/amd/test-amd-merge.yml:141 `- timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py` — AMD merge lanes collect only these four test dirs; none is tests/entrypoints or tests/dfx (these .buildkite files are unchanged by this diff). Repo-wide grep for the literal 'tests/entrypoints' under .buildkite returns only .buildkite/cuda/test-merge.yml and .buildkite/cuda/test-ready.yml, never .buildkite/amd. gh_ci_read(5713) lists `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` and `{"bucket":"pass","name":"buildkite/vllm-omni","state":"SUCCESS"}`.) + +`tests/entrypoints/test_omni_sleep_mode.py:500` [minor] — At tests/entrypoints/test_omni_sleep_mode.py:500, @pytest.mark.core_model plus the module-level pytestmark = [pytest.mark.advanced_model] (lines 26-28) gives this L4 test both markers; .buildkite/cuda/test-ready.yml:104 selects 'core_model and cuda' on l4_1 and .buildkite/cuda/test-merge.yml:68 selects 'advanced_model and cuda and L4', so the qwen_image_random cold start now runs in the ready lane too. Drop core_model if the intent was only the merge-lane H100-to-L4 move. (evidence: tests/entrypoints/test_omni_sleep_mode.py:26 `pytestmark = [` +tests/entrypoints/test_omni_sleep_mode.py:27 ` pytest.mark.advanced_model,` +tests/entrypoints/test_omni_sleep_mode.py:500 `@pytest.mark.core_model` +tests/entrypoints/test_omni_sleep_mode.py:501 `@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)` +tests/helpers/mark.py:12 ` test_platform_detail = pytest.mark.cuda` (unchanged by diff) +tests/helpers/mark.py:14 ` test_resource = pytest.mark.L4` (unchanged by diff) +.buildkite/cuda/test-ready.yml:104 ` pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'` (unchanged by diff) +.buildkite/cuda/test-merge.yml:68 ` pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"` (unchanged by diff)) + +`tests/entrypoints/test_omni_sleep_mode.py:~116` [minor] — `_ensure_awake` catches and logs every `wake_up` failure in `finally` blocks that guard class-scoped shared engines (tests/entrypoints/test_omni_sleep_mode.py:111-116; fixtures at :160 and :172). Since a failed wake leaves the shared stage asleep, a later test in the class can fail in a cascade unrelated to its own assertions. Surface the failure — e.g., re-raise only when no exception is already in flight, or assert in teardown — so the test that left the engine asleep is the one that fails, while retaining the log context. (evidence: tests/entrypoints/test_omni_sleep_mode.py:111-116 (added by this diff): +`async def _ensure_awake(engine: AsyncOmni, stage_ids: list[int]) -> None: + """Best-effort full wake so the next shared-engine test starts active.""" + try: + await engine.wake_up(stage_ids=stage_ids) + except Exception as e: + logger.warning("ensure_awake failed (stage_ids=%s): %s", stage_ids, e)` +The same file declares class-scoped shared engines at tests/entrypoints/test_omni_sleep_mode.py:160 and :172 (`@pytest_asyncio.fixture(scope="class", loop_scope="class")`) and calls the helper in `finally` blocks after each shared-engine test (e.g. :232 `await _ensure_awake(llm_engine, [0])`), so a swallowed wake failure can leave the shared engine asleep for the next test in the class.) + +`tests/entrypoints/test_omni_sleep_mode.py:~320` [minor] — Confirmed. The docstring at line 320 acknowledges the dropped 'second default-config load'. The replacement diffusion_engine fixture builds `AsyncOmni(model=MODEL, stages=stages, ...)` with `model_class_name='BagelPipeline'` and TP=2, so it does not cover default deploy-config resolution for BAGEL. `test_pure_diffusion_sleep_wake` is hardcoded `tensor_parallel_size=1` for qwen_image_random, dropping the old `[1,2]` TP parametrization. If accepted for runtime, track it explicitly; otherwise add a minimal default-config case. (evidence: tests/entrypoints/test_omni_sleep_mode.py:185 `"model_class_name": "BagelPipeline",`; tests/entrypoints/test_omni_sleep_mode.py:197 `engine = AsyncOmni(model=MODEL, stages=stages, init_timeout=600, enable_sleep_mode=True)`; tests/entrypoints/test_omni_sleep_mode.py:320 `class TP=2 BagelPipeline engine rather than a second default-config load).`; tests/entrypoints/test_omni_sleep_mode.py:515 `"tensor_parallel_size": 1,` (unchanged-by-this-diff-base had `@pytest.mark.parametrize("tp_size", [1, 2])` for test_pure_diffusion_scenario, removed by this diff).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15_smoke/pr5713.trace.json.gz b/eval/dataset/arms/copilot_v15_smoke/pr5713.trace.json.gz new file mode 100644 index 000000000..0e891b876 Binary files /dev/null and b/eval/dataset/arms/copilot_v15_smoke/pr5713.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/manifest.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/manifest.json new file mode 100644 index 000000000..7c891e16c --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v15cb_holdout4_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout4" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.cost.json new file mode 100644 index 000000000..c21790bef --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 429.3, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5608/run-20260815-083021-8151ef", + "recorded_at": "2026-08-15T08:37:30", + "attempts": [ + { + "invocation_id": "inv-b60ad863", + "outcome": "run:run-20260815-083021-8151ef", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-083021-8151ef", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-083021-8151ef", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2466, + "minutes": 7.15, + "ci_minutes": 0, + "input_tokens": 567986, + "output_tokens": 84804, + "tool_calls": 282, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3523, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.05, + "gate": 1.58, + "review": 422.98, + "report": 0.07 + }, + "time_to_first_result_s": 428.66 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5608.trace.json.gz", + "trace_bytes": 21056 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.md new file mode 100644 index 000000000..a4efcf6ac --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5608, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head de8517f6f2bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] test import failure after helper extraction — de8517f6 monkeypatches rotary_embedding module before loading patch +- [resolved] cos unsqueeze dim change — test now expects BSND (1,3,1,2) not BNSD (1,1,3,2); residual: trigger is head_dim=2 alignment not short-seq +- [resolved] vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184 still calls raw npu_rotary_mul — 310P-only path with CodePredictor seq=num_groups+1 (~33) keeps batch*heads within seq*8 at tested concurrency; residual: other raw callers (qwen3_code_predictor.py:260) on 910B not patched but longer talker seq likely avoids the constraint +- [resolved] AMD gate buildkite/vllm-omni-amd-ci red — diff touches only vllm_omni/platforms/npu/* and tests/platforms/npu/*, absent from .buildkite/amd/test-amd-ready.yml collection paths +- [resolved] checklist-3: no public API/default/protocol change; apply_rotary_pos_emb signature unchanged, behavior silent on NPU only +- [resolved] old _apply_rotary_pos_emb_npu consumers: single monkeypatch target updated; no stale importers of removed symbol +- [resolved] 'fix ci' commit de8517f6 updated cos shape assertion after helper extraction — residual: still no batch*heads case +- [resolved] de8517f6 preloads rotary_embedding in test harness required by new import at qwen3_tts_tokenizer_v2.py:12-14 +- [resolved] code_predictor/310P talker still use raw npu_rotary_mul (round-1); tokenizer hunk fixes Code2Wav crash path only +- [claim-verified] PR batch*heads<=seq*8 formula matches _bnsd_rotary_shape_is_supported at rotary_embedding.py:28 +- [claim-verified] PR Stage-1 Code2Wav crash scope matches qwen3_tts_tokenizer_v2 patch only, not talker/code_predictor +- [claim-verified] commit 504fa8cb extract preserved transpose+contiguous fallback semantics from inline _rotary_mul_npu +- [claim-verified] Repo grep shows npu_rotary_mul_with_bsnd_fallback referenced only in rotary_embedding.py and qwen3_tts_tokenizer_v2.py +- [claim-verified] PR manual 910B benchmark + WER evidence present; c=16 streaming degradation (8.33% continuity) is residual capacity/latency, not proof this fix regressed + +`tests/platforms/npu/test_310p_patches.py:~547` [minor] — This diff adds `npu_rotary_mul_with_bsnd_fallback` and updates `test_310p_patches.py:547` to expect BSND cos layout `(1, 3, 1, 2)`, but the test still drives fallback with `q=torch.zeros(1, 2, 3, 2)` where `head_dim=2` fails `_bnsd_rotary_shape_is_supported`'s alignment check (`head_dim // 2 % half_dim_alignment != 0` at `rotary_embedding.py:28`) before the motivating `batch_size * num_heads <= seq_len * 8` gate (`4*16 > 4*8` for codec short-seq shapes). Add a case with `q/k` shaped `(4, 16, 4, 64)` (and matching cos) asserting `rotary_calls[0][1].shape == (4, 4, 1, 64)` from `cos.unsqueeze(2)`, and optionally keep the misaligned toy case plus a supported fast-path case like `(1, 16, 8, 64)` asserting `(1, 1, 8, 64)` from `cos.unsqueeze(1)`. (evidence: tests/platforms/npu/test_310p_patches.py:536 `q = torch.zeros(1, 2, 3, 2)`; tests/platforms/npu/test_310p_patches.py:547 `assert rotary_calls[0][1].shape == (1, 3, 1, 2)`; vllm_omni/platforms/npu/layers/rotary_embedding.py:27-28 `half_dim_alignment = 32 // element_size` / `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8`; vllm_omni/platforms/npu/layers/rotary_embedding.py:44-55 `if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):` fast path else `cos.unsqueeze(2)` BSND fallback — for `(1,2,3,2)` float32, `head_dim//2=1` and `half_dim_alignment=8`, so `1 % 8 != 0` rejects BNSD before `batch*heads` is evaluated; for `(4,16,4,64)`, alignment passes but `4*16=64 > 4*8=32` would trigger fallback.) + +`vllm_omni/model_executor/models/common/qwen3_code_predictor.py:~260` [minor] — This diff wires `npu_rotary_mul_with_bsnd_fallback` only into `qwen3_tts_tokenizer_v2.py`, but `qwen3_code_predictor.py:257-261` still unsqueezes `cos`/`sin` at dim 1 and calls `torch_npu.npu_rotary_mul(q, cos, sin)` / `torch_npu.npu_rotary_mul(k, cos, sin)` on BNSD tensors with no helper import. Short-sequence CodePredictor NPU forwards can hit the same CANN BNSD `batch*num_heads` rejection outside the patched tokenizer path—route this callsite through the shared helper or document it as out of scope with a tracked follow-up. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/common/qwen3_code_predictor.py:257 `cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim]`; vllm_omni/model_executor/models/common/qwen3_code_predictor.py:260 `q = torch_npu.npu_rotary_mul(q, cos, sin)`; vllm_omni/model_executor/models/common/qwen3_code_predictor.py:261 `k = torch_npu.npu_rotary_mul(k, cos, sin)`; helper scope: vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:31 `npu_rotary_mul_with_bsnd_fallback(q, cos, sin, unsqueeze_dim)` (only consumer besides definition in rotary_embedding.py)) + +`vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184` [minor] — Route `_Qwen3CodePredictorAttention310P` through `npu_rotary_mul_with_bsnd_fallback` the same way `qwen3_tts_tokenizer_v2.py` does, or document this 310P talker callsite as explicitly out of scope: `_310p/patch/qwen3_tts.py:184-185` still calls raw `torch_npu.npu_rotary_mul` while the tokenizer patch uses the shared helper—same sibling gap as `qwen3_code_predictor.py:260-261`. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184 `q = torch_npu.npu_rotary_mul(q, cos, sin)`; vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:185 `k = torch_npu.npu_rotary_mul(k, cos, sin)`; changed in this PR: vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:31 `npu_rotary_mul_with_bsnd_fallback(q, cos, sin, unsqueeze_dim),`; unchanged sibling: vllm_omni/model_executor/models/common/qwen3_code_predictor.py:260 `q = torch_npu.npu_rotary_mul(q, cos, sin)`) + +`tests/platforms/npu/test_qwen3_tts_customvoice.py:1` [minor] — Add a one-line PR note that manual 910B WER/benchmark evidence used `codec_chunk_ramp` while default deploy `qwen3_tts.yaml` keeps that knob commented and this NPU e2e test lacks ramp config, so reviewers should not treat bench numbers as covering the stock deploy path without further evidence. (evidence: [claim-verified] PR manual bench used codec_chunk_ramp — deploy qwen3_tts.yaml has it commented; NPU e2e test_qwen3_tts_customvoice.py lacks ramp config) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:44` [minor] — Document or extend the guard: when `unsqueeze_dim != 1`, `npu_rotary_mul_with_bsnd_fallback` skips `_bnsd_rotary_shape_is_supported` and always calls raw `npu_rotary_mul`; no in-repo NPU caller passes non-1 today, but a future caller with `unsqueeze_dim != 1` and `batch * num_heads > seq_len * 8` would miss the BSND transpose fallback. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:44 `if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):`; unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py:362 `query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)`) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:12` [minor] — This diff gates BSND fallback on `_bnsd_rotary_shape_is_supported` (rotary_embedding.py:12-28, consumed at :44), but no test parametrizes its branches (ndim!=4, odd head_dim, alignment fail, `batch*num_heads > seq_len*8`, all-pass). Add a small CPU-marked parametrized test in `tests/platforms/npu/test_310p_patches.py` so a wrong threshold on line 28 cannot regress silently while the integration test keeps using misaligned toy shapes. (evidence: rotary_embedding.py:44 `if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):` — BSND fallback gated on shape support; rotary_embedding.py:28 `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8` — multi-branch predicate with no direct test references (grep tests/ for `_bnsd_rotary_shape_is_supported` and `npu_rotary_mul_with_bsnd_fallback`: no matches); unchanged by this diff, present in PR-time tree: tests/platforms/npu/test_310p_patches.py:536 `q = torch.zeros(1, 2, 3, 2)` — integration test only, no parametrized branch coverage) + +`tests/platforms/npu/test_310p_patches.py:~486` [nit] — This diff validates the new rotary fallback only through a mocked `torch_npu.npu_rotary_mul` in `test_310p_patches.py:483-487`, so CI (if this file were run) would prove dispatch/routing—including the `(1, 3, 1, 2)` cos layout—rather than BNSD↔BSND numerical parity; acceptable for L2 CPU plumbing, but no current `.buildkite` job collects `tests/platforms/npu/`, and NPU ready CI only runs the Qwen3-TTS e2e lane. (evidence: tests/platforms/npu/test_310p_patches.py:475-477 `def rotary_mul(hidden_states, cos, sin): rotary_calls.append((hidden_states, cos, sin)); return hidden_states + 1`; tests/platforms/npu/test_310p_patches.py:486 `npu_rotary_mul=rotary_mul,`; tests/platforms/npu/test_310p_patches.py:547 `assert rotary_calls[0][1].shape == (1, 3, 1, 2)`; unchanged by this diff, present in PR-time tree: .buildkite/npu/test-npu-ready.yml:34 `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model'`; unchanged by this diff, present in PR-time tree: grep of `.buildkite` for `tests/platforms/npu` / `test_310p_patches` returned no matches) + +`vllm_omni/platforms/npu/platform.py:~48` [nit] — This diff changes RoPE for every Qwen3-TTS 12Hz tokenizer NPU deployment: platform.py calls apply_qwen3_tts_tokenizer_v2_patch() unconditionally at init, patching modeling_qwen3_tts_tokenizer_v2.apply_rotary_pos_emb (25Hz tokenizer is unaffected). PR test evidence covers only Qwen3-TTS-12Hz-1.7B-CustomVoice on 910B3 — add a one-line PR note that the BSND fallback applies platform-wide to 12Hz tokenizer paths to help reviewers scope regressions. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/platforms/npu/platform.py:48 `apply_qwen3_tts_tokenizer_v2_patch()` — called in NPUOmniPlatform.__init__ with no model guard; vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:49 `modeling_qwen3_tts_tokenizer_v2.apply_rotary_pos_emb = _apply_rotary_pos_emb_npu` — global monkeypatch of 12Hz tokenizer RoPE; vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:30-32 `npu_rotary_mul_with_bsnd_fallback(q, cos, sin, unsqueeze_dim)` — PR-changed RoPE implementation wired through the patch) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.trace.json.gz new file mode 100644 index 000000000..65518fe4c Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5608.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.cost.json new file mode 100644 index 000000000..6720e8eab --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 480.1, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5720/run-20260815-083022-d20816", + "recorded_at": "2026-08-15T08:38:21", + "attempts": [ + { + "invocation_id": "inv-56b9b836", + "outcome": "run:run-20260815-083022-d20816", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-083022-d20816", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-083022-d20816", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4803, + "minutes": 7.98, + "ci_minutes": 0, + "input_tokens": 1345015, + "output_tokens": 106480, + "tool_calls": 401, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4687, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.6, + "gate": 1.58, + "review": 473.76, + "report": 0.06 + }, + "time_to_first_result_s": 478.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5720.trace.json.gz", + "trace_bytes": 47109 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.md new file mode 100644 index 000000000..ae325a3af --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.md @@ -0,0 +1,64 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5720, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head b4733f6bfd98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] Prior TODO on dual-DiT DLO: ref2va disables FL2VA resident staging via `_resident_dit_layers_on_device(enabled=transformer is self.transformer)` — residual: `_offload_plan.resident_dit_paths` still only `transformer`, `transformers_ref` never gets resident blocks +- [resolved] DLO staging both DiTs — resident_dit_paths keeps only transformer; diffuse() skips inactive DiT residency +- [resolved] Offloader `_dit_modules` class-vs-instance concern: pipeline sets instance list in __init__ (598-609); ModuleDiscovery.discover reads instance attribute at runtime (module_collector.py:145); residual checked — only test asserts class var. +- [resolved] CLI --task-type choices removed — serve.py now documents model-defined values; Qwen3-TTS validation enforced in config (model.py:213-218) with test coverage. +- [resolved] NPU:134 — Removing FL2VA→Ref2VA restart is correct for combined service; validated NPU evidence table covers Ref2VA without partition swap. +- [claim-verified] Default startup loads both DiTs: `_minimax_h3_partition_for_task(None)` → `combined` at pipeline_minimax_h3.py:132-134 +- [claim-verified] `--task-type fl2va|ref2va` limits downloads via `MINIMAX_H3_TASK_DOWNLOAD_PATTERNS` and single-DiT init (test_pipeline_loads_task_selected_dits) +- [claim-verified] CLI forwards `task_type` to diffusion stage (`test_serve_cli_forwards_model_defined_task_type_to_diffusion_stage`) +- [claim-verified] Shared encoder/VAE loaded once from FL2VA partition in combined mode (contract test assertions on component paths) +- [claim-verified] T2VA+Ref2VA on one endpoint via `_transformer_for_task` + request `extra_params.task` (manual PR video; unit tests for routing) +- [claim-verified] PR default combined mode — _minimax_h3_partition_for_task(None)→combined loads 2 DiTs unless --task-type set +- [claim-verified] PR shared-component-once claim — test_pipeline_loads_task_selected_dits_and_shared_components_once mocks verify single encoder/VAE/tokenizer +- [claim-verified] PR body default=combined both DiTs: `_minimax_h3_partition_for_task(None|'auto')` returns `combined` at pipeline_minimax_h3.py:133-134; `test_pipeline_loads_task_selected_dits_and_shared_components_once` expects 2 DiTs for `task_type=None`. +- [claim-verified] PR default combined mode — test_pipeline_loads_task_selected_dits(None,…) expects partition=combined and 2 DiTs (test_minimax_h3_contract.py:153-287). + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:727` [minor] — Combined-mode `_resolve_task` (lines 720–728) routes image-only requests without `extra_params.task` to `fl2va`, while a Ref2VA-only partition still defaults image-only to `ref2va` (lines 723–724); tests confirm both paths. Migrators from `.../Ref2VA` serving to `MiniMaxAI/MiniMax-H3` without `--task-type ref2va` must pass `extra_params.task=ref2va` on image-only Ref2VA calls—consider a brief recipe migration callout (examples already set `task=ref2va`). (evidence: pipeline_minimax_h3.py:723-728 `if self.partition == "ref2va": requested = "ref2va"` … `elif multi_modal_data.get("image") is not None: requested = "fl2va"`; test_minimax_h3_contract.py:129-135 `assert pipeline._resolve_task(None, {"image": object()}) == "fl2va"` (combined) and `assert pipeline._resolve_task(None, {"image": object()}) == "ref2va"` (ref2va partition).) + +`recipes/MiniMaxAI/MiniMax-H3-NPU.md:79` [minor] — NPU recipe removes `HF_HUB_OFFLINE`/`TRANSFORMERS_OFFLINE` and the `hf download --local-dir` checkpoint step while switching to repo-id on-demand download (`export MODEL=MiniMaxAI/MiniMax-H3`). Add explicit air-gapped steps: pre-download the combined layout (`FL2VA/**`, `Ref2VA/model_index.json`, `Ref2VA/transformer/**`), set `MODEL` to the local root, and restore offline env vars so NPU operators who followed the previous offline recipe are not left without guidance. (evidence: PR head recipes/MiniMaxAI/MiniMax-H3-NPU.md:27-35 `Authenticate once;` / `vllm serve` downloads the required nested components automatically:` / `export MODEL=MiniMaxAI/MiniMax-H3` — on-demand HF download, no offline env vars. PR head recipes/MiniMaxAI/MiniMax-H3-NPU.md:79-92 serve block sets `export MODEL=MiniMaxAI/MiniMax-H3` with no `HF_HUB_OFFLINE` or `TRANSFORMERS_OFFLINE`. Pre-PR (main) recipes/MiniMaxAI/MiniMax-H3-NPU.md had `hf download MiniMaxAI/MiniMax-H3 --local-dir "${MODEL_ROOT}"`, `export MODEL="${MODEL_ROOT}/FL2VA"`, and `export HF_HUB_OFFLINE=1` / `export TRANSFORMERS_OFFLINE=1`. Repo-wide recipes grep: no `HF_HUB_OFFLINE`, `TRANSFORMERS_OFFLINE`, or air-gapped guidance remains after this change.) + +`vllm_omni/diffusion/registry.py:437` [minor] — SP and torch.compile now follow runtime `_dit_modules` (registry.py:437-439; diffusion_model_runner.py:283-287). Combined H3 sets `_dit_modules` to both DiTs only when `partition == 'combined'` (pipeline_minimax_h3.py:598-609); `--task-type fl2va`/`ref2va` keeps a single entry, matching pre-PR single-transformer behavior. Confirm combined dual-DiT compile/SP is intended; scoping for single-partition servers is already in place. (evidence: registry.py:437 `transformer_attrs = getattr(model, "_dit_modules", None)` — SP iterates runtime declared DiT names; diffusion_model_runner.py:283 `transformer_attrs = getattr(self.pipeline, "_dit_modules", None)` and :286-287 `for attr_name in transformer_attrs: self._compile_transformer(attr_name)` — compile path matches; pipeline_minimax_h3.py:598 `self._dit_modules = ["transformer"]` and :599-609 `if ref2va_model_path is not None: ... self._dit_modules.append("transformers_ref")` — dual-DiT list only in combined partition.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~518` [minor] — Combined startup loads both DiTs, but `_offload_plan.resident_dit_paths` still lists only `transformer`. `diffuse()` already skips resident staging for Ref2VA (`enabled=transformer is self.transformer` at line 1477), and DLO setup in `distributed_layerwise_backend` only keeps leading blocks for names in `resident_dit_paths`, so `transformers_ref` is fully streamed. Consider a short `_offload_plan` comment so combined Ref2VA DLO perf/memory expectations are obvious. (evidence: pipeline_minimax_h3.py:513 `_dit_modules: ClassVar[list[str]] = ["transformer", "transformers_ref"]`; pipeline_minimax_h3.py:598-609 combined init sets `self._dit_modules = ["transformer"]` then appends `"transformers_ref"` when `ref2va_model_path is not None`; pipeline_minimax_h3.py:518 `resident_dit_paths=frozenset({"transformer"}),`; pipeline_minimax_h3.py:1477 `with self._resident_dit_layers_on_device(enabled=transformer is self.transformer):`; unchanged consumer in PR-time tree: distributed_layerwise_backend.py:1479 `if plan is not None and dit_name in plan.resident_dit_paths:`) + +`vllm_omni/diffusion/utils/hf_utils.py:22` [minor] — `get_diffusion_model_index` iterates `("model_index.json", "modular_model_index.json")` and returns the first match (hf_utils.py:10-25), while `omni_snapshot_download` defers eager download only when `modular_model_index.json` exists (omni_base.py:81). If a repo root has both indexes with different `_class_name`, startup skips full snapshot download but `resolve_model_class_name` / `enrich_config` can resolve the legacy index. Prefer modular when present (same gate as omni_base) and add a tmp_path dual-index test. (evidence: hf_utils.py:10-12 `DIFFUSION_MODEL_INDEX_FILES = ("model_index.json", "modular_model_index.json",)` — unchanged by this diff, present in the PR-time tree; hf_utils.py:22-25 `for filename in DIFFUSION_MODEL_INDEX_FILES: ... if isinstance(config, Mapping): return dict(config)`; omni_base.py:81 `if file_or_path_exists(model_id, "modular_model_index.json", revision=None):` — unchanged by this diff, present in the PR-time tree; data.py:551-554 `# Diffusers models: read _class_name from the pipeline index.` / `model_index = get_diffusion_model_index(model)` / `return model_index.get("_class_name")`) + +`recipes/MiniMaxAI/MiniMax-H3.md:~197` [minor] — The recipe still references `bash examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh` (recipes/MiniMaxAI/MiniMax-H3.md:197), but no `minimax_h3` path exists under `examples/`—repo-wide grep finds that script name only in this recipe. Replace with an existing script, add the script in this PR, or remove the reference so copy-paste users are not sent to a missing file. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:197 `bash examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh` — unchanged by this diff, present in PR-time tree; repo-wide grep for `run_h3_2gpu_all_tasks` matches only this file, and `examples/offline_inference/` has no `minimax_h3/` subdirectory.) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py:~63` [minor] — L4 accuracy still snapshots `FL2VA/**` and serves the partition subpath (`tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py:63-68`), while recipes now document combined `MiniMaxAI/MiniMax-H3` serving with per-request task routing. Extend that nightly test (or add a sibling) to use the repo ID, then issue Ref2VA after T2VA on the same server so CI guards the combined-service path. (evidence: unchanged by this diff, present in the PR-time tree: tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py:63-68 `snapshot_root = snapshot_download(` / `allow_patterns=["FL2VA/**"],` / `return str(Path(snapshot_root) / "FL2VA")`; tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py:179-189 `with OmniServer(` / `_model_name(),` then single POST with `"task": "t2va"` in `extra_params`; repo-wide grep found no e2e test that runs T2VA and Ref2VA on the same live server with repo-ID combined default.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1472` [minor] — Combined-mode diffuse() routes Ref2VA through _transformer_for_task to transformers_ref and gates DLO with enabled=(transformer is self.transformer) (pipeline_minimax_h3.py:1472-1477), but tests only cover _transformer_for_task and _resident_dit_layers_on_device(enabled=False) separately (test_minimax_h3_contract.py:120-140, 779-791). Consider a contract test that mocks minimax_h3_denoise_loop for partition="combined", task="ref2va", asserting the denoise loop receives transformers_ref and resident staging is skipped. (evidence: pipeline_minimax_h3.py:1472 `transformer = self._transformer_for_task(task)`; pipeline_minimax_h3.py:1477 `with self._resident_dit_layers_on_device(enabled=transformer is self.transformer):`; pipeline_minimax_h3.py:1479-1480 `video_rows, audio_rows = minimax_h3_denoise_loop(model=transformer,`; test_minimax_h3_contract.py:139-140 `assert pipeline._transformer_for_task("ref2va") is pipeline.transformers_ref`; test_minimax_h3_contract.py:787-788 `with pipeline._resident_dit_layers_on_device(enabled=False):` — no diffuse() test under tests/diffusion/models/minimax_h3/) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:694` [minor] `load_weights` requires contiguous per-DiT weight blocks: `groupby` only merges consecutive same-prefix entries, and a repeated prefix raises `MiniMax-H3 weight source ... is not contiguous`. Add a negative test in `test +- `recipes/MiniMaxAI/MiniMax-H3-MUSA.md:67` [minor] This hunk tells MUSA users to pass `--task-type fl2va` for T2VA/FL2VA, but the only serve block below is Ref2VA (`MODEL="${MODEL_ROOT}/Ref2VA"` + `--task-type ref2va`). Add a mirrored FL2VA example (`MODEL="${MODEL_ROOT} +- `recipes/MiniMaxAI/MiniMax-H3-NPU.md:118` [minor] RainFusion/INT8 examples now attach to the single combined-service command (both DiTs loaded, all tasks on one endpoint), but lines 118 and 175-176 still say both optimizations are validated for T2VA only. Either show `- + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.trace.json.gz new file mode 100644 index 000000000..653c1e2d3 Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5720.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.cost.json new file mode 100644 index 000000000..ee6460ce2 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 363.4, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5723/run-20260815-083022-4341a7", + "recorded_at": "2026-08-15T08:36:24", + "attempts": [ + { + "invocation_id": "inv-30621cb6", + "outcome": "run:run-20260815-083022-4341a7", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-083022-4341a7", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-083022-4341a7", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2097, + "minutes": 6.03, + "ci_minutes": 0, + "input_tokens": 553870, + "output_tokens": 54641, + "tool_calls": 229, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3046, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.45, + "gate": 1.76, + "review": 355.6, + "report": 0.06 + }, + "time_to_first_result_s": 361.85 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5723.trace.json.gz", + "trace_bytes": 19081 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.md new file mode 100644 index 000000000..3857c9e49 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5723, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 93c0112c114e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] Scoped AMD support via footnote instead of ✅ matches PR intent to exclude MI325X; residual: row lacks H3 marker unlike Ming 1 pattern (line 24) +- [resolved] PR ‘mark AMD supported’ — footnote instead of ✅︎ is correct for scoped SKUs; residual: no inline H3 on row :47. +- [resolved] FL2VA local path vs HF repo ID — pipeline_minimax_h3.py:143-163 accepts both; residual: ROCm section lacks MODEL_ROOT setup unlike MiniMax-H3-MUSA.md:31 +- [claim-verified] FLASH_ATTN on ROCm resolves to AITER when gfx942/gfx950: vllm_omni/platforms/rocm/platform.py:96-129 +- [claim-verified] ROCm VAE GroupNorm patch exists: vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py:3-51 +- [claim-verified] MODEL=${MODEL_ROOT}/FL2VA is valid; pipeline resolves parent root: pipeline_minimax_h3.py:148-151 +- [claim-verified] gfx950 single-GPU functional row present at recipes/MiniMaxAI/MiniMax-H3.md:481 +- [claim-refuted] PR body ‘gfx950 single + 4-GPU’ — recipe evidence table only lists 1× gfx950 (:485). +- [claim-refuted] PR body ‘FLASH_ATTN ~2× faster than SDPA’ — absent from committed recipe/docs at head. +- [claim-refuted] PR goal 'mark AMD GPU supported' — no ✅︎ in table row at supported_models.md:47 +- [claim-refuted] PR body 'gfx950 single + 4-GPU' — recipe has no gfx950 4-GPU evidence row; line 469 notes text-enc TP not exercised on gfx950 +- [validated] Prerequisites ROCm pointer at recipes/MiniMaxAI/MiniMax-H3.md:61-63 links to new section +- [validated] gfx942 T2VA/FL2VA stage sums (248.28s, 275.67s) are below quoted client E2E (267.42s, 287.07s); gap is plausible non-stage overhead +- [validated] gfx950 denoise math 40×0.73≈29.2s consistent with ~55s warmup-inclusive E2E claim + +`recipes/MiniMaxAI/MiniMax-H3.md:416` [minor] — Line 416 says the prebuilt `vllm/vllm-omni-rocm:minimax-h3` image makes “All tasks work out of the box,” but the “Validated ROCm evidence” table (477–481) only lists T2VA and FL2VA on 4×gfx942 and T2VA on 1×gfx950—no Ref2VA row and no gfx950 4-GPU row. Either narrow the claim to validated workloads or add matching evidence rows. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:10 `- Tasks: T2VA, FL2VA, and Ref2VA`; recipes/MiniMaxAI/MiniMax-H3.md:416 `Prebuilt image: \`vllm/vllm-omni-rocm:minimax-h3\`. All tasks work out of the box:`; recipes/MiniMaxAI/MiniMax-H3.md:479-481 evidence table rows are `T2VA, 1344x768... | 4x gfx942...`, `FL2VA, 1344x768... | 4x gfx942...`, and `T2VA, 832x480... | 1x gfx950...` only (no Ref2VA, no gfx950 4-GPU).) + +`docs/models/supported_models.md:~50` [minor] — This diff adds an `H3` footnote (lines 101–105) scoping MiniMax H3 AMD validation to gfx942/gfx950, but the `MiniMaxH3Pipeline` row at line 50 still has an empty AMD GPU column while the table legend (line 99) says empty cells mean “not listed as supported.” Add a scoped AMD marker (e.g. `✅︎H3`) on that row and/or attach `H3` to the Models cell (cf. `imagegen1` at line 24) so readers scanning the AMD column see qualified support instead of absence. (evidence: docs/models/supported_models.md:50 `| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |` — AMD GPU column is blank; docs/models/supported_models.md:99 `✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.`; docs/models/supported_models.md:101 `H3 MiniMax H3 AMD GPU support is validated on gfx942 (MI300X) and`) + +`recipes/MiniMaxAI/MiniMax-H3.md:~426` [minor] — ROCm single- and four-GPU serve blocks (lines 426 and 446) set `export MODEL="${MODEL_ROOT}/FL2VA"` without defining `MODEL_ROOT`, while Prerequisites (line 35) and CUDA examples use `export MODEL=MiniMaxAI/MiniMax-H3`. Add `export MODEL_ROOT=/path/to/MiniMax-H3` before those ROCm commands (as in MiniMax-H3-MUSA.md:31) or switch to the HF repo ID for copy-paste consistency. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:426 `export MODEL="${MODEL_ROOT}/FL2VA"`; recipes/MiniMaxAI/MiniMax-H3.md:446 `export MODEL="${MODEL_ROOT}/FL2VA"`; recipes/MiniMaxAI/MiniMax-H3.md:35 `export MODEL=MiniMaxAI/MiniMax-H3` (unchanged by this diff, present in PR-time tree); recipes/MiniMaxAI/MiniMax-H3.md:205 `MODEL_ROOT=/path/to/MiniMax-H3 \` (offline script only, not ROCm serve setup); unchanged by this diff, present in PR-time tree: recipes/MiniMaxAI/MiniMax-H3-MUSA.md:31 `export MODEL_ROOT=/path/to/MiniMax-H3`) + +`recipes/MiniMaxAI/MiniMax-H3.md:401` [minor] — The PR body asserts FLASH_ATTN is ~2× faster than SDPA on ROCm, but the committed recipe at head documents only FLASH_ATTN guidance and FLASH_ATTN-only timings in the ROCm section (lines 400–481) with no TORCH_SDPA baseline or speedup ratio. Add gfx942/gfx950 FLASH_ATTN-vs-TORCH_SDPA evidence beside the ROCm backend guidance here, or drop the ~2× claim from the PR description. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:400-401 `MiniMax H3 runs on AMD Instinct GPUs (gfx942 / gfx950) in BF16. Use` `--diffusion-attention-backend FLASH_ATTN`, which resolves to AITER packed varlen attention on both architectures.` — unchanged by this diff, present in PR-time tree; grep over recipes/MiniMaxAI/MiniMax-H3.md and docs/models/supported_models.md finds zero matches for SDPA, TORCH_SDPA, 2×, or 'faster than'; recipes/MiniMaxAI/MiniMax-H3.md:479 `4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile` — evidence table lists FLASH_ATTN configs only, no SDPA comparison.) + +`recipes/MiniMaxAI/MiniMax-H3.md:471` [minor] — Mirror Cosmos3-Nano.md:510-515: add an attention-backend assertion note to the ROCm evidence section (e.g., timings were collected with FLASH_ATTN/AITER, not a silent TORCH_SDPA fallback) so benchmark rows are not read without knowing which backend produced them. (evidence: unchanged by this diff, present in PR-time tree: recipes/cosmos3/Cosmos3-Nano.md:510-514 `- **Attention backend:** the numbers above are aiter FlashAttention, not SDPA. ... falls back to `TORCH_SDPA` and says so only at debug level, which is easy to miss. If you are reporting numbers, assert the choice instead of trusting the default:`; recipes/MiniMaxAI/MiniMax-H3.md:471-486 `### Validated ROCm evidence` ... table rows list `FLASH_ATTN` in Configuration ... post-table text `gfx942 figures are the mean of three requests after one excluded warmup` with no attention-backend assertion guard) + +`recipes/MiniMaxAI/MiniMax-H3.md:479` [minor] — In the gfx942 ROCm evidence table, T2VA stage totals (encode 0.09 + denoise 244.04 + decode 4.15 = 248.28 s) are ~19 s below the quoted 267.42 s client E2E, and FL2VA (275.67 s vs 287.07 s) has a similar ~11 s gap—both unlabeled. Add a footnote or column clarifying that stage sums exclude non-stage/client overhead so readers do not treat them as full end-to-end latency. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:479 `| T2VA, 1344x768, 209 frames, 50 steps | 4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile | encode 0.09 s, denoise 244.04 s, decode 4.15 s, 267.42 s client E2E; H.264 24 FPS + 32 kHz stereo AAC |`; recipes/MiniMaxAI/MiniMax-H3.md:480 `| FL2VA, 1344x768, 209 frames, 50 steps | 4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile | encode 13.98 s, denoise 257.58 s, decode 4.11 s, 287.07 s client E2E; H.264 24 FPS + 32 kHz stereo AAC |`; recipes/MiniMaxAI/MiniMax-H3.md:483 `gfx942 figures are the mean of three requests after one excluded warmup` — no note that stage sums (248.28 s / 275.67 s) omit client/non-stage overhead vs client E2E) + +`recipes/MiniMaxAI/MiniMax-H3.md:411` [nit] — The ROCm install snippet uses plain `pip install "vllm==0.26.0+rocm723"` (line 411) and `VLLM_OMNI_TARGET_DEVICE=rocm pip install -e .` (line 413), while CUDA prerequisites in the same recipe use `uv pip install` (lines 51, 58) and docs/getting_started/installation/gpu/rocm.inc.md uses `uv pip install vllm==0.26.0+rocm723` (line 16). Align both ROCm install commands with `uv pip install` for consistency. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:51 `uv pip install -e .`; recipes/MiniMaxAI/MiniMax-H3.md:58 `uv pip install -e '.[fa4]'`; recipes/MiniMaxAI/MiniMax-H3.md:411 `pip install "vllm==0.26.0+rocm723" \`; recipes/MiniMaxAI/MiniMax-H3.md:413 `VLLM_OMNI_TARGET_DEVICE=rocm pip install -e . --no-build-isolation`; unchanged by this diff, present in PR-time tree: docs/getting_started/installation/gpu/rocm.inc.md:16 `uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723`) + +`recipes/MiniMaxAI/MiniMax-H3.md:438` [nit] — The `### ROCm four GPUs` block (438+) mirrors the CUDA four-GPU serve command but its post-command prose omits the CUDA inline warmup note (`The first request includes regional compilation; warm the server once before measuring steady-state latency.`). Add the same one-liner here for parity; gfx942 evidence already excludes one warmup in the table footnote (483), but readers who stop at the command block still lack the cue present in the CUDA section. (evidence: unchanged by this diff, present in PR-time tree: recipes/MiniMaxAI/MiniMax-H3.md:247-249 `Do not add \`--enforce-eager\` to this performance configuration. The first request includes regional compilation; warm the server once before measuring steady-state latency.`; recipes/MiniMaxAI/MiniMax-H3.md:438 `### ROCm four GPUs`; recipes/MiniMaxAI/MiniMax-H3.md:467-469 `As on CUDA, H3 is CFG-distilled, so keep \`--cfg-parallel-size\` at 1, and the H3 VAE supports only its native \`tile\` mode. \`--text-encoder-tp-size\` is validated on gfx942; on gfx950 it was not exercised.` (no warmup sentence); recipes/MiniMaxAI/MiniMax-H3.md:485-486 `functional-correctness validations, not tuned throughput; the first request includes lazy regional compilation.`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.trace.json.gz new file mode 100644 index 000000000..aad342a7b Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5723.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.cost.json new file mode 100644 index 000000000..56ffd30c7 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.cost.json @@ -0,0 +1,102 @@ +{ + "wall_s": 370.8, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5756/run-20260815-083952-e256a7", + "recorded_at": "2026-08-15T08:46:03", + "attempts": [ + { + "invocation_id": "inv-87b6dfeb", + "outcome": "run:run-20260815-083952-e256a7", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-083625-3ef523", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260815-083952-e256a7", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-083952-e256a7", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3246, + "minutes": 6.17, + "ci_minutes": 0, + "input_tokens": 901999, + "output_tokens": 73647, + "tool_calls": 243, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3563, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.69, + "gate": 1.96, + "review": 363.44, + "report": 0.06 + }, + "time_to_first_result_s": 370.14 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5756.trace.json.gz", + "trace_bytes": 45714 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.md new file mode 100644 index 000000000..f0ef9ec3d --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5756, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 43592108ac79) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Prior `extra_body` form field on video POST was ignored by server (expects `extra_params`); this PR fixes the fallback path — residual: only affects models without `params_builder` passing kwargs. +- [resolved] Commit 2f9ebd04 removed conflicting `duration` knob; pipeline accepts `num_frames` when `duration` absent (`pipeline_minimax_h3.py:403-407`) — residual: recipe still documents `duration` for curl users, ComfyUI uses num_frames/fps instead. +- [resolved] Commit 43592108 fixed `lookup_model_spec` for snapshot paths like `/.../MiniMax-H3/FL2VA` — residual: patterns still keyed on substring after first `/`, not full model id. +- [resolved] OLD generate_video sent extra_body form field but server only parses extra_params (api_server.py:3009); NEW code fixes wire format—no in-repo caller relied on extra_body kwargs +- [resolved] Wan I2V: no params_builder match so task is not injected; only input_reference image sent—same effective server path as before +- [resolved] model_params type leak — api_client pops type before params_builder/fallback (api_client.py:357-359); Wan path unchanged +- [resolved] Round-1 README migration ask (nodes.py:168 image→frame) — frame/references ports documented at README.md:119-124; residual: no explicit upgraders note in Quickstart (covered by kept comment on nodes.py:168). +- [claim-verified] PR claim '20 passed' matches parametrization count: 8 image + 2 understanding + 4 TTS + 4 Wan video + 2 MiniMax ref2va = 20 (`test_comfyui_integration.py`). +- [claim-verified] CI gates clean at head per gate_report; diff is ComfyUI-only plus e2e tests, no intersecting red jobs. +- [claim-verified] gate_report clean; PR CI mergeable at head +- [claim-verified] PR 20 passed — cited in PR body; not re-run this step +- [claim-verified] gate_report clean — mergeable, no failing CI checks at head. +- [claim-refuted] PR body '20 passed' likely stale—diff adds 2 parametrized Ref2VA cases atop existing suite +- [validated] ComfyUI client multipart fields match server parser at `api_server.py:2981-3043` and `serving_video.py:145-204` (`input_reference`, `audio_reference`, `input_references`, `extra_params`). + +`tests/e2e/features/comfyui/test_comfyui_integration.py:891` [minor] — This PR routes a connected `frame` to `video_task='fl2va'` with an `input_reference` upload in `api_client.generate_video` (api_client.py:302-303, 332-336) and updates the example workflow to wire `frame` → Generate Video with a MiniMax-H3 `/FL2VA` snapshot, but new e2e coverage only exercises Ref2VA (`VIDEO_REF2VA_IMAGE_AUDIO`, `VIDEO_REF2VA_MULTI_VIDEO` at tests/e2e/features/comfyui/test_comfyui_integration.py:77-78, 862-935). Add a parametrized FL2VA case that passes `frame` + `H3_MODEL_PARAMS` (not `references`) and assert the mock receives `task='fl2va'`, `input_reference`, and `flow_shift`/`audio_flow_shift` via `_minimaxh3_params_builder`, matching recipes/MiniMaxAI/MiniMax-H3.md §2. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:302-303 `input_reference_image = frame` / `video_task = "fl2va"`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:335 `form.add_field("input_reference", ...)`; tests/e2e/features/comfyui/test_comfyui_integration.py:77-78 `VIDEO_REF2VA_IMAGE_AUDIO = "video_ref2va_image_audio"` / `VIDEO_REF2VA_MULTI_VIDEO = "video_ref2va_multi_video"` (no FL2VA kind); tests/e2e/features/comfyui/test_comfyui_integration.py:891 `async def test_video_generation_node_minimax_h3_ref2va(`; unchanged by this diff, present in PR-time tree: apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json model widgets_values ends with `.../FL2VA` and link `[29,4,0,16,0,"IMAGE"]` connects `frame`.) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:71` [minor] — This PR raises VLLMOmniClient default max_poll_duration from 60*5 to 60*30 (api_client.py:71). VLLMOmniGenerateVideo builds VLLMOmniClient(url) without overrides (nodes.py:227), and generate_video is the sole consumer of max_poll_duration (api_client.py:387), so Wan/Cosmos/H3 and any other video model now wait up to 30 minutes before timing out instead of 5. Consider scoping the longer poll window to MiniMax-H3 (or per-model) and keeping the 5-minute default elsewhere. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:71 `max_poll_duration: float = 60 * 30,` — PR head default (base at same site was `max_poll_duration: float = 60 * 5`); apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:227 `client = VLLMOmniClient(url)` — no max_poll_duration override in VLLMOmniGenerateVideo; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:387 `deadline = asyncio.get_running_loop().time() + self.max_poll_duration` — only generate_video uses this timeout) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:781` [minor] — `VLLMOmniVideoReferences` has no `VALIDATE_INPUTS`, so invalid partial reference combinations are only rejected when `api_client.generate_video` raises `ValueError` during workflow execution (api_client.py:309-328), not at ComfyUI pre-queue validation. Consider adding `VALIDATE_INPUTS` on `VLLMOmniVideoReferences` mirroring those client checks so bad graphs fail before queue. (evidence: nodes.py:781 `class VLLMOmniVideoReferences:` — class has INPUT_TYPES and get_references only; no VALIDATE_INPUTS (contrast nodes.py:181 `def VALIDATE_INPUTS(cls, url, model, frame=None, references=None, **_kwargs)` on VLLMOmniGenerateVideo). nodes.py:801-825 `def get_references(...)` builds `refs = VideoReferences()` and assigns any non-None inputs with no combination validation. api_client.py:309 `raise ValueError("references is empty; connect at least one image/audio/video.")`; api_client.py:313-316 mixed video+image/audio rejection; api_client.py:324-327 invalid-combination rejection.) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/models.py:127` [minor] — In `utils/models.py:127-130`, `lookup_model_spec` still strips leading/trailing slashes then sets `lookup_key = normalized.split('/', 1)[-1]` and matches with `pattern.search(lookup_key)`, so lookup ignores the prefix before the first `/` and is not anchored to the full model/snapshot id. Consider matching the full normalized path (or documenting/enforcing the prefix constraint) to avoid collisions between similarly named snapshot segments. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:127 `normalized = model.rstrip("/").lstrip("/")`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:128 `lookup_key = normalized.split("/", 1)[-1]`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:130 `if pattern.search(lookup_key):` — matching uses only the substring after the first `/`, not the full model path.) + +`tests/e2e/features/comfyui/test_comfyui_integration.py:927` [minor] — The new `test_video_generation_node_minimax_h3_ref2va` passes `fps=VIDEO_FPS` (8) at line 928, but the updated example workflow uses fps=24 for MiniMax-H3/FL2VA and `pipeline_minimax_h3.py` rejects any other fps. Because `mock_async_omni` stubs `AsyncOmni.generate`, these tests only prove routing and would not catch a bad fps. Use `fps=24` (or an H3-specific constant) in the H3 parametrizations and mock expectations. (evidence: tests/e2e/features/comfyui/test_comfyui_integration.py:86 `VIDEO_FPS = 8`; tests/e2e/features/comfyui/test_comfyui_integration.py:928 `fps=VIDEO_FPS,`; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:399-401 `fps = int(sampling.fps or MINIMAX_H3_FPS)` / `if fps != MINIMAX_H3_FPS:` / `raise ValueError(f"MiniMax H3 output fps is fixed at {MINIMAX_H3_FPS}")`; apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json widgets_values include `832,480,24,120` for MiniMax-H3/FL2VA) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:~168` [minor] — This PR renames the optional Generate Video input from `image` to `frame` in `INPUT_TYPES` (nodes.py:168) and `generate()` (nodes.py:199). Saved ComfyUI workflows that link to the old `image` port will disconnect on upgrade. README documents `frame` but lacks a breaking-change/migration note—add a short note telling users to rewire LoadImage → `frame`. (evidence: PR head nodes.py:168 `"frame": ("IMAGE",),` in VLLMOmniGenerateVideo.INPUT_TYPES optional inputs; nodes.py:199 `frame: torch.Tensor | None = None` in generate(). Parent revision (pre-PR, same file/location): `"image": ("IMAGE",),` and `image: torch.Tensor | None = None`. README.md:120 documents `Connect a **frame** image` with no migration/breaking-change section (grep for migration|breaking|upgrade|rewire: no matches).) + +`apps/ComfyUI-vLLM-Omni/README.md:50` [minor] — Quickstart mentions reference-conditioned video under **Generate Video** but does not list the separate **Video References** node (`__init__.py:37`, display name `__init__.py:56`) that Ref2VA workflows must connect. Mirror the **TTS** / **TTS Voice Clone** pattern (README.md:52) and add a **Video References** bullet so Quickstart lists every new top-level node. (evidence: README.md:50 `- **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video`; README.md:52 `- **TTS** and **TTS Voice Clone** for TTS tasks`; README.md:122 `> For reference-conditioned generation (MiniMax-H3 Ref2VA), connect a **Video References** node instead.`; __init__.py:37 `"VLLMOmniVideoReferences": VLLMOmniVideoReferences`; __init__.py:56 `"VLLMOmniVideoReferences": "Video References"`) + +`apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:~1` [nit] — Node 10 MarkdownNote in the updated example workflow still closes with legacy routing ('depending on whether an input image is present'), but the PR renames the optional input to `frame`, adds `references`, and README documents frame vs Video References routing. Align the note with `frame`/`references` and automatic t2va/fl2va/ref2va routing so template users are not misled. (evidence: apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:1 `"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Video\" node supports both *text-to-video*, *frame-to-video*, and *reference-to-video* generation. The node will create corresponding payload depending on whether an input image is present or not."]` — unchanged by this diff, present in PR-time tree; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:168 `"frame": ("IMAGE",),` and nodes.py:169 `"references": ("VIDEO_REFERENCES",),`; apps/ComfyUI-vLLM-Omni/README.md:120 `> Do not use \`frame\` and \`references\` together. Task routing is automatic from which inputs you connect.`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.trace.json.gz new file mode 100644 index 000000000..8354025e1 Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5756.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.cost.json new file mode 100644 index 000000000..bfa082650 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.cost.json @@ -0,0 +1,102 @@ +{ + "wall_s": 422.0, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5779/run-20260815-083953-f94b03", + "recorded_at": "2026-08-15T08:46:54", + "attempts": [ + { + "invocation_id": "inv-b4abadb1", + "outcome": "run:run-20260815-083953-f94b03", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-083731-ed2f02", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260815-083953-f94b03", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-083953-f94b03", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3099, + "minutes": 7.02, + "ci_minutes": 0, + "input_tokens": 789535, + "output_tokens": 87956, + "tool_calls": 383, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3751, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.04, + "gate": 1.63, + "review": 415.11, + "report": 0.06 + }, + "time_to_first_result_s": 420.84 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5779.trace.json.gz", + "trace_bytes": 42415 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.md new file mode 100644 index 000000000..1e3e5c459 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5779, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f092a92eb579) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] skip-softmax timestep gate: pipeline passes video_sigma via record_denoise_step(normalized_timestep=video_sigma) (pipeline_minimax_h3.py:1376-1378); matches resolve_factor convention (high=early dense) in test_trtllm_attn.py:121-126; residual: no H3 end-to-end skip-gating unit test. +- [resolved] skip-softmax gate: H3 publishes descending video_sigma via record_denoise_step(normalized_timestep=video_sigma); matches skip_softmax.md 1→0 convention (residual: not scheduler t/ntt like Wan) +- [resolved] on_step→on_step_start/on_step_end: only pipeline_minimax_h3.py caller; no stale on_step consumers repo-wide +- [resolved] AllGather-KV rejected at layer.py:96-99 when allgather_degree>1 and backend TRTLLM_ATTN +- [resolved] Issue #5771 attn_mask rejection — packed prefix mask now accepted; residual checked: non-prefix masks still raise at trtllm_attn.py:322-323 +- [resolved] Skip-softmax timestep gate on H3 — record_denoise_step(..., normalized_timestep=video_sigma) fixes prior dense-only gate; sigma descends 1→0 like doc t semantics +- [resolved] Issue #5771 attn_mask rejection on H3 packed path: trtllm_attn.py:268-275 allows mask when has_packed_metadata; residual: no test wires MiniMaxH3Attention._run_packed_attention to TrtllmAttentionImpl +- [resolved] SAGE non-finite on short token_refiner: dense fallback at trtllm_attn.py:364-371; residual: no test asserts role=minimax_h3.token_refiner path +- [claim-verified] PR fixes #5771 root cause: trtllm_attn.py:269-275 previously rejected any attn_mask; head now allows prefix-valid packed masks with cu_seqlens. +- [claim-verified] B300 manual matrix in PR body (dense/SAGE/skip-softmax combos, 37 passed unit tests) not re-run here; gate_report shows CI green at head. +- [claim-verified] PR 37 passed B300 TRTLLM tests cited; CI merge gate clean per gate_report +- [claim-verified] test_trtllm_attn.py pytestmark core_model+diffusion+cuda; runs under .buildkite/cuda/test-ready.yml diffusion cuda lane +- [claim-verified] PR B300 serve matrix (dense/SAGE/skip-softmax combos HTTP 200) — not re-run here; unit/GPU-marked tests added in diff +- [claim-verified] on_step rename — repo-wide grep: zero external callers of old minimax_h3_denoise_loop on_step kwarg + +`recipes/MiniMaxAI/MiniMax-H3.md:269` [minor] — This example gates Skip-Softmax with `disabled_until_timestep: 0.97`, but H3 publishes descending **video sigma** (`sigmas_video[step]`) via `record_denoise_step(..., normalized_timestep=video_sigma)`. Reword the recipe to say the gate uses video sigma (1→0), not Wan/global `t = timestep / num_train_timesteps`, so readers do not copy skip_softmax.md semantics verbatim. (evidence: unchanged by this diff, present in PR-time tree: recipes/MiniMaxAI/MiniMax-H3.md:269 `threshold=0.05`, after the normalized timestep reaches `0.97`:`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1376-1378 `on_step_start=lambda step, video_sigma, audio_sigma: self.record_denoise_step(step, normalized_timestep=video_sigma)`; vllm_omni/diffusion/models/minimax_h3/denoise_loop.py:208-212 `s_v, s_v_next = sigmas_video[step], sigmas_video[step + 1]` and `# Attention gates use the scheduler-style descending timestep.` / `on_step_start(step, s_v, s_a)`; vllm_omni/diffusion/forward_context.py:229-230 `if normalized_timestep is not None: _forward_context.denoise_timestep = float(normalized_timestep)`; vllm_omni/diffusion/attention/backends/trtllm_attn.py:71 `if self.gated and timestep is not None and timestep > self.disabled_until_timestep:`; docs/design/feature/skip_softmax.md:85-86 ``t` is the scheduler's own timestep divided by `num_train_timesteps`, published by the pipeline via `DenoiseProgressMixin.record_denoise_step`.``) + +`vllm_omni/diffusion/attention/layer.py:~96` [minor] — This PR adds a constructor-time ValueError in Attention.__init__ when allgather_degree > 1 and the selected backend is TRTLLM_ATTN (vllm_omni/diffusion/attention/layer.py:96–100), but the diff adds no pytest covering that configuration. Add a CPU test in tests/diffusion/attention/test_attention_config.py (stub get_attn_backend_for_role to return TRTLLM_ATTN with parallel_config.allgather_degree=2) asserting the raise, mirroring TestDiffusionKvCacheQuantization.test_diffusion_kv_cache_dtype_fp8_raises_with_ring_attention. (evidence: vllm_omni/diffusion/attention/layer.py:96 `if not skip_sequence_parallel and allgather_degree > 1 and attn_backend_cls.get_name() == "TRTLLM_ATTN":`; vllm_omni/diffusion/attention/layer.py:97-100 `raise ValueError("TRTLLM_ATTN does not support AllGather-KV sequence parallelism. ...")`; unchanged by this diff, present in PR-time tree: tests/diffusion/attention/test_attention_config.py:34 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]` and test_attention_config.py:583-594 ring+KV guard raise test — grep for AllGather error text under tests/ returns no matches) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:189` [minor] — No MiniMax H3 integration or serve-bootstrap test asserts TRTLLM is auto-selected on datacenter Blackwell without --diffusion-attention-backend. Coverage is limited to test_mask_free_allowlist_gates_trtllm_default (metadata allowlist) plus manual/L4 recipe log checks; consider a Blackwell bootstrap test that loads MiniMaxH3Pipeline and asserts TRTLLM auto-routing. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:139 `def test_packed_attention_keeps_padding_mask_for_other_backends():` — cited anchor is unrelated to TRTLLM default routing; unchanged by this diff, present in PR-time tree. tests/diffusion/attention/test_trtllm_attn.py:189-194 `def test_mask_free_allowlist_gates_trtllm_default():` … `assert meta("MiniMaxH3Pipeline").attention_mask_free is True` — only MiniMaxH3 TRTLLM-related test; checks metadata flag, not serve/bootstrap backend selection. Repo-wide under tests/diffusion/models/minimax_h3/: zero matches for TRTLLM_ATTN/TRTLLM/serve/bootstrap (grep). recipes/MiniMaxAI/MiniMax-H3.md:250 `contains \`Defaulting to diffusion attention backend TRTLLM_ATTN\` before` — manual log verification, not CI-gated.) + +`.buildkite/cuda/test-nightly.yml:434` [minor] — Nightly H3 accuracy (`.buildkite/cuda/test-nightly.yml:430-434`) runs on `h100_4` without `--diffusion-attention-backend` or `DIFFUSION_ATTENTION_BACKEND`; H100 platform default stays `FLASH_ATTN` (`cuda/platform.py` gates TRTLLM to `major == 10`), so this lane does not exercise the PR’s datacenter-Blackwell TRTLLM auto-default for MiniMax H3. (evidence: unchanged by this diff, present in PR-time tree: .buildkite/cuda/test-nightly.yml:430-434 `- label: ":full_moon: Diffusion X2V · MiniMax H3 T2VA Accuracy Test"` / `pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py` / `mirror_hardwares: h100_4`; vllm_omni/platforms/cuda/platform.py:205-211 `trtllm_attn_default_ok = (allow_trtllm_default and compute_capability is not None and compute_capability.major == 10 and head_size == 128 and trtllm_gen_available)`; vllm_omni/platforms/cuda/platform.py:243-245 `if flash_attn_supported: logger.debug("Defaulting to diffusion attention backend FLASH_ATTN")`) + +`vllm_omni/diffusion/attention/cuda/platform.py:205` [minor] — TRTLLM auto-default in vllm_omni/platforms/cuda/platform.py (205-211) requires compute_capability.major == 10 (datacenter sm_100/sm_103); workstation sm_120/sm_121 (major 12) fall through to CUDNN_ATTN. This PR already documents the split in attention_backends.md and scopes the H3 recipe to datacenter Blackwell—no further doc change required unless inline platform.py comments are desired. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/platforms/cuda/platform.py:205-211 `trtllm_attn_default_ok = ( allow_trtllm_default and compute_capability is not None and compute_capability.major == 10 and head_size == 128 and trtllm_gen_available )`; vllm_omni/platforms/cuda/platform.py:220-224 `if is_blackwell and cudnn_blackwell_ready: logger.info( "Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell %s, cuDNN %d)", sm_str, cudnn_version, )`; docs/user_guide/diffusion/attention_backends.md:114 `TRTLLM_ATTN is skipped on workstation Blackwell (sm_120 / sm_121)`) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:~408` [minor] — MiniMaxH3Pipeline is marked attention_mask_free and TrtllmAttentionImpl.forward_cuda now handles packed cu_seqlens/prefix masks, but test_minimax_h3_contract.py:97-175 still stubs backends with FakeBackend only. test_trtllm_attn.py packed tests (e.g. line 212) hand-build AttentionMetadata, and real-kernel packed parity (line 389) is @requires_trtllm_attn. Consider extending H3 contract tests to assert _run_packed_attention metadata is accepted by TrtllmAttentionImpl.forward_cuda. (evidence: unchanged by this diff, present in PR-time tree: minimax_h3_transformer.py:408 `if used < packed_total and not prefix_slice:`; test_minimax_h3_contract.py:102-106 `class FakeBackend:` / `attn_backend = FakeBackend`; test_minimax_h3_contract.py:125-135 calls `_run_packed_attention` and asserts `attn_mask is None` via FakeAttention stub only; test_trtllm_attn.py:212 `def test_packed_metadata_trims_padding_mask(monkeypatch):` builds `AttentionMetadata(...)` by hand with no H3 import; test_trtllm_attn.py:20-21 `if torch.cuda.get_device_capability()[0] < 10: return False`; test_trtllm_attn.py:388-389 `@requires_trtllm_attn` / `def test_bf16_packed_padding_matches_sdpa():`; model_metadata.py:46 `attention_mask_free=True,` on MiniMaxH3Pipeline) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~1376` [minor] — This PR wires skip-softmax timestep gating via on_step_start at pipeline_minimax_h3.py:1376–1378 (record_denoise_step(step, normalized_timestep=video_sigma)), invoked from denoise_loop.py:212 each step. Only an isolated mixin test exists at test_trtllm_attn.py:146–147; add a small minimax_h3_denoise_loop test with on_step_start asserting forward_context.denoise_timestep tracks descending video_sigma values. (evidence: pipeline_minimax_h3.py:1376 `on_step_start=lambda step, video_sigma, audio_sigma: self.record_denoise_step(`; pipeline_minimax_h3.py:1378 `normalized_timestep=video_sigma,`; denoise_loop.py:212 `on_step_start(step, s_v, s_a)`; test_trtllm_attn.py:146 `_Pipe().record_denoise_step(5, normalized_timestep=0.7)` — unchanged by this diff, present in PR-time tree: no matches for on_step_start or minimax_h3_denoise_loop under tests/) + +`tests/diffusion/attention/test_trtllm_attn.py:~29` [minor] — This PR adds `@requires_trtllm_attn`/`@requires_sage` packed parity tests (`test_bf16_packed_padding_matches_sdpa`, `test_sage_packed_non_aligned_length_matches_sdpa`) that only run on SM100+ with flashinfer. Merge-gate CI has no Blackwell lane and no step executes `tests/diffusion/attention/test_trtllm_attn.py` on L4/H100—the cuda shard is `tests/diffusion/*.py` only, and the recursive cpu sweep deselects this file’s cuda-marked tests—so real-kernel TRTLLM regressions are not validated per-PR. Please document B300 manual validation in the test plan or add a Blackwell CI step for this file. (evidence: tests/diffusion/attention/test_trtllm_attn.py:20-21 `if torch.cuda.get_device_capability()[0] < 10:`; tests/diffusion/attention/test_trtllm_attn.py:29-31 `requires_trtllm_attn = pytest.mark.skipif(`; tests/diffusion/attention/test_trtllm_attn.py:388 `@requires_trtllm_attn` on `test_bf16_packed_padding_matches_sdpa`; unchanged by this diff, present in PR-time tree: .buildkite/cuda/test-ready.yml:68 `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` with `mirror_hardwares: l4_1`; grep across `.buildkite` shows no B300/Blackwell `mirror_hardwares` preset.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.trace.json.gz new file mode 100644 index 000000000..5d8226aee Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5779.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.cost.json new file mode 100644 index 000000000..a684e091c --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.cost.json @@ -0,0 +1,102 @@ +{ + "wall_s": 450.7, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5801/run-20260815-083953-13dcb5", + "recorded_at": "2026-08-15T08:47:23", + "attempts": [ + { + "invocation_id": "inv-79c3cc7b", + "outcome": "run:run-20260815-083953-13dcb5", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-083821-d791af", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + }, + { + "run_dir": "run-20260815-083953-13dcb5", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-083953-13dcb5", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3454, + "minutes": 7.49, + "ci_minutes": 0, + "input_tokens": 826245, + "output_tokens": 111223, + "tool_calls": 372, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4028, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.22, + "gate": 2.07, + "review": 443.79, + "report": 0.07 + }, + "time_to_first_result_s": 449.13 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5801.trace.json.gz", + "trace_bytes": 31557 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.md new file mode 100644 index 000000000..5f47f6c02 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5801, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1ef0cb1562a6) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] half_head_dim API blast radius — grep RotaryEmbedding( in diffusion models: ovis/qwen/flux/hunyuan/bagel use defaults; only H3 sets False; Wan overrides own forwards +- [resolved] RMSNorm dtype blast radius — only H3 _norm passes bf16; wan/z_image/hunyuan/cosmos3 use default fp32 weight +- [resolved] H3 full-dim cos on native path — `_prepare` slices neox cos to half before `apply_rotary_emb_torch`; residual: CUDA/ROCm fused path not unit-tested with `half_head_dim=False` +- [resolved] H3 `nn.RMSNorm`→custom `RMSNorm` — only H3 passes `dtype=bf16` to diffusion RMSNorm; residual: fused CUDA `rms_norm` numerics vs `nn.RMSNorm` rely on PR visual evidence not L2 +- [resolved] H3 inline RoPE→`RotaryEmbedding` submodule — residual: production uses `forward_cuda` not the native path forced in UT; compile/fusion boundary changed but PR reports net perf gain +- [resolved] fused RMSNorm CUDA parity exists pre-PR (`test_rmsnorm_fused_native_parity_*`) — residual: uses post-init `.to(dtype)`, not new `dtype=` constructor used by H3 +- [resolved] RoPE 3D cos batch stripping — existing test_rope_broadcast regressions still apply to default `half_head_dim=True` consumers +- [claim-verified] PR fp32 native accumulation — forward_native upcasts x/weight; test_rmsnorm_dtype_matches_parameter_and_fp32_accumulation_contract +- [claim-verified] PR full-dim MindIE cos/sin — apply_rotary_emb_mindiesd skips repeat when half_head_dim=False +- [claim-verified] PR enables fused RMSNorm/RoPE for H3 — minimax_h3_transformer.py uses diffusion RMSNorm and RotaryEmbedding +- [claim-verified] PR preserves fp32 accumulation in native RMSNorm fallback — norm.py:127-133 +- [claim-verified] PR adds full-head-dim RoPE support for H3 — minimax_h3_transformer.py:335,364 + rope.py:124-126 +- [claim-verified] PR purpose fused RMSNorm/RoPE for H3 — `_norm` and `self.rope` now use diffusion layer ops +- [claim-refuted] PR -3.35% latency — calc (491014-475070)/491014*100 = 3.25%; ~0.1pp rounding gap + +`tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:73` [minor] — This diff routes H3 `_apply_rope` through `RotaryEmbedding(is_neox_style=True, half_head_dim=False)` and adds `test_h3_fused_rope_matches_reference_and_preserves_unrotated_dims`, but line 73 sets `attention.rope._forward_method = attention.rope.forward_native`, so the test never dispatches through `forward_cuda`/`forward_hip` and therefore does not exercise the fused flash-attn kernel after `_prepare_half_head_dim_cos_sin` slices full-width cos/sin—the path used on CUDA with `--diffusion-attention-backend FLASH_ATTN`. Consider a `core_model and cuda` test that calls `_apply_rope` (or `RotaryEmbedding.forward_cuda`) without overriding `_forward_method`, comparing to the same NeoX reference with duplicated freqs. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:73 `attention.rope._forward_method = attention.rope.forward_native`; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/layers/custom_op.py:33-34 `return self._forward_method(*args, **kwargs)` and vllm_omni/diffusion/layers/custom_op.py:22-23 `elif current_omni_platform.is_cuda(): return self.forward_cuda`; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335 `self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)`; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:353 `x_rot = self.rope(x_rot, cos, sin)`; vllm_omni/diffusion/layers/rope.py:137 `cos, sin = self._prepare_half_head_dim_cos_sin(cos, sin)` inside `forward_cuda`) + +`tests/diffusion/layers/test_norm.py:305` [minor] — `.buildkite/amd/test-amd-ready.yml` collects `tests/diffusion` with `-m 'core_model and cpu'`; this PR adds new tests in `tests/diffusion/layers/test_norm.py`, `tests/diffusion/layers/test_rope_broadcast.py`, and `tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py` (all module-marked `core_model` and `cpu`). If `buildkite/vllm-omni-amd-ci` is failing, confirm whether the regression is introduced here or pre-existing on main. (evidence: tests/diffusion/layers/test_norm.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]`; tests/diffusion/layers/test_rope_broadcast.py:21 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:8 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`; unchanged by this diff, present in the PR-time tree: .buildkite/amd/test-amd-ready.yml:18 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; tests/diffusion/layers/test_norm.py:305 `def test_rmsnorm_dtype_matches_parameter_and_fp32_accumulation_contract():`) + +`tests/diffusion/layers/test_norm.py:305` [minor] — H3 now builds RMSNorm via dtype= (minimax_h3_transformer.py:177), and test_rmsnorm_dtype_matches_parameter_and_fp32_accumulation_contract only checks forward_native (test_norm.py:319). On CUDA, forward_cuda defaults to _forward_fused (norm.py:95); existing test_rmsnorm_fused_native_parity_cuda still uses post-hoc weight.to(dtype) rather than the dtype= constructor. Consider extending fused/native parity to RMSNorm(..., dtype=torch.bfloat16) for bf16 activations. (evidence: tests/diffusion/layers/test_norm.py:319 `torch.testing.assert_close(norm.forward_native(x), expected, atol=0, rtol=0)` — new dtype-contract test calls only forward_native; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/layers/norm.py:95 `return self._forward_fused(x)` — CUDA path uses fused kernel; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177 `return RMSNorm(size, eps=eps, dtype=dtype)` — H3 uses dtype= constructor; tests/diffusion/layers/test_norm.py:564-565 `norm = RMSNorm(hidden_size, eps=eps).cuda()` / `norm.weight.data = norm.weight.data.to(dtype)` — existing parity test does not use dtype=.) + +`tests/diffusion/layers/test_rope_broadcast.py:242` [minor] — This diff updates RotaryEmbedding.forward_npu to pass self.half_head_dim into apply_rotary_emb_mindiesd (rope.py:175), but test_mindie_full_dim_rope_keeps_h3_cos_sin_layout only calls that helper directly (test_rope_broadcast.py:259). Add a test that instantiates RotaryEmbedding(is_neox_style=True, half_head_dim=False) with mindiesd stubbed and has_mindie enabled, then calls rope.forward_npu(x, cos, sin) so the production NPU dispatch path is covered. (evidence: tests/diffusion/layers/test_rope_broadcast.py:259 `actual = apply_rotary_emb_mindiesd(x, cos, sin, interleaved=False, half_head_dim=False)` — test calls helper only; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/layers/rope.py:175 `return apply_rotary_emb_mindiesd(x, cos, sin, self.interleaved, self.half_head_dim)` — production NPU path not exercised by this test) + +`tests/diffusion/layers/test_rope_broadcast.py:242` [minor] — This diff adds `_prepare_half_head_dim_cos_sin` and wires H3 with `half_head_dim=False`, but `test_mindie_full_dim_rope_keeps_h3_cos_sin_layout` only calls `apply_rotary_emb_mindiesd` directly. Consider adding a `tests/diffusion/layers` test for `RotaryEmbedding(is_neox_style=True, half_head_dim=False)` that feeds full-width `[T, rot_dim]` cos/sin and asserts `forward_cuda`/`forward_hip` match `forward_native` for the NeoX H3 layout (the H3 parallel test currently pins `forward_native` only). (evidence: tests/diffusion/layers/test_rope_broadcast.py:259 `actual = apply_rotary_emb_mindiesd(x, cos, sin, interleaved=False, half_head_dim=False)` — tests the MindIE helper only, not `RotaryEmbedding`; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/layers/rope.py:124-126 `if not self.half_head_dim: cos = cos[..., ::2] if self.interleaved else cos[..., : cos.shape[-1] // 2]` and rope.py:137 `cos, sin = self._prepare_half_head_dim_cos_sin(cos, sin)` in `forward_cuda`; tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:73 `attention.rope._forward_method = attention.rope.forward_native` — H3 rope test does not exercise CUDA/HIP; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335 `self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)`) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335` [minor] — Test Result pins vLLM-Omni Commit to 5215e03, but head is 1ef0cb15 after merging main. Re-run the documented E2E benchmark on 1ef0cb15 and update the PR test plan so the -3.35% latency and precision evidence describe the merged tree. (evidence: show_commit 1ef0cb15: `Merge branch 'main' into h3_opt` — frozen head is a main-merge commit, not 5215e03; PR Test Result (fetch pr_context): `**vLLM-Omni Commit:** 5215e03a91adecbb5ffece29aa74360a7569d0c5` with `E2E Latency: 491014ms -> 475070ms. **-3.35%**`) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:173` [minor] — H3 `_norm` now returns diffusion `RMSNorm(..., dtype=dtype)` (minimax_h3_transformer.py:177); `q_norm`/`k_norm` invoke it on CUDA via `_forward_fused` (norm.py:95). `test_rmsnorm_fused_native_parity_custom_weight` documents bf16 fused-vs-native gap up to atol 2e-2 (test_norm.py:640) at hidden_size 64 / eps 1e-6 — consider H3-scoped fused coverage at attention_head_dim=128 and qk_norm_eps=1e-5 now that production no longer uses `nn.RMSNorm`. (evidence: minimax_h3_transformer.py:177 `return RMSNorm(size, eps=eps, dtype=dtype)` — replaces prior nn.RMSNorm factory; minimax_h3_transformer.py:333-334 `self.q_norm = _norm(arch.attention_head_dim, eps=arch.qk_norm_eps)` / `self.k_norm = _norm(arch.attention_head_dim, eps=arch.qk_norm_eps)` with attention_head_dim=128 (line 55); minimax_h3_transformer.py:447-448 `q = self.q_norm(q)` / `k = self.k_norm(k)`; unchanged by this diff, present in PR-time tree: norm.py:95 `return self._forward_fused(x)` in `forward_cuda`; test_norm.py:640 `atol_map = {torch.float32: 1e-3, torch.float16: 1e-3, torch.bfloat16: 2e-2}` documenting fused bf16 vs fp32-native divergence) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:175` [nit] — This diff switches `_norm` from `torch.nn.RMSNorm` to diffusion `RMSNorm` (line 177), but the comment at lines 175–176 still describes `torch.nn.RMSNorm` fp32 upcasting. Update it to reference `RMSNorm.forward_native` fp32 accumulation (norm.py:130–135) and note CUDA/HIP use `_forward_fused` (norm.py:95). (evidence: minimax_h3_transformer.py:173 `def _norm(size: int, *, eps: float, dtype: torch.dtype = _BF16_DTYPE) -> RMSNorm:`; minimax_h3_transformer.py:175 `# torch.nn.RMSNorm upcasts reduced-precision inputs for the variance`; minimax_h3_transformer.py:177 `return RMSNorm(size, eps=eps, dtype=dtype)`; norm.py:131 `x = x.to(torch.float32)`; norm.py:95 `return self._forward_fused(x)`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.trace.json.gz new file mode 100644 index 000000000..73a74d03d Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5801.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.cost.json new file mode 100644 index 000000000..128a5eccb --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 311.1, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5833/run-20260815-082327-efdb02", + "recorded_at": "2026-08-15T08:28:37", + "attempts": [ + { + "invocation_id": "inv-5dbeee14", + "outcome": "run:run-20260815-082327-efdb02", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-082327-efdb02", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-082327-efdb02", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1562, + "minutes": 5.17, + "ci_minutes": 0, + "input_tokens": 367595, + "output_tokens": 51770, + "tool_calls": 174, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2556, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.71, + "gate": 2.0, + "review": 304.66, + "report": 0.07 + }, + "time_to_first_result_s": 310.42 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5833.trace.json.gz", + "trace_bytes": 14138 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.md new file mode 100644 index 000000000..097124123 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5833, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac67e8874d00) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 6 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] ar_diffusion_pipeline_capability.md not listed — doc:3-6 states 'experimental and non-normative' and 'not yet part of the stable module-design hierarchy'; residual checked: no repo references +- [resolved] Base nav duplicate vae_parallel entry removed; still listed once at docs/.nav.yml:130 +- [resolved] Base index.md was sparse (4 feature links); PR aligns index with full nav set +- [resolved] ar_diffusion_pipeline_capability.md orphan intentional — H1 marks 'experimental and non-normative' (docs/design/ar_diffusion_pipeline_capability.md:3-6) +- [resolved] Base nav duplicated vae_parallel.md twice; head lists it once under Parallelism (grep shows single nav entry at docs/.nav.yml:130) +- [resolved] Base index.md listed only 4 feature docs while nav had ~20; head index now covers the full nav set including connectors, async_chunk, prefix_caching, async_omni_architecture +- [resolved] Quantization link under Diffusion Acceleration is intentional per PR body; residual checked: overview.md documents diffusion, omni/TTS, and multi-stage diffusion scopes (docs/user_guide/quantization/overview.md:6-8) +- [claim-verified] diff_stat main..ac67e887: only docs/.nav.yml (+63/-24) and docs/design/index.md (+52/-2) changed +- [claim-verified] PR scope 'arrangement only, no design doc edits' holds — no .md under docs/design/feature|module| was modified +- [claim-verified] gate_report: mergeable, no failing checks +- [claim-verified] diff_stat main..ac67e887 — only docs/.nav.yml and docs/design/index.md changed (+91/-24) +- [claim-verified] diff_stat at head: only docs/.nav.yml and docs/design/index.md changed (2 files, +91/-24) +- [claim-verified] PR scope claim matches diff; no code, test, or CI config touched +- [claim-refuted] PR 'arrangement only' — base nav omitted metrics.md, qwen3_omni_tts_performance_optimization.md, async_diffusion_output.md, pipeline_parallel.md; head adds all four to sidebar + +`docs/.nav.yml:111` [minor] — Head adds four Design Documents sidebar entries that base nav omitted (`design/metrics.md`, `design/qwen3_omni_tts_performance_optimization.md`, `design/feature/async_diffusion_output.md`, `design/feature/pipeline_parallel.md`); the markdown files pre-exist and are unchanged. If the PR is described as arrangement-only, update the title/body to include surfacing these docs in scope. (evidence: docs/.nav.yml:111 ` - Async Diffusion Output: design/feature/async_diffusion_output.md` | docs/.nav.yml:127 ` - Pipeline Parallel: design/feature/pipeline_parallel.md` | docs/.nav.yml:138 ` - Prometheus Metrics: design/metrics.md` | docs/.nav.yml:139 ` - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md` — unchanged by this diff, absent from base nav (PR-time merge-base): base `docs/.nav.yml` Feature Design ends at `design/feature/diffusion_continuous_batching.md` with no `async_diffusion_output`, `pipeline_parallel`, `metrics.md`, or `qwen3_omni_tts_performance_optimization.md` entries; pre-existing files present in tree: `docs/design/metrics.md`, `docs/design/qwen3_omni_tts_performance_optimization.md`, `docs/design/feature/async_diffusion_output.md`, `docs/design/feature/pipeline_parallel.md`) + +`docs/.nav.yml:102` [minor] — MkDocs nav build success was reported by the author but was not re-run during review. Confirm head still passes an MkDocs/`awesome-nav` build so the regrouped `Design Documents` subtree (`docs/.nav.yml:102+`) renders correctly. (evidence: unchanged by this diff, present in PR-time tree: mkdocs.yml:73 ` - awesome-nav`; docs/.nav.yml:102 ` - Design Documents:`; docs/.nav.yml:106-107 ` - Feature Design:` / ` - Runtime and Stage Execution:`) + +`docs/design/index.md:53` [minor] — Under `### Diffusion acceleration`, `docs/design/index.md` lists Cache-DiT, TeaCache, and Diffusion Continuous Batching directly after the `#### Quantization` block (lines 49–55) with no intervening `####`, so the landing page renders them under Quantization. In `docs/.nav.yml` (lines 133–136), Quantization, Cache-DiT, TeaCache, and Diffusion Continuous Batching are sibling entries under Diffusion Acceleration. Add a separate `####` heading (e.g., Cache acceleration) before line 53 so the index taxonomy matches the sidebar. (evidence: unchanged by this diff, present in the PR-time tree: docs/design/index.md:49 `#### Quantization`; docs/design/index.md:51 `- [Quantization Overview](../user_guide/quantization/overview.md)`; docs/design/index.md:53 `- [Cache-DiT](feature/cache_dit.md)` (no new `####` before lines 54–55); docs/.nav.yml:133 ` - Quantization: user_guide/quantization/overview.md`; docs/.nav.yml:134 ` - Cache-DiT: design/feature/cache_dit.md`; docs/.nav.yml:135 ` - TeaCache: design/feature/teacache.md`; docs/.nav.yml:136 ` - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md`) + +`docs/.nav.yml:113` [nit] — `docs/.nav.yml` labels the entry `Automatic Prefix Caching`, while `docs/design/index.md` (line 20) and the target doc H1 use `Automatic Prefix Caching in Omni Models`. Align the nav label with the index link text and doc H1 for consistent search/bookmark labels. (evidence: docs/.nav.yml:113 ` - Automatic Prefix Caching: design/feature/prefix_caching.md`; docs/design/index.md:20 `- [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md)`; docs/design/feature/prefix_caching.md:1 `# Automatic Prefix Caching in Omni Models`) + +`docs/.nav.yml:146` [nit] — `docs/.nav.yml` shortens the module doc to `AsyncOmni Architecture`, while `docs/design/index.md` and `async_omni_architecture.md` H1 use `AsyncOmni Architecture (Qwen3-Omni Example)`. Match the nav label to the index link text and doc H1. (evidence: docs/.nav.yml:146 ` - AsyncOmni Architecture: design/module/async_omni_architecture.md`; docs/design/index.md:72 `- [AsyncOmni Architecture (Qwen3-Omni Example)](module/async_omni_architecture.md)`; unchanged by this diff, present in PR-time tree: docs/design/module/async_omni_architecture.md:1 `# AsyncOmni Architecture (Qwen3-Omni Example)`) + +`docs/design/index.md:13` [nit] — This diff retitles nav groups to title case (`Runtime and Stage Execution`, `Diffusion Acceleration`, `Communication`, `OmniConnector Implementations` in `docs/.nav.yml`), but matching landing-page headings in `docs/design/index.md` stay sentence case (`Runtime and stage execution`, `Diffusion acceleration`, `Communication`, `OmniConnector implementations`). Align subsection heading casing between nav and index for a consistent reader journey. (evidence: docs/.nav.yml:108 has 'Runtime and Stage Execution:'; docs/.nav.yml:123 has 'Diffusion Acceleration:'; docs/design/index.md:12 has '### Runtime and stage execution'; docs/design/index.md:33 has '### Diffusion acceleration') + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.trace.json.gz new file mode 100644 index 000000000..d2d22440c Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5833.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.cost.json new file mode 100644 index 000000000..5d0591116 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 531.2, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5864/run-20260815-084603-27aa12", + "recorded_at": "2026-08-15T08:54:54", + "attempts": [ + { + "invocation_id": "inv-30c82c47", + "outcome": "run:run-20260815-084603-27aa12", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-084603-27aa12", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-084603-27aa12", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4105, + "minutes": 8.84, + "ci_minutes": 0, + "input_tokens": 1102613, + "output_tokens": 102573, + "tool_calls": 439, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4657, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.91, + "gate": 1.65, + "review": 524.97, + "report": 0.06 + }, + "time_to_first_result_s": 530.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5864.trace.json.gz", + "trace_bytes": 42644 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.md new file mode 100644 index 000000000..81b105eed --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.md @@ -0,0 +1,64 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5864, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 07fc831e7a43) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Implicit `request_batch_max_wait_ms=500` removal is intentional — `test_engine_allows_independent_dlo_dp_requests` asserts value stays 0 (`test_diffusion_engine.py:380`); residual: `_wait_for_request_batch_admission_locked` returns immediately when wait is 0 (`diffusion_engine.py:478-480`). +- [resolved] Signal handler now calls `WorkerProc.shutdown()` (async thread sentinel + `result_mq` teardown) instead of `worker.shutdown()` only (`diffusion_worker.py:933-962,1235-1238`). +- [resolved] async output shutdown sentinel — worker uses None sentinel; test_bg_thread updated (no _running=False hack) +- [resolved] inline premarked shutdown — test_inline_shutdown_runs_after_orchestrator_premark; engine.close() before executor pool shutdown +- [resolved] num_inference_steps=None rejection removed — multiproc_executor.py uses build_request_batch_sampling_params_key; test_dlo_dp_allows_shared_default_denoise_steps passes; residual: docs still say None forbidden (cpu_offload_diffusion.md:298-309). +- [resolved] Inline shared sampling_params — inline_stage_diffusion_client.py:110 clones; StageDiffusionProc reconstructs per ZMQ message, so ZMQ serving path unaffected. +- [resolved] request_batch_max_wait_ms auto-500 removal — intentionally preserved at 0; example sets batch_wait_ms=500 explicitly; residual: callers relying on implicit 500 need explicit config +- [resolved] numpy import in ipc.py — std dep via vllm/torch stack; no new pyproject bound needed +- [resolved] kept diffusion_engine request_batch_max_wait_ms doc ask — at head init no longer injects 500ms default (diffusion_engine.py:247-254); residual: admission returns immediately when wait is 0 (diffusion_engine.py:479-480). +- [resolved] kept multiproc executor MessageQueue.shutdown ask — production shutdown still nulls queues without shutdown(); test_diffusion_ipc finally block shows intended cleanup pattern. +- [claim-verified] PR claim: rank-0-only result queue caused DP hangs — fixed via `_result_mqs` per worker, multi-pump `_start_result_pump`, and primary-rank dequeue in step_execution (`multiproc_executor.py:106-107,769-785,709-724`). +- [claim-verified] PR claim: incompatible/empty prompts dispatched before workers — preflight uses `build_request_batch_sampling_params_key`, `extra_args` JSON signature, and `_is_empty_dp_prompt` before `collective_rpc` (`multiproc_executor.py:452-474`). +- [claim-verified] PR claim: partial DP wave hangs forever — `_DLO_DP_WAVE_TIMEOUT_S` + `_fail_closed_on_dp_wave_timeout` shuts down executor and fires callbacks (`multiproc_executor.py:35,414-416,500-501`). +- [claim-verified] file_at_base main diffusion_engine.py — prior head silently set request_batch_max_wait_ms=500 when zero for DLO DP + +`vllm_omni/diffusion/diffusion_engine.py:~250` [minor] — This PR removes the `_init_runtime_state` injection that auto-set `od_config.request_batch_max_wait_ms = 500.0` when DLO DP concurrency was enabled and the value was 0. Head only sets `max_num_running_reqs=dp_size` and logs the existing wait (`diffusion_engine.py:250-253`). With default 0 (`data.py:845`, test at `test_diffusion_engine.py:380`), `_wait_for_request_batch_admission_locked` exits immediately (`diffusion_engine.py:478-480`), so bursty DP2 arrivals are unlikely to coalesce into one multi-request wave. Document that concurrent DLO DP waves need an explicit positive `request_batch_max_wait_ms`; `dlo_dp2_lifecycle.py` passes `--batch-wait-ms` 500, but HTTP/CLI/AsyncOmni defaults remain 0. (evidence: diffusion_engine.py:247-253 `if _uses_dlo_dp_concurrency(self.od_config):` / `self.scheduler.max_num_running_reqs = dp_size` / `self.dp_concurrent = True` / `logger.info("dp_concurrent: max_num_running_reqs=%d, batch_wait=%sms", dp_size, self.od_config.request_batch_max_wait_ms)` — no mutation of `request_batch_max_wait_ms`; diffusion_engine.py:478-480 `max_wait_s = self.od_config.request_batch_max_wait_ms / 1000.0` / `if max_wait_s == 0:` / `return`; tests/diffusion/test_diffusion_engine.py:380 `assert od_config.request_batch_max_wait_ms == 0`; data.py:845 `request_batch_max_wait_ms: float = 0.0`; examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py:56 `parser.add_argument("--batch-wait-ms", type=float, default=500.0)`; removed by this diff (base `_init_runtime_state`): `if getattr(self.od_config, "request_batch_max_wait_ms", 0) == 0: self.od_config.request_batch_max_wait_ms = 500.0`) + +`docs/user_guide/diffusion/stage_configs.md:385` [minor] — docs/configuration/stage_configs.md (~385-386) documents request_batch_max_wait_ms as applicable only to request-level batching pipelines with step_execution disabled, but dp_concurrent admission also reads this via diffusion_engine._wait_for_request_batch_admission_locked (diffusion_engine.py:435-436, :475-478, :495-496). Update the parameter description to cover DLO DP concurrent wave coalescing, not solely request-batch batching. (evidence: docs/configuration/stage_configs.md:385-386 `This only applies to diffusion pipelines that support request-level batching with step_execution disabled.` — unchanged by this diff, present in PR-time tree; diffusion_engine.py:435-436 `if self.supports_request_batch or self.dp_concurrent: self._wait_for_request_batch_admission_locked()`; diffusion_engine.py:475-478 `if self.step_execution or (not self.supports_request_batch and not self.dp_concurrent): return` / `max_wait_s = self.od_config.request_batch_max_wait_ms / 1000.0`; diffusion_engine.py:495-496 `# For dp_concurrent, use a longer stable window` / `if self.dp_concurrent:`) + +`tests/diffusion/test_multiproc_engine_concurrency.py:368` [minor] — `test_dlo_dp_rejects_empty_prompt_before_worker_dispatch` (lines 360-375) mocks `collective_rpc` and only proves executor preflight rejects empty prompts before dispatch—it does not exercise worker-side AllGather. Add a sibling integration test that reaches the worker path, or document in the test module that empty-prompt hang prevention is guaranteed at executor preflight only. (evidence: tests/diffusion/test_multiproc_engine_concurrency.py:368 `executor.collective_rpc = Mock()` — collective_rpc is mocked, so no worker RPC/AllGather runs; tests/diffusion/test_multiproc_engine_concurrency.py:375 `executor.collective_rpc.assert_not_called()` — test scope is preflight-only; vllm_omni/diffusion/executor/multiproc_executor.py:486-491 `empty_prompt_ids = [nr.request_id for nr in new_reqs if _is_empty_dp_prompt(nr.req.prompt)]` / `raise ValueError("DP multi-concurrency requires a non-empty prompt for every request; ...")` — rejection happens before collective_rpc at line 495; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/worker/diffusion_model_runner.py:479-480 `if req.prompt is None: raise ValueError("Cannot execute model with empty prompt")` — worker checks None only, not empty string, so AllGather-path coverage for `""` is not tested here.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:41` [minor] — `_is_empty_dp_prompt` (multiproc_executor.py:41–48) only flags `None`, empty `str`, and dicts with empty `"prompt"`; other `OmniPromptType` shapes (e.g. `TokensPrompt`/`OmniCustomPrompt` with empty `prompt_token_ids`/`prompt_ids`) hit `return False` and skip the pre-dispatch rejection at :486. Extend preflight for those encodings or document in cpu_offload_diffusion.md that only str/`{"prompt":...}` empty prompts are covered. (evidence: multiproc_executor.py:41-48 `def _is_empty_dp_prompt(prompt: object) -> bool:` / `if isinstance(prompt, dict) and "prompt" in prompt:` / `return not prompt["prompt"]` / `return False`; multiproc_executor.py:486 `empty_prompt_ids = [nr.request_id for nr in new_reqs if _is_empty_dp_prompt(nr.req.prompt)]`; inputs/data.py:151 `OmniPromptType: TypeAlias = PromptType | OmniTextPrompt | OmniTokensPrompt | OmniEmbedsPrompt | OmniCustomPrompt`) + +`vllm_omni/diffusion/worker/diffusion_worker.py:958` [minor] — This PR adds `WorkerProc.shutdown()` but only calls `result_mq.shutdown()` when the async-output thread is absent or already dead after the join timeout; if the thread is still blocked in `d2h_stream.synchronize()`, `result_mq.shutdown()` is skipped and the creator-side ring buffer may remain linked. Force `result_mq.shutdown()` in a `finally` block (or otherwise ensure teardown) even when `_async_output_thread` misses the join timeout, since the fail-closed executor path can `terminate()` workers without a successful graceful `shutdown()`. (evidence: diffusion_worker.py:911 `d2h_stream.synchronize()` — async-output thread can block here while handling a queued item; diffusion_worker.py:940-945 `self._async_output_thread.join(timeout=_ASYNC_OUTPUT_THREAD_JOIN_TIMEOUT_S)` / `if self._async_output_thread.is_alive():` / `logger.warning(` — join timeout only warns; diffusion_worker.py:958-960 `if self.result_mq is not None and (` / `self._async_output_thread is None or not self._async_output_thread.is_alive()` — `result_mq.shutdown()` runs only when the thread is gone; unchanged by this diff, present in PR-time tree: multiproc_executor.py:78-81 `alive = [proc for proc in self.processes if proc.is_alive()]` / `proc.terminate()` — fail-closed shutdown terminates stuck workers without calling `worker_proc.shutdown()` directly.) + +`docs/user_guide/diffusion/cpu_offload_diffusion.md:~299` [minor] — This PR replaces the executor's blanket `num_inference_steps` None rejection with `build_request_batch_sampling_params_key` compatibility checking (`multiproc_executor.py:469-475`; comment at 467-468: shared default None step counts are valid) and adds `test_dlo_dp_allows_shared_default_denoise_steps`, but `docs/user_guide/diffusion/cpu_offload_diffusion.md` still forbids None at lines 299-300 and 309. Update those bullets to the new contract: concurrent waves require matching resolved schedules/compatibility keys, but shared default (None) step counts are allowed. (evidence: unchanged by this diff, present in PR-time tree: docs/user_guide/diffusion/cpu_offload_diffusion.md:299 `- \`num_inference_steps\` must be specified explicitly (None is not allowed)`; docs/user_guide/diffusion/cpu_offload_diffusion.md:309 `- \`num_inference_steps=None\` is not allowed in DP multi-concurrency mode`; changed in PR: vllm_omni/diffusion/executor/multiproc_executor.py:467-470 `# and LoRA settings. Two default (None) step counts are valid and\n # resolve identically inside the same pipeline.\n compatibility_keys = [build_request_batch_sampling_params_key(nr.req) for nr in new_reqs]`; tests/diffusion/test_multiproc_engine_concurrency.py:407-408 `for new_req in scheduler_output.scheduled_new_reqs:\n new_req.req.sampling_params.num_inference_steps = None`) + +`vllm_omni/diffusion/executor/multiproc_executor.py:~480` [minor] — This PR adds DLO DP wave preflight that rejects mismatched extra_args before collective_rpc (multiproc_executor.py:476-485), but there is no unit test mirroring test_dlo_dp_rejects_incompatible_collective_wave. Add test_dlo_dp_rejects_mismatched_extra_args in tests/diffusion/test_multiproc_engine_concurrency.py that schedules two requests with different sampling_params.extra_args, asserts ValueError matching 'share identical extra_args', and verifies collective_rpc is not called. (evidence: multiproc_executor.py:480 `if len(extra_args_signatures) > 1:` — raises ValueError with 'share identical extra_args' before collective_rpc at line 495; unchanged by this diff, present in PR-time tree: tests/diffusion/test_multiproc_engine_concurrency.py:407-422 `test_dlo_dp_rejects_incompatible_collective_wave` parametrizes only num_inference_steps, guidance_scale, width with no extra_args coverage; grep of tests/diffusion/test_multiproc_engine_concurrency.py for extra_args returns no matches) + +`tests/diffusion/test_multiproc_engine_concurrency.py:451` [minor] — This PR adds `_fail_closed_on_dp_wave_timeout` to set `_is_failed=True` and call `shutdown()` (`multiproc_executor.py:427-428`), but `test_dlo_dp_partial_reply_times_out_and_fails_closed` mocks that method (`test_multiproc_engine_concurrency.py:451`) and only checks timeout errors in `runner_outputs` plus `assert_called_once()` with a `TimeoutError` arg (lines 473-475). Consider a sibling test that leaves the real method (or spies `shutdown` without replacing it) and asserts `_is_failed` and shutdown side effects on partial DP reply timeout. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/executor/multiproc_executor.py:427-428 `self._is_failed = True` / `self.shutdown()`; tests/diffusion/test_multiproc_engine_concurrency.py:451 `executor._fail_closed_on_dp_wave_timeout = Mock()`; tests/diffusion/test_multiproc_engine_concurrency.py:473-475 only assert timeout errors and `assert_called_once()` with `TimeoutError`) + +**Additional observations (beyond the comment budget):** +- `examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py:175` [minor] This PR adds `examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py` as the only AsyncOmni end-to-end coverage of the MiniMax-H3 DLO/DP2 lifecycle (asymmetric invalid wave at :175, recovery wave, `engine.close()` sh +- `tests/diffusion/test_async_output_worker.py:205` [minor] This shutdown test stubs `_async_output_thread` with a plain `queue.get()` waiter (tests/diffusion/test_async_output_worker.py:204-205), which always unblocks on the None sentinel and therefore always reaches `result_mq. +- `tests/diffusion/test_diffusion_engine.py:381` [minor] This new DLO DP test only proves init-time wiring (`dp_concurrent`, scheduler capacity, no auto 500ms wait injection) and never drives a two-request schedule through `engine.execute_fn`, unlike `test_engine_enables_batch + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.trace.json.gz new file mode 100644 index 000000000..cc2712e7f Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5864.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.cost.json new file mode 100644 index 000000000..87dc4cec7 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 380.0, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5958/run-20260815-084654-12cf56", + "recorded_at": "2026-08-15T08:53:14", + "attempts": [ + { + "invocation_id": "inv-ed689546", + "outcome": "run:run-20260815-084654-12cf56", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-084654-12cf56", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-084654-12cf56", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2338, + "minutes": 6.32, + "ci_minutes": 0, + "input_tokens": 548610, + "output_tokens": 77910, + "tool_calls": 227, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3235, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.52, + "gate": 1.74, + "review": 374.06, + "report": 0.06 + }, + "time_to_first_result_s": 379.36 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5958.trace.json.gz", + "trace_bytes": 21403 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.md new file mode 100644 index 000000000..9360b7897 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5958, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4dd07d536f34) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 9 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Prior review added `@alex-jw-brooks`/`@Gaohan123` to io paths and `@Bounty-hunter` to benchmark paths (4dd07d53); residual uncovered top-level dirs still fall to `*`. +- [resolved] Prior review: Gaohan123 missing from /worker/ — fixed in ea399ccf (.github/CODEOWNERS:46) +- [resolved] Prior review: alex-jw-brooks/Gaohan123 missing from I/O engine overrides — fixed in 4dd07d53 (.github/CODEOWNERS:53-57) +- [resolved] Prior review: Bounty-hunter missing from cli/benchmark and in-tree benchmarks — fixed in 4dd07d53 (.github/CODEOWNERS:69,100) +- [resolved] `input_output_modality_contracts.md` promotion gate 'Add explicit CODEOWNERS' — satisfied at head by `.github/CODEOWNERS:52-63` IO paths and engine file overrides; residual: `vllm_omni/utils/mm_outputs.py` still unmatched (falls to `*`). +- [resolved] merge d726ab70 in timeline; CODEOWNERS/doc parity on checked hunks still holds at head 4dd07d53 +- [claim-refuted] PR body table lists offloader after-owners as `yuanheng-zhao, Isotr0py, lishunyang12`; head CODEOWNERS:143 and `offloader.md:5-8` use `david6666666` instead of `Isotr0py`. +- [claim-refuted] PR body 'Not changed' lists /vllm_omni/lora/ but head narrows 6→2 owners; commit message says 'intentionally narrowed' (.github/CODEOWNERS:102-103) +- [claim-refuted] PR offloader table cites Isotr0py; head/doc have david6666666 per offloader.md (.github/CODEOWNERS:147) +- [claim-refuted] Test plan '23 representative paths via simulation' — no validation script/test in repo (glob *codeowner* → only .github/CODEOWNERS) +- [claim-refuted] PR body offloader table 'After: yuanheng-zhao, Isotr0py, lishunyang12' — head `.github/CODEOWNERS:145` and `offloader.md:5-7` list @david6666666, not @Isotr0py. +- [claim-refuted] PR body lists 3 design-doc updates; diff touches 13 docs/design/module files +- [validated] `ar_runtime.md:5-10` owners match `/vllm_omni/core/` and `/vllm_omni/worker/` (CODEOWNERS:45-46). +- [validated] `engine_orchestration.md`/`stage_runtime.md` owners match `/vllm_omni/engine/` (CODEOWNERS:50); io file overrides follow and win last-match (CODEOWNERS:52-56). + +`.github/CODEOWNERS:137` [major] — Regenerated from `docs/design/module/diffusion/diffusion_model_integration.md`, but `/vllm_omni/diffusion/models/` (CODEOWNERS:137) lists only `@fhfuih @Bounty-hunter @wtomin` while the doc owners include `@RuixiangMa` for `vllm_omni/diffusion/models/**`; `model_loader/` and `lora/` rules already include him. Add `@RuixiangMa` to the `diffusion/models/` line. (evidence: docs/design/module/diffusion/diffusion_model_integration.md:5-11 `owners:` includes `- "@RuixiangMa"` and `primary_code_paths:` includes `- vllm_omni/diffusion/models/**`; unchanged by this diff, present in PR-time tree: .github/CODEOWNERS:137 `/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin`; .github/CODEOWNERS:138 `/vllm_omni/diffusion/model_loader/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa`) + +`.github/CODEOWNERS:81` [major] — Under the model-integration section, `/vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi` does not match `docs/design/module/model_integration.md`, which lists `vllm_omni/plugins/**` in `primary_code_paths` with owners `@tzhouam` and `@gcanlin` only. `execution_platforms.md` does not claim `plugins/**`. Regenerate as `/vllm_omni/plugins/ @tzhouam @gcanlin`, or update the design doc first if platform owners are intended. (evidence: .github/CODEOWNERS:78 `/vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi`; unchanged by this diff, present in PR-time tree: docs/design/module/model_integration.md:5-11 `owners: - "@tzhouam" - "@gcanlin"` and `primary_code_paths: - vllm_omni/plugins/**`; unchanged by this diff, present in PR-time tree: docs/design/module/execution_platforms.md:8-10 `primary_code_paths: - vllm_omni/platforms/** - vllm_omni/attention/**` (no plugins entry)) + +`.github/CODEOWNERS:~5` [minor] — The header (lines 5–7) says rules are derived only from `owners` and `primary_code_paths`, but this PR also applies `primary_code_path_owners` (e.g. model_integration.md, execution_platforms.md → CODEOWNERS rules for model_executor/models/ and attention/) and file-level overrides (engine/*.py, entrypoints/openai/errors.py, entrypoints/cli/benchmark/, model_executor/stage_configs/). Please extend the comment so future regenerations document all three sources. (evidence: .github/CODEOWNERS:5-6 `# The vllm_omni/** source-tree rules below are derived from the \`owners\` and` / `# \`primary_code_paths\` front matter in docs/design/module/*.md.` — unchanged by this diff, present in PR-time tree; docs/design/module/model_integration.md:12-19 `primary_code_path_owners:` with `vllm_omni/model_executor/models/**` owners consumed at .github/CODEOWNERS:76 `/vllm_omni/model_executor/models/ @tzhouam @gcanlin @Sy0307 @amy-why-3459 @linyueqian`; docs/design/module/execution_platforms.md:12-18 `primary_code_path_owners:` with `vllm_omni/attention/**` owners consumed at .github/CODEOWNERS:83 `/vllm_omni/attention/ @gcanlin @tjtanaa @xuechendi @david6666666 @lishunyang12`; file-level overrides not covered by header: .github/CODEOWNERS:50-51 `# File-level overrides: input/output modality contracts own these engine files` and :65-67 `# File-level overrides within entrypoints/` / `/vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao`) + +`.github/CODEOWNERS:5` [minor] — This PR generates rules from `primary_code_paths`/`primary_code_path_owners`, but several `related_code_paths` entries still have no explicit rule and continue to hit the `*` fallback—for example `vllm_omni/tokenizers/**` (`model_integration.md`) and `vllm_omni/experimental/ar_diffusion/kv_cache/**` (`cache_management.md`). Either add matching rules or note in the header that `related_code_paths` remain on `*` intentionally. (evidence: .github/CODEOWNERS:5-6 `# The vllm_omni/** source-tree rules below are derived from the `owners` and` / `` `primary_code_paths` front matter in docs/design/module/*.md.``; .github/CODEOWNERS:11 `* @Gaohan123 @hsliuustc0106 @ywang96`; docs/design/module/model_integration.md:20-22 `related_code_paths:` / ` - vllm_omni/tokenizers/**`; docs/design/module/cache_management.md:12 ` - vllm_omni/experimental/ar_diffusion/kv_cache/**`; unchanged by this diff, present in PR-time tree: grep of `.github/CODEOWNERS` for `tokenizers|experimental` returned no matches) + +`.github/CODEOWNERS:50` [minor] — Regeneration sets `/vllm_omni/engine/` to `@tzhouam @fake0fan` (matching `owners` in `engine_orchestration.md`), but `required_reviewers` there still lists `@yinpeiqi` and `@Sy0307` (lines 14–15). Those reviewers will no longer be auto-requested for general `engine/` changes (e.g. `orchestrator.py`). Reconcile by adding them to this CODEOWNERS rule or dropping them from `required_reviewers`. (evidence: .github/CODEOWNERS:50 `/vllm_omni/engine/ @tzhouam @fake0fan`; unchanged by this diff, present in PR-time tree: docs/design/module/engine_orchestration.md:14-15 ` - "@yinpeiqi"` and ` - "@Sy0307"` under `required_reviewers:`; removed by this PR (diff only): `/vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan @Sy0307`) + +`.github/CODEOWNERS:66` [minor] — Head `/vllm_omni/entrypoints/` CODEOWNERS (line 66) auto-requests only `@alex-jw-brooks @linyueqian @NickCao`, dropping the pre-PR `@tzhouam @yinpeiqi @fake0fan`, yet `entrypoints.md:14-17` still lists `@tzhouam` under `required_reviewers`. Align CODEOWNERS with `required_reviewers` or update the design doc if those reviewers should no longer be auto-requested. (evidence: unchanged by this diff, present in PR-time tree: .github/CODEOWNERS:66 `/vllm_omni/entrypoints/ @alex-jw-brooks @linyueqian @NickCao`; docs/design/module/entrypoints.md:14-15 `required_reviewers:` / ` - "@tzhouam"`) + +`.github/CODEOWNERS:60` [minor] — Primary IO paths from `input_output_modality_contracts.md` are covered at `.github/CODEOWNERS:60-64` (and engine overrides at `:53-58`), but `vllm_omni/utils/mm_outputs.py`—used by workers and prefix cache for multimodal payload partitioning—has no matching rule and falls through to `*` (`:11`). Add an explicit owner rule under the IO section or document that default ownership is intentional. (evidence: unchanged by this diff, present in the PR-time tree: .github/CODEOWNERS:11 `* @Gaohan123 @hsliuustc0106 @ywang96`; .github/CODEOWNERS:60-64 `# Input/output modality contracts` … `/vllm_omni/inputs/` … `/vllm_omni/data_entry_keys.py` (no `/vllm_omni/utils/` rule anywhere in CODEOWNERS); docs/design/module/input_output_modality_contracts.md:18-28 `primary_code_paths` lists `vllm_omni/engine/mm_outputs.py` but not `vllm_omni/utils/mm_outputs.py`; vllm_omni/utils/mm_outputs.py:1-3 `"""Utilities for handling multimodal outputs / building multimodal output payloads`) + +`.github/CODEOWNERS:80` [minor] — This PR sets `/vllm_omni/model_extras/ @tzhouam @gcanlin @princepride` (CODEOWNERS:80), but `model_integration.md` defines only `@tzhouam` and `@gcanlin` as module owners for `vllm_omni/model_extras/**` with no `primary_code_path_owners` override for that path. Remove `@princepride` or add him to the design doc before regenerating CODEOWNERS. (evidence: .github/CODEOWNERS:80 `/vllm_omni/model_extras/ @tzhouam @gcanlin @princepride`; unchanged by this diff, present in PR-time tree: docs/design/module/model_integration.md:5-7 `owners: / - "@tzhouam" / - "@gcanlin"` and docs/design/module/model_integration.md:9-10 `primary_code_paths: / - vllm_omni/model_extras/**` with `primary_code_path_owners` only overriding `vllm_omni/model_executor/models/**` (lines 12-19), not `model_extras`.) + +**Additional observations (beyond the comment budget):** +- `.github/CODEOWNERS:102` [minor] This PR narrows `/vllm_omni/lora/` from six owners (`@princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao`) to `@SamitHuang @wtomin` with an explicit "ownership intentionally narrowed" note (`.github/CODE +- `docs/design/module/diffusion/diffusion_model_integration.md:15` [minor] CODEOWNERS cites diffusion_model_integration.md for `/vllm_omni/diffusion/lora/`, but the doc keeps `vllm_omni/diffusion/lora/**` under `related_code_paths` while the CODEOWNERS header says rules are derived from `owners + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.trace.json.gz new file mode 100644 index 000000000..6ce577689 Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5958.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.cost.json b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.cost.json new file mode 100644 index 000000000..8375bf8bb --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 438.9, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v15cb_holdout4_r1/runs/pr5978/run-20260815-084723-ca894d", + "recorded_at": "2026-08-15T08:54:42", + "attempts": [ + { + "invocation_id": "inv-b441e945", + "outcome": "run:run-20260815-084723-ca894d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-084723-ca894d", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260815-084723-ca894d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3678, + "minutes": 7.31, + "ci_minutes": 0, + "input_tokens": 920455, + "output_tokens": 108471, + "tool_calls": 414, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4066, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.38, + "gate": 1.6, + "review": 432.22, + "report": 0.06 + }, + "time_to_first_result_s": 438.24 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5978.trace.json.gz", + "trace_bytes": 35980 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.md b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.md new file mode 100644 index 000000000..3b736e160 --- /dev/null +++ b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.md @@ -0,0 +1,66 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5978, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 86854432db52) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Prior head deleted target_frame_count in prepare_reference_videos (file_at_base); head now passes it to _transcode_reference_video — residual checked for multi-ref: each ref gets the same output num_frames cap. +- [resolved] AAC priming concern: audio_frame.pts=0 yields negative first packet PTS; e2e asserts official A/V metadata parity +- [resolved] decord-vs-av concern: load_video_frames only falls back on ImportError (reference_video.py:403-410); residual unchecked if decord installed but cannot decode libx264rgb +- [resolved] max_duration_seconds<=0 raises before _dit_rank_world/collectives (pipeline_minimax_h3.py:1210-1214); test_ref2va_audio_duration_validation_precedes_rank_branch covers rank!=0 +- [resolved] Concern: contract tests deselected by cuda lane—merge cpu diffusion lane (test-merge.yml:18) still collects them +- [resolved] Profiler _encode_audio_conditions exposure: added to _PROFILER_TARGETS at pipeline_minimax_h3.py:541 with no dedicated profiler test—acceptable instrumentation-only change +- [resolved] Round-1 concern that `sample_reference_video_frames` still decodes twice (Qwen + VAE) — residual remains (pipeline ~881 vs ~1274) but is performance-only, not correctness. +- [resolved] Round-1 libx264rgb ffmpeg recipe untested in unit CI — nightly Ref2VA e2e remains the strict backstop per PR test plan. +- [claim-verified] PR bounds reference conditioning to output duration via target_frame_count in reference_video.py:376-396 and max_duration_seconds in pipeline_minimax_h3.py:1726-1728. +- [claim-verified] Lossless RGB prep uses libx264rgb/rgb24/crf 0 at reference_video.py:307-314; Qwen sampling decodes once at reference_video.py:437-440. +- [claim-verified] AAC priming fix sets audio_frame.pts/time_base in media_utils.py:191-194; test_video_server.py:253-269 asserts first_packet.pts < 0. +- [claim-verified] T2VA accuracy removed (diff_stat deletes test_minimax_h3_t2va_similarity.py); nightly now runs test_minimax_h3_i2va_ref2va_similarity.py (.buildkite/cuda/test-nightly.yml:430-433). +- [claim-verified] _encode_audio_conditions added to _PROFILER_TARGETS at pipeline_minimax_h3.py:541. +- [claim-verified] T2VA accuracy file removed and nightly job retargeted at test-nightly.yml:430-433 + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:~37` [minor] — The new nightly accuracy test downloads I2VA/Ref2VA inputs from cdn.hailuoai.com (image at lines 36-38; Ref2VA video/audio at lines 43-53) with SHA256 checks, while reference outputs use Hugging Face (lines 34-35, 41-42). Repo-wide, this is the only e2e accuracy file on that CDN; mirror inputs to MiniMaxAI/MiniMax-H3 or document/gate CDN availability so nightly failures stay diagnosable. (evidence: tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:34 `I2VA_REFERENCE_VIDEO_URL = "https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/assets/i2va.mp4"`; :36-38 `I2VA_IMAGE_URL = ("https://cdn.hailuoai.com/prod/hailuo_demo/testsets/H3_AA_I2VA/gallery/" ...)`; :41 `REF2VA_REFERENCE_VIDEO_URL = "https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/assets/r2va.mp4"`; :43-46 and :49-52 Ref2VA input video/audio URLs on `cdn.hailuoai.com`; :161 `assert digest == sha256, f"Official MiniMax H3 {label} reference changed: got {digest}, expected {sha256}"`; unchanged by this diff, present in PR-time tree: repo-wide `cdn.hailuoai.com` grep matches only this file.) + +`vllm_omni/diffusion/models/minimax_h3/reference_video.py:403` [minor] — `_transcode_reference_video` now emits `libx264rgb`/`rgb24` prepared clips (`reference_video.py:306-313`), but `load_video_frames` only uses PyAV when decord is missing (`reference_video.py:406`), not when `decord.VideoReader` fails on that stream (`reference_video.py:415-418`). Ref2VA hits this via `sample_reference_video_frames` → `load_video_frames` (`reference_video.py:440`). Extend fallback to decode failures or add an integration assertion that decord (or PyAV) can read a real prepared `libx264rgb` output. (evidence: reference_video.py:306-313 `-c:v`, `libx264rgb`, `-pix_fmt`, `rgb24` — prepared reference video is lossless RGB H.264; reference_video.py:406 `except ImportError:` — PyAV path only when decord is absent; reference_video.py:415-418 `reader = decord.VideoReader(path)` … `frames = reader.get_batch(...)` — no decode-failure fallback; reference_video.py:440 `decoded_frames = load_video_frames(prepared_path)` — Ref2VA Qwen sampling depends on this decoder for the prepared clip.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1221` [minor] — Ref2VA audio bounding uses different cut math on the two paths: standalone _encode_audio_conditions truncates with int(round(max_duration_seconds * sr)) (pipeline_minimax_h3.py:1221-1222) while embedded reference audio passes float audio_duration_seconds into load_video_audio, which trims via ffmpeg -t (pipeline_minimax_h3.py:1324-1327; reference_video.py:536-537). Align to the same sample-boundary rule so standalone and embedded conditioning do not diverge by sub-sample offsets within the nominal num_frames/fps window. (evidence: pipeline_minimax_h3.py:1221 `max_samples = int(round(max_duration_seconds * int(sample_rate)))`; pipeline_minimax_h3.py:1324-1327 `duration_seconds=item.get("audio_duration_seconds", item.get("duration_seconds"))` into load_video_audio; reference_video.py:536-537 `command.extend(["-t", f"{float(duration_seconds):.6f}"])`; reference_video.py:393-396 `"audio_duration_seconds": min(duration, float(target_frame_count) / MINIMAX_H3_FPS)`) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1535` [minor] — The standalone Ref2VA audio-bound contract test (test_minimax_h3_contract.py:1535–1558) patches FakeAudioVAE and checks truncation math on the default rank-0/world_size-1 path only; it does not exercise _encode_audio_conditions' dist.broadcast/_broadcast_tensor branch. Consider a rank≠0 contract (cf. test_distributed_video_vae_encodes_references_sequentially at :1101) or document nightly Ref2VA e2e as the distributed guard for bounded standalone audio. (evidence: pipeline_minimax_h3.py:410-411 `if not dist.is_initialized(): return None, 0, 1`; pipeline_minimax_h3.py:1217-1236 `if rank == 0: ... encoded = [self.audio_vae.encode_waveform(*audio) for audio in bounded_audios] ... if world_size > 1: dist.broadcast(lengths, src=0, group=group) ... _broadcast_tensor(rows, ...)`; test_minimax_h3_contract.py:1535-1553 `def test_ref2va_standalone_audio_condition_is_bounded_to_output_duration(): ... pipeline.audio_vae = FakeAudioVAE() ... encoded, lengths = pipeline._encode_audio_conditions([(waveform, 10)], max_duration_seconds=2.5)` (no _dit_rank_world monkeypatch); test_minimax_h3_contract.py:1525 `monkeypatch.setattr(pipeline_module, "_dit_rank_world", lambda: (None, 1, 2))` only in the validation-ordering sibling test) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~881` [minor] — This diff changes sample_reference_video_frames to decode the prepared file once via load_video_frames for Qwen indexing (reference_video.py:440; encode_prompt calls it at pipeline_minimax_h3.py:881), but _encode_video_conditions_resident still calls load_video_frames(item["prepared_path"]) for the video VAE (pipeline_minimax_h3.py:1274). On Ref2VA, both run in the same forward() pass, so each reference video is fully decoded twice in memory per request; consider caching the decoded array on the prepared dict or passing it from Qwen sampling to VAE encode. (evidence: reference_video.py:440 `decoded_frames = load_video_frames(prepared_path)` — full decode inside sample_reference_video_frames; unchanged by this diff, present in PR-time tree: pipeline_minimax_h3.py:1274 `self.video_vae.encode_video(load_video_frames(item["prepared_path"])) for item in prepared_videos` — second full decode on the same prepared_path in _encode_video_conditions_resident; pipeline_minimax_h3.py:881 `sampled = sample_reference_video_frames(item["prepared_path"])` — Qwen path entry in encode_prompt) + +`vllm_omni/diffusion/models/minimax_h3/reference_video.py:~307` [minor] — This diff switches `_transcode_reference_video` to libx264rgb/rgb24 with `-crf 0` (reference_video.py:307-313), but Ref2VA contract tests monkeypatch `_transcode_reference_video` before calling `prepare_reference_videos` (e.g. test_minimax_h3_contract.py:1235-1238, 1316-1319, 1366), so unit CI never runs the real ffmpeg subprocess. Add a small integration test on a tiny fixture that asserts decord/av can round-trip the rgb24 output, or document nightly Ref2VA accuracy as the sole guard. (evidence: reference_video.py:307 `"libx264rgb",` — production transcode uses lossless RGB x264; unchanged by this diff except the codec switch itself. reference_video.py:372 `prepared_path = _transcode_reference_video(` — sole unmocked caller in repo. test_minimax_h3_contract.py:1235-1238 `monkeypatch.setattr( + reference_video_module, + "_transcode_reference_video", + lambda source, **kwargs: transcode_calls.append((source, kwargs)) or "prepared.mp4", + )` — contract test replaces ffmpeg before exercising prepare_reference_videos.) + +`vllm_omni/diffusion/utils/media_utils.py:194` [minor] — This diff sets `audio_frame.pts = 0` and `audio_frame.time_base` in shared `mux_video_audio_bytes` (`media_utils.py:191-192`), so every path that muxes AAC through this helper—including `video_api_utils._encode_video_bytes` (HTTP `/v1/videos/sync` via `serving_video.py:372`) and direct callers in offline examples and `test_magi_human_expansion.py`—inherits the AAC priming timestamp change, not only MiniMax H3. `test_mux_video_audio_marks_aac_priming_timestamp` only guards the helper; PR e2e evidence covers MiniMax H3 I2VA/Ref2VA only. Confirm repo-wide application is intended or scope the fix if other models were validated against the old leading-silence behavior. (evidence: media_utils.py:191 `audio_frame.pts = 0`; media_utils.py:192 `audio_frame.time_base = Fraction(1, audio_sample_rate)`; unchanged consumer video_api_utils.py:524 `return mux_video_audio_bytes(`; unchanged consumer serving_video.py:372 `video_bytes = _encode_video_bytes(`; direct callers present in PR-time tree: examples/offline_inference/text_to_video/text_to_video.py:674, examples/offline_inference/image_to_video/image_to_video.py:729, examples/offline_inference/speech_to_video/speech_to_video.py:402, examples/offline_inference/x_to_video_audio/x_to_video_audio.py:268, tests/e2e/offline_inference/test_magi_human_expansion.py:153) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~1324` [minor] — This diff wires embedded reference-audio extraction to pass `duration_seconds=item.get("audio_duration_seconds", item.get("duration_seconds"))` into `load_video_audio` (`pipeline_minimax_h3.py:1324-1327`), but contract tests only assert `audio_duration_seconds` on the prepared dict (`test_minimax_h3_contract.py:1249`) and never mock `load_video_audio` at this call site. Add a contract test with `duration_seconds=10` and `audio_duration_seconds=5.17`, asserting ffmpeg extraction receives the bounded value. (evidence: pipeline_minimax_h3.py:1324-1327 `duration_seconds=item.get("audio_duration_seconds", item.get("duration_seconds")),` inside `load_video_audio(...)` in `_encode_video_audio_conditions_resident`; test_minimax_h3_contract.py:1249 `assert prepared[0]["audio_duration_seconds"] == pytest.approx(min(expected_duration, 209 / 24.0))` — only checks prepared metadata; unchanged by this diff, present in PR-time tree: ripgrep `load_video_audio` under `tests/` returns no matches) + +**Additional observations (beyond the comment budget):** +- `tests/entrypoints/openai_api/test_video_server.py:258` [minor] This diff adds `test_mux_video_audio_marks_aac_priming_timestamp`, which only demuxes AAC packets and asserts `first_packet.pts < 0`. The paired `media_utils.py` change (lines 191–194) states the goal is to avoid AAC enc + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.trace.json.gz b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.trace.json.gz new file mode 100644 index 000000000..95680938d Binary files /dev/null and b/eval/dataset/arms/copilot_v15cb_holdout4_r1/pr5978.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/manifest.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/manifest.json new file mode 100644 index 000000000..c6440c90a --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v16f_holdout4_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout4" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.cost.json new file mode 100644 index 000000000..3142ea64d --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1713.7, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5608/run-20260815-171247-f47d16", + "recorded_at": "2026-08-15T17:41:18", + "attempts": [ + { + "invocation_id": "inv-bcc06a1f", + "outcome": "run:run-20260815-171247-f47d16", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-171247-f47d16", + "llm_calls": 98, + "input_tokens": 542575, + "output_tokens": 229879, + "usd": 0.546671, + "source": "spans" + } + ], + "attempt_usd_total": 0.546671, + "llm_calls": 98, + "input_tokens": 542575, + "output_tokens": 229879, + "usd": 0.546671, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-171247-f47d16", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5467, + "minutes": 28.51, + "ci_minutes": 0, + "input_tokens": 542575, + "output_tokens": 229879, + "tool_calls": 184, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8859, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 98, + "by_role": { + "agent": { + "usd": 0.502815, + "input_tokens": 518733, + "output_tokens": 195925, + "calls": 95 + }, + "reducer": { + "usd": 0.043856, + "input_tokens": 23842, + "output_tokens": 33954, + "calls": 3 + } + }, + "cache_read_tokens": 5455872, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.09, + "gate": 1.62, + "review": 1704.97, + "report": 0.06 + }, + "time_to_first_result_s": 1710.73 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5608.trace.json.gz", + "trace_bytes": 34991 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.md new file mode 100644 index 000000000..6d03bf6f4 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.md @@ -0,0 +1,61 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5608, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head de8517f6f2bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] CI-gate red: buildkite/vllm-omni-amd-ci FAILURE while vllm-omni-npu-ci SUCCESS; touched files are NPU-gated (qwen3_tts_tokenizer_v2.py already imports torch_npu at top) and the CPU test installs a fake torch_npu before loading the real module, so no obvious in-diff AMD trigger at this head — attribution to pre-existing main needs the job log. +- [resolved] 'fix ci' commit de8517f6: the test pre-registers `vllm_omni.platforms.npu.layers.rotary_embedding` in sys.modules (test line ~495) before loading the patch module; importlib returns the full dotted name from sys.modules without importing parent packages, so the new package import in qwen3_tts_tokenizer_v2.py resolves against fakes. Residual checked: grep shows only this test loads qwen3_tts_tokenizer_v2.py from source (test line 534) — no other test needs the same registration. +- [resolved] New `vllm_omni/platforms/npu/layers/` dir has `__init__.py` (list_dir: __init__.py, fused_moe.py, rotary_embedding.py) — no wheel-packaging gap from a namespace-package subdir. +- [resolved] No public default/API/protocol change: `_apply_rotary_pos_emb_npu(q,k,cos,sin,position_ids=None,unsqueeze_dim=1)` keeps its signature; only internal dispatch changes. +- [resolved] the new helper returns `output_bsnd.transpose(1, 2)` non-contiguous; the tokenizer-v2 attention uses eager matmul (eager_attention_forward) which tolerates non-contiguous strides, so no correctness issue found. +- [claim-verified] PR's 'concurrency >= 4 crash' matches geometry: tokenizer v2 config has num_attention_heads=16, head_dim=64 (1024/16), fp16 -> aligned; B*N=16B and seq=4 gives threshold 8*4=32, so B>2 falls back — consistent with the reported repro. +- [claim-verified] BSND fallback is implemented as claimed: rotary_embedding.py:48-55 does transpose(1,2).contiguous(), cos.unsqueeze(2), fused op, then transpose back. +- [claim-verified] unsqueeze_dim != 1 keeps the original direct-call behavior (rotary_embedding.py:44-47); only BNSD (unsqueeze_dim=1) is gated, which is the correct scope. +- [claim-verified] PR's stated constraint `batch * heads <= seq_len * 8` matches the code (`batch_size * num_heads <= seq_len * 8`); crash scenario arithmetic consistent: ramp chunk seq=4, concurrency 4 × 16 heads = 64 > 4*8=32 (calc). CANN-side ground truth is not verifiable in-repo — predicate correctness rests on the author's 910B repro + post-fix benchmark. +- [claim-verified] PR benchmark table shows c=16 `Streaming continuity OK rate` = 8.33% and `Mean AUDIO_UNDERRUN (s)` = 0.48 with ramp enabled — the high-concurrency ramp is not actually healthy after the fix. +- [claim-verified] Red `buildkite/vllm-omni-amd-ci`: its ready lane runs `tests/diffusion tests/model_executor -m 'core_model and cpu'`, which does not collect this PR's only test under `tests/platforms/npu`; attribution to this PR vs main is not determinable here. +- [claim-verified] Description's example ramp `[4,4,8,16,25]` differs from the benchmark ramp `[4,4,6,8,10,12,16,20,25]`; this is a wording inconsistency, not a code defect. +- [claim-verified] benchmark table internally consistent (calc): 266.00/1.92≈138.5 vs 48*2.88476≈138.5; 266.08/6.62≈40.2 vs 12*3.32427≈39.9; 267.92/10.62≈25.2 vs 6*4.00514≈24.0; 265.76/8.46≈31.4 vs 3*9.19468≈27.6. +- [validated] Fast path retained for the real model: fp16 head_dim=64 -> head_dim//2=32, 32%16==0 aligned, so supported BNSD shapes stay on npu_rotary_mul when B*N <= seq*8. + +`tests/platforms/npu/test_310p_patches.py:~547` [major] — The only automated test for `npu_rotary_mul_with_bsnd_fallback` drives q=(1,2,3,2) fp32, whose head_dim//2=1 fails the alignment guard (`1 % 8 != 0`) before the short-sequence clause `batch * num_heads <= seq_len * 8` is ever evaluated (2 <= 24). The BSND fallback recorded at test_310p_patches.py:547 is therefore reached via the alignment rejection, not the short-sequence condition this PR fixes — removing the short-sequence clause would not make this test fail. Please add (a) an aligned short-seq trigger, e.g. q=(2,8,1,16) fp32 with cos/sin=(2,1,16) (B*N=16 > seq*8=8), asserting cos shape (2,1,1,16), and (b) an aligned in-limit BNSD fast-path case, e.g. q=(1,2,8,16) fp32 with cos/sin=(1,8,16), asserting cos shape (1,1,8,16). (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:27 `half_dim_alignment = 32 // element_size` and :28 `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8` — for fp32 q=(1,2,3,2), head_dim//2=1 and half_dim_alignment=32//4=8, so `1 % 8 != 0` returns False before the short-sequence clause (1*2=2 <= 3*8=24) is evaluated. tests/platforms/npu/test_310p_patches.py:538 `q = torch.zeros(1, 2, 3, 2)`; tests/platforms/npu/test_310p_patches.py:547 `assert rotary_calls[0][1].shape == (1, 3, 1, 2)` — the recorded cos shape proves the BSND fallback was entered via alignment rejection, not the short-sequence trigger.) + +`vllm_omni/model_executor/models/common/qwen3_code_predictor.py:~260` [minor] — The new module is documented as 'Shared NPU rotary embedding helpers', but the stage-0 code-predictor attention path still calls torch_npu.npu_rotary_mul directly on BNSD inputs (q/k transposed to (bsz, heads, seq, dim) at lines 251-252, cos/sin unsqueezed at dim 1 at lines 257-258). The 310P twin at vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184-185 has the identical direct BNSD call. With the default config (16 heads, head_dim=128, num_code_groups=32 => max_seq=33), the helper's B*N <= seq*8 gate passes batch 16 (256 <= 264) but rejects batch 32 (512 > 264). These call sites should either use npu_rotary_mul_with_bsnd_fallback or document why stage-0 cannot reach that shape. (evidence: vllm_omni/model_executor/models/common/qwen3_code_predictor.py:260 `q = torch_npu.npu_rotary_mul(q, cos, sin)` and :261 `k = torch_npu.npu_rotary_mul(k, cos, sin)` — unchanged by this diff, present in PR-time tree; :251 `q = self.q_norm(self.q_proj(hidden_states).view(hidden_shape_q)).transpose(1, 2)` proves BNSD; :257 `cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim]`. Twin call site unchanged by this diff: vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184 `q = torch_npu.npu_rotary_mul(q, cos, sin)` and :185 `k = torch_npu.npu_rotary_mul(k, cos, sin)`. New file in this diff: vllm_omni/platforms/npu/layers/rotary_embedding.py:28 `return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8`. Supporting config defaults in vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py: `num_attention_heads=16`, `head_dim=128`, `num_code_groups=32`.) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:28` [minor] — `seq_len * 8` and `32 // element_size` encode a CANN RoPE tiler contract (the PR body quotes EZ9999 from `DoRotateHalfTiling`), but this file cites neither a CANN version nor that error, and it routes every alignment-rejecting shape to BSND without stating whether the BSND fused op has its own alignment limits. The real-model hardware run only hit the `batch*num_heads <= seq_len*8` fallback (fp16, head_dim=64 is aligned); the alignment-reject branch is covered only by the CPU unit test against a mocked `torch_npu.npu_rotary_mul`, so real CANN BSND alignment behavior is still unverified. Add a comment citing the CANN version/error and the BSND op's alignment limits. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:28 ` return head_dim // 2 % half_dim_alignment == 0 and batch_size * num_heads <= seq_len * 8`; vllm_omni/platforms/npu/layers/rotary_embedding.py:24 ` if head_dim % 2 != 0 or element_size <= 0 or 32 % element_size != 0:`; vllm_omni/platforms/npu/layers/rotary_embedding.py:15-17 ` For the aligned-head-dimension branch, CANN requires + ``batch * num_heads <= seq_len * 8``. Other alignments are handled with + the BSND layout, which does not have this short-sequence limit.` — the docstring states the bound but cites no CANN version/error and asserts no BSND alignment limits.) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:~44` [minor] — Confirmed: `_bnsd_rotary_shape_is_supported` returns False for `ndim != 4`, so with `unsqueeze_dim == 1` a non-4D tensor falls into the BSND transpose path (`transpose(1, 2)` / `cos.unsqueeze(2)`), which assumes a 4D BNSD layout, instead of the pre-PR direct fused-op call. No current in-tree caller passes non-4D, so this is a latent robustness gap in a shared helper; guard `ndim == 4` in the fallback branch. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:19-20 `if hidden_states.ndim != 4:\n return False` — non-4D input is rejected by the support check; rotary_embedding.py:44 `if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):` — with unsqueeze_dim==1 and that check False, control falls through to rotary_embedding.py:51 `hidden_states_bsnd = hidden_states.transpose(1, 2).contiguous()` and rotary_embedding.py:54 `cos.unsqueeze(2),` — the BSND transpose path assumes a 4D BNSD layout, so non-4D input is transposed rather than passed directly to the fused op. The pre-PR direct call is visible in the diff: qwen3_tts_tokenizer_v2.py removed `return torch_npu.npu_rotary_mul(q, cos, sin), torch_npu.npu_rotary_mul(k, cos, sin)`.) + +`.buildkite/amd/test-amd-ready.yml:19` [minor] — `buildkite/vllm-omni-amd-ci` is red at this head while `buildkite/vllm-omni-npu-ci` is green. This diff's surface does not intersect the AMD lanes: every path-scoped AMD job targets `tests/diffusion`, `tests/model_executor`, named e2e/engine files, or `tests/e2e/features/custom_pipeline/` — never `tests/platforms/npu/`; the only pathless job (.buildkite/amd/test-amd-ready.yml:86) uses `-m "core_model and cache and diffusion and not distributed_cuda and L4"`, which deselects the touched test because it carries only `core_model, cpu` and lacks `cache`/`diffusion`/`L4`. The two runtime modules are reachable only from `NPUOmniPlatform.__init__`, and `vllm_omni/platforms/npu/platform.py` imports `vllm_ascend` at module scope, so they cannot execute on the ROCm image. Please confirm the amd-ci failure is also red on main and pre-existing, so it does not block this NPU-only fix. (evidence: Unchanged by this diff, present in the PR-time tree: .buildkite/amd/test-amd-ready.yml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; the only pathless AMD lane is .buildkite/amd/test-amd-ready.yml:86 `- timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"`; no AMD lane names `tests/platforms/npu/`. Present in the PR-time tree: tests/platforms/npu/test_310p_patches.py:21 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — it lacks `cache`, `diffusion`, `L4`, so the pathless lane deselects it. Import chain (platform.py unchanged by this diff; rotary_embedding.py added by this diff): vllm_omni/platforms/npu/platform.py:10 `from vllm_ascend.platform import NPUPlatform`; vllm_omni/platforms/npu/platform.py:42 `from vllm_omni.platforms.npu.models.qwen3_tts_tokenizer_v2 import (` (inside `NPUOmniPlatform.__init__`); vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:12 `from vllm_omni.platforms.npu.layers.rotary_embedding import (`.) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:~49` [minor] — This diff adds a per-decode-step `transpose(1, 2).contiguous()` (line 49) and a non-contiguous `transpose(1, 2)` on return (line 55) for every unsupported short chunk. The PR's own benchmark table reports ramp-enabled c=16 `Streaming continuity OK rate` = 8.33% and `Mean AUDIO_UNDERRUN (s)` = 0.48, so high-concurrency ramp is still effectively unusable; please either scope the fix claim to c<=8 (where continuity is 100%) or show that the c=16 degradation is not caused by these fallback copies. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:42 `if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):`; vllm_omni/platforms/npu/layers/rotary_embedding.py:49 `hidden_states_bsnd = hidden_states.transpose(1, 2).contiguous()`; vllm_omni/platforms/npu/layers/rotary_embedding.py:55 `return output_bsnd.transpose(1, 2)`; vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py:362 `query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)` — unchanged by this diff, present in the PR-time tree; it is monkeypatched by vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:44 `modeling_qwen3_tts_tokenizer_v2.apply_rotary_pos_emb = _apply_rotary_pos_emb_npu`, so the fallback copies run on every attention call per decode chunk. Benchmark figures cited by the comment are in the PR description body, not a repo file.) + +`tests/platforms/npu/test_310p_patches.py:548` [minor] — At tests/platforms/npu/test_310p_patches.py:477 the fake `npu_rotary_mul` returns `hidden_states + 1` and ignores `cos`/`sin`, so `torch.testing.assert_close(q_out, q + 1)` (line 548) only proves the transpose round-trip reached the mock; it cannot catch a wrong rotate-half layout or misplaced unsqueeze. Make the fake do real rotate-half math (or add a non-mocked numerical case) so the fallback's math is pinned, not just its routing. (evidence: tests/platforms/npu/test_310p_patches.py:477 `return hidden_states + 1` — the fake `npu_rotary_mul` (installed as `torch_npu.npu_rotary_mul`) ignores `cos`/`sin`. tests/platforms/npu/test_310p_patches.py:548 `torch.testing.assert_close(q_out, q + 1)` — asserts only the +1 passthrough, so the test proves routing, not numerical equivalence of the BSND vs BNSD/HF rotate-half path.) + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:44` [minor] — The `unsqueeze_dim != 1` branch at rotary_embedding.py:44 is currently unreachable: the only callers of `npu_rotary_mul_with_bsnd_fallback` go through `_apply_rotary_pos_emb_npu` (qwen3_tts_tokenizer_v2.py:31-32), whose `unsqueeze_dim` defaults to 1 and is never overridden at the sole `apply_rotary_pos_emb(query_states, key_states, cos, sin)` call (modeling_qwen3_tts_tokenizer_v2.py:362). Either document the future non-BNSD intent or drop the arm. (evidence: vllm_omni/platforms/npu/layers/rotary_embedding.py:44 (new in this diff): ` if unsqueeze_dim != 1 or _bnsd_rotary_shape_is_supported(hidden_states):`; vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:27 (changed in this diff): ` unsqueeze_dim: int = 1,`; vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:31 (changed in this diff): ` npu_rotary_mul_with_bsnd_fallback(q, cos, sin, unsqueeze_dim),`; vllm_omni/platforms/npu/models/qwen3_tts_tokenizer_v2.py:49 (present in PR-time tree, unchanged by this diff): ` modeling_qwen3_tts_tokenizer_v2.apply_rotary_pos_emb = _apply_rotary_pos_emb_npu`; vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py:362 (unchanged by this diff, present in the PR-time tree): ` query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)` — the sole call passes no unsqueeze_dim, so the default 1 is used and the `unsqueeze_dim != 1` branch is dead.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.trace.json.gz new file mode 100644 index 000000000..b3804fb1f Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5608.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.cost.json new file mode 100644 index 000000000..ac97c549f --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2188.5, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5720/run-20260815-171247-d37b3b", + "recorded_at": "2026-08-15T17:49:13", + "attempts": [ + { + "invocation_id": "inv-f065e99b", + "outcome": "run:run-20260815-171247-d37b3b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-171247-d37b3b", + "llm_calls": 168, + "input_tokens": 1728853, + "output_tokens": 266259, + "usd": 1.25831, + "source": "spans" + } + ], + "attempt_usd_total": 1.25831, + "llm_calls": 168, + "input_tokens": 1728853, + "output_tokens": 266259, + "usd": 1.25831, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-171247-d37b3b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2583, + "minutes": 36.42, + "ci_minutes": 0, + "input_tokens": 1728853, + "output_tokens": 266259, + "tool_calls": 277, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2563, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 168, + "by_role": { + "agent": { + "usd": 1.223051, + "input_tokens": 1695676, + "output_tokens": 242389, + "calls": 166 + }, + "reducer": { + "usd": 0.03526, + "input_tokens": 33177, + "output_tokens": 23870, + "calls": 2 + } + }, + "cache_read_tokens": 18467968, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.55, + "gate": 1.51, + "review": 2179.83, + "report": 0.07 + }, + "time_to_first_result_s": 2184.95 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5720.trace.json.gz", + "trace_bytes": 56549 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.md new file mode 100644 index 000000000..67335418e --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5720, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head b4733f6bfd98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] prior TODO(offload) 'reconcile static plan with modular pipeline (#5720)' was removed; resident plan stays `{'transformer'}` and diffuse gates staging — residual: combined+DLO streaming of `transformers_ref` is asserted in a comment but not tested (review comment 6). +- [resolved] DLO resident skip for ref2va: _resident_dit_layers_on_device(enabled=...) skips staging when the active transformer is transformers_ref; test_distributed_layerwise_resident_blocks_can_be_skipped covers it. Residual: _offload_plan.resident_dit_paths stays frozenset({'transformer'}), so combined-mode transformers_ref is fully streamed by design. +- [claim-verified] PR claim 'share other components once': `__init__` loads tokenizer/processor/text_encoder/VAEs only from the FL2VA path; test asserts `len(tokenizer_calls)==1`, `len(processor_calls)==1` and single entries in created dicts. +- [claim-verified] PR claim 'default initializes both DiTs': combined partition constructs `transformer` and `transformers_ref` (pipeline_minimax_h3.py:617-624). +- [claim-verified] PR claim 'task_type selects one DiT': fl2va/ref2va paths build only `transformer`; test `expected_dits==1` and `hasattr(pipeline,'transformers_ref') is False`. +- [claim-verified] docs claim 'downloads only the selected partition': `MINIMAX_H3_TASK_DOWNLOAD_PATTERNS` drives `allow_patterns`; test asserts the exact `download_calls` pattern lists. +- [claim-verified] registry alias `MiniMaxH3ModularPipeline -> MiniMaxH3Pipeline` and post-process func registered; `DiffusionModelMetadata` duplicates H3 admission limits; both asserted in test_pipeline_import_registry_and_component_discovery. +- [claim-verified] Default combined mode loads 2 DiTs and single-task modes load 1: test_pipeline_loads_task_selected_dits_and_shared_components_once asserts created['dit'] length and pipeline._dit_modules for combined/fl2va/ref2va. +- [claim-verified] Combined download patterns are FL2VA/**, Ref2VA/model_index.json, Ref2VA/transformer/** and task-scoped patterns are single-partition (pipeline_minimax_h3.py:110-121), matching the recipe claims. +- [claim-refuted] CLI help claims 'The selected model validates supported values', but only Qwen3-TTS AR and MiniMaxH3 validate; other models silently accept arbitrary task_type (review comment 1). +- [claim-refuted] 'PR docs aligned' — recipes/MiniMaxAI/MiniMax-H3-5090.md still uses `/path/to/MiniMax-H3/FL2VA` + restart and keeps a forward-looking 'after #5720 lands' note; it is the stale consumer this diff did not update. +- [validated] pipeline_minimax_h3.py:118-140 `_minimax_h3_partition_for_task` mapping (None/auto/t2va/fl2va/ref2va + local partition + unsupported) matches test_minimax_h3_contract.py:93-115. +- [validated] combined weight loader contiguity holds: diffusers_loader.py `get_all_weights` yields per-source in declared order (sources list), so `groupby` prefix keys stay contiguous; test_combined_weight_loader_routes_each_contiguous_partition exercises it. +- [validated] `_transformer_for_task` only returns `transformers_ref` when the attribute exists (pipeline_minimax_h3.py:710-713); covered by test_combined_task_inference_and_transformer_routing. + +`vllm_omni/diffusion/worker/diffusion_model_runner.py:286` [major] — diffusion_model_runner.py:283-287 now compiles every `_dit_modules` entry by default (`enforce_eager=False`, data.py:716), replacing the old hard-coded ('transformer','transformer_2'). `_compile_transformer` (lines 153-155) only skips missing attributes, so pipelines with declared DiT names outside the old tuple — e.g. HunyuanImage3 `["model"]`, MagiHuman `["dit","sr_dit"]`, SDXL `["unet"]`, SenseNovaU1 `["language_model"]`, MammothModa2 `["gen_transformer"]` — now get torch.compile attempts by default, and registry.py:437 applies the same `_dit_modules` generalization to SP hooks. The PR's tests validate only MiniMax H3; restrict this to the validated model or enumerate and validate the newly compiled/SP-hooked families. (evidence: ['vllm_omni/diffusion/worker/diffusion_model_runner.py:283-287 (changed by this diff): `transformer_attrs = getattr(self.pipeline, "_dit_modules", None)` / `if not transformer_attrs:` / ` transformer_attrs = ("transformer", "transformer_2")` / `for attr_name in transformer_attrs:` / ` self._compile_transformer(attr_name)`', 'vllm_omni/diffusion/worker/diffusion_model_runner.py:153-155 (present in PR-time tree): `model = getattr(self.pipeline, attr_name, None)` / `if model is None:` / ` return` — the only skip is for missing attributes.', 'vllm_omni/diffusion/data.py:716 (present in PR-time tree): `enforce_eager: bool = False`', 'vllm_omni/diffusion/registry.py:437 (changed by this diff): `transformer_attrs = getattr(model, "_dit_modules", None)`', 'Unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py:383 `_dit_modules: ClassVar[list[str]] = ["model"]` and vllm_omni/diffusion/models/magi_human/pipeline_magi_human.py:1685 `_dit_modules: ClassVar[list[str]] = ["dit", "sr_dit"]`']) + +`vllm_omni/data.py:546` [minor] — Restore the dropped `except Exception: model_index = None` guard around the Diffusers index fetch in `resolve_model_class_name` (vllm_omni/diffusion/data.py:552). `get_diffusion_model_index` calls `get_hf_file_to_dict` without catching, so not-found/transient HF errors propagate out of model-class resolution instead of falling back to None. (evidence: vllm_omni/diffusion/data.py:552 `model_index = get_diffusion_model_index(model)` — called without the previous try/except guard. vllm_omni/diffusion/utils/hf_utils.py:19-23 `for filename in DIFFUSION_MODEL_INDEX_FILES:\n config = get_hf_file_to_dict(filename, model_name, revision=revision)\n if isinstance(config, Mapping):\n return dict(config)` — no exception handling, so any error from `get_hf_file_to_dict` propagates to the unguarded call site.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1477` [minor] — Add a combined-mode + DLO regression test that exercises `diffuse` for a Ref2VA request routed to `transformers_ref` and asserts `_resident_dit_layers_on_device` is entered with `enabled=False`; the current `test_distributed_layerwise_resident_blocks_can_be_skipped` only covers the helper, not the diffuse gating. (evidence: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1477 `with self._resident_dit_layers_on_device(enabled=transformer is self.transformer):` — added by this diff; the immediately preceding comment states combined-mode `transformers_ref` is fully streamed and a Ref2VA request must not stage the inactive FL2VA transformer resident blocks. Existing coverage is only tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:787 `with pipeline._resident_dit_layers_on_device(enabled=False):`, which exercises the helper but not the combined-mode `diffuse` routing.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:694` [minor] — The groupby in load_weights (pipeline_minimax_h3.py:694) only merges consecutive same-prefix records and raises at line 696 on a repeated prefix, so interleaved transformer.* / transformers_ref.* streams would fail. test_combined_weight_loader_routes_each_contiguous_partition (test_minimax_h3_contract.py:318) passes an already-contiguous order and never exercises the real weights_sources producer, leaving producer contiguity unverified. Add an interleaved-order test (and/or an end-to-end load through weights_sources) to prove the producer contract. (evidence: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:694 `for prefix, grouped_weights in groupby(weights, key=source_prefix):`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:696 `raise ValueError(f"MiniMax-H3 weight source {prefix!r} is not contiguous")`; tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:318 `def test_combined_weight_loader_routes_each_contiguous_partition():` — this test feeds `pipeline.load_weights(iter([("transformer.a", ...), ("transformer.b", ...), ("transformers_ref.a", ...)]))`, an already-contiguous order, and does not exercise the weights_sources producer end-to-end.) + +`vllm_omni/entrypoints/cli/serve.py:~217` [minor] — The diff removes `choices=["CustomVoice", "VoiceDesign", "Base"]` from `--task-type` (serve.py:217) and generalizes it to a model-defined startup task type. But `OmniModelConfig._validate_startup_task_type` (model.py:215-216) returns early for every architecture except Qwen3-TTS, so any other AR model now accepts and ignores an invalid startup `task_type` (MiniMax H3 validates only inside its pipeline constructor, and only for supported values). Previously argparse rejected invalid values uniformly. Either fail fast at the entry boundary for models with no `task_type` consumer or explicitly document the silent-ignore behavior. (evidence: vllm_omni/config/model.py:213-219 `def _validate_startup_task_type(self) -> None:` / `if self.model_arch != "Qwen3TTSTalkerForConditionalGenerationARVLLM" or self.task_type is None:` / ` return` / `if self.task_type not in _QWEN3_TTS_TASK_TYPES:` / ` raise ValueError(f"Qwen3-TTS --task-type must be one of {supported}; got {self.task_type!r}")` — validation early-returns for every architecture except Qwen3-TTS, which is the only one that raises. vllm_omni/entrypoints/cli/serve.py:216-217 `default=None,` / `help="Model-defined startup task type. The selected model validates "` — the preceding `choices=[...]` line was removed by this diff, so argparse now accepts any string.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~149` [minor] — When `--model` points at a local partition directory (`/data/Ref2VA`) and an explicit contradictory `--task-type fl2va` is passed, `_minimax_h3_partition_for_task` returns `fl2va` (line 133-137), `_resolve_minimax_h3_model_root` walks the partition dir to its parent (lines 148-150), and `__init__` then computes `model_path = model_root / "FL2VA"` (line 561), silently loading the sibling partition. Add a ValueError when the requested task partition contradicts the pointed-to partition directory instead of switching models silently. (evidence: pipeline_minimax_h3.py:149 `if path.name in {"FL2VA", "Ref2VA"} and (path / "model_index.json").is_file():`; pipeline_minimax_h3.py:150 `return path.parent`; pipeline_minimax_h3.py:135-136 `if task in {"t2va", "fl2va"}:` / `return "fl2va"`; pipeline_minimax_h3.py:561 `model_path = model_root / ("Ref2VA" if self.partition == "ref2va" else "FL2VA")` — all present in the PR-time tree; lines 149-150 and 561 are inside the diff, and the task-selection lines 133-137 are also in the diff.) + +`vllm_omni/diffusion/utils/hf_utils.py:101` [minor] — The strategy-2 guard that kept model detection best-effort was removed: `is_diffusion_model` now calls `get_diffusion_model_index` unguarded (hf_utils.py:101), and the new helper (hf_utils.py:22-26) lets exceptions from `get_hf_file_to_dict` propagate. Since `OmniServeCommand.validate` calls `is_diffusion_model(model)` without a try/except, a gated-repo, not-found, or transient HF error now escapes before the friendlier access/download path. Restore the try/except around the strategy-2 call (or catch inside the helper). (evidence: Head, changed by this diff: vllm_omni/diffusion/utils/hf_utils.py:23 `config = get_hf_file_to_dict(filename, model_name, revision=revision)` (inside `get_diffusion_model_index`, no try/except). Head, changed by this diff: vllm_omni/diffusion/utils/hf_utils.py:101 `config_dict = get_diffusion_model_index(model_name)` (strategy-2 fetch, no try/except). Merge-base, removed by this diff: vllm_omni/diffusion/utils/hf_utils.py `try:` / `config_dict = get_hf_file_to_dict("model_index.json", model_name)` / ... / `except Exception as e:` — this guard no longer exists. Consumer, unchanged by this diff and present in the PR-time tree: vllm_omni/entrypoints/cli/serve.py `if model and is_diffusion_model(model):` inside `OmniServeCommand.validate`, with no surrounding exception handling.) + +`recipes/MiniMaxAI/MiniMax-H3-5090.md:~22` [minor] — This PR (#5720) changes MiniMax H3 to combined serving by default and updates the main/NPU/MUSA recipes, but `recipes/MiniMaxAI/MiniMax-H3-5090.md` is untouched and still teaches the old single-partition flow (`/path/to/MiniMax-H3/FL2VA` at line 32, stop/restart with `Ref2VA` at lines 59-60) while its line 22 note says to adopt `--task-type` only 'after #5720 lands'. Update the commands to pass `--task-type fl2va`/`--task-type ref2va`, or reword the note to state the current default. (evidence: recipes/MiniMaxAI/MiniMax-H3-5090.md:22 `> **Modular H3:** after #5720 lands, preserve this recipe's one-partition` — unchanged by this diff; recipes/MiniMaxAI/MiniMax-H3-5090.md:32 `CUDA_VISIBLE_DEVICES=0 vllm serve /path/to/MiniMax-H3/FL2VA \`; recipes/MiniMaxAI/MiniMax-H3-5090.md:59 `For Ref2VA, stop the FL2VA server and restart the same command with` and :60 `` `/path/to/MiniMax-H3/Ref2VA`. Ref2VA reference video count and prompt length ``. The diff does not modify this file (diff_stat shows only MUSA/NPU/main H3 recipes among recipes), while `pipeline_minimax_h3.py` `_minimax_h3_partition_for_task` returns `"combined"` for no/auto `task_type`, matching the PR's new combined default.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.trace.json.gz new file mode 100644 index 000000000..a89933a76 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5720.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.cost.json new file mode 100644 index 000000000..4c2ff0d08 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1224.6, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5723/run-20260815-171247-6d64ca", + "recorded_at": "2026-08-15T17:33:09", + "attempts": [ + { + "invocation_id": "inv-a923beb7", + "outcome": "run:run-20260815-171247-6d64ca", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-171247-6d64ca", + "llm_calls": 92, + "input_tokens": 654611, + "output_tokens": 160266, + "usd": 0.466484, + "source": "spans" + } + ], + "attempt_usd_total": 0.466484, + "llm_calls": 92, + "input_tokens": 654611, + "output_tokens": 160266, + "usd": 0.466484, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-171247-6d64ca", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4665, + "minutes": 20.37, + "ci_minutes": 0, + "input_tokens": 654611, + "output_tokens": 160266, + "tool_calls": 178, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7289, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 92, + "by_role": { + "agent": { + "usd": 0.432193, + "input_tokens": 639794, + "output_tokens": 132767, + "calls": 90 + }, + "reducer": { + "usd": 0.034291, + "input_tokens": 14817, + "output_tokens": 27499, + "calls": 2 + } + }, + "cache_read_tokens": 4201728, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.06, + "gate": 1.49, + "review": 1216.35, + "report": 0.06 + }, + "time_to_first_result_s": 1221.96 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5723.trace.json.gz", + "trace_bytes": 35984 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.md new file mode 100644 index 000000000..24a1a841c --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5723, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 93c0112c114e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 5 finding(s) below | + +**Validated:** +- [resolved] 'Should ROCm be a sibling file like MiniMax-H3-MUSA.md / MiniMax-H3-NPU.md?' — recipes/README.md explicitly endorses the in-file choice: 'Within a single recipe file, include different hardware support sections such as `GPU`, `ROCm`, and `NPU`'. Residual not covered: the README's Available-Recipes hardware column for MiniMax-H3.md was not updated (see comment). +- [resolved] deleted old image+audio/video Ref2VA limitation bullets are re-covered by the new combined bullet — but the new wording is factually wrong (see claim-refuted above). +- [claim-verified] Evidence-table arithmetic is consistent (calc): gfx942 T2VA 0.09+244.04+4.15=248.28s < 267.42s E2E; FL2VA 13.98+257.58+4.11=275.67s < 287.07s E2E; gfx950 1/0.73=1.37 it/s matches '~1.37 it/s'. +- [claim-verified] 'cfg-parallel-size > 1 is rejected by design': pipeline_minimax_h3.py __init__ 'if int(self.parallel_config.cfg_parallel_size) != 1: raise ValueError(...CFG-distilled...)'. +- [claim-verified] 'FLASH_ATTN resolves to AITER packed varlen on both architectures': vllm_omni/platforms/rocm/platform.py:96 'aiter currently only is supported on gfx942 and gfx950', :101 `aiter_supported = is_aiter_found_and_supported() and 90 < capability < 100`, :114 raises for FLASH_ATTN when unsupported. +- [claim-verified] 'The VAE uses AITER GroupNorm on ROCm': vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py:16 `from aiter.ops.groupnorm import GroupNorm as AiterGroupNorm`, :48 replaces on model.vae when aiter is supported. +- [claim-verified] ROCm install block matches docs/getting_started/installation/gpu/rocm.inc.md ('uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723'; 'targets ROCm 7.2.3' consistent with the recipe's 'HIP 7.2'). +- [claim-verified] gate_report clean (mergeable, no failing checks); docs-only change, no CI lane loads these files. +- [claim-refuted] PR description 'supported_models.md — mark MiniMax H3 AMD GPU supported': head table row (docs/models/supported_models.md:46) is `| `MiniMaxH3Pipeline` | ... | ✅︎ | | | |` — AMD cell empty, and `H3` is referenced nowhere in the table (only the footnote itself, line 101). +- [claim-refuted] PR body 'mark MiniMax H3 AMD GPU supported' — H3 row AMD cell still empty at head; only a footnote was appended and its H3 marker is orphaned. +- [claim-refuted] new limitation 'takes exactly one image plus one audio reference, or one or more videos with no separate audio_reference' contradicts code (_validate_ref2va_reference_counts allows <=9/<=3/<=3) and the doc's own matrix/HTTP §4. +- [claim-refuted] '--usp N --tp 1' uses a nonexistent --tp flag; DiT TP flag is --tensor-parallel-size. +- [validated] FLASH_ATTN→AITER on ROCm: vllm_omni/platforms/rocm/platform.py:82 'ROCm supports FLASH_ATTN via the aiter library'; :114 falls back only when aiter missing and capability gate '90 < capability < 100' with comment 'only is supported on gfx942 and gfx950'; docs/design/module/dit_module.md:505 'dispatches to FA2/FA3 on CUDA, AITER on ROCm'. +- [validated] VAE AITER GroupNorm claim: vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py replaces nn.GroupNorm with aiter.ops.groupnorm.GroupNorm when is_aiter_found_and_supported(). + +`docs/models/supported_models.md:101` [major] — Confirmed. In docs/models/supported_models.md the MiniMax H3 row has an empty AMD cell at line 47, while line 99 says empty cells mean 'not listed as supported on that backend' and the newly added footnote at line 101 claims AMD support. No table row references `H3`, so the legend and footnote contradict each other. Mark the AMD cell `✅︎H3` (or remove the footnote). Also reconcile the https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3 link with the in-repo recipe, since recipes/README.md shows the rendered site is a separate vllm-project/recipes publication and may not carry this PR's ROCm section. (evidence: docs/models/supported_models.md:47 `| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | |` — AMD column empty. docs/models/supported_models.md:99 `✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.` docs/models/supported_models.md:101 `H3 MiniMax H3 AMD GPU support is validated on gfx942 (MI300X) and` — added by this diff; grep confirms `H3` occurs only at line 101, so no row references it. The footnote is part of the diff, but the row it contradicts is unchanged in the diff.) + +`recipes/MiniMaxAI/MiniMax-H3.md:~729` [major] — recipes/MiniMaxAI/MiniMax-H3.md:729 — Replace this bullet with a matrix-consistent statement. The serving path accepts image-only Ref2VA, image+audio, one or more videos (with embedded soundtracks), and videos with a separate `audio_reference`, subject to images ≤9, videos ≤3, standalone audios ≤3, and total references ≤12. Do not write '9 images, 3 video clips, and 3 audio clips (12 files)': 9+3+3 is 15, and 12 is the separate total-file cap. (evidence: ['recipes/MiniMaxAI/MiniMax-H3.md (Known limitations, changed by this diff): "the current vLLM-Omni path takes exactly one image plus one audio reference, or one or more videos with no separate `audio_reference` (it uses the source soundtracks)." — the challenged sentence.', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:323 (unchanged by this diff): `if audio_count > 3:` — permits up to 3 standalone audio references, not zero-with-videos.', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:325 (unchanged by this diff): `if image_count + video_count + audio_count > 12:` — 12 is a separate total-file cap, not the sum of the per-type maxima.', 'vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1716-1717 (unchanged by this diff): `for ref_audio_t in external_audio_lengths:` / `ref_blocks.append({"kind": "audio", "ref_audio_t": ref_audio_t})` — forward path appends standalone audio alongside prepared videos.', 'vllm_omni/entrypoints/openai/api_server.py:3370-3371 (unchanged by this diff): `for audio_reference in _reference_list(request.audio_reference):` / `audio_paths.append(await decode_audio_url(audio_reference.audio_url))` — audio_reference is appended to audio_paths regardless of video presence.', 'recipes/MiniMaxAI/MiniMax-H3.md Official input matrix (same file): "images ≤9, videos ≤3, audios ≤3, total references ≤12; audio requires a visual reference", and HTTP §4: "A video may use its embedded soundtrack, a separate `audio_reference`, or both."']) + +`recipes/MiniMaxAI/MiniMax-H3.md:416` [major] — The new ROCm serve commands set `MODEL="${MODEL_ROOT}/FL2VA"` with no `--task-type` while line 415/416 claims 'All tasks work out of the box'; pipeline_minimax_h3.py auto-partitions a directory named `FL2VA` into FL2VA-only and `_resolve_task` rejects `task=ref2va`, so the started server cannot serve this recipe's own HTTP Ref2VA examples. Use the parent `${MODEL_ROOT}` or repo ID `MiniMaxAI/MiniMax-H3` to load the combined FL2VA/Ref2VA service, or explicitly mark these commands FL2VA-only and remove the all-tasks claim. (The undefined `MODEL_ROOT` itself is raised separately.) (evidence: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py, _minimax_h3_partition_for_task: 'if path.is_dir() and path.name in {"FL2VA", "Ref2VA"} and (path / "model_index.json").is_file(): return path.name.lower()' (only 'auto'/'combined' yields "combined"). _resolve_task: 'raise OmniClientError(f"checkpoint partition {self.partition!r} supports {sorted(self.supported_tasks)}, got task={task!r}")'. ref2va weights are only loaded when 'if self.partition == "combined": ref2va_model_path = model_root / "Ref2VA"'.) + +`recipes/MiniMaxAI/MiniMax-H3.md:~426` [major] — The new ROCm serve commands at lines 426 and 446 both `export MODEL="${MODEL_ROOT}/FL2VA"`, but this recipe never defines `MODEL_ROOT` in these sections — the only `MODEL_ROOT=` (line 205) is an inline environment prefix for the offline-script invocation, not an `export`. A copy-paste user gets `MODEL="/FL2VA"` and `vllm serve` fails. Define `export MODEL_ROOT=/path/to/MiniMax-H3` above each command and state it points at the local checkout containing `FL2VA/` and `Ref2VA/`, or use `export MODEL=MiniMaxAI/MiniMax-H3` like the CUDA commands. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:426 `export MODEL="${MODEL_ROOT}/FL2VA"` (new ROCm single-GPU block); recipes/MiniMaxAI/MiniMax-H3.md:446 `export MODEL="${MODEL_ROOT}/FL2VA"` (new ROCm four-GPU block); recipes/MiniMaxAI/MiniMax-H3.md:205 `MODEL_ROOT=/path/to/MiniMax-H3 \` (only MODEL_ROOT occurrence, not exported, prefix to `bash examples/offline_inference/minimax_h3/run_h3_2gpu_all_tasks.sh`); recipes/MiniMaxAI/MiniMax-H3.md:35 `export MODEL=MiniMaxAI/MiniMax-H3` (CUDA serve pattern, unchanged).) + +`docs/models/supported_models.md:102` [minor] — The new footnote (docs/models/supported_models.md:101-102) states MiniMax H3 AMD support is 'validated on gfx942 (MI300X) and gfx950 (MI350)', but the AMD evidence table (recipes/MiniMaxAI/MiniMax-H3.md:479-481) contains only T2VA and FL2VA rows for 4x gfx942 plus a single-GPU T2VA functional row for gfx950 — no Ref2VA row and no 4x gfx950 row — while the same section states 'All tasks work out of the box' (recipes/MiniMaxAI/MiniMax-H3.md:416). Scope the footnote/AMD claim to the evidenced tasks/configs, or add the missing Ref2VA and gfx950 rows. (evidence: docs/models/supported_models.md:101 `H3 MiniMax H3 AMD GPU support is validated on gfx942 (MI300X) and`; docs/models/supported_models.md:102 `gfx950 (MI350) in BF16, using the AITER `FLASH_ATTN` diffusion attention backend.`; recipes/MiniMaxAI/MiniMax-H3.md:416 `Prebuilt image: `vllm/vllm-omni-rocm:minimax-h3`. All tasks work out of the box:`; recipes/MiniMaxAI/MiniMax-H3.md:479 `| T2VA, 1344x768, 209 frames, 50 steps | 4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile | encode 0.09 s, denoise 244.04 s, decode 4.15 s, 267.42 s client E2E; H.264 24 FPS + 32 kHz stereo AAC |`; recipes/MiniMaxAI/MiniMax-H3.md:480 `| FL2VA, 1344x768, 209 frames, 50 steps | 4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile | encode 13.98 s, denoise 257.58 s, decode 4.11 s, 287.07 s client E2E; H.264 24 FPS + 32 kHz stereo AAC |`; recipes/MiniMaxAI/MiniMax-H3.md:481 `| T2VA, 832x480, ~4 s, 40 steps | 1x gfx950 (MI350), FLASH_ATTN, CPU offload | valid MP4 (H.264 + synced audio); ~0.73 s/denoise-step (~1.37 it/s), ~55 s client E2E incl. warmup |`; recipes/MiniMaxAI/MiniMax-H3.md:485 `functional-correctness validations, not tuned throughput; the first request`.) + +`recipes/MiniMaxAI/MiniMax-H3.md:405` [minor] — The ROCm intro at recipes/MiniMaxAI/MiniMax-H3.md:405 tells users to 'drop the CUDA-only FLASHINFER_DISABLE_VERSION_CHECK', but no CUDA command in this recipe sets that variable (it appears only on this line), so the clause dangles. The engine core subprocess sets it automatically and unconditionally for every stage (vllm_omni/engine/stage_engine_core_proc.py:122), so ROCm users need do nothing. Replace the clause with e.g. 'no FlashInfer-related environment variables are needed on ROCm'. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:405 `CUDA-only `FLASHINFER_DISABLE_VERSION_CHECK`, and install without the `[fa4]` extra` — this line is in the PR diff; a repo-wide grep of the recipe confirms FLASHINFER_DISABLE_VERSION_CHECK appears only here. vllm_omni/engine/stage_engine_core_proc.py:122 `os.environ.setdefault("FLASHINFER_DISABLE_VERSION_CHECK", "1")` — unchanged by this diff (diff_stat shows only docs/models/supported_models.md and recipes/MiniMaxAI/MiniMax-H3.md changed), present in the PR-time tree, and unconditional in run_stage_core.) + +`recipes/MiniMaxAI/MiniMax-H3.md:416` [minor] — The new ROCm section recommends `vllm/vllm-omni-rocm:minimax-h3`, an unversioned tag; vLLM-Omni image tags normally encode the underlying vLLM release (docs/getting_started/installation/gpu/rocm.inc.md:117 says the version of vLLM-Omni indicates the vLLM release it is based on, and sibling recipes pin e.g. `v0.26.0` / `v0.22.0`). Reference a versioned tag such as `vllm/vllm-omni-rocm:v0.26.0` or state which commit `minimax-h3` tracks so the recipe is reproducible. (evidence: In diff: recipes/MiniMaxAI/MiniMax-H3.md:416 `Prebuilt image: `vllm/vllm-omni-rocm:minimax-h3`. All tasks work out of the box:` — unversioned tag recommended by the new ROCm section. Unchanged by this diff, present in the PR-time tree: docs/getting_started/installation/gpu/rocm.inc.md:117 `The version of vLLM-Omni indicates which release of vLLM it is based on.`; docs/getting_started/installation/gpu/rocm.inc.md:133 ` vllm/vllm-omni-rocm:v0.26.0 \`; recipes/inclusionAI/Ming-omni-tts.md:132 `- Docker image: `vllm/vllm-omni-rocm:v0.22.0``.) + +`recipes/MiniMaxAI/MiniMax-H3.md:477` [minor] — Label the unaccounted residual in the ROCm timing table: the per-stage sums (T2VA 0.09+244.04+4.15=248.28s, FL2VA 13.98+257.58+4.11=275.67s) do not equal the client E2E values (267.42s, 287.07s), leaving ~19.1s/~11.4s unlabeled, and the gfx950 denoise time (~0.73 s/step × 40 steps ≈ 29.2s) sits ~25.8s below the ~55s E2E even though warmup is already mentioned. Add a note stating the residual is client overhead/warmup so the component figures are not read as a complete E2E breakdown. (evidence: recipes/MiniMaxAI/MiniMax-H3.md:479 `| T2VA, 1344x768, 209 frames, 50 steps | 4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile | encode 0.09 s, denoise 244.04 s, decode 4.15 s, 267.42 s client E2E; H.264 24 FPS + 32 kHz stereo AAC |`; recipes/MiniMaxAI/MiniMax-H3.md:480 `| FL2VA, 1344x768, 209 frames, 50 steps | 4x gfx942 (MI300X), FLASH_ATTN, USP4, text-enc TP4, VAE PP4 tile | encode 13.98 s, denoise 257.58 s, decode 4.11 s, 287.07 s client E2E; H.264 24 FPS + 32 kHz stereo AAC |`; recipes/MiniMaxAI/MiniMax-H3.md:481 `| T2VA, 832x480, ~4 s, 40 steps | 1x gfx950 (MI350), FLASH_ATTN, CPU offload | valid MP4 (H.264 + synced audio); ~0.73 s/denoise-step (~1.37 it/s), ~55 s client E2E incl. warmup |`; recipes/MiniMaxAI/MiniMax-H3.md:483 `gfx942 figures are the mean of three requests after one excluded warmup (external evidence: vllm-project/recipes#732). gfx950 figures are functional-correctness validations, not tuned throughput; the first request includes lazy regional compilation.` — this note is part of the PR diff but never attributes the encode+denoise+decode vs E2E residual to client overhead/warmup.) + +**Additional observations (beyond the comment budget):** +- `recipes/MiniMaxAI/MiniMax-H3.md:477` [minor] Confirmed. In the AMD ROCm section, the recommended four-GPU command sets --text-encoder-tp-size 4 (recipes/MiniMaxAI/MiniMax-H3.md:450) while line 468 states it 'was not exercised' on gfx950, and the Validated ROCm evid +- `recipes/MiniMaxAI/MiniMax-H3.md:442` [minor] The four-GPU ROCm description (line 442) says `no CPU offload when the model shards across the GPUs`, but the command uses `--usp 4` with no `--tensor-parallel-size`, and the same PR's new limitation (line 740) says pure +- `recipes/MiniMaxAI/MiniMax-H3.md:486` [minor] At line 486, 'MI325X (gfx942) and other MI355X SKUs' reads as if MI325X is one of the MI355X SKUs, but MI325X is gfx942 and MI355X is gfx950. Rephrase to 'MI325X (gfx942) and MI355X (gfx950) are not listed until their ow +- `recipes/README.md:48` [minor] The diff adds ROCm single-GPU and four-GPU serving commands and gfx942/gfx950 evidence to the MiniMax-H3 recipe, but recipes/README.md:48 still lists that row's hardware as '1x GPU with CPU offload / 4x B300', so readers + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.trace.json.gz new file mode 100644 index 000000000..5a476ee51 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5723.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.cost.json new file mode 100644 index 000000000..ef8a4a8cb --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1965.5, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5756/run-20260815-171247-d5a92c", + "recorded_at": "2026-08-15T17:45:30", + "attempts": [ + { + "invocation_id": "inv-ff48c161", + "outcome": "run:run-20260815-171247-d5a92c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-171247-d5a92c", + "llm_calls": 128, + "input_tokens": 1065701, + "output_tokens": 215659, + "usd": 0.850329, + "source": "spans" + } + ], + "attempt_usd_total": 0.850329, + "llm_calls": 128, + "input_tokens": 1065701, + "output_tokens": 215659, + "usd": 0.850329, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-171247-d5a92c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8503, + "minutes": 32.72, + "ci_minutes": 0, + "input_tokens": 1065701, + "output_tokens": 215659, + "tool_calls": 227, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0664, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 128, + "by_role": { + "planner": { + "usd": 0.00204, + "input_tokens": 125, + "output_tokens": 1767, + "calls": 1 + }, + "agent": { + "usd": 0.813787, + "input_tokens": 1036942, + "output_tokens": 189592, + "calls": 125 + }, + "reducer": { + "usd": 0.034503, + "input_tokens": 28634, + "output_tokens": 24300, + "calls": 2 + } + }, + "cache_read_tokens": 12050560, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.71, + "gate": 1.6, + "review": 1957.5, + "report": 0.07 + }, + "time_to_first_result_s": 1962.87 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5756.trace.json.gz", + "trace_bytes": 46961 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.md new file mode 100644 index 000000000..d3d22e15e --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5756, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 43592108ac79) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] head commit 'fix path matching': '/path/to/MiniMax-H3/FL2VA', '/path/to/MiniMax-H3/Ref2VA', 'MiniMaxAI/MiniMax-H3' all resolve via `lookup_key = normalized.split("/", 1)[-1]` + unanchored pattern.search; residual = any intermediate directory named like a registered model hijacks the spec (review comment filed). +- [resolved] extra_body→extra_params rename: /videos form endpoint declares `extra_params: str | None = Form(default=None)` (api_server.py:3009) and NO extra_body field in its full Form list (api_server.py:2981-3009) — the old client's 'extra_body' form field was silently dropped by FastAPI, so the rename fixes a dead field; in-repo nodes never passed extras, so no regression window. +- [claim-verified] 'frame and reference inputs are exclusive': enforced in nodes.py VALIDATE_INPUTS AND api_client.py ('Provide only one of frame or references, not both.'); server independently rejects mixing (api_server.py:3029-3034 'Provide input_references alone, without input_reference...'). +- [claim-verified] 'two test cases according to MiniMax H3 recipe': client combo rules (videos-only XOR exactly-1-image+1-audio) match recipes/MiniMaxAI/MiniMax-H3.md:319-320 verbatim ('Image+audio Ref2VA accepts exactly one image and one audio reference'). +- [claim-verified] PR body's two Ref2VA test cases exist in the diff: `ref2va-image-audio` and `ref2va-multi-video` params in tests/e2e/features/comfyui/test_comfyui_integration.py. +- [validated] max_poll_duration (5→30 min) has exactly one consumer — the generate_video poll deadline (`deadline = asyncio.get_running_loop().time() + self.max_poll_duration`); other nodes' client calls never poll, so blast radius is video-only. +- [validated] poll raise is numerically justified: recipe MiniMax-H3.md:306 reports 'Two-video Ref2VA, 362 frames... 784.394 s accounted model-stage mean' — old 300 s cap < 784 s would time out the very workload this PR adds; new 1800 s gives ~2.3x headroom. +- [validated] builder field routing matches server contract: flow_shift sent as direct form field (api_server.py:2998 `flow_shift: float | None = Form(default=None)`); audio_flow_shift+task inside extra_params JSON — no direct audio_flow_shift Form field exists, and recipe :246 uses the identical shape. +- [validated] test integrity: api_server fixture launches the REAL server via OmniServeCommand in a subprocess; mock is at AsyncOmni.generate (engine boundary); assertions check server-side decoded values (PIL Image.Image, base64 audio str, 2 persisted video paths) — real multipart parsing and task routing exercised, not client-authored values. +- [validated] CI selection: file has `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; test-ready.yml:28 and test-merge.yml:28 catch-all `pytest -sv tests/ -m 'core_model and cpu'` (comfyui dir not in --ignore list) — new tests run per-PR and at merge, not just locally. +- [validated] Wire contract is exercised for real: tests boot a real `vllm serve` process with real `_parse_video_form`/`_run_and_extract` and only mock AsyncOmni.generate, so multipart field names are parsed by real server code (api_server.py:2981-3073). +- [validated] `input_reference` (frame), `audio_reference` JSON data-URL and `input_references` multipart all match the server contract (api_server.py:2981-2985, 3023-3054, 3116-3118); audio_to_base64 returns a data: URL (format.py:263). +- [validated] H3 flow_shift/audio_flow_shift routing: builder sends flow_shift top-level (server Form -> serving_video.py:189-190 extra_args) and audio_flow_shift in extra_params (serving_video.py:197-204); pipeline reads both (pipeline_minimax_h3.py:1158/audio_shift). +- [sweep] indexed accesses: api_client images[0]/audios[0] guarded by the `len(images) == 1 and len(audios) == 1` branch condition; test file [0] accesses guarded by `assert len(received_sampling_params_list) == 1` / tuple-len asserts — all safe. + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:366` [major] — The Ref2VA path in `generate_video` constructs `extra_params={**extra_params, "task": video_task}` (api_client.py:366) and `_minimaxh3_params_builder` serializes it (models.py:57) without a `duration` field; `VLLMOmniGenerateVideo` exposes no duration input, so image+audio/video Ref2VA requests hit the `_resolve_shape` `num_frames` fallback (pipeline_minimax_h3.py:403-407). A reference audio/video longer than num_frames/fps is silently under-covered rather than rejected, contradicting recipes/MiniMaxAI/MiniMax-H3.md:252 ('The requested duration should cover the complete audio'). Add an optional `duration` input (or derive it from the connected reference media), pass it through `extra_params`, and extend the Ref2VA test to assert a `duration` extra_arg. (evidence: Changed by this diff: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:366 `extra_params={**extra_params, "task": video_task},`; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:56-57 `if merged_extra_params:` / `form_fields["extra_params"] = json.dumps(merged_extra_params, ensure_ascii=False)` (only `task` + `audio_flow_shift` enter `merged_extra_params`, no `duration`). Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:403 `duration = extra.get("duration")` and :404-407 `if duration is not None: ... elif int(sampling.num_frames or 1) > 1: requested_frames = int(sampling.num_frames)`. Recipe: recipes/MiniMaxAI/MiniMax-H3.md:246 `-F 'extra_params={"task":"ref2va","duration":15.0,"audio_flow_shift":3.0}' \` and :252 `The requested duration should cover the complete audio.`) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:~223` [minor] — nodes.py:223 claims `type` 'is used in api_client', but it is only discarded: api_client.py:358-359 pops `type` before model-name dispatch via `lookup_model_spec(model)` (api_client.py:361), and `_minimaxh3_params_builder` pops it again (models.py:44). The only writer is `VLLMOmniMiniMaxH3Params.get_params` (nodes.py:777), so the tag is dead. Either remove the tag, the commented-out node-side pop, and both pops, or read `type` to validate that the connected VIDEO_PARAMS node matches the resolved model spec; today mismatched Wan/H3 params only reach the logger.warning path in models.py. (evidence: ['apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:223 `# model_params["type"] is also an internal field, but is used in api_client` — the new comment claims usage that does not exist.', 'apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:777 `params["type"] = "minimax_h3"` — the only place `minimax_h3` is written.', 'apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:358-359 `model_params = dict(model_params)` / `model_params.pop("type", None)` and api_client.py:361 `spec, _ = lookup_model_spec(model)` — `type` is discarded, then dispatch is by model name only; the value is never read.', 'apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:44 `params.pop("type", None)` — second, equally dead pop in `_minimaxh3_params_builder`.', 'Repo-wide grep for `minimax_h3` in apps/ComfyUI-vLLM-Omni returns only nodes.py:777 (writer) and models.py:162 (demo `__main__` block); no consumer reads the value.']) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:~164` [minor] — The Generate Video node's fps default remains 16 (nodes.py:164), but the MiniMax-H3 pipeline hard-requires 24 fps and raises ValueError otherwise (pipeline_minimax_h3.py:83, 399-401). The node forwards fps unchanged to the server form (api_client.py:286), and the new Ref2VA tests use VIDEO_FPS = 8 with a mocked pipeline, so a real H3 run at the node default fails. Auto-correct fps to 24 for H3 in the node/client, or add explicit validation/error and document the fps=24 requirement in the README H3 section. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:164 `"fps": ("INT", {"default": 16, "min": 1}),` (in diff); apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:286 `form.add_field("fps", str(fps))` (in diff); vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:83 `MINIMAX_H3_FPS = 24` and :399-401 `fps = int(sampling.fps or MINIMAX_H3_FPS)` / `if fps != MINIMAX_H3_FPS:` / `raise ValueError(f"MiniMax H3 output fps is fixed at {MINIMAX_H3_FPS}")` (unchanged by this diff, present in PR-time tree); tests/e2e/features/comfyui/test_comfyui_integration.py:86 `VIDEO_FPS = 8` and :927 `fps=VIDEO_FPS,` (in diff).) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:71` [minor] — api_client.py:71 raises `VLLMOmniClient.max_poll_duration` from 300s to 1800s globally; api_client.py:387 applies it to every video job's poll loop, and nodes.py:227 constructs `VLLMOmniClient(url)` without passing a per-node value, so Wan and all other video models inherit 30 minutes. Scope the increase to MiniMax-H3 or expose/document a node-level poll-timeout override. (evidence: api_client.py:71 ` max_poll_duration: float = 60 * 30,` (changed by this diff from `60 * 5`); api_client.py:387 ` deadline = asyncio.get_running_loop().time() + self.max_poll_duration` (unchanged by this diff, present in PR-time tree) — used for every `generate_video` poll loop with no model branch; nodes.py:227 ` client = VLLMOmniClient(url)` (unchanged by this diff, present in PR-time tree) — the Generate Video node always uses the default, so Wan inherits the 30-minute ceiling too.) + +`tests/e2e/features/comfyui/test_comfyui_integration.py:889` [minor] — The MiniMax-H3 e2e cases are mock-backed: mock_async_omni patches vllm_omni.entrypoints.openai.api_server.AsyncOmni and replaces generate with _build_mock_outputs, while conftest stubs comfy_api.VideoInput.save_to (string path = no-op, file object = raw bytes). The tests therefore prove ComfyUI client -> real FastAPI multipart field-name plumbing only; they do not exercise real H3 pipeline semantics or real ComfyUI video encoding. If the PR's MiniMax-H3 support claim relies on these, add at least one smoke case that reaches the real AsyncOmni/pipeline, or label the suite as transport/contract tests. (evidence: tests/e2e/features/comfyui/test_comfyui_integration.py:478 `mock_async_omni_cls = mocker.patch("vllm_omni.entrypoints.openai.api_server.AsyncOmni")`; tests/e2e/features/comfyui/test_comfyui_integration.py:485 `mock_instance.generate = _build_mock_outputs(server_case.outputs, sampling_case, server_case)`; tests/e2e/features/comfyui/conftest.py:54-57 `def save_to(self, file: str | BinaryIO):` / `print("Called VideoInput.save_to with file path. Saving to a path is no-op in tests.")` / `else:` / `file.write(self._data)` — conftest.py is unchanged by this diff and present in the PR-time tree.) + +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:~128` [minor] — `lookup_model_spec` now searches the whole tail after the first slash with unanchored patterns (models.py:128 `lookup_key = normalized.split("/", 1)[-1]`; models.py:131 `if pattern.search(lookup_key)`). Because registry patterns like `Qwen2.5-Omni*` and `MiniMax-H3(/FL2VA|/Ref2VA)?` are unanchored, an intermediate directory named like a registered model hijacks a different leaf model (e.g. `/data/Qwen2.5-Omni-7B/finetunes/my-wan-model` matches the Qwen spec and gets its preprocessor, while `/ckpts/MiniMax-H3-archive/SomeOtherVideoModel` routes through `_minimaxh3_params_builder`). This function is shared with `_prepare_chat_completion_messages` (api_client.py:632), where a spurious spec match flips AR-vs-diffusion payload format. Match path components right-to-left (leaf-most wins) and anchor at component boundaries. (evidence: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:128 `lookup_key = normalized.split("/", 1)[-1]` — changed by this diff; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:129-131 `for pattern, spec in MODEL_PIPELINE_SPECS.items():` / `if pattern.search(lookup_key):` — unanchored search over the whole tail, changed by this diff; apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:68-71 `r"Qwen2.5-Omni*": {... "payload_preprocessor": _qwen25_payload_preprocessor ...}` and models.py:100 `r"MiniMax-H3(/FL2VA|/Ref2VA)?": {...}` compiled at models.py:121 `re.compile(k)` — unanchored patterns unchanged by this diff but now applied to the tail; consumer, unchanged by this diff, present in the PR-time tree: apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:632 `if (spec is None and is_single_sampling_param) or (spec is not None and spec["stages"] == ["diffusion"]):` — spurious spec match flips AR/diffusion payload format.) + +`apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:1` [minor] — apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:1 hard-codes the Generate Video model widget to the machine-specific snapshot path `/workspace/models/hub/models--MiniMaxAI--MiniMax-H3/snapshots/428986c4aa5280855b9ac26eff8d1a46b5999cd0/FL2VA` (the old workflow used `Wan-AI/Wan2.2-TI2V-5B-Diffusers`). Users importing this template on another machine cannot resolve that path. Replace it with `MiniMaxAI/MiniMax-H3` or a `${MODEL_ROOT}/FL2VA` placeholder matching recipes/MiniMaxAI/MiniMax-H3.md lines 60/91/128. (evidence: apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json:1 contains node id 16 widgets_values: `"widgets_values":["http://localhost:8000/v1","/workspace/models/hub/models--MiniMaxAI--MiniMax-H3/snapshots/428986c4aa5280855b9ac26eff8d1a46b5999cd0/FL2VA",...` — machine-specific absolute path in the PR-time tree. Convention outside the diff: recipes/MiniMaxAI/MiniMax-H3.md:60 `export MODEL="${MODEL_ROOT}/FL2VA"` and recipes/MiniMaxAI/MiniMax-H3.md:164 `export MODEL="${MODEL_ROOT}/Ref2VA"` use placeholders instead of absolute paths.) + +`tests/e2e/features/comfyui/test_comfyui_integration.py:891` [minor] — Confirmed. The only MiniMax-H3 cases in this diff are `VIDEO_REF2VA_IMAGE_AUDIO` and `VIDEO_REF2VA_MULTI_VIDEO`, both asserting `task == 'ref2va'`. `generate_video` computes `video_task = 'fl2va'`/`'t2va'`, but `task` is only forwarded when a spec `params_builder` exists (MiniMax-H3); the existing frame/no-input tests use Wan, which has no spec and drops the computed task. Add parametrized MiniMax-H3 cases asserting `task == 'fl2va'` (and ideally `task == 't2va'`). (evidence: ['tests/e2e/features/comfyui/test_comfyui_integration.py:77 ` VIDEO_REF2VA_IMAGE_AUDIO = "video_ref2va_image_audio"` and :78 ` VIDEO_REF2VA_MULTI_VIDEO = "video_ref2va_multi_video"` — the only MiniMax-H3-targeting SamplingKinds are ref2va.', 'tests/e2e/features/comfyui/test_comfyui_integration.py:868 ` served_model="MiniMaxAI/MiniMax-H3",` and :879 same — MiniMax-H3 appears only in the Ref2VA parametrize list.', 'tests/e2e/features/comfyui/test_comfyui_integration.py:414 ` "task": "ref2va",` and :442 ` "task": "ref2va",` — both MiniMax-H3 arms assert only ref2va.', 'tests/e2e/features/comfyui/test_comfyui_integration.py:929 ` references=references,` — the MiniMax-H3 test never passes `frame`, so `fl2va` never runs against a MiniMax-H3 spec.', 'apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:303 ` video_task = "fl2va"` (and :330 ` video_task = "t2va"`) — the untested routing branches.', 'apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:362 ` params_builder = spec.get("params_builder") if spec else None` and :366 ` extra_params={**extra_params, "task": video_task},` — task is only forwarded when the model spec has a params_builder (MiniMax-H3); Wan has no spec, so the else-branch at :370-375 sends only flattened model_params and discards `video_task`.']) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.trace.json.gz new file mode 100644 index 000000000..bbb0d9720 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5756.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.cost.json new file mode 100644 index 000000000..7ef285e10 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1932.3, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5779/run-20260815-173311-2b4c22", + "recorded_at": "2026-08-15T18:05:21", + "attempts": [ + { + "invocation_id": "inv-805a44c6", + "outcome": "run:run-20260815-173311-2b4c22", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-173311-2b4c22", + "llm_calls": 167, + "input_tokens": 1608121, + "output_tokens": 321186, + "usd": 1.244595, + "source": "spans" + } + ], + "attempt_usd_total": 1.244595, + "llm_calls": 167, + "input_tokens": 1608121, + "output_tokens": 321186, + "usd": 1.244595, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-173311-2b4c22", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2446, + "minutes": 32.16, + "ci_minutes": 0, + "input_tokens": 1608121, + "output_tokens": 321186, + "tool_calls": 290, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1955, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 167, + "by_role": { + "agent": { + "usd": 1.200438, + "input_tokens": 1577037, + "output_tokens": 288742, + "calls": 164 + }, + "reducer": { + "usd": 0.044157, + "input_tokens": 31084, + "output_tokens": 32444, + "calls": 3 + } + }, + "cache_read_tokens": 16929536, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.08, + "gate": 1.56, + "review": 1924.04, + "report": 0.06 + }, + "time_to_first_result_s": 1929.73 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5779.trace.json.gz", + "trace_bytes": 50252 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.md new file mode 100644 index 000000000..b8a713ff6 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.md @@ -0,0 +1,70 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5779, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f092a92eb579) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Removing FLASHINFER_DISABLE_VERSION_CHECK from the recipe is safe: stage_engine_core_proc.py:122 does os.environ.setdefault('FLASHINFER_DISABLE_VERSION_CHECK','1'). +- [claim-verified] PR says #5771 is a metadata-contract rejection before the kernel: old code raised on any attn_mask; new code admits packed prefix masks only. +- [claim-verified] H3 default flip gated by DiffusionModelMetadata.attention_mask_free=True (model_metadata.py:41-47) and consumed at layer.py:83; FluxPipeline remains False. +- [claim-verified] Issue #5771 scope is delivered: skip, quant, and combined configs are all addressed, plus denoise-timestep publishing. +- [claim-verified] PR's packed-trim design matches code: prefix-valid mask is validated, q/k/v trimmed to valid_tokens, cu_seq_lens truncated, zero-pad restored to physical shape. +- [claim-verified] PR's SAGE short-sequence fallback matches code: `use_sage = quant.enabled and all(seq_lens >= k_block_size)`; short refiner sequences use the dense kernel. +- [claim-verified] gh_ci_read: build(3.12), build(3.11), pre-commit, DCO, buildkite/vllm-omni, buildkite/vllm-omni-npu-ci, docs/readthedocs all SUCCESS. +- [claim-verified] model_metadata attention_mask_free=True flips default only for MiniMaxH3Pipeline; Wan* entries were already True pre-diff, so blast radius is H3 only. +- [claim-refuted] Recipe 'within 2%' latency claim is unsupported: PR body says TRTLLM e2e latency 'is not reported as steady-state backend performance'. +- [claim-refuted] recipe 'within 2%' is not supported by PR body, which reports only PSNR 27.10 dB / SSIM 0.8880 and explicitly declines steady-state latency — see review comment. +- [validated] trtllm_attn.py:263-268 incomplete packed metadata raises for missing keys; test_packed_metadata_must_be_complete covers it. +- [validated] trtllm_attn.py:293-300 packed metadata validation (ndim, numel, coverage) is exercised by tests test_packed_metadata_must_cover_inputs and test_packed_metadata_trims_padding_mask. +- [validated] prefix-valid mask trimming restores physical shape: test_packed_metadata_trims_padding_mask asserts captured seq_lens [6] and out[:,6:]==0. +- [validated] token refiner's trailing empty sequence cu=[0,text_len,text_len] is dropped by the while loop at trtllm_attn.py:304-311; dense attention still gets batch=1/seq_lens=[text_len]. + +`vllm_omni/diffusion/attention/layer.py:~96` [minor] — New shared-code guard in layer.py:96-99 raises ValueError for TRTLLM_ATTN with allgather_degree > 1 unless skip_sequence_parallel=True, but the diff adds no test exercising this branch (grep confirms no test references this guard string). Add a focused unit test constructing Attention with parallel_config.allgather_degree=2 and a TRTLLM-resolving role, plus a companion case that skip_sequence_parallel=True bypasses the raise. (evidence: layer.py:96 `if not skip_sequence_parallel and allgather_degree > 1 and attn_backend_cls.get_name() == "TRTLLM_ATTN":` — layer.py:97-99 `raise ValueError(` / ` "TRTLLM_ATTN does not support AllGather-KV sequence parallelism. "` / ` "Set --allgather-degree 1 or select another diffusion attention backend."` — repo-wide grep for `AllGather-KV sequence parallelism` under tests/ returns no matches, and grep for `allgather_degree` shows no reference in tests/diffusion/attention/test_trtllm_attn.py; the guard is added by this diff, and the test gap is real.) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:~300` [minor] — This diff adds per-forward device-to-host syncs in TrtllmAttentionImpl.forward_cuda on the packed path: the coverage check (trtllm_attn.py:300 and 306-307), `valid_tokens = int(flat_mask.sum().item())` (line 320), and the boundary `nonzero()`/`.item()` calls (lines 322-324). Each CUDA `.item()`/`nonzero()` serializes the GPU, and this runs once per layer per denoise step on H3's new default TRTLLM path, contradicting the stated intent at minimax_h3_transformer.py:394-396 to keep `.item()` out of the per-layer attention path. _run_packed_attention already holds Python `used`/`max_seqlen` and emits `valid_kv_length`; move the validation/trim there or pass Python ints in `extra` so the backend performs no device-to-host sync. (evidence: Added by this diff — trtllm_attn.py:300 `if int(cu_seq_lens_q[-1].item()) != q.shape[0] or int(cu_seq_lens_kv[-1].item()) != k.shape[0]:`; trtllm_attn.py:306-307 `while (` / `and int(cu_seq_lens_q[-1].item()) == int(cu_seq_lens_q[-2].item())` / `and int(cu_seq_lens_kv[-1].item()) == int(cu_seq_lens_kv[-2].item())`; trtllm_attn.py:320 `valid_tokens = int(flat_mask.sum().item())`; trtllm_attn.py:322-324 `q_boundary = (cu_seq_lens_q == valid_tokens).nonzero(as_tuple=False)` ... `q_end = int(q_boundary.item()) + 1`. Unchanged by this diff, present in PR-time tree — minimax_h3_transformer.py:394-396 `# max_seqlen is already the first (real) packed document length. Do` / `# not read the CUDA cu_seqlens scalars here: this function runs once` / `# per layer and .item() would serialize every attention launch.`) + +`vllm_omni/diffusion/model_metadata.py:~46` [minor] — vllm_omni/diffusion/model_metadata.py:46 sets `attention_mask_free=True` for MiniMaxH3Pipeline, which vllm_omni/diffusion/attention/layer.py:83 feeds into `allow_trtllm_default`; vllm_omni/platforms/cuda/platform.py (default route, unchanged by this diff) enables TRTLLM_ATTN for any compute capability major == 10 (sm_100 and sm_103). The PR body validates only B300 SXM6 AC (sm_103), and tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py:91 still pins `--diffusion-attention-backend FLASH_ATTN`, so the new default has no e2e accuracy backstop on sm_100. (evidence: Changed in this PR: vllm_omni/diffusion/model_metadata.py:46 `attention_mask_free=True,` (MiniMaxH3Pipeline). Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/attention/layer.py:83 `allow_trtllm_default = get_diffusion_model_metadata(model_class_name).attention_mask_free`; vllm_omni/platforms/cuda/platform.py:206-208 `trtllm_attn_default_ok = ( allow_trtllm_default and compute_capability is not None and compute_capability.major == 10 ... )`; tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py:91 `"FLASH_ATTN",`.) + +`recipes/MiniMaxAI/MiniMax-H3.md:247` [minor] — Confirmed with a corrected line number. recipes/MiniMaxAI/MiniMax-H3.md:247 says dense `TRTLLM_ATTN` and FA4 are 'within 2% of each other' without naming the metric or scope lock. The PR body's only TRTLLM-vs-FA4 comparison is quality (average PSNR 27.10 dB, SSIM 0.8880), and it states the dense TRTLLM run's end-to-end latency 'is not reported as steady-state backend performance'. State which metric the 2% refers to (with steps/resolution/warmup/compile state) or remove the number. (evidence: ["recipes/MiniMaxAI/MiniMax-H3.md:247 `Stable measurements with the four-GPU profile above put dense `TRTLLM_ATTN` and FA4 within 2% of each other.` — metric unspecified (line added by this PR's diff).", 'PR body (not a repo file): `Encoded-video comparison against FA4: average PSNR 27.10 dB and SSIM 0.8880.`', 'PR body (not a repo file): `This was a cold TRTLLM service run and included regional/VAE compilation plus slow CPU MP4 encoding, so its end-to-end latency is not reported as steady-state backend performance.`']) + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:305` [minor] — Add a direct unit test for the trailing-empty-suffix trim while-loop (trtllm_attn.py:303-311): pass packed metadata with cu_seqlens=[0, text_len, text_len] and a prefix-valid mask, and assert the loop drops the empty tail (attention sees batch=1/seq_lens=[text_len]). The new packed tests use cu=[0,6,8], [0,6], or [0,used,s], so this branch has no focused regression coverage. (evidence: vllm_omni/diffusion/attention/backends/trtllm_attn.py:305-311 `while ( + cu_seq_lens_q.numel() > 2 + and int(cu_seq_lens_q[-1].item()) == int(cu_seq_lens_q[-2].item()) + and int(cu_seq_lens_kv[-1].item()) == int(cu_seq_lens_kv[-2].item()) + ): + cu_seq_lens_q = cu_seq_lens_q[:-1] + cu_seq_lens_kv = cu_seq_lens_kv[:-1]` — the trim branch claimed by the comment exists verbatim. tests/diffusion/attention/test_trtllm_attn.py:216 `cu_seqlens = torch.tensor([0, 6, 8], dtype=torch.int32)` and tests/diffusion/attention/test_trtllm_attn.py:394 `cu_seqlens = torch.tensor([0, used, s], dtype=torch.int32, device="cuda")` — every packed-metadata test added by this PR uses a trailing non-empty padding sequence, never [0, text_len, text_len], so the trailing-empty-suffix trim branch has no direct unit test. These test lines are new in this PR's diff; the while-loop is also new in this PR's diff.) + +`tests/diffusion/attention/test_trtllm_attn.py:14` [minor] — The three new validation-only tests (test_packed_metadata_rejects_non_prefix_mask, test_packed_metadata_must_be_complete, test_packed_metadata_must_cover_inputs) raise before any kernel call and use only CPU tensors, but the module-level cuda marker at tests/diffusion/attention/test_trtllm_attn.py:14 keeps them out of the `core_model and cpu` merge-gate selection. Move these three tests to a cpu-marked module or add a CPU lane that globs tests/diffusion/attention/, rather than waiting for a Blackwell GPU lane. (evidence: tests/diffusion/attention/test_trtllm_attn.py:14 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cuda]`; tests/diffusion/attention/test_trtllm_attn.py:266-267 `with pytest.raises(ValueError, match="prefix-valid"):` / ` _impl().forward_cuda(q, k, v, metadata)`; tests/diffusion/attention/test_trtllm_attn.py:275-276 `with pytest.raises(ValueError, match="Incomplete packed TRTLLM attention metadata"):` / ` _impl().forward_cuda(q, k, v, metadata)`; tests/diffusion/attention/test_trtllm_attn.py:293-294 `with pytest.raises(ValueError, match="must cover all Q/K/V tokens"):` / ` _impl().forward_cuda(q, k, v, metadata)`; unchanged by this diff, present in the PR-time tree: tests/core/test_prefix_cache_async_write.py:7 `# reach tests/ select `core_model and cpu``) + +`vllm_omni/diffusion/models/minimax_h3/denoise_loop.py:210` [minor] — This hunk publishes `s_v` (the sigma/noise scale) as `denoise_timestep` via `record_denoise_step(normalized_timestep=video_sigma)`, while `SkipSoftmaxConfig.resolve_factor` and the user guide define `disabled_until_timestep` against the scheduler timestep `t` (with t = 1 - sigma). The descending direction is identical so gating works, but the absolute threshold the user sets means a different fraction of the schedule in sigma-space vs t-space (H3's recipe value 0.97 is a sigma threshold). Document the H3 sigma convention next to `disabled_until_timestep` (and on `record_denoise_step`), or pass `1.0 - s_v` if the field is contractually t. (evidence: denoise_loop.py:204: "s_v, s_v_next = sigmas_video[step], sigmas_video[step + 1]"; attention_backends.md:143: "Holds early, high-noise steps dense; skip turns on once the normalized timestep `t ≤ D`."; forward_context.py:229: "_forward_context.denoise_timestep = float(normalized_timestep)".) + +`recipes/MiniMaxAI/MiniMax-H3.md:~595` [minor] — The diff removes `--diffusion-attention-backend FLASH_ATTN` from the four-GPU serving block, and the new Attention Backends section says datacenter Blackwell defaults to dense BF16 `TRTLLM_ATTN`, but the 'Validated four-GPU evidence' row at line 595 still shows the pre-change `86.964 s mean HTTP client latency` with no backend label, so it can be misread as the new default. Relabel the row with the backend it was actually measured on (FA4); note the PR reports no steady-state dense-TRTLLM end-to-end latency (the dense-TRTLLM 50-step run was cold and includes compilation/MP4 encoding), so there is no direct replacement number to substitute. (evidence: PR-time tree, recipes/MiniMaxAI/MiniMax-H3.md:239 `On datacenter Blackwell GPUs, MiniMax H3 defaults to dense BF16` (section then states `TRTLLM_ATTN`). The four-GPU serving block ends at recipes/MiniMaxAI/MiniMax-H3.md:227-228 ` --vae-parallel-mode tile \` / ` --vae-use-tiling` with no `--diffusion-attention-backend` flag (unchanged from this diff's removal of `--diffusion-attention-backend FLASH_ATTN`). Evidence table row unchanged by this diff and present in the PR-time tree: recipes/MiniMaxAI/MiniMax-H3.md:595 `| FL2VA, 209 frames, 1248x768 | no offload, U4, VPP4 tile, regional compile | 86.964 s mean HTTP client latency |`. PR body: `This was a cold TRTLLM service run and included regional/VAE compilation plus slow CPU MP4 encoding, so its end-to-end latency is not reported as steady-state backend performance.`) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/attention/backends/trtllm_attn.py:364` [minor] When SAGE is enabled, line 364 performs a GPU→CPU .item() sync on every attention forward. seq_lens is a CUDA tensor in both paths — trtllm_attn.py:341 `seq_lens = (cu_seq_lens_kv[1:] - cu_seq_lens_kv[:-1]).to(dtype=torc +- `tests/diffusion/attention/test_trtllm_attn.py:14` [minor] tests/diffusion/attention/test_trtllm_attn.py is marked `core_model + diffusion + cuda` (line 14, no `cpu` marker) and is not selected by any CI lane: the merge-gate diffusion lane runs `tests/diffusion -m 'core_model an +- `tests/diffusion/attention/test_trtllm_attn.py:459` [minor] `test_sage_packed_non_aligned_length_matches_sdpa` (tests/diffusion/attention/test_trtllm_attn.py:459) asserts `rel < 0.2` while the dense SAGE parity test `test_sage_quant_matches_sdpa` (line 435) asserts `rel < 0.1`, w + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.trace.json.gz new file mode 100644 index 000000000..ac7f0e009 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5779.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.cost.json new file mode 100644 index 000000000..c4fe59b64 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1461.2, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5801/run-20260815-174120-f1d329", + "recorded_at": "2026-08-15T18:05:39", + "attempts": [ + { + "invocation_id": "inv-25c43462", + "outcome": "run:run-20260815-174120-f1d329", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-174120-f1d329", + "llm_calls": 119, + "input_tokens": 673832, + "output_tokens": 255389, + "usd": 0.707385, + "source": "spans" + } + ], + "attempt_usd_total": 0.707385, + "llm_calls": 119, + "input_tokens": 673832, + "output_tokens": 255389, + "usd": 0.707385, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-174120-f1d329", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7074, + "minutes": 24.31, + "ci_minutes": 0, + "input_tokens": 673832, + "output_tokens": 255389, + "tool_calls": 253, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8922, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 119, + "by_role": { + "agent": { + "usd": 0.674104, + "input_tokens": 654535, + "output_tokens": 229911, + "calls": 117 + }, + "reducer": { + "usd": 0.033281, + "input_tokens": 19297, + "output_tokens": 25478, + "calls": 2 + } + }, + "cache_read_tokens": 9056384, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.75, + "gate": 1.79, + "review": 1453.11, + "report": 0.06 + }, + "time_to_first_result_s": 1458.69 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5801.trace.json.gz", + "trace_bytes": 39653 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.md new file mode 100644 index 000000000..c8f44e586 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.md @@ -0,0 +1,73 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5801, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 1ef0cb1562a6) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Concern 'fused CUDA rms_norm may mishandle bf16 weight': base tests tests/diffusion/layers/test_norm.py:564+ already call `norm._forward_fused` with `norm.weight.data = norm.weight.data.to(dtype)` for bf16 and compare vs native, so the bf16 fused path is already exercised at base. +- [resolved] Merge revalidation: feature commit 97fb4442 briefly added RMSNorm.self.dtype with self.dtype accumulation; head norm.py has no self.dtype and forward_native hardcodes float32, consistent with the PR purpose and the new dtype test. +- [claim-verified] test_norm.py:322-337 npu_rms_norm mock test asserts bf16 gamma (`captured['gamma'].dtype == torch.bfloat16`) reaches forward_npu. +- [claim-refuted] E2E latency math: (491014-475070)/491014 = 3.247%, not -3.35% (the 3.35% matches 15944/475070 = 3.356%, i.e. divided by the new value). +- [claim-refuted] PR body says E2E latency 491014ms -> 475070ms = -3.35%; calc gives (491014-475070)/491014 = 3.247%, while 15944/475070 = 3.356% — the author divided by the opt value instead of the baseline. +- [claim-refuted] PR body '-3.35%' uses new value as denominator: (491014-475070)/475070 = 3.356%; relative-to-baseline is (491014-475070)/491014 = 3.247%. Recompute or label the metric. +- [claim-refuted] PR body says E2E latency improved -3.35%, but (475070-491014)/491014 = -3.247%. +- [claim-refuted] PR body cites vLLM-Omni Commit 5215e03a91, but head is merge 1ef0cb1562a6 (show_commit) which merged 51 files of main incl. H3 pipeline/offloader changes; the reported measurement may not describe the merged state. +- [validated] test_minimax_h3_parallel.py:64-87: duplicated-half freqs produce actual==expected atol=0/rtol=0 on the forced-native path; pass-through dims 96: asserted equal. +- [validated] file_at_base contrast: for half_head_dim=True the new `_prepare_half_head_dim_cos_sin` reproduces base forward_cuda/hip/native cos[0] behavior exactly. +- [validated] H3 native RoPE semantics: rope.py `_prepare_half_head_dim_cos_sin` neox first-half slice matches MiniMaxH3Rope's symmetric `torch.cat((half, half), dim=-1)` (minimax_h3_transformer.py:230), so the atol=0/rtol=0 reference in test_minimax_h3_parallel.py:72-85 is a real semantic check of forward_native. +- [validated] RMSNorm dtype default=float32 preserves old weight dtype for every existing diffusion RMSNorm caller (grep RMSNorm( in vllm_omni: only minimax_h3 `_norm` passes dtype); no stale caller. +- [validated] RotaryEmbedding half_head_dim default=True preserves old behavior for all keyword callers; the only `half_head_dim=False` producers are minimax_h3 (neox) and RotaryEmbeddingWan (overrides all forward_* methods). +- [validated] RMSNorm new `dtype` default torch.float32 preserves old `torch.ones(hidden_size)`; all in-repo importers (indextts2 gpt_fast, wan2_2, z_image, hunyuan_image3, cosmos3 subclass) construct without dtype, so no stale consumer. + +`tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:73` [minor] — This diff adds `test_h3_fused_rope_matches_reference_and_preserves_unrotated_dims`, but line 73 pins `attention.rope._forward_method = attention.rope.forward_native`, so the new `half_head_dim=False` full-dim slicing is validated only on the native fallback. Note `forward_native` shares `_prepare_half_head_dim_cos_sin`, so the helper's slicing branch is exercised, but the production H3 CUDA/HIP dispatch `RotaryEmbedding.forward_cuda`/`forward_hip` -> `_prepare_half_head_dim_cos_sin` -> `vllm.vllm_flash_attn.layers.rotary.apply_rotary_emb` has no automated coverage (grep shows no `RotaryEmbedding.forward_cuda`/`forward_hip` test). Add a CUDA/HIP test that calls `RotaryEmbedding(is_neox_style=True, half_head_dim=False).forward_cuda(x, cos, sin)` with full-dim `[T,96]` cos/sin and asserts the reference output, or mock `apply_rotary_emb` and assert the sliced cos/sin; otherwise rename the test to `..._native_...` and state where the fused path is covered. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py:73 ` attention.rope._forward_method = attention.rope.forward_native` — the new test forces the native fallback. vllm_omni/diffusion/layers/rope.py:137 ` cos, sin = self._prepare_half_head_dim_cos_sin(cos, sin)` (inside `forward_cuda`) and vllm_omni/diffusion/layers/rope.py:204 ` cos, sin = self._prepare_half_head_dim_cos_sin(cos, sin)` (inside `forward_native`) — both routes use the same helper, but the fused `forward_cuda`/`forward_hip` dispatch introduced for H3's `half_head_dim=False` is not invoked by any test; grep of `tests/diffusion` for `forward_cuda` returns only norm/attention tests, none for `RotaryEmbedding`.) + +`vllm_omni/diffusion/layers/rope.py:125` [minor] — In rope.py, `_prepare_half_head_dim_cos_sin` silently truncates full-dim cos/sin when `half_head_dim=False` (NeoX keeps the first half; interleaved keeps even indices) before CUDA/native kernels consume them. The only in-repo constructor with `half_head_dim=False` is MiniMaxH3Attention's NeoX rope, and its full-dim table is the duplicated `torch.cat((half, half), dim=-1)` from MiniMaxH3Rope.forward, so truncation is currently correct. Any independent full-dim table would be silently mis-rotated, and the interleaved full-dim arm is unreachable in-repo (RotaryEmbeddingWan overrides every forward; RotaryEmbeddingS2V passes half_head_dim=True). Suggest asserting that the discarded half equals the kept half in the non-interleaved branch, documenting the required duplicated-half layout, and explicitly testing or removing the interleaved full-dim arm. (evidence: vllm_omni/diffusion/layers/rope.py:125 (added by this diff): ` cos = cos[..., ::2] if self.interleaved else cos[..., : cos.shape[-1] // 2]`; vllm_omni/diffusion/layers/rope.py:126 (added by this diff): ` sin = sin[..., ::2] if self.interleaved else sin[..., : sin.shape[-1] // 2]`; vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:335 (added by this diff): ` self.rope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)`; unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:227 ` return torch.cat((half, half), dim=-1) # [S, 96]`. Repo-wide grep for `half_head_dim` shows no other in-repo RotaryEmbedding constructed with `half_head_dim=False`; RotaryEmbeddingWan uses `half_head_dim=False` but overrides forward_cuda/forward_hip/forward_npu/forward_native, so it never reaches `_prepare_half_head_dim_cos_sin`.) + +`tests/diffusion/layers/test_rope_broadcast.py:259` [minor] — Confirmed. The diff routes H3's NPU RoPE into the pre-existing half_head_dim=False MindIE path (forward_npu now passes self.half_head_dim), which for interleaved=False skips the 4D reshape and sends raw 2D [S, D] cos/sin to rotary_position_embedding in rotated_mode='rotated_half'. The new test only monkeypatches a stub that returns x and asserts tensor identity, so it does not prove the real MindIE kernel accepts the full-dim 2D layout. Validate on a real NPU lane (MiniMax-H3 NPU smoke) or add an explicit rank/layout assertion/documentation for this branch. (evidence: vllm_omni/diffusion/layers/rope.py:62-66 (unchanged block, newly exercised by this diff's forward_npu change): +``` + if half_head_dim: + seqlen = cos.shape[0] + sin = sin.unsqueeze(0).unsqueeze(2).repeat(1, 1, 1, 2) + cos = cos.unsqueeze(0).unsqueeze(2).repeat(1, 1, 1, 2) + return rotary_position_embedding(x, cos, sin, rotated_mode="rotated_half", head_first=False, fused=True) +``` +When half_head_dim=False the reshape is skipped and 2D cos/sin pass directly to the kernel call. tests/diffusion/layers/test_rope_broadcast.py (draft-cited lines 246-248): +``` + def rotary_position_embedding(x, cos, sin, **kwargs): + captured.update(x=x, cos=cos, sin=sin, kwargs=kwargs) + return x +``` +plus `assert actual is x` and `assert captured["cos"] is cos` — proving only forwarding identity, not real-kernel rank acceptance.) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:362` [minor] — PR body: 'E2E Latency: 491014ms -> 475070ms' reports -3.35%, but the baseline-relative decrease is (491014-475070)/491014 = 3.247%. The -3.35% figure approximates dividing by the optimized value (15944/475070 = 3.356%). Please recompute as -3.25% or label the metric as relative-to-optimized. (evidence: PR #5801 body (via gh_pr_view): 'E2E Latency: 491014ms -> 475070ms. **-3.35%**'. Computation: 491014-475070=15944; 15944/491014=3.247%; 15944/475070=3.356%. Cited line is unrelated code: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:362 ' sin = torch.sin(freqs).to(x.dtype)'.) + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:362` [minor] — PR body binds the perf result to 'vLLM-Omni Commit: 5215e03a91adecbb5ffece29aa74360a7569d0c5', but the merged head is 1ef0cb1562a6 ('Merge branch main into h3_opt'). Re-run the E2E benchmark at 1ef0cb1562a6 or explicitly label the quoted latency as not revalidated at the merged state. (evidence: PR body (gh_pr_view #5801): '**vLLM-Omni Commit:**\n5215e03a91adecbb5ffece29aa74360a7569d0c5' — the quoted latency is bound to 5215e03a. show_commit 1ef0cb1562a6: 'commit 1ef0cb1562a67a0ecb1d773fb582865f386fd8c5\nMerge: aaeaa000 9235b0ae\n... Merge branch 'main' into h3_opt' — the PR head is a distinct merge commit. Anchor line in the diff: vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:362 ' cos = torch.cos(freqs).to(x.dtype) # [T, rot_dim]' (changed by this PR).) + +`tests/diffusion/layers/test_norm.py:322` [minor] — This NPU test replaces `torch_npu` with a `SimpleNamespace` stub whose `npu_rms_norm` only records its arguments and returns `(x,)`, then asserts `captured['gamma'].dtype == torch.bfloat16`. It proves the NPU dispatch path forwards the bf16 gamma, not that the real torch_npu kernel handles bf16 gamma. Add a real torch_npu semantic check on an NPU lane or rename/document the test as routing-only. (evidence: tests/diffusion/layers/test_norm.py:333 `monkeypatch.setitem(sys.modules, "torch_npu", types.SimpleNamespace(npu_rms_norm=npu_rms_norm))` — torch_npu is stubbed. tests/diffusion/layers/test_norm.py:337-340 `assert norm.forward_npu(x) is x` / `assert captured["gamma"] is norm.weight` / `assert captured["gamma"].dtype == torch.bfloat16` / `assert captured["epsilon"].item() == pytest.approx(1e-5)` — only dispatch identity and forwarded argument dtype/epsilon are checked; no real torch_npu kernel executes. The real dispatch path in vllm_omni/diffusion/layers/norm.py (unchanged import and `torch_npu.npu_rms_norm(x, gamma=self.weight, epsilon=self.variance_epsilon)[0]`) is bypassed by the stub.) + +`vllm_omni/diffusion/layers/norm.py:70` [minor] — This diff stores RMSNorm.weight in a caller-supplied dtype (norm.py:70) and H3's _norm now returns this custom RMSNorm for every norm slot (minimax_h3_transformer.py:177), so on CUDA H3 hits the fused vllm._custom_ops.rms_norm path instead of torch.nn.RMSNorm. The _norm comment (lines 174-176) still claims torch.nn.RMSNorm/fp32 accumulation, and the added test only exercises forward_native with an expected expression that repeats forward_native's own formula; the bf16 fused path is documented at test_norm.py:624 as computing in bf16 and is never compared with the old layer. Update the _norm comment to state that fp32 accumulation holds only for the native fallback, and add a CUDA parity test of RMSNorm(hidden_size, eps, dtype=torch.bfloat16) against torch.nn.RMSNorm on H3's q/k norm size (or report the measured numeric delta). (evidence: ['vllm_omni/diffusion/layers/norm.py:70 ` self.weight = nn.Parameter(torch.ones(hidden_size, dtype=dtype))` — changed by this diff.', 'vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:177 ` return RMSNorm(size, eps=eps, dtype=dtype)` — changed by this diff.', 'vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:174 ` # RMSNorm uses fp32 accumulation with bf16 inputs and outputs.` (line 175 still says `# torch.nn.RMSNorm upcasts reduced-precision inputs for the variance`) — stale comment present in PR-time tree.', 'vllm_omni/diffusion/layers/norm.py:134 ` out = self.weight.to(torch.float32) * out` — native fallback only; unchanged by this diff, present in the PR-time tree.', 'tests/diffusion/layers/test_norm.py:624 ` Note: bfloat16 requires wider tolerance because the fused CUDA kernel` — documents bf16-internal fused kernel; unchanged by this diff, present in the PR-time tree.']) + +`tests/diffusion/layers/test_norm.py:305` [minor] — `buildkite/vllm-omni-amd-ci` is red at this head, and this diff's new/updated tests are collected by its `core_model and cpu` lane: `.buildkite/amd/test-amd-ready.yml:16/19` runs `timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' ...`, and each touched test file carries those markers (test_norm.py:12, test_rope_broadcast.py:21). Before treating the red check as unrelated, confirm whether it reproduces on main or is introduced by these tests, and point to the specific failing assertion in the AMD job log. (evidence: .buildkite/amd/test-amd-merge.yml:21 ` - "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; tests/diffusion/layers/test_rope_broadcast.py:21 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.trace.json.gz new file mode 100644 index 000000000..64f2a2a30 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5801.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.cost.json new file mode 100644 index 000000000..e67b6a489 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 836.5, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5833/run-20260815-174532-bd9a9e", + "recorded_at": "2026-08-15T17:59:26", + "attempts": [ + { + "invocation_id": "inv-13e9ede2", + "outcome": "run:run-20260815-174532-bd9a9e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-174532-bd9a9e", + "llm_calls": 59, + "input_tokens": 117947, + "output_tokens": 96644, + "usd": 0.184596, + "source": "spans" + } + ], + "attempt_usd_total": 0.184596, + "llm_calls": 59, + "input_tokens": 117947, + "output_tokens": 96644, + "usd": 0.184596, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-174532-bd9a9e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1846, + "minutes": 13.9, + "ci_minutes": 0, + "input_tokens": 117947, + "output_tokens": 96644, + "tool_calls": 116, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4798, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 59, + "by_role": { + "agent": { + "usd": 0.161399, + "input_tokens": 111128, + "output_tokens": 77271, + "calls": 57 + }, + "reducer": { + "usd": 0.023196, + "input_tokens": 6819, + "output_tokens": 19373, + "calls": 2 + } + }, + "cache_read_tokens": 1720064, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.55, + "gate": 1.59, + "review": 828.69, + "report": 0.06 + }, + "time_to_first_result_s": 833.88 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5833.trace.json.gz", + "trace_bytes": 20540 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.md new file mode 100644 index 000000000..4d081f2e8 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5833, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac67e8874d00) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] docs/design/ar_diffusion_pipeline_capability.md is intentionally absent from nav — its own text says 'experimental and non-normative ... not yet part of the stable module-design hierarchy' (file:3-5); no restore needed. +- [claim-verified] diff_stat reports exactly 2 changed files (docs/.nav.yml, docs/design/index.md; 91 insertions / 24 deletions) — matches PR body's 'only two files' claim. +- [claim-verified] 'does not add or modify design documents' — only the index landing page changed; H1 of every linked doc matches its nav label closely enough. +- [claim-verified] diff_stat shows only docs/.nav.yml and docs/design/index.md changed (91+/24-), matching the PR scope claim; no design documents are added or modified. +- [claim-verified] diff_stat shows only docs/.nav.yml (41+/22-) and docs/design/index.md (50+/2-) changed; PR claim 'diff contains only these two files' is true. +- [claim-verified] gh_ci_read(5833) shows pre-commit, DCO, buildkite/omni-release and docs/readthedocs.org:vllm-omni all SUCCESS; gate_report 'gates clean' matches. +- [claim-verified] PR claim 'does not add or modify design documents' holds: diff_stat touches no docs/design/feature/* or docs/design/module/* file. +- [validated] Every nav target in the reorganized tree exists: design/feature/* (list_dir), design/feature/omni_connectors/{6 md files} (list_dir), design/module/* (list_dir), user_guide/quantization/overview.md (list_dir). +- [validated] No pre-existing feature doc dropped from nav: old flat list fully maps into new sections; the duplicated vae_parallel.md entry was removed while one copy remains. +- [validated] omni_connectors directory contains exactly the 6 md files now listed explicitly in the nav — no connector doc left orphaned by replacing the directory entry. +- [validated] Newly promoted nav targets (async_diffusion_output.md, pipeline_parallel.md, metrics.md, qwen3_omni_tts_performance_optimization.md) all exist and are pre-existing files since diff touches only 2 files. +- [validated] Every nav/index target exists at head: list_dir docs/design, docs/design/feature, docs/design/feature/omni_connectors, docs/design/module, docs/user_guide/quantization confirmed overview.md and all 6 connector pages. +- [validated] Newly-linked docs have content and H1 titles: async_diffusion_output.md ('# Async Diffusion Output'), pipeline_parallel.md ('# Pipeline Parallel'), metrics.md ('# Prometheus Metrics Design'), qwen3_omni_tts_performance_optimization.md (Qwen3 speech perf title). +- [validated] index.md relative link '../user_guide/quantization/overview.md' resolves to docs/user_guide/quantization/overview.md (file exists). + +`docs/design/index.md:53` [minor] — At docs/design/index.md:49-55, Cache-DiT, TeaCache, and Diffusion Continuous Batching immediately follow the Quantization Overview bullet with no intervening heading, so markdown renderers nest all three under `#### Quantization`. The sidebar (docs/.nav.yml:133-136) keeps them as siblings of Quantization under `Diffusion Acceleration`. Add a `#### Caching and Continuous Batching` heading before line 53 or move the bullets out of the Quantization subsection. (evidence: docs/design/index.md:49 `#### Quantization` +docs/design/index.md:51 `- [Quantization Overview](../user_guide/quantization/overview.md)` +docs/design/index.md:53 `- [Cache-DiT](feature/cache_dit.md)` +docs/design/index.md:54 `- [TeaCache](feature/teacache.md)` +docs/design/index.md:55 `- [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)` +docs/.nav.yml:133 `- Quantization: user_guide/quantization/overview.md` +docs/.nav.yml:134 `- Cache-DiT: design/feature/cache_dit.md`) + +`docs/.nav.yml:133` [minor] — Confirmed. docs/.nav.yml:133 adds `- Quantization: user_guide/quantization/overview.md` inside the `Diffusion Acceleration` subtree, while docs/user_guide/quantization/overview.md:3 states the same `quantization_config` entrypoint is used across diffusion-only, multi-stage omni/TTS, and multi-stage diffusion models, and the same page is already linked under `Features -> Quantization -> Overview` at docs/.nav.yml:76. Moving the entry out of `Diffusion Acceleration` or retitling it as `Quantization User Guide` / `Quantization (all model types)` would avoid implying the overview is diffusion-only. (evidence: docs/.nav.yml:133 ` - Quantization: user_guide/quantization/overview.md`; docs/.nav.yml:76 ` - Overview: user_guide/quantization/overview.md`; docs/user_guide/quantization/overview.md:3-5 `vLLM-Omni exposes quantization through the unified `quantization_config` path. The same configuration entrypoint is used across diffusion-only models, multi-stage omni/TTS models, and multi-stage diffusion models, but each model type has a different quantization scope.`) + +`docs/design/index.md:57` [nit] — The diff renames `## Infrastructure Design Documents` to `## Infrastructure and Performance` at `docs/design/index.md:57`. MkDocs heading anchors derive from heading text, so the old `#infrastructure-design-documents` fragment will no longer resolve. Grep found no in-repo links to the old anchor, so only external links are affected; either keep the old heading, add an explicit anchor preserving the old fragment, or accept the fragment change deliberately. (evidence: docs/design/index.md:57 `## Infrastructure and Performance` (changed by this diff); docs/design/index.md base:17 `## Infrastructure Design Documents` (old heading, changed by this diff); grep for `infrastructure-design-documents` over docs returned no matches, so no in-repo link to the old fragment exists.) + +`docs/design/index.md:33` [nit] — docs/design/index.md headings are sentence case (`### Runtime and stage execution`, `#### OmniConnector implementations`, `### Diffusion acceleration`) while docs/.nav.yml sidebar labels use title case (`Runtime and Stage Execution`, `OmniConnector Implementations`, `Diffusion Acceleration`); align the landing-page headings with the sidebar labels so the reorganized docs scan consistently. (evidence: docs/design/index.md:13 `### Runtime and stage execution` vs docs/.nav.yml:107 ` - Runtime and Stage Execution:`; docs/design/index.md:24 `#### OmniConnector implementations` vs docs/.nav.yml:115 ` - OmniConnector Implementations:`; docs/design/index.md:33 `### Diffusion acceleration` vs docs/.nav.yml:122 ` - Diffusion Acceleration:` — all six lines are in PR 5833's diff (docs/.nav.yml and docs/design/index.md are both modified).) + +`docs/.nav.yml:109` [nit] — docs/.nav.yml:109 and docs/design/index.md:16 relabel the link as 'Ray-based Execution', but docs/design/feature/ray_based_execution.md:1 is still '# Distributed utils' and its first paragraph says the utilities support both Ray and Multiprocessing backends. Align the page H1 with the new label, or keep the link label matching the existing page title, so the title does not promise a narrower scope than the page delivers. (evidence: docs/.nav.yml:109 ` - Ray-based Execution: design/feature/ray_based_execution.md`; docs/design/index.md:16 `- [Ray-based Execution](feature/ray_based_execution.md)`; docs/design/feature/ray_based_execution.md:1 `# Distributed utils` (unchanged by this diff, present in the PR-time tree); docs/design/feature/ray_based_execution.md:3 `...utilities for distributed execution in vllm-omni, supporting both **Ray** and **Multiprocessing** backends.`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.trace.json.gz new file mode 100644 index 000000000..e3ef5f264 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5833.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.cost.json new file mode 100644 index 000000000..4e5a63785 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2011.3, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5864/run-20260815-174915-b805a4", + "recorded_at": "2026-08-15T18:22:44", + "attempts": [ + { + "invocation_id": "inv-8a9d11fb", + "outcome": "run:run-20260815-174915-b805a4", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-174915-b805a4", + "llm_calls": 162, + "input_tokens": 1417089, + "output_tokens": 316237, + "usd": 1.170323, + "source": "spans" + } + ], + "attempt_usd_total": 1.170323, + "llm_calls": 162, + "input_tokens": 1417089, + "output_tokens": 316237, + "usd": 1.170323, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-174915-b805a4", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.1703, + "minutes": 33.48, + "ci_minutes": 0, + "input_tokens": 1417089, + "output_tokens": 316237, + "tool_calls": 245, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1897, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 162, + "by_role": { + "agent": { + "usd": 1.132984, + "input_tokens": 1381828, + "output_tokens": 290988, + "calls": 160 + }, + "reducer": { + "usd": 0.037339, + "input_tokens": 35261, + "output_tokens": 25249, + "calls": 2 + } + }, + "cache_read_tokens": 16290688, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.22, + "gate": 2.08, + "review": 2002.6, + "report": 0.07 + }, + "time_to_first_result_s": 2008.94 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5864.trace.json.gz", + "trace_bytes": 57271 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.md new file mode 100644 index 000000000..47b97af34 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.md @@ -0,0 +1,83 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5864, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 07fc831e7a43) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Concern 'example breaks after auto-wait removal' — the reproducer passes `request_batch_max_wait_ms=args.batch_wait_ms` (default 500, dlo_dp2_lifecycle.py:72); residual = other default-config DLO DP deployments are now silently serialized. +- [resolved] asymmetric AllGather hang on empty prompt: _is_empty_dp_prompt + `if empty_prompt_ids:` raise before dispatch (multiproc_executor.py:466-470); residual: only str/dict-with-'prompt' empties are detected. +- [resolved] partial DP reply now bounded by _DLO_DP_WAVE_TIMEOUT_S and fails closed via _fail_closed_on_dp_wave_timeout (multiproc_executor.py:414-435). +- [resolved] Old `result_handle = data.get('result_handle')` rank-0-only collection is replaced; every rank must supply a handle or init fails (multiproc_executor.py:346-348). +- [resolved] Removed 500ms auto-set of request_batch_max_wait_ms leaves the scheduler with an immediate-dispatch default; greps show no deploy YAML in repo sets request_batch_max_wait_ms, so no config depends on the old value. +- [resolved] collect_rank_status semantics change is safe at this head: only executor execute_request (DLO path) calls collective_rpc with unique_reply_rank=None; all other exec_all_ranks callers pass unique_reply_rank=0. +- [resolved] per-worker result_handle now required for every rank; _launch_workers raises on a missing rank handle instead of silently accepting rank0 only. +- [claim-verified] Base `_launch_workers` only read rank-0's result handle (`if i == 0: result_handle = data.get("result_handle")` via file_at_base); head collects a handle from every rank and `_init_executor` builds `_result_mqs` + one pump per queue (multiproc_executor.py:343-352, 120-130). +- [claim-verified] PR claim 'result path assumed one rank-0 queue' is fixed at head: per-worker MessageQueue creation in WorkerProc.__init__ and one pump thread per queue in `_start_result_pump`. +- [claim-refuted] PR body architecture text 'RequestScheduler — Collect up to DP-size requests' overstates the default after this diff removes the auto `request_batch_max_wait_ms=500` (see comment A). +- [validated] Worker side always creates/exported a per-rank result queue and `worker_main` sends `result_handle` for every rank (diffusion_worker.py:1105-1114); executor now consumes all of them. +- [validated] DP reply gate: `_execute_rpc` computes `output_rank=None`/`collect_rank_status=False` for multi-rank reply and only the primary rank of each replica enqueues a reply (diffusion_worker.py:960-985); executor sorts replies by `dp_rank` (multiproc_executor.py:748-750). +- [validated] Wave preflight happens before `collective_rpc`: compatibility keys via `build_request_batch_sampling_params_key`, `extra_args` signature, and `empty_prompt_ids` (multiproc_executor.py:464-492); tests `test_dlo_dp_rejects_*` assert `collective_rpc.assert_not_called()`. +- [validated] Fail-closed path: partial DP reply blocks until `_DLO_DP_WAVE_TIMEOUT_S` deadline, raises TimeoutError, and `_fail_closed_on_dp_wave_timeout` sets `_is_failed` + `shutdown()` + failure callbacks (multiproc_executor.py:411-431, 497); test `test_dlo_dp_partial_reply_times_out_and_fails_closed` covers it. + +`vllm_omni/diffusion/diffusion_engine.py:247` [major] — Confirmed. _init_runtime_state no longer sets request_batch_max_wait_ms=500.0 for DLO DP, while data.py:845 keeps the default 0.0. _wait_for_request_batch_admission_locked returns immediately when max_wait_s == 0, so the busy loop schedules the first waiting request as a one-request wave instead of coalescing two compatible DLO DP requests; the per-replica wave path in MultiprocDiffusionExecutor.execute_request requires len(new_reqs) > 1 and is therefore not reached reliably with default args. Restore an effective coalescing wait (computed in the wait function, not by mutating shared config) or document/warn when DLO DP runs with request_batch_max_wait_ms == 0, and add the requested scheduler-level two-request wave test. (evidence: PR-time tree, unchanged by this diff: vllm_omni/diffusion/data.py:845 `request_batch_max_wait_ms: float = 0.0`; vllm_omni/engine/arg_utils.py:190 `request_batch_max_wait_ms: float = 0.0`. Decisive PR-time code: vllm_omni/diffusion/diffusion_engine.py:478-480: + max_wait_s = self.od_config.request_batch_max_wait_ms / 1000.0 + if max_wait_s == 0: + return +vllm_omni/diffusion/diffusion_engine.py:247-254 (PR-time _init_runtime_state; the diff removed the former auto-set to 500.0, and only logging remains): + if _uses_dlo_dp_concurrency(self.od_config): + self.scheduler.max_num_running_reqs = dp_size + self.dp_concurrent = True + logger.info( + "dp_concurrent: max_num_running_reqs=%d, batch_wait=%sms", + dp_size, + self.od_config.request_batch_max_wait_ms, + )) + +`vllm_omni/diffusion/executor/multiproc_executor.py:47` [minor] — The guard `_is_empty_dp_prompt` (vllm_omni/diffusion/executor/multiproc_executor.py:41-49) misses empty list/tuple token prompts: `[]` falls through to `return False`, and `OmniSingletonPrompt` (vllm_omni/inputs/data.py:148) permits `list[int]`, so an empty token-list wave would not be rejected at the dispatch check (multiproc_executor.py:486). Extend the helper to return True for empty `list`/`tuple` (and ideally empty `OmniTokensPrompt`/`OmniCustomPrompt` token collections) and add a test for `_is_empty_dp_prompt([])`. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:49 `return False` — the fallback of `_is_empty_dp_prompt` after handling None/str/dict, so `[]` is treated as non-empty. Unchanged by this diff, present in the PR-time tree: vllm_omni/inputs/data.py:148 `OmniSingletonPrompt: TypeAlias = str | list[int] | OmniTextPrompt | OmniTokensPrompt | OmniEmbedsPrompt` — list[int] is an accepted prompt type. vllm_omni/diffusion/executor/multiproc_executor.py:486 `empty_prompt_ids = [nr.request_id for nr in new_reqs if _is_empty_dp_prompt(nr.req.prompt)]` — dispatch relies solely on the helper.) + +`docs/user_guide/diffusion/cpu_offload_diffusion.md:~299` [minor] — docs/user_guide/diffusion/cpu_offload_diffusion.md lines 299 and 309 still state that num_inference_steps must be explicit and that None is not allowed in DP multi-concurrency. This diff's MultiprocDiffusionExecutor.execute_request now uses build_request_batch_sampling_params_key for DLO DP wave validation and explicitly accepts two default (None) step counts when every request in the wave resolves to the same pipeline default (verified by test_dlo_dp_allows_shared_default_denoise_steps). Update the DP Multi-concurrency requirements and Limitations sections to say None is allowed when all requests in the wave resolve identically, while shape/CFG/output count/LoRA must still match. (evidence: docs/user_guide/diffusion/cpu_offload_diffusion.md:299 `- `num_inference_steps` must be specified explicitly (None is not allowed)` — unchanged by this diff, present in the PR-time tree; docs/user_guide/diffusion/cpu_offload_diffusion.md:309 `- `num_inference_steps=None` is not allowed in DP multi-concurrency mode` — unchanged by this diff, present in the PR-time tree; vllm_omni/diffusion/executor/multiproc_executor.py:466 `# and LoRA settings. Two default (None) step counts are valid and` (full comment: `Two default (None) step counts are valid and resolve identically inside the same pipeline.`); tests/diffusion/test_multiproc_engine_concurrency.py:397 `def test_dlo_dp_allows_shared_default_denoise_steps(self):`) + +`tests/diffusion/test_diffusion_engine.py:380` [minor] — tests/diffusion/test_diffusion_engine.py:380 only asserts `od_config.request_batch_max_wait_ms == 0` after init, which proves config preservation but not the coalescing behavior the removed auto-500 provided. Add an engine-level test that starts the busy loop, submits two compatible DLO DP requests with a non-zero `request_batch_max_wait_ms`, and asserts the scheduler/executor sees both requests in a single wave. (evidence: tests/diffusion/test_diffusion_engine.py:380 `assert od_config.request_batch_max_wait_ms == 0` — present in the PR-time tree; the enclosing `test_engine_allows_independent_dlo_dp_requests_for_single_request_pipeline` only checks engine state and never submits requests, so it proves config preservation only. Removed by this diff from `vllm_omni/diffusion/diffusion_engine.py::_init_runtime_state` (absent at PR-time head): `- if getattr(self.od_config, "request_batch_max_wait_ms", 0) == 0:` / `- self.od_config.request_batch_max_wait_ms = 500.0`. Present in the PR-time tree, unchanged by this diff, in `vllm_omni/diffusion/diffusion_engine.py::_wait_for_request_batch_admission_locked`: `max_wait_s = self.od_config.request_batch_max_wait_ms / 1000.0` then `if max_wait_s == 0: return` — with the default 0 the engine performs no admission wait, so two arriving DLO DP requests can be scheduled serially and the test gap the comment identifies is real.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:726` [minor] — The step_execution-gated per-worker queue mapping in collective_rpc (multiproc_executor.py:726-735) has no production caller in this diff: the only multi-rank-reply producer is execute_request's DLO DP wave (step_execution=False), whose dequeue path reads the pump-fed _sync_result_buffer (multiproc_executor.py:188-198); the branch is currently only exercised by test_step_execution_reads_each_dp_primary_result_queue. Consider deleting the branch/test or adding a real step-mode caller that validates the dp_rank * replica_size rank layout. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:726-735 (PR-time head, inside this diff): `if self.od_config.step_execution:` ... `primary_ranks = [dp_rank * replica_size for dp_rank in range(num_responses)]` ... `result_mqs = [all_result_mqs[rank] for rank in primary_ranks]`. vllm_omni/diffusion/executor/multiproc_executor.py:499-502: `results = self.collective_rpc( + "execute_model", + timeout=_DLO_DP_WAVE_TIMEOUT_S, + args=(reqs_list, self.od_config, scheduler_output.kv_prefetch_job), + unique_reply_rank=None, + exec_all_ranks=True, + )`. vllm_omni/diffusion/executor/multiproc_executor.py:188-198: `if not self.od_config.step_execution: + try: + return self._sync_result_buffer.get(timeout=chunk_timeout) + except queue.Empty: + if self._is_failed or self._closed: + raise EngineDeadError() + continue` — proving the request-mode multi-rank path reads `_sync_result_buffer` and not the per-worker queue indices. The step-mode branch's only caller is the added test in tests/diffusion/test_multiproc_engine_concurrency.py (`test_step_execution_reads_each_dp_primary_result_queue`).) + +`tests/diffusion/test_diffusion_ipc.py:71` [minor] — The new per-worker result-queue/pump/SHM coverage is exercised only by mock/object.__new__ tests (tests/diffusion/test_diffusion_ipc.py:101 `executor = object.__new__(MultiprocDiffusionExecutor)`) plus a manual example, and tests/diffusion/test_diffusion_ipc.py is marked cpu (line 84), so `.buildkite/cuda/test-merge.yml:18` and `test-ready.yml:18` (`pytest -sv tests/diffusion -m 'core_model and cpu'`) are the only lanes that run it; no CI lane executes a real 2-GPU DLO DP2 forward (grep finds no `dlo_dp2_lifecycle` or DLO/DP2 references under .buildkite). Add a gated GPU regression running `CUDA_VISIBLE_DEVICES=0,1 examples/offline_inference/minimax_h3/dlo_dp2_lifecycle.py --model /path/to/MiniMax-H3/FL2VA --mode dlo-dp2`, or document DLO DP2 as manual-only and run the reproducer against a real checkpoint before merge. (evidence: tests/diffusion/test_diffusion_ipc.py:101 `executor = object.__new__(MultiprocDiffusionExecutor)` (added by this diff). tests/diffusion/test_diffusion_ipc.py:84 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]` (unchanged by this diff, present in PR-time tree). .buildkite/cuda/test-merge.yml:18 `"timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (unchanged by this diff, present in PR-time tree). Grep for `dlo_dp2_lifecycle` under .buildkite returned no matches, so the new 2-GPU DLO DP2 reproducer is not wired into any CI lane.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:~864` [minor] — `_return_result`'s docstring still says `"""Reply to client, only on rank 0."""`, but DLO DP multi-concurrency now lets each DP primary reply. Update it to reflect the per-primary-reply contract gated in `_execute_rpc`. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:864 `"""Reply to client, only on rank 0."""` — unchanged by this diff, present in the PR-time tree. vllm_omni/diffusion/executor/multiproc_executor.py:662 `multi_rank_reply = unique_reply_rank is None and exec_all_ranks` and :673 `"output_rank": None if multi_rank_reply else (unique_reply_rank if unique_reply_rank is not None else 0),` — changed in this diff. vllm_omni/diffusion/worker/diffusion_worker.py:1011 `should_reply = is_primary_in_replica and self.result_mq is not None` — unchanged by this diff, present in the PR-time tree; multiple DP primaries satisfy this gate.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:~958` [minor] — The new `WorkerProc.shutdown` joins the async output thread with `_ASYNC_OUTPUT_THREAD_JOIN_TIMEOUT_S` (10s) and only shuts down/unlinks `result_mq` when `self._async_output_thread is None or not self._async_output_thread.is_alive()`, while `self.mq.shutdown()` in the same finally block is unconditional. If the thread survives the join, the worker-owned result ring buffer is left open. Since a still-running thread could still be enqueueing to `result_mq`, keep the gate but add a one-line comment explaining that the queue is intentionally left open for the surviving thread (or verify unconditional cleanup is safe). (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:66 `_ASYNC_OUTPUT_THREAD_JOIN_TIMEOUT_S = 10.0`; :940 `self._async_output_thread.join(timeout=_ASYNC_OUTPUT_THREAD_JOIN_TIMEOUT_S)`; :951 `self.mq.shutdown()` (unconditional in finally); :958-963 `if self.result_mq is not None and (self._async_output_thread is None or not self._async_output_thread.is_alive()): result_mq = self.result_mq; self.result_mq = None; result_mq.shutdown(); del result_mq; gc.collect()` — changed by this diff, present in PR-time tree.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.trace.json.gz new file mode 100644 index 000000000..2195e627d Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5864.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.cost.json new file mode 100644 index 000000000..0b2519a45 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1194.7, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5958/run-20260815-175929-096d3e", + "recorded_at": "2026-08-15T18:19:21", + "attempts": [ + { + "invocation_id": "inv-085f8489", + "outcome": "run:run-20260815-175929-096d3e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-175929-096d3e", + "llm_calls": 87, + "input_tokens": 143950, + "output_tokens": 175674, + "usd": 0.317658, + "source": "spans" + } + ], + "attempt_usd_total": 0.317658, + "llm_calls": 87, + "input_tokens": 143950, + "output_tokens": 175674, + "usd": 0.317658, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-175929-096d3e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3177, + "minutes": 19.87, + "ci_minutes": 0, + "input_tokens": 143950, + "output_tokens": 175674, + "tool_calls": 231, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6519, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 87, + "by_role": { + "planner": { + "usd": 0.001325, + "input_tokens": 32, + "output_tokens": 1131, + "calls": 1 + }, + "agent": { + "usd": 0.270451, + "input_tokens": 123574, + "output_tokens": 137892, + "calls": 83 + }, + "reducer": { + "usd": 0.045882, + "input_tokens": 20344, + "output_tokens": 36651, + "calls": 3 + } + }, + "cache_read_tokens": 3168512, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.32, + "gate": 1.99, + "review": 1186.61, + "report": 0.07 + }, + "time_to_first_result_s": 1191.96 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5958.trace.json.gz", + "trace_bytes": 30180 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.md new file mode 100644 index 000000000..49e843e02 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5958, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4dd07d536f34) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] lora and transformers_utils carry explicit 'no dedicated design doc' in-file comments, so their non-derivation is disclosed — but the lora narrowing itself is still undocumented in the PR body. +- [claim-verified] Coverage-gap paths exist at head: vllm_omni/{inputs,outputs,model_extras,plugins,attention,benchmarks} plus errors.py, request.py, data_entry_keys.py (list_dir). +- [claim-verified] PR body 'before' columns for engine/worker/entrypoints/deploy match base CODEOWNERS (file_at_base). +- [claim-refuted] PR body offloader 'After' = yuanheng-zhao, Isotr0py, lishunyang12, but offloader.md owners and CODEOWNERS:148 are yuanheng-zhao, david6666666, lishunyang12. +- [claim-refuted] PR body worker 'After' omits @Gaohan123, present at CODEOWNERS:47 and added in ar_runtime.md. +- [claim-refuted] PR body lists /vllm_omni/lora/ under 'Not changed', but the diff narrows it 6→2 owners (base line vs CODEOWNERS:106). +- [claim-refuted] model_executor/models 9→5 owner shift (drops ZeldaHuang, princepride, yuanheng-zhao, Bounty-hunter, NickCao) is absent from the 'Conflicts resolved' table. +- [claim-refuted] 'Design docs updated' section names only 3 of the 12 touched docs; offloader.md actually swaps Isotr0py→david6666666, which the summary hides. +- [claim-refuted] PR conflict table says offloader after = 'yuanheng-zhao, Isotr0py, lishunyang12' but head CODEOWNERS:148 is '@yuanheng-zhao @david6666666 @lishunyang12' (Isotr0py removed, david added). +- [claim-refuted] PR conflict table worker 'after' omits @Gaohan123; head worker rule carries all 5 ar_runtime.md owners (tzhouam, yinpeiqi, fake0fan, Sy0307, Gaohan123). +- [claim-refuted] PR 'Not changed' says lora is unchanged, but base lora rule had 6 owners and head has 2 (@SamitHuang @wtomin); ownership was narrowed. +- [claim-refuted] PR 'Design docs updated' list omits most doc changes: ar_runtime, benchmarking, cache_management, diffusion_model_integration owner swap, diffusion_runtime, execution_platforms, input_output, observability, parallelism. +- [claim-refuted] PR body offloader 'After' = 'yuanheng-zhao, Isotr0py, lishunyang12', but head CODEOWNERS:148 and offloader.md:7 have @david6666666 and no @Isotr0py. +- [claim-refuted] PR body worker 'After' omits @Gaohan123; ar_runtime.md:10 adds him and CODEOWNERS:47 carries 5 owners. + +`.github/CODEOWNERS:~75` [major] — CODEOWNERS:75 `/vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan` overreaches its cited source: omni_connector.md:12 lists only `vllm_omni/distributed/omni_connectors/**`, while the directory also contains `omni_coordinator/` (owned in engine_orchestration.md:30 and related in stage_runtime.md:30) and `ray_utils/` (stage_runtime.md:31). With no more-specific CODEOWNERS rules for those two subdirs and last-match-wins in effect, coordinator/ray_utils reviews route to connector owners with no design-doc basis, contradicting the header's claim that rules derive from `primary_code_paths`. Narrow this rule to `/vllm_omni/distributed/omni_connectors/` and add explicit rules for the coordinator/ray_utils paths, or expand omni_connector.md's primary paths. (evidence: In the PR-time tree, .github/CODEOWNERS:75 is exactly `/vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan` (unchanged by this diff from the pre-PR broad rule, but re-presented under the new derivation header). .github/CODEOWNERS:3-5 states: 'The vllm_omni/** source-tree rules below are derived from the `owners` and `primary_code_paths` front matter in docs/design/module/*.md. When updating ownership, update the design doc first, then regenerate the matching rule here. Last-match-wins: more specific rules must come AFTER broader ones.' docs/design/module/omni_connector.md:12: `- vllm_omni/distributed/omni_connectors/**`. docs/design/module/engine_orchestration.md:30: `- vllm_omni/distributed/omni_coordinator/**`. docs/design/module/stage_runtime.md:30-31: `- vllm_omni/distributed/omni_coordinator/**` and `- vllm_omni/distributed/ray_utils/**`. Repo grep confirms CODEOWNERS contains no rules for `omni_coordinator` or `ray_utils`, so the broad line 75 rule is the effective owner for those paths. Directory listing of vllm_omni/distributed shows `omni_connectors/`, `omni_coordinator/`, and `ray_utils/`.) + +`.github/CODEOWNERS:137` [major] — .github/CODEOWNERS:137 `/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin` omits @RuixiangMa, while the cited source docs/design/module/diffusion/diffusion_model_integration.md:9 lists him as an owner and its primary_code_paths cover `vllm_omni/diffusion/models/**` as well as `model_loader/**`. Because this file declares last-match-wins, the more-specific models/ rule replaces the umbrella `/vllm_omni/diffusion/` rule and removes @RuixiangMa from auto-assignment for diffusion model files. Add @RuixiangMa to line 137 or update the doc's owners list. (evidence: Present in the PR-time tree and in the PR diff — .github/CODEOWNERS:137 `/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin`; .github/CODEOWNERS:138 `/vllm_omni/diffusion/model_loader/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa`; docs/design/module/diffusion/diffusion_model_integration.md:9 ` - "@RuixiangMa"`.) + +`.github/CODEOWNERS:81` [major] — This diff adds `vllm_omni/plugins/` → @gcanlin @tjtanaa @xuechendi and `vllm_omni/model_extras/` → @tzhouam @gcanlin @princepride, but neither owner set is derivable from the cited model_integration.md, whose owners are only @tzhouam and @gcanlin (docs/design/module/model_integration.md:6-7). model_integration.md lists model_extras/** and plugins/** under primary_code_paths with no per-path owner override, so regenerating from the doc yields {tzhouam, gcanlin} for both. plugins/ drops @tzhouam and adds @tjtanaa/@xuechendi (the execution_platforms owners), and model_extras adds @princepride — both unexplained by the cited doc. Since the CODEOWNERS header declares rules are derived from the design docs' `owners` and `primary_code_paths` front matter, this breaks the PR's own authoritative-doc contract. Fix by either updating model_integration.md's owner sets to match these rules, or changing the CODEOWNERS rules to the doc's {tzhouam, gcanlin} (plus scoped primary_code_path_owners); if plugins really belongs to execution platforms, add `vllm_omni/plugins/**` to execution_platforms.md primary_code_paths and cite that doc. (evidence: .github/CODEOWNERS:79-82 (verbatim): `# Model integration — docs/design/module/model_integration.md` / `/vllm_omni/model_executor/ @tzhouam @gcanlin` / `/vllm_omni/model_executor/models/ @tzhouam @gcanlin @Sy0307 @amy-why-3459 @linyueqian` / `/vllm_omni/model_extras/ @tzhouam @gcanlin @princepride` / `/vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi` — docs/design/module/model_integration.md:6-7 (verbatim): `owners:` / ` - "@tzhouam"` / ` - "@gcanlin"` — docs/design/module/model_integration.md:9-11 (verbatim): `primary_code_paths:` / ` - vllm_omni/model_executor/**` / ` - vllm_omni/model_extras/**` / ` - vllm_omni/plugins/**` — docs/design/module/execution_platforms.md:6-7 (verbatim, only occurrence of @tjtanaa in docs/design): `owners:` / ` - "@gcanlin"` / ` - "@tjtanaa"` / ` - "@xuechendi"` (this doc's primary_code_paths cover platforms/** and attention/**, not plugins/**).) + +`.github/CODEOWNERS:69` [minor] — entrypoints.md:23 `- vllm_omni/entrypoints/cli/**` still owns the benchmark subtree, but its only `primary_path_exceptions` entry is `vllm_omni/entrypoints/openai/errors.py` (owner: error_contracts.md). benchmarking.md:10 `- vllm_omni/entrypoints/cli/benchmark/**` declares overlapping ownership, and CODEOWNERS encodes it via line 69 after the parent entrypoints rule, so last-match-wins gives benchmark owners the subtree without a matching design-doc exception. Add a `primary_path_exceptions` entry in entrypoints.md mapping `vllm_omni/entrypoints/cli/benchmark/**` to `benchmarking.md` so the design docs remain the single source. (evidence: CODEOWNERS:66 `/vllm_omni/entrypoints/ @alex-jw-brooks @linyueqian @NickCao`; CODEOWNERS:69 `/vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter`; entrypoints.md:23 ` - vllm_omni/entrypoints/cli/**`; entrypoints.md:29-31 `primary_path_exceptions:\n - path: vllm_omni/entrypoints/openai/errors.py\n owner: error_contracts.md`; benchmarking.md:10 ` - vllm_omni/entrypoints/cli/benchmark/**`) + +`.github/CODEOWNERS:148` [minor] — PR body description is out of sync with the merged CODEOWNERS/design-doc diff: the conflict table says offloader becomes 'yuanheng-zhao, Isotr0py, lishunyang12' with only '@lishunyang12' added, but .github/CODEOWNERS:148 is '/vllm_omni/diffusion/offloader/ @yuanheng-zhao @david6666666 @lishunyang12' and docs/design/module/diffusion/offloader.md:7 adds '@david6666666' while removing '@Isotr0py'. The body also omits ar_runtime.md's @Gaohan123 addition (.github/CODEOWNERS:47) and the lora narrowing from 6 owners to '@SamitHuang @wtomin' (.github/CODEOWNERS:106), while falsely listing lora under 'Not changed'. Please sync the description with the merged diff. (evidence: .github/CODEOWNERS:148 `/vllm_omni/diffusion/offloader/ @yuanheng-zhao @david6666666 @lishunyang12`; docs/design/module/diffusion/offloader.md:7 `- "@david6666666"` (head owners omit @Isotr0py; base was `@yuanheng-zhao`, `@Isotr0py`); .github/CODEOWNERS:47 `/vllm_omni/worker/ @tzhouam @yinpeiqi @fake0fan @Sy0307 @Gaohan123`; .github/CODEOWNERS:106 `/vllm_omni/lora/ @SamitHuang @wtomin` — all present in the PR-time tree; PR body retrieved via gh_pr_view states offloader becomes 'yuanheng-zhao, Isotr0py, lishunyang12', 'add @lishunyang12' only, and lists lora under 'Not changed'.) + +`.github/CODEOWNERS:5` [nit] — Header cites only `owners` and `primary_code_paths`, but the `/vllm_omni/attention/` and `/vllm_omni/model_executor/models/` rules also incorporate `primary_code_path_owners` from execution_platforms.md and model_integration.md. Name all three fields in the regeneration instruction so a future regeneration keeps the scoped owners. (evidence: ['.github/CODEOWNERS:5-6 `# The vllm_omni/** source-tree rules below are derived from the `owners` and` / `# `primary_code_paths` front matter in docs/design/module/*.md. When updating` — header names only two source fields.', '.github/CODEOWNERS:87 `/vllm_omni/attention/ @gcanlin @tjtanaa @xuechendi @david6666666 @lishunyang12` — @david6666666 and @lishunyang12 come from `primary_code_path_owners`, not `owners`/`primary_code_paths`.', 'docs/design/module/execution_platforms.md:12-18 `primary_code_path_owners:` ... `- paths:` ... `- vllm_omni/attention/**` ... `owners:` ... `- "@david6666666"` ... `- "@lishunyang12"` — source of the attention scoped owners.', '.github/CODEOWNERS:79 `/vllm_omni/model_executor/models/ @tzhouam @gcanlin @Sy0307 @amy-why-3459 @linyueqian` — @Sy0307/@amy-why-3459/@linyueqian come from `primary_code_path_owners`, not `owners`/`primary_code_paths`.', 'docs/design/module/model_integration.md:12-17 `primary_code_path_owners:` ... `- paths:` ... `- vllm_omni/model_executor/models/**` ... `owners:` ... `- "@Sy0307"` ... `- "@amy-why-3459"` ... `- "@linyueqian"` — source of the model_executor/models scoped owners.']) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.trace.json.gz new file mode 100644 index 000000000..89c697ea2 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5958.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.cost.json b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.cost.json new file mode 100644 index 000000000..c0a518bb1 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1700.9, + "split": "holdout4", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout4_r1/runs/pr5978/run-20260815-180524-c040be", + "recorded_at": "2026-08-15T18:33:42", + "attempts": [ + { + "invocation_id": "inv-b0e3b725", + "outcome": "run:run-20260815-180524-c040be", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-180524-c040be", + "llm_calls": 152, + "input_tokens": 1209017, + "output_tokens": 257882, + "usd": 0.939351, + "source": "spans" + } + ], + "attempt_usd_total": 0.939351, + "llm_calls": 152, + "input_tokens": 1209017, + "output_tokens": 257882, + "usd": 0.939351, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-180524-c040be", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9394, + "minutes": 28.31, + "ci_minutes": 0, + "input_tokens": 1209017, + "output_tokens": 257882, + "tool_calls": 309, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0387, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 152, + "by_role": { + "agent": { + "usd": 0.909438, + "input_tokens": 1182930, + "output_tokens": 237129, + "calls": 150 + }, + "reducer": { + "usd": 0.029913, + "input_tokens": 26087, + "output_tokens": 20753, + "calls": 2 + } + }, + "cache_read_tokens": 12194304, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.59, + "gate": 1.58, + "review": 1692.19, + "report": 0.06 + }, + "time_to_first_result_s": 1698.41 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5978.trace.json.gz", + "trace_bytes": 51066 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.md b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.md new file mode 100644 index 000000000..7d467148b --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.md @@ -0,0 +1,64 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5978, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 86854432db52) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 10 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [claim-verified] New e2e file defines SSIM_THRESHOLD=0.97 and PSNR_THRESHOLD=34.0 shared by both I2VA and Ref2VA (lines 62-64), matching the PR body's 'one accuracy gate' claim. +- [claim-verified] e2e constants WIDTH=1344, HEIGHT=768, FPS=24, I2VA_NUM_FRAMES=192, REF2VA_NUM_FRAMES=124 and expected_audio {'aac',32000,2} match the PR body's contract claims. +- [claim-verified] diff_stat confirms t2va test deletion, i2va_ref2va test addition, nightly label/command swap, and no other files added/removed. +- [claim-verified] 'one accuracy gate SSIM>=0.97 PSNR>=34' — SSIM_THRESHOLD=0.97 and PSNR_THRESHOLD=34.0 in the new test file. +- [claim-verified] Ref2VA 124-frame assertion is consistent: minimax_h3_align_frame_count(round(5.0*24)=120) == 124 (time_request.py:94-100), so duration 5.0 -> 124 frames. +- [claim-verified] common gate SSIM>=0.97/PSNR>=34 is encoded as SSIM_THRESHOLD=0.97 / PSNR_THRESHOLD=34. +- [claim-verified] PR numbers checked with calc: I2VA SSIM 0.975190 vs 0.97 = +0.535% margin, PSNR 35.981286 vs 34 = +1.98 dB; Ref2VA +1.09%/+8.65 dB; e2e frame constants 192/124 match PR body. +- [claim-verified] av==18.0.0 is pinned in pyproject dev extras (line 54), so the new PyAV calls (audio_frame.pts/time_base, av.open/demux) have a bounded version; no new lower-bound issue. +- [claim-verified] T2VA still supported: pipeline_minimax_h3.py:592 includes t2va in FL2VA partition defaults and :757 resolves text-only requests to t2va; its only accuracy gate was removed. +- [claim-verified] hardware_test(res={'cuda':'H100'}, num_cards=4) in tests/helpers/mark.py only adds H100/cuda/distributed marks plus device_count skipif; it does not verify SKU, so a B300 run is possible while nightly pins h100_4. +- [claim-verified] PR claim 'Expose standalone audio encoding in the MiniMax H3 profiler' = the one-line _PROFILER_TARGETS addition at pipeline:538 — present and correct. +- [claim-refuted] PR body says 'Hardware: 4x NVIDIA B300' but the added tests decorate `res={"cuda": "H100"}` (lines 228/304) and the nightly step uses mirror_hardwares: h100_4 (test-nightly.yml:434); the reported SSIM/PSNR were not produced on the gate hardware. +- [claim-refuted] PR body says 'Hardware: 4x NVIDIA B300' but the new tests are `@hardware_test(res={"cuda": "H100"}, num_cards=4)` and the nightly step uses `mirror_hardwares: h100_4`. +- [validated] pipeline._encode_audio_conditions truncation is exercised by test_ref2va_standalone_audio_condition_is_bounded_to_output_duration (40 samples -> 25) and validation-before-rank by test_ref2va_audio_duration_validation_precedes_rank_branch. + +`vllm_omni/diffusion/models/minimax_h3/reference_video.py:~440` [minor] — Confirmed: `sample_reference_video_frames` decodes the full prepared stream via `load_video_frames(prepared_path)` (reference_video.py:440) and `load_video_frames` materializes every frame with `reader.get_batch(list(range(len(reader))))` (reference_video.py:418), then only the sampled `indices` (typically 3 frames) are used. The same prepared stream is decoded again by the video VAE at pipeline_minimax_h3.py:1274. For the I2VA/Ref2VA gate this is bounded by `target_frame_count` (209 in tests), but it is a large RGB array (roughly 209*1344*768*3 ≈ 618 MB) materialized in the same path. Consider passing `indices` to decord (`reader.get_batch(indices)`) or adding a bounded-load helper. (evidence: reference_video.py:440 `decoded_frames = load_video_frames(prepared_path)`; reference_video.py:441 `frames = [np.asarray(decoded_frames[source_index]) for source_index in indices]`; reference_video.py:418 `frames = reader.get_batch(list(range(len(reader))))`; pipeline_minimax_h3.py:1274 `self.video_vae.encode_video(load_video_frames(item["prepared_path"])) for item in prepared_videos` (the pipeline consumer is unchanged by this diff and present in the PR-time tree).) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:228` [minor] — I2VA/Ref2VA accuracy tests are gated on H100 (`@hardware_test(res={"cuda": "H100"}, num_cards=4)` at :228 and :304) and the nightly step pins `mirror_hardwares: h100_4`, but the PR body's passing evidence was measured on 4x NVIDIA B300. I2VA's SSIM margin over the 0.97 gate is only ~0.005 (0.975190), so please validate both cases on the H100 mirror hardware and update the reported numbers, or explicitly state that the B300 measurements are not gate evidence. (evidence: tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:228 `@hardware_test(res={"cuda": "H100"}, num_cards=4)` (and :304 for Ref2VA); .buildkite/cuda/test-nightly.yml:434 `mirror_hardwares: h100_4` — unchanged by this diff (diff context line), present in PR-time tree; PR body via gh_pr_view: `**Hardware:** 4x NVIDIA B300, BF16, Ulysses 4, ring 1, HSDP shard size 4, VAE patch parallel 4, tiled VAE, FlashAttention, 50 steps, seed 0.` and `- I2VA: SSIM `0.975190`, PSNR `35.981286 dB`, passing the common gate.`; same file:183-184 `SSIM_THRESHOLD = 0.97` / `PSNR_THRESHOLD = 34.0`) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:30` [minor] — This new file keeps MODEL_REVISION = "main" (line 31), and _model_name() passes revision=MODEL_REVISION to snapshot_download at line 113, so the I2VA/Ref2VA nightly test pulls moving model weights under the strict SSIM>=0.97/PSNR>=34 gate (lines 101-102). Pin the exact revision that produced the reported measurements, or require the VLLM_TEST_MINIMAX_H3_FL2VA_MODEL / VLLM_TEST_MINIMAX_H3_REF2VA_MODEL env-var overrides; otherwise an upstream MiniMax H3 update can flip the nightly gate with no code change. (evidence: tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:31 `MODEL_REVISION = "main"`; same file:113 `revision=MODEL_REVISION,` inside `_model_name()`'s `snapshot_download(repo_id=MODEL_REPO_ID, ...)`; same file:101 `SSIM_THRESHOLD = 0.97`; same file:102 `PSNR_THRESHOLD = 34.0`; same file:29 `MODEL_ENV_VAR = "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL"` and same file:30... actually line 32 `REF2VA_MODEL_ENV_VAR = "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL"`. All lines are in the new file added by this diff.) + +`tests/entrypoints/openai_api/test_video_server.py:253` [minor] — Confirmed anchor and scope. `test_mux_video_audio_marks_aac_priming_timestamp` is added at tests/entrypoints/openai_api/test_video_server.py:253 under `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, and the new minimax_h3 contract cases are under `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]` (test_minimax_h3_contract.py:14). Both are in scope of the failing lanes: .buildkite/cuda/test-merge.yml:18/23 and .buildkite/amd/test-amd-merge.yml:21, and gh_ci_read shows buildkite/vllm-omni and buildkite/vllm-omni-amd-ci as FAILURE. Please confirm via the Buildkite logs whether the failures are pre-existing on main or introduced by these newly collected tests. (evidence: tests/entrypoints/openai_api/test_video_server.py:41 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; tests/entrypoints/openai_api/test_video_server.py:253 `def test_mux_video_audio_marks_aac_priming_timestamp():`; tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:14 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`; .buildkite/cuda/test-merge.yml:18 `- "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/cuda/test-merge.yml:23 `- "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/amd/test-amd-merge.yml:21 `- "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (all unchanged-by-this-diff CI pipeline files present in the PR-time tree; the FAILURE states themselves come from the GitHub checks API, gh_ci_read, not from code).) + +`.buildkite/cuda/test-nightly.yml:~430` [minor] — The nightly MiniMax H3 accuracy step (.buildkite/cuda/test-nightly.yml:430) now runs only test_minimax_h3_i2va_ref2va_similarity.py, and tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py was deleted, leaving no e2e pixel-level T2VA accuracy coverage. T2VA remains supported (pipeline_minimax_h3.py:592 supported_tasks defaults to {"t2va", "fl2va"} for the FL2VA partition; pipeline_minimax_h3.py:757 defaults to task "t2va"), so consider keeping a T2VA accuracy case at a documented lower threshold or filing a tracking issue for the gap. (evidence: tests/e2e/accuracy/minimax_h3 now contains only `test_minimax_h3_i2va_ref2va_similarity.py` (directory listing). .buildkite/cuda/test-nightly.yml:430 `- label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"` with command `pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py`. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:592 `supported_tasks = {"ref2va"} if partition == "ref2va" else {"t2va", "fl2va"}` and :757 `requested = "t2va"`.) + +`tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:228` [minor] — Add a runtime GPU-SKU guard to both I2VA and Ref2VA accuracy tests (e.g. `assert 'H100' in torch.cuda.get_device_name()` in each test body), since `@hardware_test(res={'cuda':'H100'}, num_cards=4)` only adds pytest marks and a `device_count` skipif and never verifies the GPU model; the nightly `h100_4` mirror is a scheduling hint, not a hardware assertion. (evidence: tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py:228 `@hardware_test(res={"cuda": "H100"}, num_cards=4)` (also line 304); tests/helpers/mark.py:85-90 (unchanged by this diff) `test_skipif = pytest.mark.skipif(current_platform.device_count() < num_cards, reason=f"Need at least {num_cards} CUDA GPUs to run the test.")` — the decorator path adds only marks plus this device-count skipif, with no `get_device_name`/SKU verification, so a 4-GPU B300 node can satisfy the gate while nightly labels the lane `h100_4`.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1237` [minor] — Add a fast unit test that actually invokes `_transcode_reference_video` (or at least asserts the constructed argv) on a tiny fixture video, so the new lossless ffmpeg args (`-frames:v`, `libx264rgb`, `-crf 0`, `-preset veryfast`, `rgb24`) are covered in merge CI. Current contract tests monkeypatch `_transcode_reference_video` (lines 1237, 1318, 1366), leaving only the nightly full_model e2e to exercise these args. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:1237 ` "_transcode_reference_video",` — monkeypatch target inside test_r7_r8_ref2va_video_segment_matrix (unchanged context, present in PR-time tree); identical monkeypatch targets at :1318 and :1366 ` monkeypatch.setattr(reference_video_module, "_transcode_reference_video", lambda source, **kwargs: "prepared.mp4")`. Because the function is replaced in unit tests, the new args added by this diff in vllm_omni/diffusion/models/minimax_h3/reference_video.py (` "-frames:v",`, ` str(int(target_frame_count)),`, ` "libx264rgb",`, ` "-crf",`, ` "0",`, ` "-preset",`, ` "veryfast",`, ` "rgb24",`) are never constructed in merge CI; only the new nightly-only tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py (pytestmark includes full_model; wired into .buildkite/cuda/test-nightly.yml) drives them.) + +`vllm_omni/diffusion/utils/media_utils.py:194` [minor] — Do not rely on the MagiHuman e2e to validate this shared mux change: its _validate_mp4 only checks audio stream presence (and video dimensions/frame count), so it cannot catch an A/V-sync regression from audio_frame.pts=0 / time_base=1/audio_sample_rate. Add a sync assertion to at least one consumer (decode the muxed output and verify audio starts at PTS 0 and aligns with video duration); the negative-PTS unit check alone is insufficient. (evidence: In diff: vllm_omni/diffusion/utils/media_utils.py:194 `audio_frame.pts = 0`; :195 `audio_frame.time_base = Fraction(1, audio_sample_rate)`. Unchanged by this diff, present in PR-time tree: tests/e2e/offline_inference/test_magi_human_expansion.py:58 `a_streams = [s for s in container.streams if s.type == "audio"]`; :59 `assert len(a_streams) >= 1, "No audio stream found in MP4"`; :66 `for frame in container.decode(video=0):` — audio is only presence-checked and never decoded/PTS/duration-checked. In diff: tests/entrypoints/openai_api/test_video_server.py:268 `assert first_packet.pts < 0` — the added unit test checks only the negative priming PTS, not A/V alignment or duration.) + +**Additional observations (beyond the comment budget):** +- `tests/entrypoints/openai_api/test_video_server.py:269` [minor] This diff adds `test_mux_video_audio_marks_aac_priming_timestamp`, which only asserts `first_packet.pts < 0` (tests/entrypoints/openai_api/test_video_server.py:269). The production intent in vllm_omni/diffusion/utils/med +- `vllm_omni/diffusion/utils/media_utils.py:194` [minor] This diff changes the shared `mux_video_audio_bytes` (vllm_omni/diffusion/utils/media_utils.py:194-195) used by the production video+audio response path (vllm_omni/entrypoints/openai/video_api_utils.py:524) and offline e +- `vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1325` [minor] Confirmed. pipeline_minimax_h3.py:1324-1327 uses item.get("audio_duration_seconds", item.get("duration_seconds")); prepare_reference_videos (the sole producer) unconditionally sets audio_duration_seconds at reference_vid + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.trace.json.gz new file mode 100644 index 000000000..aee021d9a Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout4_r1/pr5978.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/manifest.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/manifest.json new file mode 100644 index 000000000..3509a4203 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v16f_holdout5_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout5" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.cost.json new file mode 100644 index 000000000..edd535272 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1352.5, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr5676/run-20260816-003553-289492", + "recorded_at": "2026-08-16T00:58:23", + "attempts": [ + { + "invocation_id": "inv-fab6281f", + "outcome": "run:run-20260816-003553-289492", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-003553-289492", + "llm_calls": 152, + "input_tokens": 2091061, + "output_tokens": 216926, + "usd": 1.416773, + "source": "spans" + } + ], + "attempt_usd_total": 1.416773, + "llm_calls": 152, + "input_tokens": 2091061, + "output_tokens": 216926, + "usd": 1.416773, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-003553-289492", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.4168, + "minutes": 22.49, + "ci_minutes": 0, + "input_tokens": 2091061, + "output_tokens": 216926, + "tool_calls": 238, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0911, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 152, + "by_role": { + "agent": { + "usd": 1.383616, + "input_tokens": 2043626, + "output_tokens": 198467, + "calls": 150 + }, + "reducer": { + "usd": 0.033157, + "input_tokens": 47435, + "output_tokens": 18459, + "calls": 2 + } + }, + "cache_read_tokens": 22724736, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.21, + "gate": 1.5, + "review": 1343.68, + "report": 0.07 + }, + "time_to_first_result_s": 1349.44 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5676.trace.json.gz", + "trace_bytes": 51228 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.md new file mode 100644 index 000000000..d04279dd1 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5676, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 44e60c00090e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] diff_stat confirms 18 changed files (9 non-test) matching the PR's touched-file list. +- [claim-refuted] PR body says request isolation was validated, but its E2E commands only cover TI2V-5B T2V/I2V; S2V/VACE batched forward is mock-only and has no real-engine row. +- [validated] tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py:27 and test_wan22_i2v_pipeline.py:22 run the real post_process_func with `sampling_params.output_type='latent'` and assert identity — not mocked. +- [validated] tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py:158/166/223 assert split outputs and generator/latent collation through real Wan22Pipeline.forward with stubbed encode/prepare/diffuse. +- [sweep] Read vllm_omni/diffusion/worker/request_batch.py: `split_diffusion_output_by_request` validates num_outputs_per_prompt>0 and slices tensors/tuples/lists per request; S2V re-tuples audio after split. +- [sweep] Read vllm_omni/diffusion/sched/interface.py + request_scheduler.py: new sample_solver/flow_shift/condition_key fields are excluded from bulk getattr and populated explicitly via `_normalize_explicit_sample_solver`/`_normalize_explicit_flow_shift`. +- [sweep] CI selection: .buildkite/cuda/test-merge.yml:18 and test-ready.yml:18 run `pytest -sv tests/diffusion -m 'core_model and cpu'`; every changed diffusion test file carries `pytestmark=[core_model,cpu,...]`. +- [sweep] tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py:17 has `pytestmark=[core_model,cpu]` (no diffusion marker) but is still matched by `core_model and cpu`. +- [sweep] Read pipeline_wan2_2.py forward: prompt_embeds/negative_prompt collation, generator/latent collation, batched image collation, and split output are present; `common = sampling_params_list[0]` reads batch-wide params from request 0. +- [sweep] Read pipeline_wan2_2_i2v.py forward + prepare_latents: batched image/last_image collation and first_frame_mask batch-size fix are present. +- [sweep] Read pipeline_wan2_2_s2v.py forward/diffuse/_make_clip_generators: found `generator=clip_generator if isinstance(clip_generator, torch.Generator) else None` drops batched generator list in scheduler.step. +- [sweep] Read pipeline_wan2_2_vace.py forward: batched condition contexts and reference-count equality check are present; VACE inherits `supports_request_batch=True` from Wan22Pipeline. +- [sweep] tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py:244+ asserts batched prepare_latents shapes; forward tests stub encode/prepare/diffuse (plumbing-level for the denoise loop). +- [sweep] tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py:246 `pipeline.diffuse = MagicMock(side_effect=lambda **kwargs: kwargs['latents'])` — the changed real diffuse loop is not executed by the batched test. + +`tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py:86` [minor] — The test at line 86 asserts `generators[2] is explicit_generator`, and `_make_clip_generators` (pipeline_wan2_2_s2v.py:85) returns the request generator unchanged whenever `seed is None and request_generator is not None`. For a multi-clip S2V batch this reuses the same torch.Generator object at every `clip_index`, producing a serial continuation of one RNG stream rather than independent per-clip streams. Please either fork/derive a fresh per-clip generator for explicit generators and assert per-clip freshness, or explicitly document why serial RNG continuation is the intended deterministic invariant; the test should not enshrine the identity reuse without justification. (evidence: tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py:86 ` assert generators[2] is explicit_generator` — test pins identity reuse of the explicit generator. vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py:84-85 ` if seed is not None or request_generator is None` / ` else request_generator` — returns the same generator object for every clip_index when no seed is supplied. Both lines are added by this diff.) + +`vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py:~1081` [minor] — forward always passes a list `clip_generators` into `diffuse` (`clip_generator=clip_generators`), and `_make_clip_generators` always returns a list, so inside `diffuse` the expression `generator=clip_generator if isinstance(clip_generator, torch.Generator) else None` collapses it to `None` for every scheduler step, including single-request S2V. Index/route a matching per-clip generator into `scheduler.step`, or document that these S2V schedulers are deterministic and ignore `generator`; the batched S2V test mocks `pipeline.diffuse` with a lambda and does not catch this. (evidence: vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py:1081 `generator=clip_generator if isinstance(clip_generator, torch.Generator) else None,`; vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py:1390 `clip_generator=clip_generators,`; vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py:1317 `clip_generators = _make_clip_generators(seeds, generators, r, device)`; vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py:78-85 `_make_clip_generators` returns a `list[torch.Generator | None]` (`return [ ... for seed, request_generator in zip(seeds, request_generators)]`).) + +`vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:~571` [minor] — Wan22Pipeline.forward derives all batch-wide parameters (height, width, num_frames, num_inference_steps, guidance, boundary_ratio, max_sequence_length, output_type) from `common = sampling_params_list[0]` and never verifies that the remaining requests in `req.sampling_params_list` agree. Scheduler admission guards this in normal flows, but the request-batch API is callable directly and the new tests construct DiffusionRequestBatch by hand; a mismatched batch would silently use the first request's values. Add an assertion over the batch-wide fields in `req.sampling_params_list`, or explicitly document the scheduler-enforced precondition. (evidence: vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:571 `common = sampling_params_list[0]`; vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:593 `height = common.height or 480`. In the PR-time tree the same function also derives width, num_frames, num_inference_steps, guidance_scale, boundary_ratio, max_sequence_length and output_type from `common`, and no comparison/assertion over the remaining `sampling_params_list` entries precedes them.) + +`vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:~239` [minor] — vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:239 sets `request.batch_compatibility_key = ("wan22_image_condition", has_image)` in the T2V preprocess path, gating batch admission on present/absent/empty-list image conditions, but no test references `wan22_image_condition` while the I2V `wan22_i2v_last_image` key has dedicated tests. Add mirrored T2V tests for present-image, absent-image, and empty-list-image cases. (evidence: vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:239 `request.batch_compatibility_key = ("wan22_image_condition", has_image)` — present in the PR-time tree; recursive grep of tests/ for `wan22_image_condition` returns no matches.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.trace.json.gz new file mode 100644 index 000000000..fe6aee2d9 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5676.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.cost.json new file mode 100644 index 000000000..df9b21049 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1040.0, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr5732/run-20260816-003554-7570f8", + "recorded_at": "2026-08-16T00:53:11", + "attempts": [ + { + "invocation_id": "inv-f87628e0", + "outcome": "run:run-20260816-003554-7570f8", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-003554-7570f8", + "llm_calls": 73, + "input_tokens": 313717, + "output_tokens": 177161, + "usd": 0.339919, + "source": "spans" + } + ], + "attempt_usd_total": 0.339919, + "llm_calls": 73, + "input_tokens": 313717, + "output_tokens": 177161, + "usd": 0.339919, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-003554-7570f8", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3399, + "minutes": 17.28, + "ci_minutes": 0, + "input_tokens": 313717, + "output_tokens": 177161, + "tool_calls": 156, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6182, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 73, + "by_role": { + "planner": { + "usd": 0.001452, + "input_tokens": 1958, + "output_tokens": 839, + "calls": 1 + }, + "agent": { + "usd": 0.308847, + "input_tokens": 298687, + "output_tokens": 152641, + "calls": 70 + }, + "reducer": { + "usd": 0.02962, + "input_tokens": 13072, + "output_tokens": 23681, + "calls": 2 + } + }, + "cache_read_tokens": 2234752, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.66, + "gate": 1.55, + "review": 1031.22, + "report": 0.08 + }, + "time_to_first_result_s": 1036.48 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5732.trace.json.gz", + "trace_bytes": 28723 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.md new file mode 100644 index 000000000..d4d3f3ca7 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5732, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 82a142ad19c2) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] d145581e lost mixed-dtype promotion and misdetected RGBA on 2D width-4; 82a142ad restored both (`len(frame_shape) == 3`, `common_dtype = np.result_type(...)`). Residual: a hypothetical mixed uint8+float direct call now keeps uint8 values verbatim instead of stack-promoting, but that path is unreachable via all production normalizers. +- [claim-verified] pre-commit and DCO CI buckets are SUCCESS (gh_ci_read). +- [claim-verified] Perf numbers not self-refuting: 362*1344*768*3 float32 stack = 4.18 GiB and old peak temporaries ~8.35 GiB vs claimed RSS delta 5.8 GiB (14.86->9.06) is plausible given RSS/allocation ordering; latency not independently run. +- [claim-refuted] PR body '11 passed' is stale: head file collects 13 items (12 functions, test_channel_last parametrized [3,4] counts twice); d145581e added 1 test (->11), 82a142ad added 2 more without updating the plan. +- [claim-refuted] PR test plan says '11 passed' but head collects 13 tests (12 functions, one parametrized ×2; .pytest_cache/v/cache/nodeids lists 13 node IDs) — the two bugfix-commit tests were never reflected in the plan. +- [claim-refuted] Test plan '11 passed' is stale: base file has 10 collected cases, head has 13 (12 functions, one parametrized x2); 11 matches neither. +- [validated] frames[0].shape at video_api_utils.py:431 is guarded by `if not frames: raise ValueError("No frames found to encode.")` (428-429) and every frame is an np.ndarray from _coerce_video_to_frames. +- [validated] Shape-mismatch branch at video_api_utils.py:441 raises ValueError, matching old np.stack's failure; the equal-shape arm proceeds. +- [validated] Consumer census: only video_api_utils.py:472 (_encode_video_bytes) and :497 (FragmentedMP4VideoEncoder.encode) call _coerce_video_to_uint8_frames; serving_video.py:281/283 goes through encode_video_base64 and the (F,H,W,3) shape/dtype contract is preserved. +- [validated] RGBA parity: old `frames_np.ndim == 4 and frames_np.shape[-1] == 4` is equivalent to new `len(frame_shape) == 3 and frame_shape[-1] == 4` for stacked frames; the new 2D width-4 test locks the fix. +- [validated] dtype-promotion parity: `common_dtype = np.result_type(*(frame.dtype for frame in frames))` matches np.stack promotion; the mixed float16+float32 test compares directly against np.stack+np.rint. +- [validated] rounding parity: np.round (round-half-even) and np.rint agree; test's expected[0,0,0,0]==25 checks 0.1f16->float32 0.0999756*255=25.49 -> rint 25. +- [validated] input immutability: np.clip returns a new array and the float test asserts `np.testing.assert_array_equal(frame, original)`. +- [validated] Indexed access video_api_utils.py:431 `frame_shape = frames[0].shape` is guarded by the preceding `if not frames: raise ValueError("No frames found to encode.")` at 427-429, so frames[0] is safe. + +`vllm_omni/entrypoints/openai/video_api_utils.py:~444` [minor] — The per-frame `if frame.dtype == np.uint8: frames_u8[index] = frame; continue` fast path at line 444 cannot fire through production entrypoints: `frames` comes from `_coerce_video_to_frames` (line 427), and every integer input is normalized to float32 before this loop — torch tensors via `video_tensor.to(torch.float32) / 255.0` (line 317), numpy arrays via `video_array.astype(np.float32) / 255.0` (line 340), and list frames via `frame_array.astype(np.float32) / 255.0` (line 374) — while floating inputs keep floating dtype. Remove the dead branch or assert a float dtype; as written it also advertises a mixed uint8+float copy-through path inconsistent with the np.stack promotion semantics this change preserves. (evidence: vllm_omni/entrypoints/openai/video_api_utils.py:444 `if frame.dtype == np.uint8:` — new fast path in the diff; vllm_omni/entrypoints/openai/video_api_utils.py:427 `frames = _coerce_video_to_frames(video)` — unchanged by this diff, present in PR-time tree, the only producer feeding the loop; vllm_omni/entrypoints/openai/video_api_utils.py:317 `video_tensor = video_tensor.to(torch.float32) / 255.0` — integer torch input normalized to float32; vllm_omni/entrypoints/openai/video_api_utils.py:340 `video_array = video_array.astype(np.float32) / 255.0` — integer numpy input normalized to float32; vllm_omni/entrypoints/openai/video_api_utils.py:374 `frame_array = frame_array.astype(np.float32) / 255.0` — integer list frame normalized to float32.) + +`vllm_omni/entrypoints/openai/video_api_utils.py:431` [minor] — Gate report shows `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` FAILING at head 82a142ad19c2. This diff rewrites `_coerce_video_to_uint8_frames`, and the AMD ready lane exercises it: `.buildkite/amd/test-amd-ready.yml:124` runs `tests/e2e/online_serving/test_wan22_t2v.py`, whose `/v1/videos` response flows through `serving_video.py:30` (`encode_video_base64` -> `_encode_video_bytes` -> `_coerce_video_to_uint8_frames`). The new unit tests run only in the CPU entrypoints lane, so they cannot catch an AMD regression. Confirm the AMD failure is pre-existing on main (and the NPU failure unrelated) before treating the gate as clean. (evidence: vllm_omni/entrypoints/openai/video_api_utils.py:431 ` frame_shape = frames[0].shape` (inside this PR's diff; anchor exists). Unchanged by this diff, present in the PR-time tree: .buildkite/amd/test-amd-ready.yml:124 ` pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'`. Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/serving_video.py:30 `from vllm_omni.entrypoints.openai.video_api_utils import _encode_video_bytes, encode_video_base64`. gh_ci_read(pr=5732): `buildkite/vllm-omni-amd-ci` state FAILURE; `buildkite/vllm-omni-npu-ci` state FAILURE.) + +`vllm_omni/entrypoints/openai/video_api_utils.py:451` [minor] — The final head (82a142ad) rescales via `np.clip(frame.astype(common_dtype, copy=False), 0.0, 1.0)`, while the d145581e perf run measured plain `np.clip(frame, 0.0, 1.0)`; the PR-body latency/RSS/checksum numbers are therefore not tied to this head and no benchmark config/result JSON is attached. Attach the benchmark config + result JSON, state the measured commit, and re-establish 'output checksum: identical' at final head. (evidence: PR-time tree, video_api_utils.py:451 ` scaled = np.clip(frame.astype(common_dtype, copy=False), 0.0, 1.0)`. Commit d145581e (perf) diff shows `+ scaled = np.clip(frame, 0.0, 1.0)`; commit 82a142ad (bugfix) diff shows `- scaled = np.clip(frame, 0.0, 1.0)` / `+ scaled = np.clip(frame.astype(common_dtype, copy=False), 0.0, 1.0)`. PR body reports 'conversion latency: 79.17 s -> 0.97 s', 'peak process RSS: 14.86 GiB -> 9.06 GiB', 'output checksum: identical' with no artifact link or measured-commit statement.) + +`vllm_omni/entrypoints/openai/video_api_utils.py:435` [minor] — The diff computes `common_dtype = np.result_type(*(frame.dtype for frame in frames))` and scales each frame into the uint8 buffer, moving dtype promotion from `np.stack` to this loop. This helper is the shared output encoder for all video models (non-streaming via serving_video.py:281 `encode_video_base64`, streaming via serving_video_output_stream.py:469 `create_streaming_video_encoder`), but the PR's e2e/benchmark evidence covers only MiniMax H3 float32 1344x768; the streaming chunk path and float16 arithmetic are not exercised through a real mux. Add a float16 mux smoke (e.g. `video_api_utils._encode_video_bytes` over a list of float16 frames — the mixed-dtype unit test already proves the arithmetic) or scope the performance claim to float32. (evidence: video_api_utils.py:435 `common_dtype = np.result_type(*(frame.dtype for frame in frames))` — anchor line present in changed `_coerce_video_to_uint8_frames`. Unchanged consumers in PR-time tree: serving_video.py:281 `encode_video_base64(video, fps=artifacts.output_fps, video_codec_options=video_codec_options)`; serving_video_output_stream.py:469 `encoder = create_streaming_video_encoder(` (its encoder.encode calls `_coerce_video_to_uint8_frames` at video_api_utils.py:497). New tests only assert `_coerce_video_to_uint8_frames` directly (test_video_api_utils.py:65,76,91), not `_encode_video_bytes`/mux; PR body benchmark: 'Measured with a 362-frame 1344x768 MiniMax H3 float32 output'.) + +`tests/entrypoints/openai_api/test_video_api_utils.py:73` [minor] — PR description's test plan reports `pytest -q tests/entrypoints/openai_api/test_video_api_utils.py (11 passed)`, but final head has 12 test functions and `test_channel_last_video_tensor_preserves_channel_sized_frame_count` is parametrized over `[3, 4]`, so the file collects 13 items. Re-run the command at final head and report 13 passed. (evidence: tests/entrypoints/openai_api/test_video_api_utils.py:51 `def test_float_frames_are_converted_without_stacking_full_video(monkeypatch):`; tests/entrypoints/openai_api/test_video_api_utils.py:73 `def test_two_dimensional_width_four_frames_are_not_treated_as_rgba():`; tests/entrypoints/openai_api/test_video_api_utils.py:83 `def test_mixed_float_dtypes_preserve_stacked_rounding_semantics():`; tests/entrypoints/openai_api/test_video_api_utils.py:97 `@pytest.mark.parametrize("frame_count", [3, 4])`; PR body (not a file line): `- [x] \`pytest -q tests/entrypoints/openai_api/test_video_api_utils.py\` (11 passed)`) + +`tests/entrypoints/openai_api/test_video_api_utils.py:79` [minor] — `_coerce_video_to_uint8_frames` still promises `(F, H, W, 3)` output (video_api_utils.py:426), but `has_alpha = len(frame_shape) == 3 and frame_shape[-1] == 4` (video_api_utils.py:432) leaves 2D `(H, W)` frames untouched, producing `(F, H, W)` — the very shape the new test asserts at tests/entrypoints/openai_api/test_video_api_utils.py:79. Both downstream consumers reject that shape: `FragmentedMP4Muxer.mux_video_frames` raises `ValueError("video_frames must have shape (T, H, W, 3).")` (media_utils.py:60), and `mux_video_audio_bytes` would fail when indexing 2D slices for `av.VideoFrame.from_ndarray(..., format='rgb24')`. Reject `ndim == 2` in `_coerce_video_to_uint8_frames` or change the docstring and downstream contract to explicitly allow rank-preserving `(F, H, W)` output. (evidence: vllm_omni/entrypoints/openai/video_api_utils.py:426 `"""Convert a video payload into contiguous uint8 frames shaped (F, H, W, 3)."""`; vllm_omni/entrypoints/openai/video_api_utils.py:432 `has_alpha = len(frame_shape) == 3 and frame_shape[-1] == 4`; tests/entrypoints/openai_api/test_video_api_utils.py:79 `assert frames.shape == (2, 2, 4)`; vllm_omni/diffusion/utils/media_utils.py:60 `raise ValueError("video_frames must have shape (T, H, W, 3).")` (media_utils.py and the media_utils consumer line are unchanged by this diff and present in the PR-time tree).) + +`vllm_omni/entrypoints/openai/video_api_utils.py:453` [minor] — Add a float16-only unit test for `_coerce_video_to_uint8_frames` (feed a list of float16 frames so `common_dtype` stays float16) that exercises the `scaled *= 255.0; np.rint(scaled, out=scaled)` in-place rint path. The existing mixed float16+float32 test promotes to float32, so it never runs `np.rint` with a float16 `out`; the float16 endpoint's rint support remains unverified. (evidence: vllm_omni/entrypoints/openai/video_api_utils.py:435 `common_dtype = np.result_type(*(frame.dtype for frame in frames))`; vllm_omni/entrypoints/openai/video_api_utils.py:453 `np.rint(scaled, out=scaled)`; tests/entrypoints/openai_api/test_video_api_utils.py:85 `np.full((1, 1, 3), 0.1, dtype=np.float16),` — only float16 occurrence in the test file, paired with a float32 frame in the mixed test, so no pure-float16 path reaches line 453.) + +`vllm_omni/entrypoints/openai/video_api_utils.py:~434` [nit] — The per-frame loop in `_coerce_video_to_uint8_frames` only avoids the full-video float stack; `_coerce_video_to_frames` still returns `list(video_array)` (video_api_utils.py:384 and :390), i.e. frame views over the full normalized float32 video, so that ~4.18 GiB buffer (362×1344×768×3×4 bytes) remains live alongside the new `frames_u8` output while the loop runs. This is pre-existing input materialization, not a regression; if end-to-end peak memory is the goal, confirm the 9.06 GiB measurement includes it and consider per-frame normalization/coercion as a follow-up. (evidence: ['vllm_omni/entrypoints/openai/video_api_utils.py:434 `frames_u8 = np.empty((len(frames), *output_shape), dtype=np.uint8)` — in this diff.', 'vllm_omni/entrypoints/openai/video_api_utils.py:429 `frames = _coerce_video_to_frames(video)` — the conversion loop consumes this list.', 'vllm_omni/entrypoints/openai/video_api_utils.py:384 `return list(video_array)` — unchanged by this diff, present in the PR-time tree; torch path returns frame views over the full normalized float video.', 'vllm_omni/entrypoints/openai/video_api_utils.py:390 `return list(video_array)` — unchanged by this diff, present in the PR-time tree; ndarray ndim==4 path likewise returns views over the full array.']) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.trace.json.gz new file mode 100644 index 000000000..6c8b93afe Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5732.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.cost.json new file mode 100644 index 000000000..168d37317 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2139.1, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr5752/run-20260816-003553-d1ebbe", + "recorded_at": "2026-08-16T01:11:30", + "attempts": [ + { + "invocation_id": "inv-ade49364", + "outcome": "run:run-20260816-003553-d1ebbe", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-003553-d1ebbe", + "llm_calls": 179, + "input_tokens": 2055988, + "output_tokens": 359463, + "usd": 1.551918, + "source": "spans" + } + ], + "attempt_usd_total": 1.551918, + "llm_calls": 179, + "input_tokens": 2055988, + "output_tokens": 359463, + "usd": 1.551918, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-003553-d1ebbe", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.5519, + "minutes": 35.61, + "ci_minutes": 0, + "input_tokens": 2055988, + "output_tokens": 359463, + "tool_calls": 282, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3341, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 179, + "by_role": { + "agent": { + "usd": 1.493844, + "input_tokens": 1995525, + "output_tokens": 321575, + "calls": 176 + }, + "reducer": { + "usd": 0.058074, + "input_tokens": 60463, + "output_tokens": 37888, + "calls": 3 + } + }, + "cache_read_tokens": 22273792, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.05, + "gate": 1.6, + "review": 2130.93, + "report": 0.06 + }, + "time_to_first_result_s": 2136.62 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5752.trace.json.gz", + "trace_bytes": 73656 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.md new file mode 100644 index 000000000..9d958cdcc --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.md @@ -0,0 +1,73 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5752, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 57bcbfe1bcc1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 4 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] T2VA aspect_ratio requirement is documented in recipes and enforced, but it is enforced even when explicit width/height are supplied — residual breaking change for existing T2VA callers (see FP8 test comment). +- [resolved] Uploaded multipart file cleanup: _persist_uploaded_media_references unlinks on exception, and _cleanup_video_references now deletes reference_audio.cleanup_paths in both async finally and sync finally paths. +- [resolved] api_server.py imports `from numbers import Integral` (line 20) and uses it in `_read_upload_limited` (line 3027); no NameError. +- [claim-verified] Diff delivers stated scope: _resolve_fl2va_keyframe_indices, _validate_ref2va_reference_counts, start_time_seconds/long-ref, duration/aspect/short_edge, num_outputs fanout, recipes and model metadata all present (pipeline_minimax_h3.py, reference_video.py, api_server.py, docs). +- [claim-verified] accuracy test MODEL_REVISION='main' + blob->resolve URL rewrite + SHA256 pin are coherent; digest assert guards asset drift (test_minimax_h3_t2va_similarity.py:98-112). +- [claim-verified] num_outputs_per_prompt 1..10 validation and seed+output_index fanout implemented and tested (test_g1_fanout_uses_incrementing_output_seeds). +- [claim-verified] Author test count 104 passed/1 skipped is consistent with 403 new contract lines + 113 new video-server lines; not re-run (read-only). +- [claim-verified] max_multimodal_image_inputs=9 for MiniMaxH3Pipeline is asserted by test_minimax_h3_advertises_the_official_ref2va_image_limit. +- [claim-verified] CI: buildkite/vllm-omni SUCCESS, pre-commit/DCO/builds SUCCESS; buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci FAILURE (from gh_ci_read + gate_report). +- [claim-refuted] PR '104 passed, 1 skipped' does not reconcile: test_minimax_h3_e2e.py has two skipif(MODEL_ENV) tests (ulysses8 + fp8), so with MODEL_ENV unset both skip (2 skips); more importantly the FP8 smoke is now broken when actually run (review comment). +- [claim-refuted] Docs say HEIC/HEIF are supported (recipes/MiniMaxAI/MiniMax-H3.md:370 and MINIMAX_H3_REFERENCE_IMAGE_FORMATS) but repo-wide grep found no pillow-heif dependency; Pillow's Image.open will raise UnidentifiedImageError for those formats. +- [validated] _resolve_output_canvas area clamp re-derived: 16:9->(768,1344), 21:9->(672,1536), 9:16->(1344,768) match new contract tests (pipeline_minimax_h3.py:422-445). +- [validated] minimax_h3_align_frame_count(96)=107 matches the ulysses8 smoke 107-frame assertion (time_request.py:5-11). +- [validated] condition_noise full_t=max(target_latent_t+imgvid_cond_num_frames, latent_t) keeps the short-ref RNG stream identical and only widens long refs; noise still sliced [:, :, :latent_t] (condition_noise.py:86-92). + +`vllm_omni/entrypoints/openai/api_server.py:~3330` [blocker] — The shared typed-reference branch in `_parse_video_form` (api_server.py:3330-3331 and 3350-3351) appends `media_data.source_path` to `video_paths` for every model, and `decode_video_url` (video_api_utils.py:226) now always returns `VideoFrames(source_path=)` for URL/data-URL video. Only multipart `input_references` is gated on `supports_mixed_reference_inputs`; typed `video_reference` is not. Generic models (e.g., Wan VACE, Cosmos3) that consume `multi_modal_data["video"]` as decoded frames now receive a list of .mp4 path strings, which PIL cannot open. Gate the source_path branch on `supports_mixed_reference_inputs` and keep `ReferenceVideo(data=list(media_data))` frames for generic models. (evidence: ['vllm_omni/entrypoints/openai/api_server.py:3330-3331 (changed by this diff): `if media_data.source_path is not None:` / ` video_paths.append(media_data.source_path)` — unguarded typed-video_reference branch; the only `supports_mixed_reference_inputs` gate in this function is the multipart `input_references` branch (line 3295).', 'vllm_omni/entrypoints/openai/video_api_utils.py:226 (changed by this diff): `path = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4").name` — `decode_video_url` writes every data:/http(s) video to a temp .mp4 and returns `VideoFrames(source_path=path)`, so generic models receive .mp4 path strings instead of decoded frames.']) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:288` [blocker] — Making `aspect_ratio` mandatory for T2VA breaks the in-repo ComfyUI client: `generate_video` sets `video_task = "t2va"` for the no-frame/no-references path, always sends `width`/`height`/`num_frames`/`fps`, and `_minimaxh3_params_builder` only forwards `flow_shift`/`audio_flow_shift`/`task` — never `aspect_ratio` or `duration`. `_resolve_shape` resolves the aspect ratio before using the supplied width/height, so every ComfyUI T2VA request now fails with `t2va requires an explicit aspect_ratio` (and `num_frames < 96` also falls below the 4-second floor). Emit `aspect_ratio`/`duration` in the client or accept a T2VA default when explicit `width`/`height` are supplied before merging. (evidence: ['vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:290 — in this PR\'s diff: `if value is None:` / `if task == "t2va":` / `raise OmniClientError("t2va requires an explicit aspect_ratio")`.', "vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py (in `_resolve_shape`, this PR's diff; ~line 652) — `aspect_ratio = _resolve_minimax_h3_aspect_ratio(task, aspect_ratio, image)` is called before the `if height is None or width is None:` branch, so supplied width/height do not bypass the mandatory ratio.", 'apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py:330 — unchanged by this diff, present in the PR-time tree: `else:` / `video_task = "t2va"`; earlier in the function it always adds `form.add_field("width", str(width))`, `height`, `num_frames`, `fps`.', 'apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:160-169 — unchanged by this diff: `_minimaxh3_params_builder` only does `form_fields["flow_shift"] = params.pop("flow_shift")` and, for `("audio_flow_shift",)`, `merged_extra_params[key] = params.pop(key)`; repo grep for `aspect_ratio` under apps/ComfyUI-vLLM-Omni returned no matches.']) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:~118` [major] — `test_minimax_h3_t2va_fp8_single_gpu_smoke` still passes `num_frames=29` with no `duration` and no `aspect_ratio`. Under the new `_resolve_shape`, `_resolve_minimax_h3_aspect_ratio("t2va", None, None)` raises `OmniClientError` at pipeline_minimax_h3.py:290, and even with a ratio, `29/24=1.208s` fails the 4-15s duration gate at line 636. When MODEL_ENV is set this test errors instead of generating. Mirror the ulysses8 smoke: add `"duration": 4.0, "aspect_ratio": "16:9"` and update the 39-frame expectation to 107. (evidence: unchanged by this diff, present in the PR-time tree: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:118 ` num_frames=29,` with no `duration`/`aspect_ratio` in the following extra_args (lines 120-122: ` "task": "t2va",` / ` "flow_shift": 12.0,` / ` "audio_flow_shift": 3.0,`). Changed by this diff: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:290 ` raise OmniClientError("t2va requires an explicit aspect_ratio")` and vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:636 ` if not MINIMAX_H3_MIN_OUTPUT_SECONDS <= requested_frames / fps <= MINIMAX_H3_MAX_OUTPUT_SECONDS:`) + +`vllm_omni/entrypoints/openai/api_server.py:3290` [major] — Apply the same `_read_upload_limited` size cap used for multipart uploads to `decode_video_url`/`decode_audio_url`: these URL/data-URL paths currently buffer the whole HTTP response with no cap before H3 pipeline validation, so an oversized video/audio reference can exhaust memory before any declared/actual-size guard runs. (evidence: [resource] decode_video_url/decode_audio_url buffer whole HTTP responses with no size cap before H3 pipeline validation, unlike multipart which uses _read_upload_limited.) + +`tests/entrypoints/openai/test_video_server.py:210` [minor] — test_r5 (tests/entrypoints/openai_api/test_video_server.py:520) and test_r10 (model_class_name set at :1262) run through the test_client fixture whose `_engine_client` is `FakeAsyncOmni`; both tests merely flip `_engine_client.model_class_name = 'MiniMaxH3Pipeline'` on the fake and assert against its captured fields. The fake's `generate` re-reads `Path(item).read_bytes()` from the temp files the serving layer persisted, so `b'ftyp'` only proves an MP4 was written, not that MiniMaxH3Pipeline consumed typed video/audio references. Add a pipeline-level consumer test or narrow the coverage claim to serving-layer routing. (evidence: tests/entrypoints/openai_api/test_video_server.py:520 `def test_r5_mixed_image_video_audio_references_and_fanout(test_client, mocker: MockerFixture):` ; tests/entrypoints/openai_api/test_video_server.py:525 `test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"` ; tests/entrypoints/openai_api/test_video_server.py:1262 `test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"` — both tests set a model name on the FakeAsyncOmni engine client rather than instantiating MiniMaxH3Pipeline, and `FakeAsyncOmni.generate` only records `self.captured_prompt`/`self.captured_sampling_params_list` and re-reads the persisted temp files, so the real pipeline consumer is never exercised.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:123` [minor] — PR-body '104 passed, 1 skipped' is unreconcilable: test_minimax_h3_e2e.py has two MODEL_ENV skipif tests (ulysses8 at :30-34 and fp8 at :85-89), so MODEL_ENV unset yields ≥2 skips; with MODEL_ENV set, the fp8 smoke's t2va extra_args (test_minimax_h3_e2e.py:123-127) lacks aspect_ratio and now raises 't2va requires an explicit aspect_ratio' (pipeline_minimax_h3.py:290). Recompute counts after fixing the fp8 smoke. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:30-34 `@pytest.mark.skipif( + not os.environ.get(MODEL_ENV), ...) +def test_minimax_h3_t2va_ulysses8_smoke():`; :85-89 `@pytest.mark.skipif( + not os.environ.get(MODEL_ENV), ...) +def test_minimax_h3_t2va_fp8_single_gpu_smoke():`; :123-127 (fp8, unchanged by this diff but present in PR-time tree) `extra_args={ + "task": "t2va", + "flow_shift": 12.0, + "audio_flow_shift": 3.0, + },`; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:290 `raise OmniClientError("t2va requires an explicit aspect_ratio")`, reached from :652 `aspect_ratio = _resolve_minimax_h3_aspect_ratio(task, aspect_ratio, image)` with aspect_ratio=None in the fp8 path.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~1441` [minor] — Standalone audio validation and decoding run on every DiT rank with no rank gate: `pipeline_minimax_h3.py:1441-1443` calls `validate_reference_audio_files(raw_audio)` and `_load_audios(raw_audio)` unconditionally. `validate_reference_audio_files` shells out to `ffprobe` per path (`reference_video.py:201`) and `_load_audios`/`_load_audio`/`load_audio_file` fully decodes path-backed files, while the video path is explicitly rank-gated (`_prepare_reference_videos` returns `None` on `rank != 0`). In a multi-node deployment the API-server temp file exists only on the rank-0 host, so non-zero ranks would raise. Gate the standalone-audio validation/load to rank 0 and let `_encode_audio_conditions` broadcast the result (it already broadcasts on `rank == 0`), or document the single-node assumption. (evidence: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1441 `if raw_audio is not None:` ; :1442 `validate_reference_audio_files(raw_audio)` ; :1443 `standalone_audios = _load_audios(raw_audio) if raw_audio is not None else []` — no rank check around these calls. Contrast with the video path in the same function: `prepared_videos = self._prepare_reference_videos(...)` and, at lines 924-925, `if rank != 0:` / `return None` inside `_prepare_reference_videos`. `validate_reference_audio_files` (vllm_omni/diffusion/models/minimax_h3/reference_video.py:201) runs `subprocess.run(["ffprobe", ...])` per path.) + +`vllm_omni/entrypoints/openai/api_server.py:~3353` [minor] — For MiniMax-H3, a video uploaded through the singular `input_reference` field is decoded by `decode_input_reference(None, None, input_reference_bytes, ...)`, which returns `VideoFrames` with `source_path=None` because only `decode_video_url` materializes a temp path. api_server.py:3353 then stores `list(media_data)` (PIL frames) as the reference video, and H3's `prepare_reference_videos` rejects any non-path item at reference_video.py:341 with `MiniMax H3 Ref2VA video references require file paths`. Either write `input_reference_bytes` to a temp file and set `source_path` when `supports_mixed_reference_inputs` and the upload is a video, or reject video-in-`input_reference` for H3 with a clear 400. (evidence: vllm_omni/entrypoints/openai/api_server.py:3353 `video_frames = list(media_data)` (inside the `else` of `if media_data.source_path is not None`); vllm_omni/entrypoints/openai/video_api_utils.py:335 `return _decode_media_bytes(` — `decode_input_reference` sends `input_reference_bytes` through `_decode_media_bytes`, which calls `_decode_video_bytes(media_bytes, source=source, max_frames=max_video_frames, keep=video_keep)` without a `source_path` argument, so the returned `VideoFrames.source_path` remains None; vllm_omni/diffusion/models/minimax_h3/reference_video.py:341 `f"MiniMax H3 Ref2VA video references require file paths, got item {index}: {type(value)!r}"` — rejects the non-path PIL frame items.) + +**Additional observations (beyond the comment budget):** +- `recipes/MiniMaxAI/MiniMax-H3.md:367` [minor] recipes/MiniMaxAI/MiniMax-H3.md:367 advertises HEIC/HEIF reference images, and vllm_omni/entrypoints/openai/api_server.py:166 plus vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:104 whitelist 'heic'/'heif', +- `vllm_omni/entrypoints/openai/serving_video.py:180` [minor] serving_video.py:180-187 writes extra_args['duration'], 'aspect_ratio', 'short_edge', and 'start_time_seconds' for every diffusion model whenever those fields are supplied, with no model gate. pipeline_minimax_h3.py:615 +- `.buildkite/amd/test-amd-merge.yml:21` [minor] The buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci checks are red. The new `test_minimax_h3_contract.py` tests carry `pytestmark = [core_model, cpu, diffusion]`, so the AMD lane's `.buildkite/amd/test-amd-merg +- `vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1532` [minor] The new `num_outputs_per_prompt` fanout loop (diffuse + decode + `torch.cat`) at pipeline_minimax_h3.py:1532 is not exercised by any test: `test_g1_fanout_uses_incrementing_output_seeds` only asserts the helper returns ` + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.trace.json.gz new file mode 100644 index 000000000..86b844d50 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5752.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.cost.json new file mode 100644 index 000000000..dea7e166f --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 892.2, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr5871/run-20260816-003553-5d80cf", + "recorded_at": "2026-08-16T00:50:43", + "attempts": [ + { + "invocation_id": "inv-bb5968f0", + "outcome": "run:run-20260816-003553-5d80cf", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-003553-5d80cf", + "llm_calls": 53, + "input_tokens": 254365, + "output_tokens": 113623, + "usd": 0.264992, + "source": "spans" + } + ], + "attempt_usd_total": 0.264992, + "llm_calls": 53, + "input_tokens": 254365, + "output_tokens": 113623, + "usd": 0.264992, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-003553-5d80cf", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.265, + "minutes": 14.83, + "ci_minutes": 0, + "input_tokens": 254365, + "output_tokens": 113623, + "tool_calls": 141, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5374, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 53, + "by_role": { + "agent": { + "usd": 0.239731, + "input_tokens": 222745, + "output_tokens": 98457, + "calls": 51 + }, + "reducer": { + "usd": 0.025261, + "input_tokens": 31620, + "output_tokens": 15166, + "calls": 2 + } + }, + "cache_read_tokens": 2641792, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 31.02, + "gate": 1.92, + "review": 856.69, + "report": 0.07 + }, + "time_to_first_result_s": 889.67 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5871.trace.json.gz", + "trace_bytes": 53262 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.md new file mode 100644 index 000000000..014b9b276 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5871, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6c00fecff92e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] design-contracts.md has an absent-page fallback: 'Use current code and tests when a page is draft, deferred, stale, absent, or conflicts with the frozen implementation' — but the metadata resolution steps (2–3) still no-op on this repo. +- [claim-verified] 33 files / 1639 insertions equals PR body's '31 references + 1,639 total lines': diff_stat lists 33 changed files, 1639 insertions (SKILL.md 231, openai.yaml 4). +- [claim-verified] '231-line core workflow': SKILL.md diff stat shows 231 added lines. +- [claim-verified] Five shallow groups, no folder indexes/deeper nesting: references/ contains only checks/, delivery/, features/, modules/, process/ (31 .md files; no nested dirs). +- [claim-verified] No personal workflow scripts/baseline: grep 'hsliuustc' and 'vllm-omni-skills' over .claude/skills -> 0 matches; diff adds only .md/.yaml. +- [claim-verified] 'review-pr already listed in .claude/skills/readme.md': readme.md Available Skills has 'review-pr: provides a structured workflow for reviewing pull requests'. +- [claim-verified] 500-LOC RFC policy is real: docs/contributing/README.md:165 '>500 LOC excluding kernel/data/config/test ... GitHub issue (RFC)'. +- [claim-verified] recipes links resolve: recipes/TEMPLATE.md and recipes/README.md exist; README.md:33 '## Available Recipes'. +- [claim-verified] CODEOWNERS exists with last-match path rules incl. /.claude/: .github/CODEOWNERS:1 and :13. +- [claim-verified] Relative links resolve: grep of '](references/' and '](../' shows every target exists (31 refs; sibling skills add-tts-model, diffusion-perf-opt, vllm-omni-test, quantization, vllm-omni-npu-upgrade; recipes/*). +- [claim-verified] Taxonomy arithmetic: 16 module refs + diffusion.md's 5 submodule designs = 21; features 5(runtime)+6(comm)+13(diff-accel)+2(infra)=26 — matches '21/21 + 26/26 represented'. +- [claim-verified] CI green: gh_ci_read 5871 -> pre-commit, build (3.12), build (3.11), DCO, docs/readthedocs all SUCCESS. +- [claim-verified] PR body '1,639 total lines': summing all 33 new-file hunk line counts = 1639 (calc). +- [claim-verified] PR body '231-line core workflow': SKILL.md hunk header is @@ -0,0 +1,231 @@. + +`.claude/skills/review-pr/SKILL.md:2` [minor] — PR body Test Plan references `quick_validate.py`, but a repo-wide grep finds no such file in the PR-time tree, so the 'Skill validation: passed' step cannot be reproduced from the branch. Either commit the validator (with source/version pinned) or drop that step and rely on the passing pre-commit check-yaml gate. (evidence: .claude/skills/review-pr/SKILL.md:2 `name: review-pr` — present in the PR-time tree and in the diff. PR 5871 body (metadata, not a repo file): Test Plan `- Validate skill frontmatter and structure with `quick_validate.py`.` and Test Result `- Skill validation: passed.` — verbatim. Repo-wide grep for `quick_validate` in the PR-time tree returns 0 matches; gh_ci_read 5871 shows `pre-commit` state SUCCESS.) + +`.claude/skills/review-pr/references/process/review-routing.md:59` [minor] — Normalize the skill-invocation convention in `review-routing.md`: the table rows and `Use [...]` lines write skill names bare (`add-tts-model`, `diffusion-perf-opt`, `vllm-omni-test`, `quantization`), but line 59 writes `$vllm-omni-npu-model-runner-upgrade` with a literal `$` prefix. Pick one convention and apply it consistently across the routing table so sibling-skill references are unambiguous. (evidence: .claude/skills/review-pr/references/process/review-routing.md:58 `Use [`quantization`](../../../quantization/SKILL.md) for a quantization module` versus .claude/skills/review-pr/references/process/review-routing.md:59 `change and [`$vllm-omni-npu-model-runner-upgrade`](../../../vllm-omni-npu-upgrade/SKILL.md)`; also line 51 `| New or expanded model, loader, processor, registry, or stage config | [model-addition-checklist.md](../checks/model-addition-checklist.md) | [`add-tts-model`](../../../add-tts-model/SKILL.md) or [`add-diffusion-model`](../../../add-diffusion-model/SKILL.md) |` — all in the PR-time tree and part of the added file.) + +`.claude/skills/review-pr/references/process/design-contracts.md:16` [minor] — design-contracts.md:16-18 tells reviewers to read `primary_code_paths`/`status`/`architecture_state`/`last_verified_commit` from the reviewed head's docs/design pages, but at this head `docs/design/index.md` has exactly four module docs (ar_module, dit_module, entrypoint_module, async_omni_architecture) and none define that metadata — grep shows these keys appear only in the new design-contracts.md. The same gap affects the `owners`/`required_reviewers` lookup in review-requests.md, so steps 2-3 no-op on every PR in this repo. Either ship the updated design pages/index in this PR, or mark the module table/routing as published-latest-only discovery and add an explicit head-side mapping. (evidence: NEW IN DIFF: .claude/skills/review-pr/references/process/design-contracts.md:16 `2. Match changed behavior against each module page's `primary_code_paths`,`, :18 `3. Read `status`, `architecture_state`, `last_verified_commit`, and any linked`. UNCHANGED BY DIFF, present in PR-time tree: docs/design/index.md:62 `## Module Design Documents` followed by exactly four entries (lines 66-72): `- [AR Module](module/ar_module.md)`, `- [DIT Module](module/dit_module.md)`, `- [Entrypoint Module](module/entrypoint_module.md)`, `- [AsyncOmni Architecture (Qwen3-Omni Example)](module/async_omni_architecture.md)`. docs/design/module/ar_module.md:1 `# AutoRegressive (AR) Module` — plain prose; repo-wide grep finds `primary_code_paths`, `last_verified_commit`, and `architecture_state` only in the new design-contracts.md, and `required_reviewers` only in the new review-requests.md, so the metadata the skill tells reviewers to consume does not exist at this head.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.trace.json.gz new file mode 100644 index 000000000..d3f6f63dc Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5871.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.cost.json new file mode 100644 index 000000000..9df210a74 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1166.6, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr5946/run-20260816-005045-e3b6fe", + "recorded_at": "2026-08-16T01:10:09", + "attempts": [ + { + "invocation_id": "inv-8d11ab18", + "outcome": "run:run-20260816-005045-e3b6fe", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-005045-e3b6fe", + "llm_calls": 107, + "input_tokens": 675684, + "output_tokens": 175491, + "usd": 0.596372, + "source": "spans" + } + ], + "attempt_usd_total": 0.596372, + "llm_calls": 107, + "input_tokens": 675684, + "output_tokens": 175491, + "usd": 0.596372, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-005045-e3b6fe", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5964, + "minutes": 19.4, + "ci_minutes": 0, + "input_tokens": 675684, + "output_tokens": 175491, + "tool_calls": 198, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7666, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 107, + "by_role": { + "agent": { + "usd": 0.566479, + "input_tokens": 665369, + "output_tokens": 150885, + "calls": 105 + }, + "reducer": { + "usd": 0.029893, + "input_tokens": 10315, + "output_tokens": 24606, + "calls": 2 + } + }, + "cache_read_tokens": 8181376, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.53, + "gate": 2.01, + "review": 1158.31, + "report": 0.06 + }, + "time_to_first_result_s": 1163.89 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5946.trace.json.gz", + "trace_bytes": 29530 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.md new file mode 100644 index 000000000..c19da4bd2 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.md @@ -0,0 +1,67 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5946, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 2e4207726863) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Removed sibling 'Modular H3' note (commit bc265de5) is not a defect at this head: _minimax_h3_partition_for_task auto-branch still detects FL2VA/Ref2VA from the serve path (pipeline_minimax_h3.py), so one-partition behavior holds; residual: if #5720 later changes the auto default, this recipe (like 5090) would need --task-type. +- [claim-verified] 'Denoise dominates at 96% of end-to-end': 2088/2169.4 = 96.2% (calc). +- [claim-verified] Variance 'up to 23%': (490-397)/397 = 23.4% (calc). +- [claim-refuted] Recipe line 130 states the VLLM_OMNI_VIDEO_SYNC_TIMEOUT default is 1800 s; the code default is 600.0 s (vllm_omni/entrypoints/openai/api_server.py:3023). +- [claim-refuted] Per-step 42.61 s vs Denoise 2088 s / 50 steps: 42.61*50 = 2130.5 != 2088; 2088/49 = 42.612, so the column uses a steps-1 denominator that the methodology section never states. +- [claim-refuted] VLLM_OMNI_VIDEO_SYNC_TIMEOUT default is 600.0 s (api_server.py:3023), not the 1800 s the doc states at line 130. +- [claim-refuted] Latency table: 50*42.61=2130.5 != Denoise 2088 s; 2088/50=41.76 while 42.61=2088/49 — because denoise loop runs len(sigmas)-1=49 iterations for 50 sigma points (denoise_loop.py:201; time_request.py:47). +- [claim-refuted] '128 GB (~121 GiB usable)': 128 GB decimal = 119.2 GiB binary, so 121 GiB overstates even raw capacity (calc: 128e9/2**30=119.21). +- [claim-refuted] 'A BF16 partition is 135 GiB and does not fit in 121 GiB' reuses disk storage (main recipe: ~134 GiB safetensors/~135 GiB on disk, MiniMax-H3.md:72); as memory, 135-31=104 GiB FP8 weights > 97.7 GiB reported peak, impossible. +- [validated] t2va aspect_ratio gate exists verbatim: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:353 `raise OmniClientError("t2va requires an explicit aspect_ratio")`; form `aspect_ratio` parsed at api_server.py:3203 and mapped into extra_args at serving_video.py:182-183. +- [validated] Recipe's width/height precedence claim matches _resolve_shape: width/height are read before ratio resolution and used directly when both supplied (pipeline_minimax_h3.py _resolve_shape), while ratio is still validated for t2va. +- [validated] X-Stage-Durations and X-Peak-Memory-MB are set on POST /v1/videos/sync: vllm_omni/entrypoints/openai/api_server.py:3566-3567 inside create_video_sync. +- [validated] X-Peak-Memory-MB semantics match recipe: diffusion_model_runner.py:369-376 documents max_memory_reserved CUDA pool high-water and samples it per request after reset_peak_memory_stats. +- [validated] CLI flags exist: vllm_omni/entrypoints/cli/serve.py:295 (--init-timeout), :739 (--vae-parallel-mode), :549 (--diffusion-attention-backend); --quantization/--vae-use-tiling/--usp/--ring/--text-encoder-tp-size/--enable-cpu-offload/--enable-distributed-layerwise-offload appear in sibling recipes and pipeline config. + +`recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:130` [minor] — recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:130 states the default for VLLM_OMNI_VIDEO_SYNC_TIMEOUT is 1800 s, but the API server reads `float(os.environ.get('VLLM_OMNI_VIDEO_SYNC_TIMEOUT', 600.0))`, so the real default is 600 s. Change 'default 1800 s' to 'default 600 s'; the guidance to set the timeout above the measured 2169.4 s 50-step run, and the already exported 7200 s in this recipe's serve command, remain correct. (evidence: recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:130 `Set `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` well above the expected request time — the default 1800 s is shorter than a 50-step run on this platform.`; vllm_omni/entrypoints/openai/api_server.py:3023 (unchanged by this diff, present in the PR-time tree) `VIDEO_SYNC_TIMEOUT_S = float(os.environ.get("VLLM_OMNI_VIDEO_SYNC_TIMEOUT", 600.0))`) + +`recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:104` [minor] — Line 104's `Steps 50` / `Per step 42.61 s` are consistent only as 49 denoise iterations (2088/49 = 42.61), because denoise_loop.py:201 runs `num_steps = len(sigmas_video) - 1` and a 50-point sigma schedule yields 49 loop iterations. Annotate the row as '50 requested steps -> 49 denoise iterations; 42.61 s/iteration' (do not relabel Per step as 41.76 s). Also state the geometry for 'Repeated identical 10-step requests': 397-490 s denoise implies the 960x576/8 s 10-step run, not the 4 s smoke shape. (evidence: ['unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/denoise_loop.py:201 ` num_steps = len(sigmas_video) - 1`', 'unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/time_request.py:47 ` base = torch.linspace(` (linspace(1.0, 0.0, int(num_steps))); with num_steps=50 this yields 50 sigma points and the later guard appends 0.0 only when shifted[-1] > 0.0, which is false, so the schedule stays at 50 points -> 49 loop iterations', "calc: 2088/49 = 42.61224489795919 (matches the doc's 42.61 s); 2088/50 = 41.76; 50*42.61 = 2130.5"]) + +`recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:108` [minor] — Confirmed. The serve command (lines 43–50) never passes `--enable-diffusion-pipeline-profiler`, and with the flag off `setup_diffusion_pipeline_profiler` returns before creating `_stage_durations` (diffusion_pipeline_profiler.py:90-92), so MiniMaxH3Pipeline.forward emits `stage_durations={}` (pipeline_minimax_h3.py:1801). A user following the command verbatim cannot reproduce the latency table. Add the flag to the serve command, or explicitly state that reproducing the measurements requires starting the server with `--enable-diffusion-pipeline-profiler`. (evidence: This diff: recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:108 `- Stage times come from \`--enable-diffusion-pipeline-profiler\`, read from the`; the serve command at :43-50 ends `--quantization fp8 --enforce-eager \` / `--diffusion-attention-backend CUDNN_ATTN` with no profiler flag. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/profiler/diffusion_pipeline_profiler.py:90-92 `if not enable_diffusion_pipeline_profiler:` / ` self.enable_diffusion_pipeline_profiler = enable_diffusion_pipeline_profiler` / ` return` — returns before `self._stage_durations: dict[str, float] = {}` is created. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1801 `stage_durations=(self.stage_durations if hasattr(self, "_stage_durations") else {}),` — with the flag off `_stage_durations` is absent, so the emitted value is `{}`.) + +`recipes/README.md:33` [minor] — Add [`MiniMaxAI/MiniMax-H3-Spark-GB10.md`](./MiniMaxAI/MiniMax-H3-Spark-GB10.md) to the 'Available Recipes' table in recipes/README.md. The new GB10 recipe is not listed; MiniMax-H3-5090.md and MiniMax-H3-NPU.md are also omitted, so this is a non-blocking discoverability polish item. (evidence: recipes/README.md:33 `## Available Recipes`; recipes/README.md:48 `| [`MiniMaxAI/MiniMax-H3.md`](./MiniMaxAI/MiniMax-H3.md) | T2VA, FL2VA, image+audio Ref2VA, and multi-video Ref2VA serving | 1x GPU with CPU offload / 4x B300 |`; recipes/README.md:49 `| [`MiniMaxAI/MiniMax-H3-MUSA.md`](./MiniMaxAI/MiniMax-H3-MUSA.md) | MiniMax H3 T2VA, FL2VA, and Ref2VA on MUSA | 4x MTT S5000 (TP4 Ref2VA profile) |` — the PR-time index contains no MiniMax-H3-Spark-GB10 row even though the diff adds recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md (new file).) + +`recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:43` [minor] — Pin the one-partition behavior explicitly in the GB10 recipe: either document that `_minimax_h3_partition_for_task` auto-detects FL2VA/Ref2VA from the serve path, or pass `--task-type fl2va` (and `--task-type ref2va` for the Ref2VA restart) so the recipe does not break if #5720 later changes the auto default. (evidence: recipe is the PR-added file; the pipeline code is unchanged by this diff but present in the PR-time tree. recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:43 `vllm serve /path/to/MiniMax-H3/FL2VA \` — the command passes no `--task-type`. vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:134 `task = str(task_type or "auto").lower()` ; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:135 `if task == "auto" and model is not None:` ; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:136 `path = Path(model)` ; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:137 `if path.is_dir() and path.name in {"FL2VA", "Ref2VA"} and (path / "model_index.json").is_file():` ; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:138 `return path.name.lower()` ; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:572-574 `self.partition = _minimax_h3_partition_for_task(` / `getattr(od_config, "task_type", None),` / `str(od_config.model),`.) + +`recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:28` [minor] — Line 28 says 'A BF16 partition is 135 GiB and does not fit in 121 GiB', but 135 GiB is the on-disk/checkpoint-storage figure (table line 13; MiniMax-H3.md:72-73: ~134 GiB of BF16 safetensors, ~135 GiB on disk). For a resident-memory argument, state the BF16 resident estimate instead: 97.7 GiB measured FP8 peak + ~31 GiB DiT FP8 saving = ~128.7 GiB (still > 121 GiB), or note that the BF16 weights alone are ~134 GiB. (evidence: recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:28 `- **`--quantization fp8` is mandatory.** A BF16 partition is 135 GiB and does not` ; recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:29 ` fit in 121 GiB. Online FP8 quantizes the DiT only (62 GiB to ~31 GiB); the` ; recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:13 `| Checkpoint storage | 135 GiB per partition |` ; recipes/MiniMaxAI/MiniMax-H3.md:72-73 (unchanged by this diff, present in the PR-time tree) `H3 checkpoint partition (`FL2VA` or `Ref2VA`) contains about **134 GiB** of` / `BF16 safetensors (about **135 GiB** on disk). Keeping both partitions`) + +`recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:147` [nit] — Confirmed. The in-tree guard rejects only DLO+AllGather+online quantization (diffusers_loader.py:382-389, and the same check in distributed_layerwise_backend.py:1415-1426), and the error explicitly suggests `--dlo-no-use-allgather` or disabling online quantization. Please align the line 147 paragraph with this dtype/layout guard and its workaround rather than a broad 'incompatible with layerwise offload' Cutlass stride claim. (evidence: vllm_omni/diffusion/model_loader/diffusers_loader.py:382-389 (unchanged by this diff, present in the PR-time tree): `if _dist_offload and _use_ag and _has_online_quant: + raise ValueError( + "Online quantization is incompatible with DLO+AllGather: " + "the sharding + AllGather mechanism flattens weights by " + "dtype, which breaks quantized weight/scale layouts. " + "Please use --dlo-no-use-allgather or disable online " + "quantization." + )`; vllm_omni/diffusion/offloader/distributed_layerwise_backend.py:1415-1426 (unchanged by this diff, present in the PR-time tree): `if self.config.dlo_use_allgather and self.dp_size > 1:` ... `raise ValueError( + "Online quantization is incompatible with DLO+AllGather: " ... "Please use --dlo-no-use-allgather or disable online " + "quantization.")`; recipes/MiniMaxAI/MiniMax-H3-Spark-GB10.md:147: `- Online FP8 is incompatible with layerwise offload — the offload path produces a + weight stride the Cutlass FP8 kernel rejects. This is not a practical`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.trace.json.gz new file mode 100644 index 000000000..388f6e4e8 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5946.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.cost.json new file mode 100644 index 000000000..0ce19cb9d --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1120.2, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr5983/run-20260816-005313-e33220", + "recorded_at": "2026-08-16T01:11:51", + "attempts": [ + { + "invocation_id": "inv-bf4ba8c3", + "outcome": "run:run-20260816-005313-e33220", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-005313-e33220", + "llm_calls": 88, + "input_tokens": 504637, + "output_tokens": 201503, + "usd": 0.477054, + "source": "spans" + } + ], + "attempt_usd_total": 0.477054, + "llm_calls": 88, + "input_tokens": 504637, + "output_tokens": 201503, + "usd": 0.477054, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-005313-e33220", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4771, + "minutes": 18.63, + "ci_minutes": 0, + "input_tokens": 504637, + "output_tokens": 201503, + "tool_calls": 173, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7036, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 88, + "by_role": { + "agent": { + "usd": 0.446732, + "input_tokens": 489685, + "output_tokens": 177648, + "calls": 86 + }, + "reducer": { + "usd": 0.030322, + "input_tokens": 14952, + "output_tokens": 23855, + "calls": 2 + } + }, + "cache_read_tokens": 4412928, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.94, + "gate": 2.08, + "review": 1111.5, + "report": 0.07 + }, + "time_to_first_result_s": 1117.57 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5983.trace.json.gz", + "trace_bytes": 33424 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.md new file mode 100644 index 000000000..14593170a --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5983, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 8ec04e6c8dd2) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] `concurrent.futures.Future[_ResultT]` runtime subscription is safe: existing code already annotates `dict[str, concurrent.futures.Future[AsyncDiffusionOutput]]` in multiproc_executor.py, so the generic alias works on supported Python. +- [resolved] The PR's claim 'applied at every call site in _result_pump that resolves a future pulled from the shared dicts' holds within multiproc_executor.py; the gap is the separate DiffusionEngine resolver, not a missed _result_pump site. +- [resolved] Worker-process liveness is already monitored (_start_worker_monitor, multiproc_executor.py:368) but only watches p.sentinel, not the DiffusionResultPump threads — the observability gap survives the fix (minor comment below). +- [resolved] New module import is safe: future_utils.py has no `from __future__ import annotations`, but concurrent.futures.Future defines __class_getitem__=classmethod(types.GenericAlias) (Python 3.12 _base.py:687), so Future[_ResultT] evaluates at import time on 3.12; min-python 3.10 support is assumed (same stdlib feature). +- [claim-verified] diff_stat confirms exactly 3 files changed (109 insertions, 7 deletions): tests/diffusion/test_result_pump.py, vllm_omni/diffusion/executor/multiproc_executor.py, vllm_omni/diffusion/utils/future_utils.py. +- [claim-verified] PR 'left untouched the two set_result calls on freshly-created local Future()' — multiproc_executor.py:915 and :940 are fresh-local; :913 is a fresh-local set_exception; none shared pre-resolve. +- [claim-verified] gh_ci_read: buildkite/vllm-omni SUCCESS and buildkite/vllm-omni-intel-ci SUCCESS at head; amd-ci and npu-ci FAILURE. +- [claim-refuted] diffusion_engine.py _process_rpc_queue has the identical TOCTOU: its own comment at :571-574 says cancellation 'would kill the busy loop', yet it still does `if not fut.done(): fut.set_exception/set_result`; callers cancel task.future at :990 (sync) and :1014 (async). +- [claim-refuted] PR body says 'With the fix reverted: all 3 fail' — only test_batch_split_racing_cancel_does_not_crash_pump would fail; the two single-future tests assert only `fut.cancelled()` (true regardless) and _feed_one_msg_to_pump never inspects the pump thread's exception, so they pass even when the pump raises InvalidStateError. +- [claim-refuted] PR body names tests/diffusion/test_multiproc_result_pump.py; the diff actually modifies tests/diffusion/test_result_pump.py and no test_multiproc_result_pump.py exists (list_dir confirmed). +- [claim-refuted] PR body says tests were added in tests/diffusion/test_multiproc_result_pump.py; diff_stat/repo listing show they were appended to tests/diffusion/test_result_pump.py, and test_multiproc_result_pump.py does not exist at this head. +- [claim-refuted] 'With the fix reverted: all 3 fail' is not supported by the code: test_compute_done_racing_cancel (assert at :452) and test_output_ready_racing_cancel (assert at :467) only assert the already-set fut.cancelled(), and _feed_one_msg_to_pump joins the daemon pump thread without re-raising (test_result_pump.py:64), so an InvalidStateError in the thread would not fail those two tests; only test_batch_split_racing_cancel fails on revert (healthy_fut never resolved). +- [claim-refuted] PR says 'With the fix reverted: all 3 fail with InvalidStateError': only test_batch_split_racing_cancel (healthy_fut.done() at 486) fails on revert; test_compute_done (443) and test_output_ready (454) assert only fut.cancelled() (452/467), which is true before the pump ever runs. +- [claim-refuted] PR Testing section names tests/diffusion/test_multiproc_result_pump.py; list_dir(tests/diffusion) shows only test_result_pump.py, and the helper names are try_set_result/try_set_exception (no leading underscore). + +`vllm_omni/diffusion/diffusion_engine.py:~576` [major] — This diff closes the race in MultiprocDiffusionExecutor._result_pump, but DiffusionEngine._process_rpc_queue (busy-loop thread) still resolves futures with the same vulnerable check-then-act pattern: `if not fut.done(): fut.set_exception(exc)` at line 576 and `if not fut.done(): fut.set_result(result)` at line 579, plus `_fail_pending_rpcs`'s `if not task.future.done(): task.future.set_exception(exc)` at line 588. `collective_rpc` and `async_collective_rpc` cancel the future on timeout (lines 990/1014), so a cancel landing between done() and set_exception/set_result raises InvalidStateError and kills the singleton busy loop — the exact #5793 race class this PR names. Reuse try_set_exception/try_set_result here (and in the timeout branch at line 568), or file a tracked follow-up. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/diffusion_engine.py:576 `fut.set_exception(exc)` inside `if not fut.done():`; vllm_omni/diffusion/diffusion_engine.py:579 `fut.set_result(result)` inside `if not fut.done():`; vllm_omni/diffusion/diffusion_engine.py:588 `task.future.set_exception(exc)` inside `if not task.future.done():`; vllm_omni/diffusion/diffusion_engine.py:990 `task.future.cancel()` and :1014 `task.future.cancel()`; vllm_omni/diffusion/diffusion_engine.py:573-574 comment `# running. Setting state on a cancelled/done future raises` / `# InvalidStateError, which would kill the busy loop.`) + +`tests/diffusion/test_result_pump.py:~452` [major] — Confirmed: tests/diffusion/test_result_pump.py:452 and :467 only assert `fut.cancelled()` on a future cancelled before the pump ran, and `_feed_one_msg_to_pump` at :62-64 starts a daemon pump thread and only `t.join(timeout=2.0)` without propagating thread exceptions. On reverted code, `_result_pump` raises `InvalidStateError` in that daemon thread, is printed by `threading.excepthook`, and both tests still pass; only the batch-split test at :493 (`assert healthy_fut.done()`) catches the crash. Strengthen the two single-future tests to observe pump survival or successful resolution. (evidence: tests/diffusion/test_result_pump.py:62-64 `t = threading.Thread(target=executor._result_pump, daemon=True)` / `t.start()` / `t.join(timeout=2.0)` — thread target exceptions are not propagated; tests/diffusion/test_result_pump.py:426-428 `def done(self) -> bool: return False` — the racy future always reports not done; tests/diffusion/test_result_pump.py:431-433 `fut = _RacyFuture()` / `fut.cancel()` / `assert fut.cancelled()` — the future is cancelled before the pump; tests/diffusion/test_result_pump.py:452 `assert fut.cancelled()` and :467 `assert fut.cancelled()` — the only assertions in the two single-future tests; tests/diffusion/test_result_pump.py:493 `assert healthy_fut.done()` — the assertion that actually catches pump death. Reverted pre-PR code shown by this diff: vllm_omni/diffusion/executor/multiproc_executor.py:874 `fut.set_exception(RuntimeError(msg.error))` / :876 `fut.set_result(msg)` and :907 `pending.set_exception(exc)` / :909 `pending.set_result(output_result)` — these raise `InvalidStateError` on a cancelled future and kill the pump thread.) + +`tests/diffusion/test_result_pump.py:~16` [minor] — This diff adds the `TestResultPumpCancelledFutureRace` tests to `tests/diffusion/test_result_pump.py`, whose module-level marker is `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (line 16). The AMD ready/merge lanes run `pytest -sv tests/diffusion -m 'core_model and cpu'`, so the red `buildkite/vllm-omni-amd-ci` check at head directly intersects this diff; the NPU lane has no in-tree `tests/diffusion` job, so its red is likely unrelated. Please confirm whether the AMD failure is pre-existing on main or introduced by these new tests before dismissing it by surface area. (evidence: tests/diffusion/test_result_pump.py:16 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — unchanged-by-this-diff module marker that selects the added race tests. .buildkite/amd/test-amd-ready.yml:31 `pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml` and .buildkite/amd/test-amd-merge.yml:32 `pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-diffusion-$$BUILDKITE_PARALLEL_JOB.xml` — both present in the PR-time tree (CI config, outside the diff). grep of `.buildkite/npu` for `tests/diffusion` returned no matches (present in PR-time tree).) + +`vllm_omni/diffusion/executor/multiproc_executor.py:~876` [minor] — This diff correctly fixes the cancelled-future race by replacing the pump's check-then-act `fut.set_result`/`set_exception` with `try_set_result`/`try_set_exception` (multiproc_executor.py:876), but the pump loop only wraps `dequeue` in try/except. The post-dequeue dispatch — `self._deliver_batch_split(per_req_map, msg.output, msg.error)` at multiproc_executor.py:887 — is unguarded, so any other uncaught exception still kills the daemon pump thread. `check_health()` (multiproc_executor.py:973-980) only checks `_is_failed` and worker-process liveness; `_pump_running` is assigned but never read, so a dead pump is not surfaced. Either add pump liveness to the health check, or split the observability/recovery half of #5793 into a tracked follow-up instead of claiming `Fixes #5793` closes it. (evidence: multiproc_executor.py:876 `try_set_result(fut, msg)` — narrow InvalidStateError guard only. multiproc_executor.py:887 `self._deliver_batch_split(per_req_map, msg.output, msg.error)` — outside the dequeue try/except, so an exception here exits the pump thread. multiproc_executor.py:973-980 (unchanged by this diff, present in the PR-time tree) `def check_health(self) -> None: if self._is_failed: raise EngineDeadError() ... for p in self._processes: if not p.is_alive(): self._is_failed = True; raise EngineDeadError(f"Worker process {p.name} is dead")` — no pump-thread liveness check. future_utils.py:17-20 `try: fut.set_result(result) except concurrent.futures.InvalidStateError: logger.debug("Dropping late result for already-resolved/cancelled future")` — the new helper guards only InvalidStateError, not other pump-dispatch exceptions.) + +`tests/diffusion/test_result_pump.py:1` [minor] — PR #5983's Testing section cites `tests/diffusion/test_multiproc_result_pump.py`, but that file does not exist at this head. The diff actually adds the regression tests to `tests/diffusion/test_result_pump.py` (e.g. `TestResultPumpCancelledFutureRace`). Please correct the cited paths to `tests/diffusion/test_result_pump.py`. (evidence: PR body (GitHub PR #5983, body field): "Added `tests/diffusion/test_multiproc_result_pump.py` (CPU-only, no GPU needed)" and "With the fix: `tests/diffusion/test_multiproc_result_pump.py` — 3 passed." The PR-time tree listing for `tests/diffusion/` contains `test_result_pump.py` but no `test_multiproc_result_pump.py`; the diff header is `+++ b/tests/diffusion/test_result_pump.py`. The actual file exists at tests/diffusion/test_result_pump.py:3: "Unit tests for MultiprocDiffusionExecutor async result pump and wait_output_ready.") + +`vllm_omni/diffusion/utils/future_utils.py:~20` [minor] — In vllm_omni/diffusion/utils/future_utils.py, both try_set_result and try_set_exception catch InvalidStateError and debug-log a bare 'Dropping late...' message at lines 20 and 27. Include the caught exception (exc) and repr(fut) in those logs so an operator can distinguish an intended cancelled-drop from a double-resolve bug this silent catch would otherwise hide. (evidence: vllm_omni/diffusion/utils/future_utils.py:20 `logger.debug("Dropping late result for already-resolved/cancelled future")`; vllm_omni/diffusion/utils/future_utils.py:27 `logger.debug("Dropping late exception for already-resolved/cancelled future")` — both handlers catch `concurrent.futures.InvalidStateError` without including the caught exception or `repr(fut)` in the log message.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.trace.json.gz new file mode 100644 index 000000000..ae23f8881 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5983.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.cost.json new file mode 100644 index 000000000..9250a367a --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1165.7, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr5991/run-20260816-005826-ee9c40", + "recorded_at": "2026-08-16T01:17:49", + "attempts": [ + { + "invocation_id": "inv-09d3524e", + "outcome": "run:run-20260816-005826-ee9c40", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-005826-ee9c40", + "llm_calls": 120, + "input_tokens": 965244, + "output_tokens": 223968, + "usd": 0.771862, + "source": "spans" + } + ], + "attempt_usd_total": 0.771862, + "llm_calls": 120, + "input_tokens": 965244, + "output_tokens": 223968, + "usd": 0.771862, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-005826-ee9c40", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7719, + "minutes": 19.38, + "ci_minutes": 0, + "input_tokens": 965244, + "output_tokens": 223968, + "tool_calls": 255, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8328, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 120, + "by_role": { + "agent": { + "usd": 0.731609, + "input_tokens": 944943, + "output_tokens": 192398, + "calls": 118 + }, + "reducer": { + "usd": 0.040253, + "input_tokens": 20301, + "output_tokens": 31570, + "calls": 2 + } + }, + "cache_read_tokens": 9810432, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.16, + "gate": 2.01, + "review": 1156.75, + "report": 0.06 + }, + "time_to_first_result_s": 1162.97 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5991.trace.json.gz", + "trace_bytes": 39948 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.md new file mode 100644 index 000000000..59bce66e4 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5991, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ab661161928c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] ClassVar `_base_schedule_by_partition={}` keeps pre-existing `test_h3_prepares_resolved_cache_state_immediately_before_denoise` passing (no attr set → `{}`→None→uniform). +- [resolved] zip(..., strict=False) is Python 3.10+; pyproject.toml:14 requires-python = ">=3.10,<3.14", so the new code is compatible. +- [resolved] No in-repo MiniMax H3 deploy/stage config sets a default num_inference_steps (grep found such defaults only for soulxsinger/mammoth_moda2/hunyuan_image_3_moe/glm_image). +- [claim-verified] base_schedule is read from `_minimax_h3.base_schedule`: pipeline_minimax_h3.py:617 `release = model_index.get("_minimax_h3") or {}` then :621 `_read_base_schedule(release)` -> sigma_schedule.from_metadata key `base_schedule`. +- [claim-verified] both video/audio sigma schedules receive base_schedule: pipeline_minimax_h3.py:1512-1522. +- [claim-verified] step count inferred as intervals (len-1): sigma_schedule.py:68-70; forward uses it at pipeline_minimax_h3.py:1790. +- [claim-verified] mismatched explicit num_inference_steps rejected with OmniClientError: pipeline_minimax_h3.py:1793-1798. +- [claim-verified] no DiT weight-loading or denoise-solver changes: diff_stat lists only 6 files; denoise_loop.py and weight_utils untouched. +- [claim-verified] legacy 50-step path preserved when base_schedule is absent: pipeline_minimax_h3.py:1783-1785 keeps `int(sampling.num_inference_steps or 50)` and `base_schedule=None`. +- [claim-verified] `--task-type t2va` routes to FL2VA partition: `_minimax_h3_partition_for_task` maps t2va->fl2va (pipeline_minimax_h3.py). +- [claim-verified] PR claim 'read _minimax_h3.base_schedule from FL2VA/model_index.json': pipeline 613 + 621 confirmed. +- [claim-verified] PR claim 'no changes required to DiT weight loading or denoising solver': diff touches only time_request/pipeline/sched/tests. +- [claim-verified] 'No changes required to DiT weight loading or denoising solver': diff_stat shows only the 6 listed files; no weight/denoise files. +- [claim-verified] Shift math reproduces the PR's asserted values: 12*0.7/(1+11*0.7)=0.9655172414 and 3*0.7/(1+2*0.7)=0.875. + +`vllm_omni/diffusion/sched/sigma_schedule.py:63` [minor] — `DMD2SigmaSchedule.from_metadata` uses `raw = metadata.get(key); if raw is None: return None`, so an explicit `"base_schedule": null` is treated exactly like an absent key and silently falls back to the legacy uniform schedule. If a distilled checkpoint must never contain null, reject a present-but-`None` value (e.g. `if key not in metadata: return None` then validate the present value); otherwise reword the docstring to say null is treated as absent. (evidence: vllm_omni/diffusion/sched/sigma_schedule.py:58-60 `An absent key means the release is not distilled and keeps the legacy uniform schedule. An explicitly empty value is a malformed contract and is rejected rather than silently falling back.` and vllm_omni/diffusion/sched/sigma_schedule.py:62-65 `raw = metadata.get(key)\n if raw is None:\n return None\n return cls.from_positions(raw)` — present in this diff, so `base_schedule: null` returns None instead of being rejected.) + +`tests/diffusion/sched/test_dmd2_sigma_schedule.py:7` [minor] — The diff's new/updated CPU tests are collected by both failing lanes. Note the AMD lane runs `.buildkite/amd/test-amd-ready.yml:19` `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py`, not the exact `tests/diffusion`-only command quoted; confirm whether the two failures are pre-existing on main, and if the newly selected cases introduced them, attach the failing job log and fix. (evidence: tests/diffusion/sched/test_dmd2_sigma_schedule.py:7 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:14 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]`; unchanged by this diff, present in the PR-time tree: .buildkite/cuda/test-ready.yml:18 `"timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; unchanged by this diff, present in the PR-time tree: .buildkite/amd/test-amd-ready.yml:19 `"timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1793` [minor] — In MiniMaxH3Pipeline.forward, the distilled-schedule mismatch check at pipeline_minimax_h3.py:1793 runs only after encode_prompt (line 1714) and the visual/audio condition encoding block, so a request with a wrong explicit num_inference_steps pays full encoder/VAE cost before raising OmniClientError. Move the check earlier (after task resolution) to fail before expensive encoding. (evidence: pipeline_minimax_h3.py:1714 `text_embeddings, text_tags = self.encode_prompt(` (precedes the check in forward); pipeline_minimax_h3.py:1793 `if requested_steps is not None and int(requested_steps) != num_steps:` (evaluated after the encoding block)) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:~1791` [minor] — Confirmed: distilled checkpoints treat num_steps as denoising intervals (pipeline_minimax_h3.py:1791 -> sigma_schedule.py:70 len(base_schedule)-1), while the legacy branch treats num_steps as sigma boundaries (pipeline_minimax_h3.py:1782 -> time_request.py:59-66 torch.linspace(1.0, 0.0, int(num_steps))) and then denoise_loop.py:201 runs len(sigmas)-1 iterations. So num_inference_steps=4 distilled means 4 steps but num_inference_steps=50 undistilled means 49 steps, and quality_policy.resolve(num_inference_steps=num_steps) receives 50 for 49 legacy steps. Reconcile the unit or document the per-checkpoint meaning. (evidence: vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1791 `num_steps = sigma_schedule.num_inference_steps` (added by this diff); vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1782 `num_steps = int(sampling.num_inference_steps or 50)` (legacy branch in this diff); vllm_omni/diffusion/sched/sigma_schedule.py:70 `return len(self.base_schedule) - 1` (new file in this diff); unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/time_request.py:59-66 `torch.linspace(1.0, 0.0, int(num_steps))`; unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/minimax_h3/denoise_loop.py:201 `num_steps = len(sigmas_video) - 1`) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1793` [minor] — The new pinned-step guard at pipeline_minimax_h3.py:1792-1798 rejects any non-None `num_inference_steps` that differs from the distilled step count. In the OpenAI video path, `gen_params` is initialized from `_resolve_default_sampling_params()` (serving_video.py:145) before the request-level override at serving_video.py:201, so a stage/default sampling param that pre-fills `num_inference_steps` would be treated as a client-provided mismatch. No MiniMax H3 stage/default sampling params set this field in-repo today; document that constraint near the check (or tolerate default-injected values) to prevent future breakage. (evidence: Unchanged by this diff, present in the PR-time tree — serving_video.py:145 `gen_params = self._resolve_default_sampling_params()` seeds params before serving_video.py:201 `if "num_inference_steps" in provided_fields and request.num_inference_steps is not None:` applies the request override; pipeline_minimax_h3.py:1792-1794 `requested_steps = sampling.num_inference_steps` / `if requested_steps is not None and int(requested_steps) != num_steps:` then raises the mismatch.) + +`tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:~565` [minor] — Confirmed: the new distilled-forward tests stub `pipeline.diffuse` with `def diffuse(**kwargs): diffuse_calls.append(kwargs)` (test_minimax_h3_contract.py:565), so they only assert `diffuse_calls[0]["num_steps"] == 4` and never execute the real `MiniMaxH3Pipeline.diffuse` body that passes `base_schedule=base_schedule` (pipeline_minimax_h3.py:1515) into `minimax_h3_denoise_loop`. The actual 5-boundaries-to-4-iterations conversion is `num_steps = len(sigmas_video) - 1` (denoise_loop.py:201) and is uncovered by this diff. A regression test that keeps the real diffuse (or calls minimax_h3_denoise_loop directly) with 5 distilled sigmas and a call-counting fake transformer would close the gap; deleting `base_schedule=base_schedule` should fail it. (evidence: tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:565 `def diffuse(**kwargs):` (stub that replaces the real method in _distilled_pipeline) — added by this diff; vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:1515 `base_schedule=base_schedule,` — added by this diff but inside the real `diffuse` body, which the new tests bypass; vllm_omni/diffusion/models/minimax_h3/denoise_loop.py:201 `num_steps = len(sigmas_video) - 1` — unchanged by this diff, present in the PR-time tree, and the conversion the tests fail to exercise.) + +`vllm_omni/diffusion/sched/sigma_schedule.py:74` [minor] — In `DMD2SigmaSchedule.shifted_sigmas` (sigma_schedule.py:74), `if shift_scale <= 0:` lets NaN (and +inf) pass, because NaN comparisons are False and +inf <= 0 is False; line 77 then returns all-NaN sigmas. The distilled path reads `flow_shift`/`audio_flow_shift` via `float(extra.get(..., default))` (pipeline_minimax_h3.py:1798-1799) with no finite check. `math` is already imported in sigma_schedule.py, so use `if not math.isfinite(shift_scale) or shift_scale <= 0:` to fail fast. (evidence: sigma_schedule.py:74 `if shift_scale <= 0:` — in Python, `float('nan') <= 0` is False, so a NaN shift passes this guard. sigma_schedule.py:76-77 `shift = float(shift_scale)` / `return [shift * u / (1 + (shift - 1) * u) for u in self.base_schedule]` — a NaN shift makes every returned sigma NaN. Consumer present in the PR-time tree but unchanged by this diff: pipeline_minimax_h3.py:1798 `video_shift = float(extra.get("flow_shift", self.default_video_shift))` — no `math.isfinite` check before use.) + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:623` [minor] — In combined mode, line 623 starts pinning a distilled Ref2VA schedule from `ref2va_release`, but `default_video_shift`/`default_audio_shift` are still sourced only from the FL2VA `release` at line 614. `forward` then applies those FL2VA shift scales to the Ref2VA base_schedule (lines 1798-1799), so a distilled Ref2VA whose metadata has its own `sigma_shift_scales` would silently run the wrong sigma trajectory. Either read and store `ref2va_release["sigma_shift_scales"]` per partition or remove the line-623 Ref2VA schedule read and keep this PR scoped to t2va/fl2va. (evidence: pipeline_minimax_h3.py:614 ` shifts = release.get("sigma_shift_scales") or {}` (release is the FL2VA index in combined mode); pipeline_minimax_h3.py:623 ` self._base_schedule_by_partition["ref2va"] = _read_base_schedule(ref2va_release)` (per-partition schedule, but no per-partition shifts); pipeline_minimax_h3.py:1798 ` video_shift = float(extra.get("flow_shift", self.default_video_shift))`; pipeline_minimax_h3.py:1799 ` audio_shift = float(extra.get("audio_flow_shift", self.default_audio_shift))` — both shift scales remain global FL2VA values and are applied to the Ref2VA base_schedule in `diffuse`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.trace.json.gz new file mode 100644 index 000000000..6de739570 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr5991.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.cost.json new file mode 100644 index 000000000..be13850fd --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1736.2, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr6023/run-20260816-011012-4a6a19", + "recorded_at": "2026-08-16T01:39:06", + "attempts": [ + { + "invocation_id": "inv-ad28312f", + "outcome": "run:run-20260816-011012-4a6a19", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-011012-4a6a19", + "llm_calls": 137, + "input_tokens": 1084580, + "output_tokens": 294796, + "usd": 0.950685, + "source": "spans" + } + ], + "attempt_usd_total": 0.950685, + "llm_calls": 137, + "input_tokens": 1084580, + "output_tokens": 294796, + "usd": 0.950685, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-011012-4a6a19", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9507, + "minutes": 28.89, + "ci_minutes": 0, + "input_tokens": 1084580, + "output_tokens": 294796, + "tool_calls": 237, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0512, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 137, + "by_role": { + "agent": { + "usd": 0.913492, + "input_tokens": 1063378, + "output_tokens": 266229, + "calls": 135 + }, + "reducer": { + "usd": 0.037193, + "input_tokens": 21202, + "output_tokens": 28567, + "calls": 2 + } + }, + "cache_read_tokens": 12354560, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.5, + "gate": 1.56, + "review": 1727.47, + "report": 0.07 + }, + "time_to_first_result_s": 1733.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6023.trace.json.gz", + "trace_bytes": 44646 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.md new file mode 100644 index 000000000..67a1e9451 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.md @@ -0,0 +1,69 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6023, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head a240368b50b5) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] early OUTPUT_READY race: pump caches the batch output under batch_id when no split map exists (multiproc_executor.py:915), and execute_batch pops it under _futures_lock (650) before registering a map; race closed at this head. +- [resolved] single-request future race: pump now unpacks before `_output_futures.pop`/`_completed_outputs` (890-915), so wait_output_ready cannot orphan a future during unpack; residual is the test that mocks unpack (see review comment). +- [claim-verified] PR defect #1: file_at_base ipc.py:123 had `if val.nelement() * val.element_size() > _SHM_TENSOR_THRESHOLD`; head uses `max(view_bytes, storage_bytes)`. +- [claim-verified] PR defect #2: base `_result_pump` cached the whole batch under batch_id when no map existed; head adds early adoption at multiproc_executor.py:649-653. +- [claim-verified] PR defect #3: grep shows the only `self.result_mq.enqueue` in the worker is diffusion_worker.py:904 inside `_enqueue_result`; all writes are serialized. +- [claim-verified] PR defect #4: new single-request path unpacks before the `_futures_lock` pop/set at multiproc_executor.py:887-915, closing the old waiter race. +- [claim-verified] CI all green per gh_ci_read: build 3.12/3.11, pre-commit, DCO, buildkite/vllm-omni, readthedocs. +- [claim-verified] all four claimed defect scopes are present in the diff (pack threshold, split-map race, result_mq lock, single-request atomic resolution) plus the timeout-diagnostics addition. +- [claim-verified] PR body defect #4 matches the diff: old `_result_pump` popped `_output_futures` before `unpack_diffusion_output_shm`; new code unpacks first then resolves/caches under `_futures_lock` (multiproc_executor.py:883-916). +- [claim-verified] pickle allowlist needed: test_diffusion_ipc.py has a local `import pickle` that matches `PICKLE_RE` `^\s*import pickle` and the file is in ALLOWED_FILES, so pre-commit scan (which reads every line) passes. +- [claim-verified] LTX doc row: registry maps LTX2DistilledPipeline to two-stage with supports_request_batch=False (pipeline_ltx2_two_stage.py:30) while LTX2Pipeline has True (pipeline_ltx2.py:36); the LTX-2.3 row is plausible but under-specified. +- [claim-refuted] docs/design/feature/async_diffusion_output.md:149 states LTX-2.3 `supports_request_batch=False`, but the one-stage `LTX2Pipeline` declares `supports_request_batch = True` (pipeline_ltx2.py:36); only two-stage/distilled entries are False. +- [validated] ipc.py:136 `if max(view_bytes, storage_bytes) > _SHM_TENSOR_THRESHOLD` and `_tensor_to_shm` copies only the view (`cpu.copy_(t)` / `.cpu().contiguous()`), so SHM payload does not grow. +- [validated] `describe_pending_state` exists only on MultiprocDiffusionExecutor (grep), and diffusion_engine.py:345-350 uses getattr with an 'unavailable' fallback. + +`vllm_omni/diffusion/executor/multiproc_executor.py:~915` [major] — In `_result_pump`, the batch-vs-single decision is made at line 879 (`per_req_map = self._batch_split_map.pop(batch_id, None)`) before SHM unpack. When the map is missing, the pump unpacks outside `_futures_lock`, so `execute_batch` can register `self._batch_split_map[batch_id] = per_req_map` at line 651 before the pump re-acquires the lock; the pump then stores only `self._completed_outputs[batch_id] = fut` at line 915. Per-request `{batch_id}/{request_id}` waiters are never resolved and each request hangs until `_ASYNC_OUTPUT_TIMEOUT`. Re-acquire `_futures_lock` after unpack and atomically re-check/re-pop `_batch_split_map` before falling back to caching (or call `_deliver_batch_split` with the newly found map), and add a regression test that gates `unpack_diffusion_output_shm` until after `execute_batch` registers the map and asserts per-request futures resolve. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:879 `per_req_map = self._batch_split_map.pop(batch_id, None) if batch_id else None` — pump pops the map before unpacking; vllm_omni/diffusion/executor/multiproc_executor.py:651 `self._batch_split_map[batch_id] = per_req_map` — execute_batch can register the map while the pump is still unpacking; vllm_omni/diffusion/executor/multiproc_executor.py:915 `self._completed_outputs[batch_id] = fut` — pump then caches only the batch-id future without re-checking the map, so per-request `{batch_id}/{request_id}` futures are never delivered.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:~887` [major] — In the batch branch of `_result_pump`, an `unpack_diffusion_output_shm(msg.output)` failure is logged and then ignored; the code still calls `_deliver_batch_split(per_req_map, msg.output, msg.error)` (line 887). Because `_deliver_batch_split` copies `req_output.result` and resolves each future via `pending.set_result(per_req_result)` (lines 913-924), a failed unpack resolves waiting requests successfully with `DiffusionOutput` fields still holding `{'__tensor_shm__': True, ...}` handle dicts. On unpack failure, set `exc` from the exception and fail each per-request future (or deliver `DiffusionOutput(error=...)`), mirroring the single-request branch at lines 895-908. (evidence: multiproc_executor.py:884-887 `try: + unpack_diffusion_output_shm(msg.output) +except Exception: + logger.exception("SHM unpack failed for batch %s", batch_id) +self._deliver_batch_split(per_req_map, msg.output, msg.error)`; multiproc_executor.py:913-924 `req_output = batch_output.get_request_output(req_id) if batch_output is not None else None` / `per_req_result = req_output.result` / `pending.set_result(per_req_result)`; multiproc_executor.py:896-903 single-request contrast `except Exception as e: + logger.exception("SHM unpack failed in result pump") + exc = e` then `pending.set_exception(exc)`. ipc.py:334-336 unchanged supporting detail: `if isinstance(val, dict) and val.get("__tensor_shm__"): + return _tensor_from_shm(val)` and `_tensor_from_shm` raises on a bad segment, leaving the handle dict in place.) + +`vllm_omni/diffusion/diffusion_engine.py:~818` [minor] — The diff adds `describe_pending_state` timeout diagnostics only to `step_streaming`'s async wait (diffusion_engine.py:345), but the other async-output wait in `add_req_and_wait_for_response` still runs bare: `output = fut.result(timeout=_ASYNC_OUTPUT_TIMEOUT)` (line 818), and that is the path used by `_dummy_run`/warmup (line 914). The doc sentence at docs/design/feature/async_diffusion_output.md:128 says every async-wait timeout logs executor bookkeeping, so either add the same `getattr(self.executor, 'describe_pending_state', None)` logging to this wait or scope the doc sentence to `step_streaming`; otherwise a startup-warmup stall still reports only a raw timeout. (evidence: diffusion_engine.py:345 `describe = getattr(self.executor, "describe_pending_state", None)` — diagnostic added only to step_streaming's wait. diffusion_engine.py:818 `output = fut.result(timeout=_ASYNC_OUTPUT_TIMEOUT)` — second async-output wait with no diagnostic; unchanged by this diff, present in the PR-time tree. diffusion_engine.py:914 `output = self.add_req_and_wait_for_response(req)` — confirms _dummy_run/warmup uses the bare wait path. docs/design/feature/async_diffusion_output.md:128 `If a timeout occurs, the executor logs detailed diagnostic bookkeeping (pending futures, cached outputs, and batch split maps) to identify whether the stall occurred in the worker, pump, or waiter.` — broader than actual diff coverage.) + +`docs/design/feature/async_diffusion_output.md:149` [minor] — The docs table row `| **LTX-2.3** | Video / Audio | `False` |` (docs/design/feature/async_diffusion_output.md:149) is ambiguous: the one-stage `LTX2Pipeline` registered under key 'LTX2Pipeline' declares `supports_request_batch = True` (vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py:36), while the two-stage/distilled entries declare `False` (vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py:30). Please state which registry key/checkpoint 'LTX-2.3' refers to, or split the row, so the table does not contradict the one-stage flag. (evidence: docs/design/feature/async_diffusion_output.md:149 `| **LTX-2.3** | Video / Audio | `False` |` (changed by this PR). Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/ltx2/pipeline_ltx2.py:36 ` supports_request_batch = True`; vllm_omni/diffusion/models/ltx2/pipeline_ltx2_two_stage.py:30 ` supports_request_batch = False`; vllm_omni/diffusion/registry.py:69-72 ` "LTX2Pipeline": (` / ` "ltx2",` / ` "pipeline_ltx2",` / ` "LTX2Pipeline",`.) + +`vllm_omni/diffusion/worker/diffusion_worker.py:1162` [minor] — Add direct call-order coverage for the busy-loop `{'type':'sleep'}` drain site at diffusion_worker.py:1162-1163. The only existing assertion of drain-before-sleep order, `test_sleep_rpc_drains_before_releasing_memory`, drives `_execute_rpc` with method `handle_sleep_task`; the busy-loop path that handles a `{'type':'sleep'}` dict message remains an unasserted drain entry point. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:1162-1163 `if isinstance(msg, dict) and msg.get("type") == "sleep":` / `self.drain_async_outputs()` — busy-loop drain site added by this PR. vllm_omni/diffusion/worker/diffusion_worker.py:1100-1101 `if method in _MEMORY_RELEASING_METHODS:` / `self.drain_async_outputs()` — the separate RPC drain site. tests/diffusion/test_async_output_worker.py: `def test_sleep_rpc_drains_before_releasing_memory` calls `proc._execute_rpc({"method": "handle_sleep_task", ...})` and asserts `assert call_order == ["drain", "sleep"]`; repo grep shows no `_worker_busy_loop` or busy-loop sleep-message test in that file, confirming the second drain entry point is unasserted.) + +`docs/design/feature/async_diffusion_output.md:122` [minor] — Make the drain-before-sleep blast radius explicit in the Reliability section (docs/design/feature/async_diffusion_output.md:122): `_execute_rpc` drains for every request-mode pipeline's sleep path via `_MEMORY_RELEASING_METHODS` (diffusion_worker.py:73), and `_worker_busy_loop` drains on sleep messages too (diffusion_worker.py:1163), but this PR's end-to-end validation covered only Qwen-Image and LTX-2.3 (CUDA). The Model Coverage section (line 141) already says other request-mode models are supported but not yet verified; mirror that scoping in the Reliability section or add a follow-up smoke pass on another request-mode pipeline. (evidence: vllm_omni/diffusion/worker/diffusion_worker.py:73 `_MEMORY_RELEASING_METHODS = frozenset({"sleep", "handle_sleep_task"})` (in diff). vllm_omni/diffusion/worker/diffusion_worker.py:1100-1101 ` if method in _MEMORY_RELEASING_METHODS:` / ` self.drain_async_outputs()` (in diff, inside generic `_execute_rpc`). vllm_omni/diffusion/worker/diffusion_worker.py:1163 ` self.drain_async_outputs()` (in diff, `_worker_busy_loop` sleep branch). docs/design/feature/async_diffusion_output.md:141 `All models running in request-mode (`step_execution=False`, the default) automatically use async output.` (unchanged by this diff, present in PR-time tree) — the broad scope exists in Model Coverage but is not repeated for the new Reliability section at line 122.) + +`vllm_omni/diffusion/executor/multiproc_executor.py:~1008` [minor] — `_result_pump` populates `_completed_outputs` on both the single-output path (`_completed_outputs[batch_id] = fut`, line 915) and per-request batch delivery (`_completed_outputs[per_req_id] = fut`, line 940), but `shutdown()` clears `_rpc_futures`, `_output_futures`, and `_batch_split_map` (lines 1007-1008) without clearing `_completed_outputs`. Requests that never reach `wait_output_ready` leave resolved futures holding unpacked `DiffusionOutput` tensors until executor GC. Add `self._completed_outputs.clear()` next to the other lifecycle clears. (evidence: vllm_omni/diffusion/executor/multiproc_executor.py:915 `self._completed_outputs[batch_id] = fut`; vllm_omni/diffusion/executor/multiproc_executor.py:940 `self._completed_outputs[per_req_id] = fut`; vllm_omni/diffusion/executor/multiproc_executor.py:1007 `self._rpc_futures.clear()`; vllm_omni/diffusion/executor/multiproc_executor.py:1008 `self._batch_split_map.clear()` (preceded by line 1006 `self._output_futures.clear()`), with no `_completed_outputs.clear()` in the `_futures_lock` block.) + +`vllm_omni/diffusion/ipc.py:136` [minor] — `_pack_tensor_if_large` now sizes the pack/inline decision on `max(view_bytes, storage_bytes)` (ipc.py:136), and this same `pack_diffusion_output_shm` is reached by the synchronous step-execution fallback (`_return_result` at diffusion_worker.py:931; async path only when `not self.od_config.step_execution` at :915). Step-mode outputs that are small views onto a >1MB storage therefore now take the SHM path, yet the test plan only covers request-mode Qwen-Image and LTX-2.3. Please confirm step-mode outputs are always contiguous (storage_bytes == view_bytes, unchanged behavior) or add a step-mode (Helios/streaming) round-trip smoke/perf check for this sync path. (evidence: vllm_omni/diffusion/ipc.py:136 ` if max(view_bytes, storage_bytes) > _SHM_TENSOR_THRESHOLD:` (changed by this diff); vllm_omni/diffusion/worker/diffusion_worker.py:915 ` if not self.od_config.step_execution and isinstance(output, (DiffusionOutput, BatchRunnerOutput)):` (unchanged by this diff, present in the PR-time tree — async path gate) and :931 ` pack_diffusion_output_shm(output)` (unchanged sync fallback, present in the PR-time tree).) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/test_result_pump.py:270` [minor] `test_output_ready_atomic_resolution_when_future_already_waiting` is meant to prove the single-request atomic-resolution fix, but the autouse `_mock_unpack` fixture (tests/diffusion/test_result_pump.py:58-60) replaces `u + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.trace.json.gz new file mode 100644 index 000000000..47e3a6d50 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6023.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.cost.json new file mode 100644 index 000000000..425bb8233 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1700.2, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr6070/run-20260816-011132-23afa1", + "recorded_at": "2026-08-16T01:39:50", + "attempts": [ + { + "invocation_id": "inv-8fb6b594", + "outcome": "run:run-20260816-011132-23afa1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-011132-23afa1", + "llm_calls": 154, + "input_tokens": 1749203, + "output_tokens": 288103, + "usd": 1.456832, + "source": "spans" + } + ], + "attempt_usd_total": 1.456832, + "llm_calls": 154, + "input_tokens": 1749203, + "output_tokens": 288103, + "usd": 1.456832, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-011132-23afa1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.4568, + "minutes": 28.29, + "ci_minutes": 0, + "input_tokens": 1749203, + "output_tokens": 288103, + "tool_calls": 256, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2012, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 154, + "by_role": { + "agent": { + "usd": 1.386997, + "input_tokens": 1642159, + "output_tokens": 250954, + "calls": 151 + }, + "reducer": { + "usd": 0.069835, + "input_tokens": 107044, + "output_tokens": 37149, + "calls": 3 + } + }, + "cache_read_tokens": 24727168, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.05, + "gate": 2.26, + "review": 1690.23, + "report": 0.06 + }, + "time_to_first_result_s": 1697.59 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6070.trace.json.gz", + "trace_bytes": 104686 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.md new file mode 100644 index 000000000..7b7822374 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.md @@ -0,0 +1,70 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6070, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9df34bba154e) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] T2V path: LTX2Pipeline/_LTX2TwoStageBase set unified_text_image_entry=True (pipeline_ltx2.py, pipeline_ltx2_two_stage.py), so image=None delegates to LTXRuntime prepare_latents — no 'image must be provided' break for LTX-2.5 T2V. +- [resolved] one_stage now validates checkpoint kind 'regular' via resolve_ltx_checkpoint_kind, but LTX2TwoStagePipeline already validated 'regular' against the same checkpoints and DMD2 returns None, so no new break; validate_ltx_checkpoint confirmed in ltx2_request.py:57-89. +- [claim-verified] Four LTX-2.5 pipeline entries exist: registry.py adds LTX2DistilledOneStagePipeline + LTX2DistilledTwoStagePipeline; LTX2Pipeline/LTX2TwoStagePipeline already cover 2.5 via metadata; alias LTX2DistilledPipeline = LTX2DistilledTwoStagePipeline (pipeline_ltx2_two_stage.py). +- [claim-verified] docs/models/supported_models.md and docs/user_guide/diffusion_features.md updated with the four LTX-2.5 rows, CRF/FP8/negative-prompt markers consistent with code. +- [claim-verified] PR's 'BF16 cuDNN official parity' for LTX-2.5 is backed by new nightly jobs in .buildkite/cuda/test-nightly.yml (LTX-2.5 online + parity rows) and tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py. +- [claim-refuted] PR body 'HSDP2, DLO DP2, whole-model offload, and VAE slicing qualified as memory/capacity paths' — tests/e2e/online_serving/test_ltx25.py only adds --enable-layerwise-offload on single-card marks; no LTX-2.5 HSDP/DLO/VAE-slicing test or config in diff (hsdp grep hits only pre-existing tests/e2e/online_serving/test_ltx.py:61 distilled_hsdp2 for LTX-2/2.3). +- [claim-refuted] No evidence the existing LTX-2/2.3 official-parity gates (tests/e2e/accuracy/ltx/test_ltx_official_similarity.py) were re-run after the shared noising/cache-dit/tokenization changes. +- [claim-refuted] PR description [x] 'HSDP2, DLO DP2, whole-model offload, and VAE slicing qualified' has no test/benchmark in the diff; grep tests/e2e/accuracy/ltx + recipe for HSDP|DLO|fully_shard|vae_slicing = no matches; only --enable-layerwise-offload is exercised. +- [claim-refuted] 'BF16 cuDNN official parity': attention_backend metadata is a hardcoded label (run_ltx25_reference.py:412/566); assertions compare it to the same constant (test_ltx25_official_similarity.py:547/550/704/707), so the actually-selected backend is never observed. +- [validated] data.py enrich_config resolves model_class_name (data.py:1198-1199) before calling get_transformer_config_subfolder (data.py:1216-1219), so LTX-2.5 transformer_full is selected; test_ltx25_full_config_enrichment_uses_selected_transformer_subfolder pins it. +- [validated] cudnn_attn.forward_cuda degenerate-shape MATH route is exercised by test_cudnn_routes_single_token_attention_to_math driving the real CuDNNAttentionImpl.forward_cuda; enable_gqa is computed pre-permute. +- [validated] _ltx25_full_sigmas re-derived: sigma_shift=2.05, exp=7.7679, res[1]=1-(1/(29*exp+1))/(29/(0.9*(exp+29)))=0.994957 — matches test_ltx25_full_sigma_schedule assertion 0.9949570. +- [validated] serving_video.preserves_reference_image_size uses get_diffusion_od_config() synchronously (async_omni.py:220 is a plain def, async_omni_engine.py:270 is a plain def), so the cached_property cannot cache a coroutine — no async poisoning bug. +- [validated] skip_if_gated_repo_inaccessible signature change (tests/helpers/__init__.py:14) is backward-compatible: all 3 in-repo callers (test_stable_audio_expansion.py:83, test_ltx25.py:30, test_ltx25_official_similarity.py:316/328) pass compatible args. + +`requirements/common.txt:12` [minor] — The gate report shows buildkite/vllm-omni-npu-ci FAILING at this head. The NPU ready lane runs MiniCPM-o/Qwen3-TTS tests and its build installs this file, so the new transformers floor/cap is the only in-diff dependency surface it plausibly loads. Please confirm whether the failure is pre-existing on main or introduced by the `>=5.10.1,<5.15` pin (e.g. the NPU image already had a transformers version outside this range), and if introduced, fix the lane/pin rather than leaving the gate red. (evidence: `.buildkite/npu/test-npu-ready.yml:12` runs `pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py ... -m 'core_model and npu and A3 and omni'`; `requirements/common.txt:12` `transformers >= 5.10.1, < 5.15` is the changed dependency constraint.) + +`tests/e2e/online_serving/test_ltx25.py:~45` [minor] — PR body claims 'HSDP2, DLO DP2, whole-model offload, and VAE slicing qualified as memory/capacity paths', but the only new LTX-2.5 online-serving test (tests/e2e/online_serving/test_ltx25.py) passes just --enable-layerwise-offload and pins every case to SINGLE_CARD_MARKS (H100). No LTX-2.5 HSDP2 / distributed-layerwise DP2 / VAE-slicing test exists in this diff (the only distilled_hsdp2 runtime case is the pre-existing tests/e2e/online_serving/test_ltx.py:61 for LTX-2/2.3). Either add LTX-2.5 cases for those paths or drop the qualification. (evidence: PR body (gh_pr_view, not a repo file): 'HSDP2, DLO DP2, whole-model offload, and VAE slicing qualified as memory/capacity paths.' — tests/e2e/online_serving/test_ltx25.py:24 `SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})`; tests/e2e/online_serving/test_ltx25.py:45 `"--enable-layerwise-offload",`; tests/e2e/online_serving/test_ltx25.py:51 `pytest.param(_server("LTX2Pipeline"), 30, id="full_one_stage", marks=SINGLE_CARD_MARKS),`; unchanged by this diff, present in the PR-time tree: tests/e2e/online_serving/test_ltx.py:61 `id="distilled_hsdp2",`.) + +`vllm_omni/diffusion/models/ltx2/ltx2_transformer.py:1536` [minor] — This hunk flips `LTX2VideoTransformer3DModel._cache_dit_adapter_config.has_separate_cfg` from `True` to `False` (ltx2_transformer.py:1536), and backend.py:219 forwards that value into `BlockAdapter(..., has_separate_cfg=...)`, so step accounting changes for all LTX-2/LTX-2.3 one-stage cache_dit users: `LTX2_ONE_STAGE_RECIPE` (ltx2_recipes.py:122) and `LTX23_ONE_STAGE_RECIPE` (ltx2_recipes.py:126) still declare `supports_cache_dit=True`. The added cache coverage is only the static assertion `assert adapter_config.has_separate_cfg is False` (test_cache_dit.py:130). Note the pre-existing `test_ltx2_cache_dit_receives_audio_as_encoder` does run a tiny LTX2 transformer through the CacheDiT backend, but it checks audio-as-encoder routing, not `has_separate_cfg` step accounting. Please add a functional LTX-2/2.3 one-stage cache-dit test that exercises the flipped accounting, or scope the flag per model version. (evidence: vllm_omni/diffusion/models/ltx2/ltx2_transformer.py:1536 `has_separate_cfg=False,` — changed by this PR; vllm_omni/diffusion/cache/cachedit/backend.py:219 `has_separate_cfg=adapter_config.has_separate_cfg,` — unchanged by this diff, present in the PR-time tree; vllm_omni/diffusion/models/ltx2/ltx2_recipes.py:122 `supports_cache_dit=True,` (LTX2_ONE_STAGE_RECIPE) and :126 `supports_cache_dit=True,` (LTX23_ONE_STAGE_RECIPE) — unchanged by this diff; tests/diffusion/cache/test_cache_dit.py:130 `assert adapter_config.has_separate_cfg is False` — added by this PR.) + +`vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py:~164` [minor] — The rewritten `_get_gemma_prompt_embeds` (ltx2_conditioning.py:164) is shared by LTX-2/2.3/2.5 through `LTXRuntime(LTXRequestMixin, LTXTextConditioningMixin, ...)`. The only new coverage, `test_gemma_tokenization_matches_official_bos_and_left_padding` (test_ltx25_pipeline.py:824), uses a `FakeTokenizer` whose `__call__` returns BOS-less ids by construction (line 837), so it verifies the new prepend/pad logic, not parity with the real Gemma3 tokenizer. Run the existing LTX-2.3 official-similarity test (or add a real-tokenizer regression) before merging. (evidence: Shared MRO, unchanged by this diff but present in the PR-time tree: vllm_omni/diffusion/models/ltx2/ltx2_runtime.py:116-118 `class LTXRuntime(\n LTXRequestMixin,\n LTXTextConditioningMixin,` — the rewritten mixin serves every LTX variant. Changed code: vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py:164-165 ` if not ids or ids[0] != bos_token_id:\n ids = [bos_token_id, *ids][:max_sequence_length]` — new per-text BOS prepend in the shared path. New test authors its own tokenizer behavior: tests/diffusion/models/ltx2/test_ltx25_pipeline.py:837 ` ids = [7, 8, 9, self.eos_token_id] if text == "long" else [6, self.eos_token_id]` — the fake returns BOS-less ids by construction, so it cannot catch drift in the real Gemma3 tokenizer.) + +`vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py:615` [minor] — The unpacked-latents (Stage-2 I2V) branch at ltx2_conditioning.py:615 unconditionally switches all LTX versions to `create_conditioned_noised_state` (noise lerp → bf16 intermediate → clean/noised lerp), while `is_ltx25` (line 561) only gates the fresh-init branch at lines 669–670. For LTX-2/LTX-2.3 this changes the previous single fp32 lerp path without an official-parity row. Gate this branch on `model_version == "2.5"`, or add/run an LTX-2.3 two-stage I2V parity test showing the 2/2.3 gates still pass. (evidence: vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py:561 ` is_ltx25 = getattr(self, "model_version", None) == "2.5"`; vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py:615 ` latents = latent_ops.create_conditioned_noised_state(` — inside the `if unpacked_latents:` block with no `is_ltx25` check; vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py:669 ` if is_ltx25:` and vllm_omni/diffusion/models/ltx2/ltx2_conditioning.py:670 ` latents = latent_ops.create_conditioned_noised_state(` — the only `is_ltx25`-gated use is the fresh-init branch, not the unpacked-latents branch.) + +`vllm_omni/entrypoints/openai/serving_video.py:112` [minor] — The new `preserves_reference_image_size` cached_property reads `revision` from `od_config`, but the out-of-process fallback `AsyncOmniEngine.get_diffusion_od_config()` returns a `SimpleNamespace` without `model` or `revision` (async_omni_engine.py:281-286). For an out-of-process LTX-2.5 server started with a non-default `--revision`, `revision` is None, so `ltx_preserves_reference_image_size` calls `detect_ltx_model_version(model, revision=None)` and may mis-detect 2.5 at the default revision, causing the server to LANCZOS-resize the reference image the pipeline owns. Thread `revision`/`model` into the engine view or resolve the policy from the worker's config. (evidence: Changed by this diff, `vllm_omni/entrypoints/openai/serving_video.py:111-115`: `model = model or self.model_name` / `revision = getattr(od_config, "revision", None) if od_config is not None else None` / `return should_preserve_reference_image_size(model_class_name, model=None if model is None else str(model), revision=revision)` — so when `od_config` lacks `revision`, the policy call passes None. Unchanged by this diff, `vllm_omni/entrypoints/async_omni.py:240-241`: `if saw_diffusion_stage:` / ` return self.engine.get_diffusion_od_config()` — the out-of-process fallback. Unchanged by this diff, `vllm_omni/engine/async_omni_engine.py:279-286`: `model_class_name = resolve_model_class_name(self.model)` and `self._diffusion_od_config_view = SimpleNamespace(model_class_name=model_class_name, supports_multimodal_inputs=metadata.supports_multimodal_inputs, max_multimodal_image_inputs=metadata.max_multimodal_image_inputs, supports_mixed_reference_inputs=metadata.supports_mixed_reference_inputs)` — no `model` and no `revision`, confirming the revision is dropped on this path.) + +`tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py:~550` [minor] — omni_metadata["attention_backend"] is written as the hardcoded string "torch_sdpa_cudnn_bf16" inside _run_omni (run_ltx25_reference.py:566), identical to the ATTENTION_BACKEND constant this assertion compares against (test_ltx25_official_similarity.py:85), so the check can only fail if the label is edited. It never observes which SDPA backend the runtime actually selected; the official side has the same hardcoded label at run_ltx25_reference.py:412. Capture the real dispatch (e.g. assert/log the Omni diffusion_attention_config backend and the official PytorchAttention priority) or soften the claim. (evidence: ['tests/e2e/accuracy/ltx/run_ltx25_reference.py:566 ` "attention_backend": "torch_sdpa_cudnn_bf16",` — hardcoded omni metadata label, not runtime-observed', 'tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py:85 `ATTENTION_BACKEND = "torch_sdpa_cudnn_bf16"` — constant used for the comparison', 'tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py:550 ` assert omni_metadata["attention_backend"] == ATTENTION_BACKEND` — tautological assertion']) + +`.buildkite/cuda/test-nightly.yml:496` [minor] — `.buildkite/cuda/test-nightly.yml:497` gives this single `h100_1` step 180 minutes for `test_ltx25_distilled_two_stage_matches_official` (2 task params) and `test_ltx25_full_matches_official` (2 task x 2 pipeline_mode params) = 6 cases, each running official and Omni 22B subprocesses with `--enable-layerwise-offload` (12 heavy runs). Consider splitting distilled and full into separate steps or documenting measured runtime so a timeout isolates which pipeline regressed. (evidence: .buildkite/cuda/test-nightly.yml:496 ` - label: ":full_moon: Diffusion X2V · LTX-2.5 Official Parity Test"` +.buildkite/cuda/test-nightly.yml:497 ` timeout_in_minutes: 180` +.buildkite/cuda/test-nightly.yml:499 ` - pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_distilled_two_stage_matches_official -m full_model --run-level full_model` +.buildkite/cuda/test-nightly.yml:500 ` - pytest -s -v tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py::test_ltx25_full_matches_official -m full_model --run-level full_model` +.buildkite/cuda/test-nightly.yml:501 ` mirror_hardwares: h100_1` +tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py:470 `@pytest.mark.parametrize("task", ["t2v", "i2v"])` (distilled test -> 2 cases) +tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py:598 `@pytest.mark.parametrize("task", ["t2v", "i2v"])` +tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py:599 `@pytest.mark.parametrize("pipeline_mode", ["full_one_stage", "full_two_stage"])` (full test -> 4 cases) +tests/e2e/accuracy/ltx/test_ltx25_official_similarity.py:488 ` "--enable-layerwise-offload",` (runner_args passed to both the official and Omni subprocess calls in each test body)) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/models/ltx2/ltx2_components.py:25` [minor] vllm_omni/diffusion/models/ltx2/ltx2_components.py:25 imports `AutoModelForImageTextToText` unguarded at module top, while only `Gemma4UnifiedForConditionalGeneration` is try/except-guarded below (lines 52-54) and requir + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.trace.json.gz new file mode 100644 index 000000000..6588d35b4 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6070.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.cost.json b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.cost.json new file mode 100644 index 000000000..3a2d16986 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1258.7, + "split": "holdout5", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_holdout5_r1/runs/pr6075/run-20260816-011153-28c58d", + "recorded_at": "2026-08-16T01:32:50", + "attempts": [ + { + "invocation_id": "inv-f630cccb", + "outcome": "run:run-20260816-011153-28c58d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-011153-28c58d", + "llm_calls": 123, + "input_tokens": 990445, + "output_tokens": 183100, + "usd": 0.774244, + "source": "spans" + } + ], + "attempt_usd_total": 0.774244, + "llm_calls": 123, + "input_tokens": 990445, + "output_tokens": 183100, + "usd": 0.774244, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-011153-28c58d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7742, + "minutes": 20.94, + "ci_minutes": 0, + "input_tokens": 990445, + "output_tokens": 183100, + "tool_calls": 254, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8618, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 123, + "by_role": { + "agent": { + "usd": 0.738126, + "input_tokens": 953842, + "output_tokens": 159288, + "calls": 121 + }, + "reducer": { + "usd": 0.036117, + "input_tokens": 36603, + "output_tokens": 23812, + "calls": 2 + } + }, + "cache_read_tokens": 11311616, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.2, + "gate": 1.59, + "review": 1250.74, + "report": 0.08 + }, + "time_to_first_result_s": 1256.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr6075.trace.json.gz", + "trace_bytes": 57650 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.md b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.md new file mode 100644 index 000000000..61e694e30 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 6075, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7ec78360bb95) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Old DLO design path: grep found only the mkdocs.yml:87 redirect; no stale `design/feature/distributed_layerwise_offload.md` links remain in the tree. +- [resolved] Legacy anchors preserved: attention_backends.md keeps `## TRTLLM_ATTN Backend and Skip-Softmax`, `## RAINFUSION_ATTN Backend and Block-Sparse Video Attention`, Sage/Hub headings; cpu_offload.md keeps model-level/layerwise/distributed headings. +- [claim-verified] Benchmark table numbers are carried over unchanged (HV 147.05/73.02/127.84 s etc.); no new fitted formula/threshold introduced to re-derive. +- [claim-verified] mkdocs.yml adds redirects for design/feature/{cpu_offloading,model_level_offload,layerwise_offload,distributed_layerwise_offload}.md -> offloader pages (mkdocs.yml:84-87). +- [claim-verified] Legacy section anchors preserved via 'Compatibility anchors' headings in attention_backends.md and cpu_offload.md (read both at head). +- [claim-refuted] PR body says 'InferMatrix direct review (no remaining findings)' — I found two broken command families and one wrong model-table row (see review_comments). +- [validated] Selector resolution order matches docs: vllm_omni/diffusion/attention/selector.py:112-132 resolves exact per_role[role] -> per_role[role_category] -> default -> platform default and returns (class, None) only on platform default. +- [validated] Selector cache/log claims: selector.py:47-61 `_cached_get_backend_cls(backend_name, head_size, allow_trtllm_default)` and a separate `_log_backend_resolution(role, role_category, backend_name, source)` cache. +- [validated] Registry override semantics: registry.py:116-150 `_DIFFUSION_ATTN_OVERRIDES` consulted in `get_path()`; `register_diffusion_backend()` mutates overrides without changing enum values. +- [validated] Offload priority/derive claims: offloader/base.py:69-104 selects DISTRIBUTED_LAYER_WISE > LAYER_WISE > MODEL_LEVEL, logs higher-priority disabling, derives dp_size from parallel_config. +- [validated] get_offload_backend contract: offloader/__init__.py:60-103 returns None for NONE/unsupported before any enable(), so no partially installed hooks. +- [validated] XPU blocking-copy claim: sequential_backend.py:80-82 `non_blocking = not self.use_hsdp and not current_omni_platform.is_xpu()` with the cache-eviction comment. +- [validated] --dlo-resident-layers flag exists and no-AllGather requirement is real: base.py:104-110 raises when set with dlo_use_allgather; CLI serve.py:726; OffloadPlan.resident_dit_paths at offload_plan.py:51. +- [validated] attention_backend_selection.md contract matches selector.py: resolution order per_role[role]->category->default->platform (selector.py:97-146), cache keyed by backend_name/head_size/allow_trtllm_default (_cached_get_backend_cls). + +`docs/user_guide/diffusion/attention_backends.md:55` [major] — The split backend pages show `vllm-omni serve --diffusion-attention-backend ...` without `--omni` (attention_backends.md:55,74,79; dense_backends.md:17,29,52,77; sage.md:32,54; trtllm.md:26,42,80; huggingface_hub.md:28; rainfusion.md:23). vllm_omni/entrypoints/cli/main.py dispatches to upstream vLLM when `--omni` is absent, and `--diffusion-attention-backend`/`--diffusion-attention-config` are only registered inside OmniServeCommand.subparser_init (serve.py:600), so these copied commands reach the upstream parser and reject the unknown flag. Add `--omni` to each command, or use the repo-standard `vllm serve --omni --diffusion-attention-backend ...`. (evidence: docs/user_guide/diffusion/attention_backends.md:55 `vllm-omni serve --diffusion-attention-backend FLASH_ATTN` — changed by this diff, missing `--omni`. Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/cli/main.py:12 `if "--omni" not in sys.argv:` / vllm_omni/entrypoints/cli/main.py:13 `from vllm.entrypoints.cli.main import main as vllm_main` / vllm_omni/entrypoints/cli/main.py:15 `vllm_main()` / vllm_omni/entrypoints/cli/main.py:16 `return`; and vllm_omni/entrypoints/cli/serve.py:600 `"--diffusion-attention-backend",` (inside `OmniServeCommand.subparser_init`, only reached on the `--omni` branch).) + +`docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md:31` [major] — In `docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md:31` the primary AllGather usage block runs `--data-parallel-size 4` under `--omni`. `vllm_omni/entrypoints/cli/serve.py` rejects this: `prohibited_with_omni` (lines 147-156) maps `data_parallel_size` to `--data-parallel-size`, and the raise at 157-165 executes before the diffusion-model early return at 189-195, so the documented command fails at parse time. Point users at per-stage YAML `parallel_config.data_parallel_size` via `--deploy-config`, or the `--usp 4` example shown below. (evidence: New file in this diff: `docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md` contains ` --data-parallel-size 4` in the primary AllGather usage block. Decisive code is unchanged by this diff and present in the PR-time tree: `vllm_omni/entrypoints/cli/serve.py:148` ` "data_parallel_size": "--data-parallel-size",`; `vllm_omni/entrypoints/cli/serve.py:157-165` ` offenders = sorted(flag for dest, flag in prohibited_with_omni.items() if dest in explicit_cli_keys)` / ` if offenders:` / ` raise ValueError(` (message: 'The following CLI args are not supported under --omni: ...'); and the diffusion early return comes AFTER this check at `vllm_omni/entrypoints/cli/serve.py:190-195` ` from vllm_omni.diffusion.utils.hf_utils import is_diffusion_model` ... ` if model and is_diffusion_model(model):` / ` logger.info("Detected diffusion model: %s", model)` / ` return`.) + +`docs/user_guide/diffusion/attention_backends.md:28` [minor] — The new Backend options table links to anchors such as `attention_backends/dense_backends.md#torch_sdpa`, whose targets are headings written as `## `TORCH_SDPA``. Run a MkDocs build before merge to confirm the default slugify strips backticks and lowercases as expected, rather than relying on the assumption that the heading targets exist. (evidence: docs/user_guide/diffusion/attention_backends.md:28 `| `TORCH_SDPA` | Dense | Conservative reference; always available | [Dense Backends](attention_backends/dense_backends.md#torch_sdpa) |` — the anchor assumes a backtick-stripped, lowercased slug; docs/user_guide/diffusion/attention_backends/dense_backends.md:10 `## `TORCH_SDPA`` — the actual heading includes backticks, so resolution depends on MkDocs slugify behavior (present in the PR-time tree, added/rewritten by this diff).) + +`docs/user_guide/diffusion/attention_backends/trtllm.md:11` [minor] — Reword `## Requirements` as automatic-selection prerequisites: explicit `TRTLLM_ATTN` selection in `vllm_omni/platforms/cuda/platform.py:190-201` validates only datacenter-Blackwell compute capability (major == 10) and `flashinfer_available`; `head_size == 128` and `trtllm_gen_available` are checked only in the auto-route default branch at lines 205-211, and `AttentionBackend.supports_head_size()` has no callers. State explicitly that an explicit selection on `head_dim != 128` or a masked model is not rejected at this selection point. (evidence: Unchanged by this diff, present in the PR-time tree: `vllm_omni/platforms/cuda/platform.py:190` `trtllm_attn_supported = compute_capability is not None and compute_capability.major == 10`; `vllm_omni/platforms/cuda/platform.py:197` `if not flashinfer_available:` — the explicit branch has no head_size or trtllm-gen check. Auto-route only: `vllm_omni/platforms/cuda/platform.py:205-211` `trtllm_attn_default_ok = (allow_trtllm_default and compute_capability is not None and compute_capability.major == 10 and head_size == 128 and trtllm_gen_available)`. `vllm_omni/diffusion/attention/layer.py:83` `allow_trtllm_default = get_diffusion_model_metadata(model_class_name).attention_mask_free` — passed only into the platform-default call. `vllm_omni/diffusion/attention/backends/abstract.py:80` `def supports_head_size(cls, head_size: int) -> bool:` — repo-wide grep finds no caller.) + +`docs/user_guide/diffusion/cpu_offload.md:46` [minor] — The copied supported-models table in cpu_offload.md lists OvisImage's layerwise block attribute as `transformer`, but vllm_omni/diffusion/models/ovis_image/ovis_image_transformer.py:369 declares `_layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]` and the class instantiates only `self.transformer_blocks` and `self.single_transformer_blocks` (no `self.transformer`). Update the row to `transformer_blocks`, `single_transformer_blocks`. (evidence: docs/user_guide/diffusion/cpu_offload.md:46 `| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | — | yes | — | `transformer` |` — changed by this diff. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/ovis_image/ovis_image_transformer.py:369 `_layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]`; vllm_omni/diffusion/models/ovis_image/ovis_image_transformer.py:399 `self.transformer_blocks = nn.ModuleList(` and the subsequent `self.single_transformer_blocks = nn.ModuleList(` with no `self.transformer` container.) + +`docs/design/feature/offloader/README.md:17` [nit] — The strategy-selection list at docs/design/feature/offloader/README.md:17-18 renders `DISTRIBUTED_LAYERWISE` and `LAYERWISE`, but vllm_omni/diffusion/offloader/base.py:19-20 defines the actual `OffloadStrategy` members as `LAYER_WISE = "layer_wise"` and `DISTRIBUTED_LAYER_WISE = "distributed_layer_wise"`. Use the real enum member spellings so symbol grep resolves. (evidence: In this diff (new file): docs/design/feature/offloader/README.md:17 `1. \`DISTRIBUTED_LAYERWISE\`;` and :18 `2. \`LAYERWISE\`;`. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/offloader/base.py:19 `LAYER_WISE = "layer_wise" # Block-level` and :20 `DISTRIBUTED_LAYER_WISE = "distributed_layer_wise" # Block-level with DP sharding + H2D/AllGather overlap`.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.trace.json.gz b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.trace.json.gz new file mode 100644 index 000000000..4fe045f82 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_holdout5_r1/pr6075.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_smoke/manifest.json b/eval/dataset/arms/copilot_v16f_smoke/manifest.json new file mode 100644 index 000000000..31fadb74d --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_smoke/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v16f_smoke", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_smoke/pr4870.cost.json b/eval/dataset/arms/copilot_v16f_smoke/pr4870.cost.json new file mode 100644 index 000000000..d3218d2f0 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_smoke/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1709.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_smoke/runs/pr4870/run-20260815-153037-9cc8eb", + "recorded_at": "2026-08-15T15:59:04", + "attempts": [ + { + "invocation_id": "inv-de30342e", + "outcome": "run:run-20260815-153037-9cc8eb", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-153037-9cc8eb", + "llm_calls": 143, + "input_tokens": 1585636, + "output_tokens": 216878, + "usd": 1.011026, + "source": "spans" + } + ], + "attempt_usd_total": 1.011026, + "llm_calls": 143, + "input_tokens": 1585636, + "output_tokens": 216878, + "usd": 1.011026, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-153037-9cc8eb", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.011, + "minutes": 28.46, + "ci_minutes": 0, + "input_tokens": 1585636, + "output_tokens": 216878, + "tool_calls": 286, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0659, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 143, + "by_role": { + "agent": { + "usd": 0.966859, + "input_tokens": 1561792, + "output_tokens": 182645, + "calls": 140 + }, + "reducer": { + "usd": 0.044167, + "input_tokens": 23844, + "output_tokens": 34233, + "calls": 3 + } + }, + "cache_read_tokens": 12753280, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.01, + "gate": 1.56, + "review": 1701.77, + "report": 0.11 + }, + "time_to_first_result_s": 1707.4 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 39551 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_smoke/pr4870.md b/eval/dataset/arms/copilot_v16f_smoke/pr4870.md new file mode 100644 index 000000000..0d23b854f --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_smoke/pr4870.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] The #4527 full-payload starvation guard is intact at this head: the non-async-chunk branch still ships the full payload to the next stage (comment citing PR #4792 unchanged); this PR only changes the split length fed into those payloads. Residual: the new unit test monkeypatches `_should_accumulate_full_payload_output` to False, so the accumulate path itself is exercised only via the shared pre-split payload code, not directly. +- [resolved] No stale `seq_len=` kwarg remains: grep of `_build_omni_model_runner_output_from_snapshot` lists only the 2027 source call and test calls; the diff removes the kwarg at each site, so the removed parameter cannot cause a TypeError. +- [claim-verified] tests/worker/test_gpu_ar_model_runner.py:438 test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only drives hidden_states shape (1,1) vs codes shape (3,16) and asserts per-request slices codes[0:1]/[1:2]/[2:3]; this would fail under the old seq_len=1 (whole-batch clone). +- [claim-verified] tests/entrypoints/openai_api/test_serving_speech.py:3453 and :3473 add the two serving tests named in the PR body (final_only for qwen3, delta for non-qwen3). +- [claim-verified] PR body claim that the E2E probe file is not included in the PR diff is confirmed by diff_stat. +- [claim-verified] 'The probe file was used only for validation and is not included in this PR diff' — diff_stat lists exactly 4 files (2 test files, serving_speech.py, gpu_ar_model_runner.py); no probe file present at head 231b3ddb. +- [claim-verified] Prefix-cache splitter unaffected: core/prefix_cache.py:597 calls `to_payload_element` with its own seq_len, and the mm_outputs.py docstring states the prefix-cache passthrough already passes 'the total scheduled token count' — consistent with the new contract. +- [claim-verified] CI pre-commit SUCCESS is consistent with the PR's ruff claim; diff_stat confirms exactly the 4 claimed files. +- [claim-verified] PR body states the E2E probe file is not included in the diff; confirmed no e2e probe file appears in the 4-file diff, so the FINAL_ONLY response mode has no committed end-to-end coverage. +- [claim-verified] Worker change is not qwen3-only: it alters the shared `needs_pooler_payload` path for every model; for normal async_chunk=True models old `seq_len` equaled `hidden_states.shape[0]` == total scheduled tokens, so behavior there is equivalent, with tail-only-hidden models (qwen3_tts and any sibling emitting batch-aligned codes) now sliced correctly. +- [validated] gpu_ar_model_runner.py:1731-1787 read in situ: mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) now feeds to_payload_element(seq_len=...), replacing the removed hidden_states.shape[0]-derived seq_len. +- [validated] entrypoints/utils.py:759 coerce_param_message_types maps is_streaming=True->DELTA and False->FINAL_ONLY, so the new False branch correctly keeps FINAL_ONLY. +- [validated] Fix mechanism confirmed: vllm_omni/utils/mm_outputs.py:131-133 `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len: return element[start:end]` else falls through to `element.clone()` — a mismatched split length duplicates the whole batched tensor into every request (the #4851 corrupted-audio mechanism). +- [validated] Producer contract: qwen3_tts_talker.py `make_omni_output` does `audio_codes = torch.cat(audio_codes_list, dim=0)` over per-request info dicts → `codes.audio` dim-0 = total scheduled tokens (unpadded); new `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` matches this, while old `seq_len = hidden_states.shape[0]` was the (possibly padded / tail-only) forward length and could never match for tail-only hidden. + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:~905` [major] — NPU twin still slices multimodal payloads with `seq_len = hidden_states.shape[0]` (npu_ar_model_runner.py:905, passed at :1144). The GPU runner changed this to `int(scheduler_output.total_num_scheduled_tokens)` (gpu_ar_model_runner.py:1734). For Qwen3-TTS with `requires_full_prefix_cached_hidden_states = False` (qwen3_tts_talker.py:311), `codes.audio` dim0 is the total scheduled-token count while `hidden_states.shape[0]` differs (tail-only or padded), so `to_payload_element`'s equality check (mm_outputs.py:131) fails and the whole batched tensor is cloned into every request — the same cross-request contamination the GPU fix eliminates. NPU ships qwen3_tts patches (platforms/npu/platform.py:37-41), so apply the same `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` fix on NPU and add a regression test. (evidence: CHANGED by this diff: vllm_omni/worker/gpu_ar_model_runner.py:1734 ` mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`. UNCHANGED by this diff, present in the PR-time tree: vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905 ` seq_len = hidden_states.shape[0]`; :1144 ` seq_len=seq_len,`. UNCHANGED: vllm_omni/utils/mm_outputs.py:131 ` if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len:` — when false, the tensor falls through to the same function's `elif isinstance(element, torch.Tensor): return element.clone()` branch, cloning the entire batch tensor per request. UNCHANGED: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:311 ` self.requires_full_prefix_cached_hidden_states = False`; vllm_omni/platforms/npu/platform.py:37 ` from vllm_omni.platforms.npu.models.qwen3_tts_code2wav import (` ... :41 ` apply_qwen3_tts_code2wav_patch()`.) + +`vllm_omni/worker/gpu_ar_model_runner.py:~1790` [major] — This hunk changes the mm split key from `hidden_states.shape[0]` (padded forward length) to the unpadded `scheduler_output.total_num_scheduled_tokens` (gpu_ar_model_runner.py:1734,1790). `to_payload_element` slices only when `element.shape[0] == seq_len`; any other tensor is `clone()`d whole, so forward-axis producers whose dim 0 is the padded length now ship the entire batch to every request. qwen3_omni thinker's `captured_layer_dict` (qwen3_omni.py:497) is such a producer: it is cloned from padded forward hidden states in Qwen3MoeLLMModel.forward and is passed untrimmed, unlike the talker which truncates to `span_len`. Please trim forward-axis producers to the scheduled total before `make_omni_output`, or gate the slice for known per-token keys when `element.shape[0] >= seq_len`, and run the make_omni_output census under non-eager padding before merge. (evidence: gpu_ar_model_runner.py:1734 (added by this diff): ` mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`; gpu_ar_model_runner.py:1790: ` seq_len=mm_seq_len,`. vllm_omni/utils/mm_outputs.py:134 (unchanged by this diff, present in PR-time tree): ` if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len:` -> ` return element[start:end].contiguous()`; the tensor else-branch returns ` return element.clone()` (whole batched tensor per request). vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py:497 (unchanged by this diff, present in PR-time tree): ` multimodal_outputs: OmniPayload = captured_layer_dict if captured_layer_dict is not None else {}`. vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py:552 (unchanged by this diff, present in PR-time tree): ` layers[layer_idx] = hidden_states.clone().view(-1, hidden_states.shape[-1])` — captures the padded forward input, so dim 0 is num_tokens_padded, not total_num_scheduled_tokens.) + +`vllm_omni/entrypoints/openai/serving_speech.py:~3509` [minor] — The FINAL_ONLY flip for qwen3_tts streaming requests with async_chunk=False is only unit-tested at the coerce_param_message_types argument level, and the PR's real-serving E2E validation used stream=False, so the streaming response path is not E2E-covered. Please run tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py::test_voice_001[no_async_chunk] (stream=True, stream_format='audio', lines 58/77-78) on the remote H20 setup and add the result to the test table. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload` (in this diff; for qwen3_tts with async_chunk=False, qwen3_full_payload=True so coerce_param_message_types receives False → FINAL_ONLY). tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py:58 `id="no_async_chunk",`; :77 `"stream": True,`; :78 `"stream_format": "audio",` (unchanged by this diff, present in the PR-time tree). PR body: 'The E2E probe used real remote H20 serving with Qwen3-TTS CustomVoice, `--no-async-chunk`, `stream=False`, and `request_num=3`.') + +`vllm_omni/worker/gpu_ar_model_runner.py:~1734` [minor] — The `buildkite/vllm-omni-amd-ci` check is failing at this head (confirmed via gh_ci_read: state=FAILURE). The diff is not obviously inert for that lane: it edits the shared output builder in `vllm_omni/worker/gpu_ar_model_runner.py` (removing the `seq_len` argument and deriving `mm_seq_len` at line 1734) and `tests/worker/test_gpu_ar_model_runner.py`, and the AMD PR lane selects `test-amd-ready.yaml` (`.buildkite/bootstrap-amd-omni.sh:185`). Please confirm whether this failure is pre-existing on main or first appears on this branch by checking the same job on a recent main build. (evidence: gh_ci_read for PR 4870 returns `{"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` — the check is failing at this head. `vllm_omni/worker/gpu_ar_model_runner.py:1734` `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` — the anchor exists in the PR-time tree. `.buildkite/bootstrap-amd-omni.sh:185` `TEST_YAML="test-amd-ready.yaml"` — confirmed (unchanged by this diff): the AMD PR lane selects the ready suite. The diff itself modifies `vllm_omni/worker/gpu_ar_model_runner.py` (removing the `seq_len` parameter from `_build_omni_model_runner_output_from_snapshot` and its callers) and `tests/worker/test_gpu_ar_model_runner.py`.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3508` [minor] — PR body's E2E table validates only the CustomVoice non-streaming concurrent path (test_voice_001_nonstream_concurrent_no_async_chunk[no_async_chunk]), while issue #4851 also lists the Base-model test_response_format_001[no_async_chunk]. The runner fix at gpu_ar_model_runner.py:1734 is model-agnostic, so it likely repairs both, but please either run/record tests/e2e/online_serving/test_qwen3_tts_base_expansion.py::test_response_format_001[no_async_chunk] (and, if available, test_voice_002[no_async_chunk]) or state explicitly why the Base failure is out of scope. (evidence: Anchor in diff: vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`. Fix is in the shared worker and model-agnostic (in diff): vllm_omni/worker/gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`. PR body (gh_pr_view, GitHub metadata, not a repo file) E2E table has only one row: `| `test_voice_001_nonstream_concurrent_no_async_chunk[no_async_chunk]` | `ff3cbb4c` | After fix | Passed | ...`. Issue #4851 body (gh_issue_view, GitHub metadata): 'Affected tests from the attached logs:\n- `tests/e2e/online_serving/test_qwen3_tts_base_expansion.py::test_response_format_001[no_async_chunk]`\n- `tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py::test_voice_002[no_async_chunk]`'.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3509` [minor] — Update the `create_speech` docstring at serving_speech.py:4034: it still says raw streaming 'yields each Code2Wav chunk as raw bytes as soon as it is', but the new qwen3_tts async_chunk=False + stream=True mode sets `is_streaming_request = request.is_streaming() and not qwen3_full_payload` (line 3509) to False, i.e. FINAL_ONLY, so this mode emits the final waveform once rather than per Code2Wav chunk. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3508 (changed by this diff): ` qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; vllm_omni/entrypoints/openai/serving_speech.py:3509 (changed by this diff): ` is_streaming_request = request.is_streaming() and not qwen3_full_payload`; vllm_omni/entrypoints/openai/serving_speech.py:4034 (unchanged by this diff, present in the PR-time tree): ` Raw audio streaming yields each Code2Wav chunk as raw bytes as soon as it is`) + +`tests/worker/test_gpu_ar_model_runner.py:438` [minor] — Add a sibling test for the accumulate branch: monkeypatch `_should_accumulate_full_payload_output` to True with a tail-only multi-request batch and assert `accumulate_full_payload_output` receives per-request payloads sliced by scheduled tokens. As written, `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` monkeypatches the accumulate gate to False and proves only the non-accumulate split path. (evidence: tests/worker/test_gpu_ar_model_runner.py:449 `monkeypatch.setattr(GPUARModelRunner, "_should_accumulate_full_payload_output", lambda self: False)` — the new test explicitly forces the non-accumulate branch; vllm_omni/worker/gpu_ar_model_runner.py:1805 (unchanged by this diff, present in PR-time tree) `if pooler_inter and self._should_accumulate_full_payload_output():` — this branch is skipped whenever the gate is False, so the test cannot catch a break in the accumulate path.) + +`tests/entrypoints/openai_api/test_serving_speech.py:~3453` [minor] — This test only asserts `coerce_param_message_types` is called with `False`; it never exercises `_generate_audio_chunks` (serving_speech.py:2700), the path that turns a FINAL_ONLY engine generator into streamed bytes. The qwen3 `async_chunk=False` change (serving_speech.py:3508-3509) therefore leaves its streaming delivery path uncovered by this PR. Consider a test feeding a FINAL_ONLY generator into `_generate_audio_chunks` (or a `create_speech` streaming smoke) asserting one full chunk + done. (evidence: tests/entrypoints/openai_api/test_serving_speech.py:3471 (PR-time tree) `mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, False)` — the test's only assertion; it never calls `_generate_audio_chunks`. vllm_omni/entrypoints/openai/serving_speech.py:3508-3509 (changed by this diff) `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` and `is_streaming_request = request.is_streaming() and not qwen3_full_payload` — the behavior change that needs downstream coverage. Unchanged by this diff, present in the PR-time tree: serving_speech.py:2700 `async def _generate_audio_chunks(` (definition) with streaming call sites at 2883, 3723, and 4071; the new test does not reach any of them.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_smoke/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v16f_smoke/pr4870.trace.json.gz new file mode 100644 index 000000000..f194195f7 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_smoke/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/manifest.json b/eval/dataset/arms/copilot_v16f_train_r1/manifest.json new file mode 100644 index 000000000..fe4aa319b --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v16f_train_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4804.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4804.cost.json new file mode 100644 index 000000000..a370e781d --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1876.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4804/run-20260815-155951-d92aae", + "recorded_at": "2026-08-15T16:31:05", + "attempts": [ + { + "invocation_id": "inv-7493bb61", + "outcome": "run:run-20260815-155951-d92aae", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-155951-d92aae", + "llm_calls": 151, + "input_tokens": 1676336, + "output_tokens": 285086, + "usd": 1.319711, + "source": "spans" + } + ], + "attempt_usd_total": 1.319711, + "llm_calls": 151, + "input_tokens": 1676336, + "output_tokens": 285086, + "usd": 1.319711, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-155951-d92aae", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.3197, + "minutes": 31.23, + "ci_minutes": 0, + "input_tokens": 1676336, + "output_tokens": 285086, + "tool_calls": 300, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2054, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 151, + "by_role": { + "agent": { + "usd": 1.269167, + "input_tokens": 1615280, + "output_tokens": 254164, + "calls": 149 + }, + "reducer": { + "usd": 0.050544, + "input_tokens": 61056, + "output_tokens": 30922, + "calls": 2 + } + }, + "cache_read_tokens": 20500224, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.56, + "gate": 1.6, + "review": 1867.41, + "report": 0.06 + }, + "time_to_first_result_s": 1873.62 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 76673 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4804.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4804.md new file mode 100644 index 000000000..bc81ae29e --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4804.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] MOSS-local runaway generation: pipeline.py now pins `"stop_token_ids": [151645]` for the local variant AND compute_logits warns 'still running after im_end_token_id=%d was forced once' — residual: id is hardcoded and delay/realtime variants are not pinned +- [resolved] warn-instead-of-raise weight loading: load_weights now raises 'MOSS Audio Tokenizer weights were not fully loaded' on any missing/skipped/shape-mismatch — residual: the raise sits OUTSIDE _build_codec's try, so a wrong-class construction can no longer reach the legacy fallback (see comment on _build_codec) +- [claim-verified] 'make async chunk indeed work instead of just accumulating': old accumulate-everything logic (`chunk_frames: int = 1 << 30`, emit on finish) survives only on the delay path; local/realtime now emit real 1→15-frame chunks +- [claim-refuted] PR description 'Fix some bugs: - TODO' is literally unfilled while the diff carries rider behavior changes (stop_token_ids pin, seed unification for ALL tts model types, strict codec loading) — see nit comment +- [validated] Seed-key rename complete: grep `qwen3_tts_request_seed` → 0 matches repo-wide; runner reads `extra_args.get("tts_local_seed")` (gpu_model_runner.py:1826) and both tests updated +- [validated] Old processor name fully retired: grep `talker2codec_async_chunk` → 0 matches; pipeline.py wires `talker2codec_delay_async_chunk` (delay) and `talker2codec_raw_async_chunk` (realtime, local) +- [validated] Initial-chunk threshold works: chunk_transfer_adapter.py:339 `self.put_req_chunk[external_req_id] += 1` after each put, so `emitted_any` flips after the 80ms first chunk and steady state uses codec_chunk_frames=15 +- [validated] GPU real path satisfies the new hard contract: gpu_generation_model_runner.py:316 `model_kwargs["seq_token_counts"] = tokens`; dummy path patched at :799 +- [validated] Abort release wired on GPU: gpu_generation_model_runner.py:81-82 calls `self.model.on_requests_finished(scheduler_output.finished_req_ids)`; codec hook clears slot, pending codes, starved set +- [validated] moss_tts_local.yaml self-consistent: `codec_stream_slots: 8` == stage-1 `max_num_seqs: 8`; `initial_codec_chunk_frames: 1` yields the advertised 80 ms first packet +- [validated] `AudioResampler` from vllm.multimodal.audio is an established dependency (10+ pre-existing call sites incl. benchmarks/patch/patch.py:964) — no version-floor risk from the new call +- [validated] fish_speech talker_mtp honors the new kwarg (`generator=kwargs.get("generator")`, fish_speech_slow_ar.py); qwen3_omni.talker_mtp swallows it via `**kwargs` (no 'generator' anywhere in qwen3_omni.py) — a seeded request there pays the scalar-decode path with zero determinism gain (endpoint reachability unverified, kept out of comments) +- [validated] All six shipped MOSS deploy yamls set `async_chunk: true` (moss_tts:4, moss_ttsd:4, moss_sound_effect:6, moss_voice_generator:4, moss_tts_realtime:5, moss_tts_local:6), so the broken-by-this-diff sync `talker2codec` path is only reachable via custom configs (comment scoped to latent) +- [validated] tests/worker/test_omni_gpu_model_runner.py:10 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; .buildkite/test-merge.yml:29 runs `pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor`, so the touched test file IS selected at merge time (not deselected). + +`vllm_omni/deploy/moss_tts_realtime.yaml:~42` [major] — Confirmed. The realtime stage-0 now routes through talker2codec_raw_async_chunk (pipeline.py:65), which stamps codec_streaming=True on every payload (stage_input_processors/moss_tts.py:301 and :334). moss_tts_realtime.yaml is unchanged: stage-0 max_num_seqs: 8 (line 24) vs stage-1 max_num_seqs: 1 (line 42), with no codec_stream_slots. _stream_slots defaults to 0 (modeling_moss_tts_codec.py:176), so _ensure_stream_session falls back to scheduler_config.max_num_seqs = 1 and creates one slot (line 444). Under concurrency, requests buffer; a request whose finished packet arrives while the single slot is held drops its buffered codes via _finish_stream_request(..., None) (lines 479-483) and completes with empty/partial audio. Raise stage-1 max_num_seqs and add codec_stream_slots (as done for moss_tts_local.yaml), or make starved-finish a hard error, and add a realtime concurrency test. (evidence: vllm_omni/model_executor/models/moss_tts/pipeline.py:65 `async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_raw_async_chunk"),` (MOSS_TTS_REALTIME_PIPELINE stage 0; changed by this diff). vllm_omni/model_executor/stage_input_processors/moss_tts.py:334 `codec_streaming=True,` (normal emit; also line 301 for the finish sentinel; changed by this diff). vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:176 `self._stream_slots: int = self._connector_int("codec_stream_slots", default=0)` (changed by this diff). vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:444 `slots = int(getattr(scheduler_cfg, "max_num_seqs", 1) or 1)` (changed by this diff). vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:479-483 `"MOSS codec stream request %s finished before a codec stream slot became available; " / "dropping buffered codes instead of offline-decoding a non-delta waveform."` then `self._finish_stream_request(request_id, session, None)` (changed by this diff). Unchanged by this diff, present in the PR-time tree: vllm_omni/deploy/moss_tts_realtime.yaml:24 `max_num_seqs: 8` and :42 `max_num_seqs: 1`, with no `codec_stream_slots` key (grep finds `codec_stream_slots` only in moss_tts_local.yaml:20 and modeling_moss_tts_codec.py:176).) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:~729` [major] — `_build_codec` (modeling_moss_tts_codec.py:727) tries the vendored v2 config/model first for every MOSS variant and falls back to the legacy codec only if that try block raises. `MossAudioTokenizerV2Config` is permissive (`sampling_rate: int = 48000`, `number_channels: int = 2`, `kwargs.pop("model_type", None)`, legacy aliases), and it shares `model_type = "moss-audio-tokenizer"` with the legacy config, so a v1 mono/24 kHz checkpoint parses as v2 and skips fallback. The strict `load_weights` raise (modeling_moss_tts_codec.py:680) is outside that try/except, so weight-name/shape mismatches hard-fail instead of reaching the legacy path. Gate v1-vs-v2 on an explicit checkpoint marker rather than try/except, and verify both a delay/realtime v1 checkpoint and a Local v2 checkpoint load through their intended codec classes. (evidence: modeling_moss_tts_codec.py:727-736 `def _build_codec(self, codec_path: str) -> tuple[Any, nn.Module]: try: codec_cfg = MossAudioTokenizerV2Config.from_pretrained(codec_path); codec = MossAudioTokenizerV2Model(codec_cfg); ... return codec_cfg, codec; except Exception: logger.exception(...); codec_cfg = MossAudioTokenizerConfig.from_pretrained(codec_path); codec = MossAudioTokenizerModel(codec_cfg); return codec_cfg, codec` — v2-first, fallback only on exception. configuration_moss_audio_tokenizer_v2.py:89 `sampling_rate: int = 48000,`; :94 `number_channels: int = 2,`; :105 `kwargs.pop("model_type", None)` — permissive, no v2 marker required. modeling_moss_tts_codec.py:680 `raise RuntimeError("MOSS Audio Tokenizer weights were not fully loaded: ...")` — inside `load_weights`, after `_build_codec` returns, so it cannot fall back. audio_tokenizer.py:43 `sampling_rate: int = 24000,` and :48 `number_channels: int = 1` — legacy v1 codec defaults whose fields are all accepted by the v2 config, so a v1 checkpoint parses as v2 and never triggers fallback.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1324` [major] — No test exercises the new Local-v1.5 streaming path: `MossTTSLocalTalkerForGeneration` sets `talker_mtp_accepts_req_infos = True` (modeling_moss_tts_talker.py:1324), which gates runner-side `req_ids`/`req_infos` passing, but the only changed runner test uses `runner.model = SimpleNamespace(talker_mtp_output_key=("codes", "audio"))` (test_omni_gpu_model_runner.py:159) without that attribute and `CaptureTalkerMTP.forward` accepts no `req_ids`/`req_infos`, so the gated branch cannot execute. grep under `tests/` also returns no matches for `MossTTSLocalTalkerForGeneration`, `talker2codec_raw_async_chunk`, or `_MossCodecStreamSession`. Add the proposed weight-free `make_omni_output` routing test and `talker2codec_raw_async_chunk` threshold/finish tests. (evidence: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1324 ` self.talker_mtp_accepts_req_infos = True` (added by this diff). tests/worker/test_omni_gpu_model_runner.py:159 ` runner.model = SimpleNamespace(talker_mtp_output_key=("codes", "audio"))` (unchanged by this diff, present in the PR-time tree; `SimpleNamespace` has no `talker_mtp_accepts_req_infos`, so the runner-side `req_ids`/`req_infos` branch is never taken). Grep of `tests/` for `MossTTSLocalTalkerForGeneration`, `talker2codec_raw_async_chunk`, and `_MossCodecStreamSession` returns no matches.) + +`vllm_omni/worker/gpu_generation_model_runner.py:81` [major] — GPUGenerationModelRunner gains on_requests_finished and dummy-run seq_token_counts, but NPUGenerationModelRunner still lacks both. Because this diff makes MossTTSCodecDecoder hard-raise when seq_token_counts is absent, a MOSS codec stage under the NPU generation runner now fails profile/warmup and leaks stream slots on aborts. Update NPU _update_request_states and _dummy_run to match; this strict requirement is a shared-contract change, and the NPU gap is newly introduced by this diff, not pre-existing. (evidence: gpu_generation_model_runner.py:81-82 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)` (in this diff). gpu_generation_model_runner.py:799 `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]` (in this diff). modeling_moss_tts_codec.py:260-264 `token_counts = self._normalize_seq_token_counts(kwargs.get("seq_token_counts"))` / `if token_counts is None:` / `raise RuntimeError(` / `"MossTTS codec requires seq_token_counts; otherwise concatenated "` / `"codec tokens cannot be split per request."` (in this diff). Unchanged by this diff, present in the PR-time tree: npu_generation_model_runner.py:48-51 `def _update_request_states(self, scheduler_output: SchedulerOutput):` / `# remove requests` / `for req_id in scheduler_output.finished_req_ids:` / `self.input_batch.remove_request(req_id)` — no on_requests_finished call anywhere in this file. Unchanged by this diff: npu_generation_model_runner.py:780-782 injects only `get_dummy_runtime_additional_information` into model_kwargs; repo-wide grep shows no `seq_token_counts` assignment inside NPUGenerationModelRunner._dummy_run (the file's only seq_token_counts set is line 299 in execute_model, not _dummy_run).) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:~1148` [minor] — The talker emission now sends only the newest code frame per step (`per_req_codes[i] = new_codes.unsqueeze(0)` in the realtime talker; the Local `talker_mtp` path marks the current frame `emit: True`), but all three pipelines still wire `sync_process_input_func=talker2codec`, whose contract is the full `(T, NQ)` sequence. With `async_chunk: false` (note: the deploy default is actually `true` — `stage_config.py:371` — so this only fires on an explicit opt-out; shipped MOSS yamls stay async), Stage-1 would receive at most one frame, and the Local stop step's empty `multimodal_outputs={}` becomes silence. Replace/remove the sync processor for the local/realtime pipelines, keep full accumulation when `async_chunk` is off, or fail loudly in that mode. (evidence: ['vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1142 `per_req_codes[i] = new_codes.unsqueeze(0)` — in-diff; realtime talker emits a single current frame per request instead of the accumulated (T, n_vq) sequence.', 'vllm_omni/model_executor/models/moss_tts/pipeline.py:48 `sync_process_input_func=f"{_PROC}.talker2codec",` — unchanged by this diff, present in the PR-time tree; also line 77 (realtime) and line 109 (local).', 'vllm_omni/model_executor/stage_input_processors/moss_tts.py:44 `Stage 0 output contains ``codes["audio"]`` shaped ``(T, NQ)`` where T is the number of generated audio frames` — unchanged by this diff; the sync processor then does `codes_nq_t = audio_codes.transpose(0, 1).contiguous()` and flattens whatever it receives.', "vllm_omni/config/stage_config.py:371 `async_chunk: bool = True` (via grep) — corrects the comment's 'async_chunk: false (the default)' premise; the default is true."]) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:~1524` [minor] — The new `talker_mtp` (modeling_moss_tts_talker.py:1524-1527) declares `temperature`, `top_k`, and `top_p` but never reads them: it hardcodes `audio_temperature = 1.7`, `audio_top_k = 25`, `audio_top_p = 0.8`, and only honors `do_sample` (`do_sample_val = True if do_sample is None else bool(do_sample)`). `gpu_model_runner.py:1817` sources those kwargs from `model_config.subtalker_sampling_params`, so configured subtalker temperature/top_k/top_p are silently dead for MOSS-TTS-Local. Honor non-None passed values with these model-card fallbacks, or remove the ignored parameters from the signature and log/raise when `subtalker_sampling_params` sets them for this model. (evidence: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1524 `audio_temperature = 1.7` (and 1525-1527 `audio_top_k = 25` / `audio_top_p = 0.8` / `do_sample_val = True if do_sample is None else bool(do_sample)`) — the `talker_mtp` signature added by this diff accepts `temperature`, `top_k`, `top_p`, but the body calls `generate_frame(... temperature=audio_temperature, top_k=audio_top_k, top_p=audio_top_p)`, so those parameters are never read. vllm_omni/worker/gpu_model_runner.py:1817 `subtalker_params = getattr(self.vllm_config.model_config, "subtalker_sampling_params", None)` — unchanged by this diff, present in the PR-time tree as the source of the ignored kwargs.) + +`vllm_omni/benchmarks/patch/patch.py:~1251` [minor] — Confirmed: `patch.py:1251` falls back to `output.tts_output_pcm_bytes = bytes(pcm_capture)` after normalization fails, but the field is documented as "Raw PCM s16le mono at 24 kHz for Seed-TTS WER" and `seed_tts_eval.py` consumers assume 24 kHz mono (`pcm_s16le_mono_to_wav_bytes(pcm, sample_rate=24000)`; `_pcm_s16le_to_f32_16k(pcm, pcm_sample_rate=24000)`). With 48 kHz stereo capture, a normalization failure produces wrongly-interpreted WER/SIM/UTMOS audio while the row is still reported as success. Leave `tts_output_pcm_bytes` unset on failure so the existing no-PCM/failed counters catch it, or mark the row failed. (evidence: patch.py:1251 (this diff, inside the new `except Exception` branch): ` output.tts_output_pcm_bytes = bytes(pcm_capture)`; seed_tts_eval.py consumer (unchanged by this diff, present in PR-time tree): `path.write_bytes(pcm_s16le_mono_to_wav_bytes(pcm, sample_rate=24000))` in `_save_seed_tts_eval_audio`, and `def _pcm_s16le_to_f32_16k(pcm: bytes, pcm_sample_rate: int = 24000) -> np.ndarray:` — both treat the field as 24 kHz mono.) + +`vllm_omni/benchmarks/patch/patch.py:140` [minor] — This diff adds `_pcm_s16le_to_seed_tts_wer_bytes` (vllm_omni/benchmarks/patch/patch.py:140, 48 kHz/2ch -> 24 kHz mono downmix + resample) and `_save_seed_tts_eval_audio` (vllm_omni/benchmarks/data_modules/seed_tts_eval.py:100), which compute the PR's Seed-TTS WER artifacts, but no test covers either helper (repo-wide grep returns only the definition and caller for each). Add a unit test feeding a synthetic stereo 48 kHz int16 buffer and asserting int16 mono, sample count ~input//2, and preserved L/R mean; also cover the 24 kHz pass-through branch and the save helper's filename sanitization. (evidence: vllm_omni/benchmarks/patch/patch.py:140 `def _pcm_s16le_to_seed_tts_wer_bytes(` — new in this diff; repo-wide grep finds only this definition and its caller at patch.py:1244 `output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(` and no test references. vllm_omni/benchmarks/data_modules/seed_tts_eval.py:100 `def _save_seed_tts_eval_audio(` — new in this diff; repo-wide grep finds only this definition and its caller at seed_tts_eval.py:581 `audio_path = _save_seed_tts_eval_audio(` and no test references.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:27` [minor] audio_tokenizer_v2.py:27 imports `PreTrainedAudioTokenizerBase` unconditionally at module top level, and modeling_moss_tts_codec.py:23 imports that module at module top level, so an ImportError at transformers==5.5.3 wou +- `tests/worker/test_omni_gpu_model_runner.py:322` [minor] tests/worker/test_omni_gpu_model_runner.py:347-348 only assert the generators exist and differ, never that the per-request `tts_local_seed` values (11, 22) were applied. Since `_explicit_talker_seed` reads `extra_args.ge + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4804.trace.json.gz new file mode 100644 index 000000000..a9b0fa92a Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4817.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4817.cost.json new file mode 100644 index 000000000..b81a3f44f --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4817.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1257.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4817/run-20260815-162256-6a7124", + "recorded_at": "2026-08-15T16:43:51", + "attempts": [ + { + "invocation_id": "inv-b7957265", + "outcome": "run:run-20260815-162256-6a7124", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-162256-6a7124", + "llm_calls": 62, + "input_tokens": 250373, + "output_tokens": 152408, + "usd": 0.296981, + "source": "spans" + } + ], + "attempt_usd_total": 0.296981, + "llm_calls": 62, + "input_tokens": 250373, + "output_tokens": 152408, + "usd": 0.296981, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-162256-6a7124", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.297, + "minutes": 20.92, + "ci_minutes": 0, + "input_tokens": 250373, + "output_tokens": 152408, + "tool_calls": 147, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6585, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 62, + "by_role": { + "planner": { + "usd": 0.005164, + "input_tokens": 3, + "output_tokens": 4631, + "calls": 1 + }, + "agent": { + "usd": 0.244098, + "input_tokens": 232371, + "output_tokens": 108877, + "calls": 58 + }, + "reducer": { + "usd": 0.047719, + "input_tokens": 17999, + "output_tokens": 38900, + "calls": 3 + } + }, + "cache_read_tokens": 2286336, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.36, + "gate": 1.67, + "review": 1250.04, + "report": 0.07 + }, + "time_to_first_result_s": 1255.12 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4817.trace.json.gz", + "trace_bytes": 29341 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4817.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4817.md new file mode 100644 index 000000000..3217cf668 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4817.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4817, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head fc423e5a9e86) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Log-flood concern from the PR repro ('log flooded by Quack FP8 GEMM failed'): at this head the handler already uses `logger.warning_once("Quack FP8 GEMM failed (%s); using FlashInfer.", exc)` (quack_fp8.py:123) — residual NOT covered: the per-GEMM exception+fallback COST survives under the `=1` override → review comment (minor) +- [resolved] Docs no longer claim quack auto-enables on every Blackwell — the hunk now says 'datacenter Blackwell only (`sm_100` / `sm_101` / `sm_103`, compute capability `10.x`)'; residual: cc 11.x is not addressed. +- [claim-verified] Old gate was `torch.cuda.get_device_capability()[0] >= 10` (file_at_base vllm_omni/quantization/quack_fp8.py:22), matching the PR's stated bug. +- [claim-verified] Diff narrows to `== 10` at vllm_omni/quantization/quack_fp8.py:32; sm_120/121 now return False, sm_100/101/103 return True. +- [claim-verified] '`>= 10` matched cc 12.x too': calc [(12>=10)=True, (12==10)=False, (10==10)=True, (11==10)=False] — fix does what the body claims; sm_100/103 unchanged, 12.x now off +- [claim-verified] 'env escape hatch preserved': quack_enabled() checks os.environ.get first (quack_fp8.py:38-40); docs `=1` claim matches _TRUTHY={'1','true','yes','on'} with .lower(); test covers 'TRUE'/'On' and ''→off +- [claim-verified] 'hardware-free unit test': test monkeypatches exactly the two probes _is_quack_capable reads (is_available, get_device_capability) and drives both real entry points (quack_enabled + _is_quack_capable); reverting the gate to >=10 fails the (12,0) case — the test CAN fail +- [claim-verified] PR claim 'test fails on old >= 10 gate and passes on fix': old gate returns True for (12,0) while the test expects False, so it would fail pre-fix. +- [claim-verified] 'No behavior change on datacenter Blackwell (sm_100.x)': for major==10 both `>= 10` and `== 10` evaluate True. +- [claim-refuted] PR body root cause says `tcgen05.mma` targets `sm_100a / sm_101a / sm_103a / sm_110a`, but `== 10` excludes cc 11.x (sm_110a); code, docstring, and description disagree on 11.x scope. +- [claim-refuted] PR claim the test 'runs on CI without a Blackwell GPU': the file has no pytestmark, CI lanes use `-m` selection (e.g. .buildkite/test-merge.yml:13,29; test-ready.yml:13,29), and no buildkite file references tests/quantization — the test is collected then deselected. +- [validated] Env override is checked before capability: vllm_omni/quantization/quack_fp8.py:37-41 (`if override is not None: return override.lower() in _TRUTHY`), so `=1`/`=0` still win. +- [validated] Rename `_is_blackwell` has no remaining references repo-wide (grep `_is_blackwell` -> no matches). +- [validated] New test drives the real entry points `quack_fp8._is_quack_capable()` and `quack_fp8.quack_enabled()` (test lines 50-51, 57-58, 67, 75), not a re-authored copy of the gate. + +`tests/quantization/test_quack_fp8_gate.py:23` [major] — The new regression guard tests/quantization/test_quack_fp8_gate.py sets no `pytestmark`. In the merge gate, the only step that collects tests/quantization/ is .buildkite/test-merge.yml:29, which runs `-m 'core_model and cpu'`; the step at line 13 only covers tests/diffusion and tests/model_executor. Repo-wide there is no `pytest_collection_modifyitems` auto-marking hook, so this file is collected and then deselected — the regression guard never runs in PR CI. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` after the imports, matching the repo convention (e.g. tests/test_outputs.py:11). (evidence: Unchanged by this diff, present in the PR-time tree: .buildkite/test-merge.yml:29 `"timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — the only merge-gate step that collects tests/quantization/; .buildkite/test-merge.yml:13 runs only `tests/diffusion tests/model_executor`. New file tests/quantization/test_quack_fp8_gate.py (read in full, 75 lines) contains no `pytestmark` anywhere; its imports end at lines 18–23: `import pytest` / `import torch` / `from vllm_omni.quantization import quack_fp8` / `_ENV = "VLLM_OMNI_USE_QUACK_FP8"`. Repo-wide grep for `pytest_collection_modifyitems` and `def pytest_collection` returned no matches, and pyproject.toml:236 `"core_model: L1&L2 tests (run in each PR)"` declares the marker without any auto-add mechanism.) + +`tests/quantization/test_quack_fp8_gate.py:18` [minor] — Confirmed: this diff adds tests/quantization/test_quack_fp8_gate.py and flips the gate to `== 10`. buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are red, but the diff surface is inert on those platforms: the AMD lane (.buildkite/test-amd.yaml:10) runs only explicit pytest files under tests/e2e/tests/diffusion/tests/engine/tests/entrypoints on gfx942, the NPU ready lane (.buildkite/test-npu-ready.yaml:21) runs only test_qwen3_tts_customvoice.py, and no .buildkite config collects tests/quantization. For cc<10 both the old `>= 10` and new `== 10` return False. Please confirm these two red lanes are pre-existing on main rather than introduced by this PR. (evidence: tests/quantization/test_quack_fp8_gate.py:18 `import pytest` (anchor present). vllm_omni/quantization/quack_fp8.py:32 ` return torch.cuda.get_device_capability()[0] == 10` (changed by this PR). .buildkite/test-amd.yaml:10 ` - export GPU_ARCHS=gfx942` and all pytest invocations in that file target explicit files under tests/e2e, tests/diffusion, tests/engine, tests/entrypoints — unchanged by this diff; grep for tests/quantization in .buildkite has no matches. .buildkite/test-npu-ready.yaml:21 ` - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'` (only test in lane) — unchanged by this diff. gh_ci_read reports buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci as FAILURE while buildkite/vllm-omni is SUCCESS.) + +`vllm_omni/quantization/quack_fp8.py:~79` [minor] — The predicate is still platform-blind: `_is_quack_capable()` only checks `torch.cuda.is_available()` before `torch.cuda.get_device_capability()[0] == 10`. On ROCm this API reports the gfx architecture (see `RocmOmniPlatform.get_diffusion_attn_backend_cls`, which uses it for gfx90x/95x detection), so gfx10xx RDNA2 cards return major 10 and `quack_enabled()` returns True, triggering a doomed `_load_quack()` and the 'Quack FP8 unavailable, using FlashInfer' warning (line 79). Add `current_platform.is_cuda()` (or `torch.version.hip is None`) to the guard and add a HIP gfx10xx test case; the cited warning line is 79, not 80. (evidence: Changed in this diff — `vllm_omni/quantization/quack_fp8.py:32`: `return torch.cuda.get_device_capability()[0] == 10` (guarded only by `torch.cuda.is_available()` at line 30, with no CUDA-platform check). Unchanged by this diff, present in the PR-time tree — `vllm_omni/platforms/rocm/platform.py:84`: `compute_capability = torch.cuda.get_device_capability()` and `:87`: `aiter_supported = is_aiter_found_and_supported() and 90 < capability < 100`; this repo code already relies on `torch.cuda.get_device_capability()` returning the ROCm gfx arch, so gfx10xx (RDNA2) yields major 10 and passes the `== 10` gate.) + +`vllm_omni/quantization/quack_fp8.py:32` [minor] — vllm_omni/quantization/quack_fp8.py:32 — the `== 10` gate does exclude cc 11.x (sm_110a) while the PR body bullet lists sm_110a as a tcgen05.mma target; please reconcile by either documenting that quack's CuteDSL still can't run there (CUTLASS hardcodes admissible_archs=[Arch.sm_100a]) or widening the gate. Separately, the no-arg `torch.cuda.get_device_capability()` probe reads only the current/default device but is evaluated once at import to install the patch process-wide (patch.py:419), so on heterogeneous nodes the gate can be wrong for non-device-0 GEMMs — worth a per-tensor device guard/log, or at minimum a log when the GEMM device's capability differs from the probed device. (evidence: vllm_omni/quantization/quack_fp8.py:32 ` return torch.cuda.get_device_capability()[0] == 10` — the changed gate (diff line) excludes cc 11.x and uses the no-arg probe that reads only the current/default device. PR body (gh_pr_view 4817, unchanged by diff): 'PTX `tcgen05.mma` targets `sm_100a / sm_101a / sm_103a / sm_110a`; `ptxas` rejects `tcgen05` on `sm_120`' — lists sm_110a as a tcgen05 target. vllm_omni/patch.py:419 `_patch_fp8_use_quack_fused_bias()` — module-level import-time call that installs the quack patch process-wide; its body calls patch.py:414 `install_quack_fp8_patch()`.) + +`docs/user_guide/quantization/fp8.md:47` [minor] — This hunk correctly narrows the auto-enable doc to 'datacenter Blackwell only (sm_100/sm_101/sm_103, cc 10.x)', matching the new `== 10` gate — but the section heading and intro still pitch the feature for all Blackwell: fp8.md:31 '### Faster FP8 GEMM on Blackwell (quack)' and fp8.md:33 'On Blackwell (SM 100+), vLLM runs FP8 linears through the FlashInfer kernel...'. An RTX 5090/RTX PRO 6000 (sm_120, which IS 'SM 100+') reader who stops at the install commands will install the extra and, by design of this PR, get no speedup and no signal why. Since this PR already rewrites the paragraph below (fp8.md:47), align the frame in the same pass: retitle to 'Faster FP8 GEMM on datacenter Blackwell (quack)' and qualify the intro (e.g. 'Installing the optional `quack` kernel on datacenter Blackwell (see gate below) lets vLLM-Omni fuse ...'). (evidence: docs/user_guide/quantization/fp8.md:31 `### Faster FP8 GEMM on Blackwell (quack)` — unchanged by this diff, present in the PR-time tree; docs/user_guide/quantization/fp8.md:33 `On Blackwell (SM 100+), vLLM runs FP8 linears through the FlashInfer kernel, which` — unchanged by this diff, present in the PR-time tree; docs/user_guide/quantization/fp8.md:47 `It is enabled automatically once installed (no flag needed) and is **datacenter` — added by this diff; vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10` — the gate excludes sm_120/121 despite them satisfying 'SM 100+'.) + +`vllm_omni/quantization/quack_fp8.py:~123` [nit] — Default gate fix is correct, but the per-GEMM fallback pathology it documents still exists behind the `VLLM_OMNI_USE_QUACK_FP8=1` override this PR now advertises. Once `install_quack_fp8_patch()` is installed and `_load_quack()` has cached a valid `gemm_interface`, `apply_scaled_mm` calls `quack_scaled_fp8_mm` on every invocation; if `gemm.gemm_tuned` rejects the arch (sm_120/121), the except path only `warning_once`-logs and falls back to FlashInfer without marking quack dead, so the exception-plus-fallback repeats on every GEMM of the denoise loop. Add a sticky circuit breaker in the except path (e.g. `_gemm_interface = False`) so a wrong override degrades to one warning + native FlashInfer instead of per-GEMM overhead. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/quantization/quack_fp8.py:57-58 ` if _gemm_interface is not None:` / ` return _gemm_interface or None` (cached interface returned on every call); vllm_omni/quantization/quack_fp8.py:92-97 ` gemm = _load_quack()` / ` if gemm is None:` / ` return None` / ` out = torch.empty(a.shape[0], b.shape[1], device=a.device, dtype=out_dtype)` / ` alpha = scale_a.reshape(1).float() * scale_b.reshape(1).float()` / ` gemm.gemm_tuned(a, b, out, bias=bias, alpha=alpha)` (quack GEMM attempted every call); vllm_omni/quantization/quack_fp8.py:122-124 ` except Exception as exc: # noqa: BLE001` / ` logger.warning_once("Quack FP8 GEMM failed (%s); using FlashInfer.", exc)` / ` return original(self, A=A, B=B, out_dtype=out_dtype, As=As, Bs=Bs, bias=bias, output_shape=output_shape)` (no sticky-disable, so the next GEMM retries the failing quack path).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4817.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4817.trace.json.gz new file mode 100644 index 000000000..988770d3a Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4817.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4859.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4859.cost.json new file mode 100644 index 000000000..571da5f0a --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4859.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1703.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4859/run-20260815-163107-9caff4", + "recorded_at": "2026-08-15T16:59:28", + "attempts": [ + { + "invocation_id": "inv-21fae817", + "outcome": "run:run-20260815-163107-9caff4", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-163107-9caff4", + "llm_calls": 124, + "input_tokens": 1080235, + "output_tokens": 218092, + "usd": 0.826856, + "source": "spans" + } + ], + "attempt_usd_total": 0.826856, + "llm_calls": 124, + "input_tokens": 1080235, + "output_tokens": 218092, + "usd": 0.826856, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-163107-9caff4", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8269, + "minutes": 28.35, + "ci_minutes": 0, + "input_tokens": 1080235, + "output_tokens": 218092, + "tool_calls": 260, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9982, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 124, + "by_role": { + "agent": { + "usd": 0.768417, + "input_tokens": 1055271, + "output_tokens": 171159, + "calls": 121 + }, + "reducer": { + "usd": 0.058439, + "input_tokens": 24964, + "output_tokens": 46933, + "calls": 3 + } + }, + "cache_read_tokens": 10936704, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.76, + "gate": 1.76, + "review": 1695.18, + "report": 0.07 + }, + "time_to_first_result_s": 1700.75 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4859.trace.json.gz", + "trace_bytes": 40405 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4859.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4859.md new file mode 100644 index 000000000..2f179d8fe --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4859.md @@ -0,0 +1,68 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4859, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6ff4ce59ed87) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] removed include_language kwarg has no stale in-repo caller; flash path already called with default include_voice=False and is unaffected. +- [claim-verified] PR test plan env `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100` matches Ming-TTS `SAMPLE_RATE = 44100` (constants.py), so the capture-side duration/rtf math is correct. +- [claim-refuted] PR body's 'This PR, vLLM-Omni benchmark: 1.83% WER' is computed on corrupted audio: capture resamples 44.1k→24k (patch.py:118 `Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER.`), then eval re-resamples that 24k PCM as if it were 44100 (seed_tts_eval.py:616 + 188-193). +- [claim-refuted] PR WER table (1.83%) was measured 'based on 0455bffc' before the merge commit 6ff4ce59; at merged head the WER decode path changes whenever the test plan's own `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100` is set (44100/24000 = 1.8375x mis-speed), so the numbers do not describe head. +- [claim-refuted] PR test plan env VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100: WER decode double-resamples the 24k-normalized PCM (44100/24000=1.8375), so the 1.83% WER is measured on wrong-speed audio. +- [validated] seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)` is the sole caller; `pcm` is `out.tts_output_pcm_bytes`, documented at patch.py:427 as 24 kHz mono s16le. +- [validated] metrics/definitions.py:326/332 `if raw_sr:`/`if raw_ch:` parse env with try/int and the function clamps `max(sample_rate,1), max(channels,1)` — no div-by-zero. +- [validated] file_at_base(seed_tts_eval.py) confirms the pre-PR default was `pcm_sample_rate: int = 24000` (hardcoded), so the eval 24k→16k resample was correct before this PR. +- [validated] definitions.py:326/332 env branches mirror the removed `_audio_pcm_format` (warn + default on ValueError) and add `max(..., 1)` clamps — an improvement: old code allowed env '0' through, which divided by zero at patch.py `output.audio_duration = total_samples / sample_rate`. +- [validated] patch.py logger/import reorder plus the `defs.stream_pcm_format_from_env()` swap in `async_request_openai_audio_speech` is behavior-preserving for duration/RTF/continuity stats (same env names, same defaults). +- [validated] `import torch.nn.functional as F` exists at audio_vae.py:6, so the new `F.softplus` call resolves; base code's `torch.chunk(h, 2, dim=1)[0]` discarded the scale half that the new code consumes. +- [validated] `from vllm_omni.metrics.definitions import stream_pcm_format_from_env` is wired in seed_tts_eval.py:64. +- [validated] diff_stat (merge-base a8b2bf11d2f2) confirms exactly the 6 files in the PR's stated scope; no unrelated artifacts ride along. +- [validated] tts_output_pcm_bytes producers normalize to 24k mono: patch.py:130 (audio-speech) and patch.py:946 (chat-omni) both resample to 24000. + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:189` [major] — seed_tts_eval.py:189 makes `_pcm_s16le_to_f32_16k` read `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE/CHANNELS` when called without args, but its only caller (line 616) passes no args and decodes `tts_output_pcm_bytes`, which both capture paths already normalize to 24 kHz mono: patch.py:1215 via `_pcm_s16le_to_seed_tts_wer_bytes`, and the chat-omni path via `AudioResampler(target_sr=24000)` / `load_audio(..., sr=24000, mono=True)` at patch.py:948/957. With `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100` this resamples 24 kHz audio as 44.1 kHz into Whisper (44100/24000 ≈ 1.84x speed error), and channels=2 halves a mono buffer, while `_save_seed_tts_eval_audio` still writes the bytes as a 24 kHz WAV. Revert this function to the fixed 24 kHz/mono contract and drop the new module-docstring env claim; the env override belongs only at capture. (Retain the note that the benchmark's WER number must be re-established at the merged head.) (evidence: seed_tts_eval.py:189 `env_sample_rate, env_channels = stream_pcm_format_from_env()` (in this diff) makes the no-arg decode path read env rate/channels; seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)` is the only call and passes neither. The bytes it decodes are produced as 24 kHz mono elsewhere in the PR-time tree (unchanged by this diff): patch.py:113 `"""Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER."""`, patch.py:1215 `output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(`, patch.py:948 `output.tts_output_pcm_bytes = (pcm_f32 * 32767).astype(np.int16).tobytes()` (after `resampler = AudioResampler(target_sr=24000)`), patch.py:957 `output.tts_output_pcm_bytes = (waveform * 32767).astype(np.int16).tobytes()` (after `load_audio(..., sr=24000, mono=True)`).) + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:351` [major] — This hunk changes shared `AudioVAE.encode_latent` from the deterministic posterior mean to stochastic reparameterized sampling (`audio_vae.py:351`). Both consumers inherit it: ming_tts's per-request prompt encoder (`prompt_encoder.py:137`) and ming_flash_omni's `VoicePresetRegistry._build_wav_embeddings` (`voice_presets.py:257`), whose result is stored in `self.registered[voice_name]["prompt_wav_lat"]` — so every Ming-flash preset registration (and server restart) now yields different cached voice conditioning, with no flash-side parity evidence in this PR. Sampling uses `torch.randn_like` with no generator, so even identical eager-mode requests are not reproducible unless an external global seed is set. Either confirm the Ming-flash HF reference also samples (cite the line) or scope sampling to the ming_tts path (e.g. a `sample: bool` arg defaulting per caller), and thread a generator for reproducibility. (evidence: ['Changed by this diff, present in PR-time tree: vllm_omni/model_executor/models/common/ming/audio_vae.py:351 ` latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)` (diff replaced `mean = torch.chunk(h, 2, dim=1)[0]` / `latent = mean.transpose(1, 2)` with `mean, scale = torch.chunk(h, 2, dim=1)` / `std = F.softplus(scale) + 1e-4` and this sampling line).', 'Unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/ming_flash_omni/voice_presets.py:257 ` prompt_wav_lat, _ = self._audio_vae.encode_latent(` inside `_build_wav_embeddings`; register() then stores `self.registered[voice_name] = {"prompt_wav_lat": prompt_wav_lat, ...}`.', 'Unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/ming_tts/prompt_encoder.py:137 ` latents, _ = encoder.encode_latent(waveform, waveform_length)`.']) + +`examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py:~106` [major] — Confirmed: this PR deletes the `language → 方言` mapping in `_parse_ming_instruction_fields` (serving_speech.py: the diff removes `include_language=True` and the `instruction_dict["方言"] = request.language` branch), but the in-repo ming client still sends `payload["language"] = args.dialect` (openai_speech_client.py:106, `--dialect` help at :192 still says 'Maps to Ming `方言`') and README.md:101 still documents `language → 方言`. After this diff the instruction path no longer applies that control. Please update the client/README (or restore the mapping), and confirm whether the generic language validation at serving_speech.py:1647-1651 rejects non-empty `language` for ming dense so this is not a silent no-op. (evidence: Unchanged by this diff, present in the PR-time tree: examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py:106 ` payload["language"] = args.dialect`; examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py:192 ` parser.add_argument("--dialect", default=None, help="Maps to Ming `方言`")`; examples/online_serving/text_to_speech/ming_tts/README.md:101 `| `language` | Ming `方言` control |`. Changed by this diff: vllm_omni/entrypoints/openai/serving_speech.py deletes `- if include_language and request.language not in (None, "", "Auto"):` / `- instruction_dict["方言"] = request.language` and drops `- include_language=True,` from the `_parse_ming_instruction` call, leaving only `include_voice=True` and `plain_text_passthrough=True`.) + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:141` [minor] — In Encoder.__init__ (patch_size != -1 path), `config.num_hidden_layers = 4` at line 141 mutates the same Qwen2Config object already used by `self.encoder = Qwen2Model(config)` at line 131, so `self.encoder.config.num_hidden_layers` permanently reports 4 for an encoder with a different real depth. The removed base code built a fresh non-aliasing config via `_qwen2_config({**encoder_args, "num_hidden_layers": 4})`. Restore the separate config (or deep-copy before mutating). (evidence: vllm_omni/model_executor/models/common/ming/audio_vae.py:131 `self.encoder = Qwen2Model(config)` and :141 `config.num_hidden_layers = 4` and :142 `self.aggregator = Qwen2Model(config)` — encoder and aggregator share the single `config` object, so the mutation rewrites the encoder's config too (line 131 is unchanged context in the PR-time tree, outside the diff). The base code that the PR replaced created a non-aliasing copy (file_at_base): `aggregator_config = _qwen2_config({**encoder_args, "num_hidden_layers": 4})` / `self.aggregator = Qwen2Model(aggregator_config)`.) + +`vllm_omni/model_executor/models/ming_tts/patch_emission.py:151` [minor] — The +1→+2 bump at patch_emission.py:130 and :151 moves the earliest stop-head step from 4 to 5 (default STOP_HEAD_MIN_STEPS=3) and raises the validation floor: with ming_max_decode_steps=4 (OpenAI max_new_tokens=4, accepted since _TTS_MAX_NEW_TOKENS_MIN=1), _validate_ming_decode_window now raises in MingLLMModel.forward. No tests reference _resolve_ming_stop_decision or _validate_ming_decode_window, so this semantic constant ships with zero coverage and no comment citing the HF rationale. Please add a unit test pinning the earliest-stop step and the max_decode_steps == min_stop_step + 1 rejection boundary, plus a one-line comment referencing the upstream behavior. (evidence: Changed by this diff: vllm_omni/model_executor/models/ming_tts/patch_emission.py:130 `min_required = max(min_stop_step + 2, min_steps)`; patch_emission.py:132 `if max_steps < min_required:` followed by `raise ValueError(` at :134; patch_emission.py:151 `min_required_decode_steps = max(min_stop_step + 2, min_decode_steps)`; patch_emission.py:158 `should_stop_head = ((step + 1) >= min_required_decode_steps) and stop_prob > stop_threshold`. Unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/ming_tts/constants.py:51 `STOP_HEAD_MIN_STEPS = 3`; vllm_omni/entrypoints/openai/serving_speech.py:153 `_TTS_MAX_NEW_TOKENS_MIN = 1`; vllm_omni/entrypoints/openai/serving_speech.py:2679 `runtime_controls[KEY_MAX_DECODE_STEPS] = request.max_new_tokens`. Repo grep for `_resolve_ming_stop_decision|_validate_ming_decode_window` matches only patch_emission.py and ming_tts_llm.py (defs/imports/calls at 46, 50, 119, 134, 140, 223, 478); no test file references either symbol.) + +`tests/model_executor/models/ming_tts/test_ming_shared_modules.py:~27` [minor] — buildkite/vllm-omni-amd-ci is red at this head. The cpu/tts suite in tests/model_executor/models/ming_tts/test_ming_shared_modules.py imports the touched serving_speech.py and audio_vae.py, but no in-diff path affects it: _parse_ming_instruction_fields (serving_speech.py:2619) no longer accepts include_language, the test never passed it (lines 88-99), the dense case's language='Auto' was excluded by the removed guard too, and no test constructs Encoder or calls encode_latent. Please confirm whether the AMD failure pre-exists on main or first appears at this head. (evidence: tests/model_executor/models/ming_tts/test_ming_shared_modules.py:27 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.tts]` (anchor exists one line above the cited 28). tests/model_executor/models/ming_tts/test_ming_shared_modules.py:91-99 `dense_with_fields = serving._parse_ming_instruction(SimpleNamespace(instructions="calm", language="Auto", voice="灵小甄"))` / `flash_fields = serving._parse_ming_instruction_fields(SimpleNamespace(instructions="calm", language="粤语", voice="灵小甄"))` / `assert flash_fields == {"风格": "calm"}` — no include_language passed. vllm_omni/entrypoints/openai/serving_speech.py:2619-2624 `def _parse_ming_instruction_fields(self, request, *, include_voice=False, plain_text_passthrough=False):` — include_language no longer exists in the PR-time tree. Removed by this diff (previously excluded language='Auto'): `- if include_language and request.language not in (None, "", "Auto"):`. Repo-wide grep for `include_language` returns 0 hits; grep for `encode_latent` under tests/ and for `Encoder(` under tests/model_executor/models/ming_tts both return no matches.) + +`vllm_omni/metrics/definitions.py:337` [minor] — definitions.py:337 `return max(sample_rate, 1), max(channels, 1)` changes base behavior for `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=0`/negative from a loud ZeroDivisionError to silently clamped 1. The only bench consumer `async_request_openai_audio_speech` at patch.py:1194 then computes `output.audio_duration = total_samples / sample_rate`, inflating audio_duration (~24000x for 0→1) and deflating audio_rtf/underrun with no log. The replacement docstring also drops the model-mapping (e.g. MOSS-TTS-Local-Transformer-v1.5 48 kHz stereo) and the 'silently skews audio_duration/audio_rtf/underrun stats' warning. Please (a) warn and fall back to the default on parsed non-positive values instead of clamping to 1, and (b) restore the operational guidance in the helper docstring. (evidence: vllm_omni/metrics/definitions.py:337 ` return max(sample_rate, 1), max(channels, 1)` — new in this diff. vllm_omni/benchmarks/patch/patch.py:1163 ` sample_rate, channels = defs.stream_pcm_format_from_env()` — new in this diff. vllm_omni/benchmarks/patch/patch.py:1194 ` output.audio_duration = total_samples / sample_rate` — unchanged by this diff, present in the PR-time tree; consumes the clamped value. Merge-base vllm_omni/benchmarks/patch/patch.py `_audio_pcm_format` returned `return sample_rate, channels` with no clamp, so `sample_rate=0` reached the consumer and raised ZeroDivisionError.) + +`tests/model_executor/models/ming_tts/test_ming_shared_modules.py:91` [minor] — Confirmed. tests/model_executor/models/ming_tts/test_ming_shared_modules.py:91-94 exercises _parse_ming_instruction with language="Auto"; the removed guard `if include_language and request.language not in (None, "", "Auto")` also skipped "Auto", so this case (and the language=None case) produces the same dict before and after the removal. Add a dense case with a real dialect (e.g. language="粤语") asserting the returned dict has no "方言" key so the removed mapping is pinned by a regression test. (evidence: tests/model_executor/models/ming_tts/test_ming_shared_modules.py:91-94 + dense_with_fields = serving._parse_ming_instruction( + SimpleNamespace(instructions="calm", language="Auto", voice="灵小甄") + ) + assert dense_with_fields == {"IP": "灵小甄", "风格": "calm"} + +Removed by this PR's diff (old code, no longer in the PR-time tree): + if include_language and request.language not in (None, "", "Auto"): + instruction_dict["方言"] = request.language +Since "Auto" is in the exclusion tuple, the old dense path also returned only {"IP": "灵小甄", "风格": "calm"}, so this test cannot detect the removal.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4859.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4859.trace.json.gz new file mode 100644 index 000000000..f8e11fc66 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4859.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4870.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4870.cost.json new file mode 100644 index 000000000..434d56307 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1655.6, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4870/run-20260815-155951-3acb07", + "recorded_at": "2026-08-15T16:27:24", + "attempts": [ + { + "invocation_id": "inv-43dd1f32", + "outcome": "run:run-20260815-155951-3acb07", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-155951-3acb07", + "llm_calls": 144, + "input_tokens": 1740228, + "output_tokens": 204519, + "usd": 1.038511, + "source": "spans" + } + ], + "attempt_usd_total": 1.038511, + "llm_calls": 144, + "input_tokens": 1740228, + "output_tokens": 204519, + "usd": 1.038511, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-155951-3acb07", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0385, + "minutes": 27.55, + "ci_minutes": 0, + "input_tokens": 1740228, + "output_tokens": 204519, + "tool_calls": 269, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0617, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 144, + "by_role": { + "agent": { + "usd": 0.994984, + "input_tokens": 1717712, + "output_tokens": 170545, + "calls": 141 + }, + "reducer": { + "usd": 0.043527, + "input_tokens": 22516, + "output_tokens": 33974, + "calls": 3 + } + }, + "cache_read_tokens": 12728832, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.16, + "gate": 1.64, + "review": 1647.06, + "report": 0.06 + }, + "time_to_first_result_s": 1652.91 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 38856 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4870.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4870.md new file mode 100644 index 000000000..758228aaf --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4870.md @@ -0,0 +1,62 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] checklist concern 'async_chunk defaults to False, a gate keyed on it hits every async_chunk=False model' — head scopes it: `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` (serving_speech.py:3508) plus test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3; residual: other full-payload models (moss_tts, voxcpm2, ...) intentionally keep DELTA (unchanged from today), and the voxtral test cannot detect a broken async_chunk read because the model-type check short-circuits +- [resolved] Engine-level FINAL_ONLY streaming still emits a single full waveform: _generate_audio_chunks treats a non-list audio_val as one tensor and yields it once; residual = no real-path streaming validation (raised as minor). +- [resolved] `seq_len` removal does not break internal `_build_omni_mm_payload`/`_build_omni_pooler_payload` calls at gpu_ar_model_runner.py:879/923; those are a separate local `seq_len` variable and remain valid. +- [resolved] concern 'does the fixture state match production?' for the five edited legacy tests: each SimpleNamespace scheduler_output already carried total_num_scheduled_tokens equal to the deleted seq_len kwarg (3/6/3/2/2), which is why the PR could delete the argument without touching any assert — checked all five call sites in situ +- [claim-verified] diff_stat: exactly 4 files changed, matching the PR body's 'four files changed'. +- [claim-verified] Commit timeline has merge commits (fc40c1b2, 88fa5ff1, 231b3ddb); per REPO briefing head matches the squashed diff, so merged state was reviewed. +- [claim-verified] PR body 'four files changed' — diff_stat confirms exactly 4 files (2 tests, 2 prod) +- [claim-verified] PR claim 'the output builder used hidden_states.shape[0] as the multimodal split length' — confirmed: removed `seq_len = hidden_states.shape[0]` in sample_tokens and to_payload_element gates slicing on `element.shape[0] == seq_len` +- [claim-verified] PR body's 'four files changed' matches the diff (serving_speech.py, gpu_ar_model_runner.py, two test files). +- [claim-verified] PR's remote E2E row explicitly used stream=False; no streaming async_chunk=False row exists, so the newly changed streaming cell is untested beyond mocks. +- [claim-verified] PR claim 'changes the multimodal payload split length to scheduler_output.total_num_scheduled_tokens' matches the diff exactly (one derivation at 1734, one use at 1790, parameter removed); both tests named in the test plan exist in the diff at the claimed locations +- [validated] Central fix: gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` flows to `to_payload_element(..., seq_len=mm_seq_len)` at :1790; mm_outputs.py:108-145 slices any tensor with dim0==seq_len by start/end, so per-request codes.audio slicing is restored. +- [validated] New worker regression test exercises the bug: codes (3,16), hidden (1,1), three 1-token requests, asserts slices codes[0:1]/[1:2]/[2:3]. +- [validated] New serving tests exercise the new routing: qwen3 no-async-chunk asserts coerce(..., False) and voxtral asserts coerce(..., True). + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:~905` [major] — Confirmed. This PR fixes the CUDA runner to split multimodal outputs by scheduler_output.total_num_scheduled_tokens, but the NPU twin still derives seq_len from hidden_states.shape[0] (line 905) and feeds it into to_payload_element (line 1144). With tail-only hidden states, mm_outputs.py's `element.shape[0] == seq_len` check fails and falls through to `element.clone()`, giving every request the whole batched codes tensor. The non-async-chunk branch (line 1156) aliases pooler_output to both inter-stage and client outputs, so contamination propagates. Apply the same total_num_scheduled_tokens fix to npu_ar_model_runner.py or file a tracking issue for deferred NPU coverage. (evidence: vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905 `seq_len = hidden_states.shape[0]` — unchanged by this diff, present in the PR-time tree. vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:1144 `seq_len=seq_len` is passed to `to_payload_element(...)`. vllm_omni/utils/mm_outputs.py:131 `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len:` returns `element[start:end].contiguous()`, else mm_outputs.py:151 `return element.clone()` — whole batched tensor per request. Contrast: vllm_omni/worker/gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` and 1790 `seq_len=mm_seq_len`.) + +`vllm_omni/entrypoints/openai/serving_speech.py:~3507` [minor] — The new gate reads `async_chunk` via `getattr(self.model_config, "async_chunk", True)` at serving_speech.py:3507, but the pre-existing consumer `_generate_audio_bytes` reads `bool(getattr(self.engine_client.model_config, "async_chunk", False))` at serving_speech.py:3823, and OmniModelConfig's declared default is False (vllm_omni/config/model.py:121). Both new tests mutate `engine_client.model_config.async_chunk`. Read the flag through the same accessor with the same fallback so the streaming-mode gate and cumulative-audio-list handling cannot diverge. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3507 `async_chunk = getattr(self.model_config, "async_chunk", True)`; vllm_omni/entrypoints/openai/serving_speech.py:3823 `async_chunk = bool(getattr(self.engine_client.model_config, "async_chunk", False))` (unchanged by this diff, present in the PR-time tree); vllm_omni/config/model.py:121 `async_chunk: bool = False`) + +`vllm_omni/entrypoints/openai/serving_speech.py:~3509` [minor] — This hunk makes qwen3_tts + async_chunk=False streaming requests (both stream_format=audio and SSE, since is_streaming() covers both) skip DELTA coercion and use FINAL_ONLY; the diff only adds mock-level assertions of the coerce flag and does not exercise a real streaming no_async_chunk Qwen3-TTS path. Please run one real streaming request against a --no-async-chunk Qwen3-TTS server and confirm _generate_audio_chunks emits exactly one non-empty final waveform — the regression risk is an HTTP-200 stream that is empty or duplicates the cumulative snapshot. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` (changed by this diff); vllm_omni/entrypoints/openai/serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload` (changed by this diff); unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/protocol/audio.py:302 `def is_streaming(self) -> bool:` and :303 `return self.is_raw_audio_stream() or self.is_sse_stream()`; tests/entrypoints/openai_api/test_serving_speech.py:3468 `mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, False)` (added by this diff; mock-level only).) + +`vllm_omni/entrypoints/openai/serving_speech.py:~3508` [minor] — The red buildkite/vllm-omni-amd-ci gate is in scope: test-amd-ready.yaml:173 runs `timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"` on mi325_1, and that suite includes a streaming case (test_text_to_audio_002, stream=True). OmniModelConfig.async_chunk defaults to False (vllm_omni/config/model.py:121), so this streaming case passes through the new qwen3_full_payload gate at serving_speech.py:3508 (provided qwen3_tts.yaml does not override async_chunk=True). The changed unit tests (tests/entrypoints, tests/worker) are not collected by this lane. Please confirm which AMD step failed and whether it is pre-existing on main before merge. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` (added by this diff). Unchanged by this diff, present in the PR-time tree: vllm_omni/config/model.py:121 `async_chunk: bool = False`. Unchanged by this diff, present in the PR-time tree: .buildkite/test-amd-ready.yaml:173 `- timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"`. Unchanged by this diff, present in the PR-time tree: tests/e2e/online_serving/test_qwen3_tts_customvoice.py:97 `"stream": True,`.) + +`vllm_omni/worker/gpu_ar_model_runner.py:~1734` [minor] — PR body says 'Fixes #4851', but #4851 reports two failures — Base `test_response_format_001[no_async_chunk]` and CustomVoice `test_voice_002[no_async_chunk]` — while the E2E table validates only CustomVoice `test_voice_001_nonstream_concurrent_no_async_chunk[no_async_chunk]` with request_num=3. The worker fix (`mm_seq_len = scheduler_output.total_num_scheduled_tokens`) corrects per-request slicing only for batched requests, so if the Base failure ran with a single in-flight request it is not explained by this change. Please add Base E2E evidence reproducing the same batched mechanism, or scope the 'Fixes #4851' claim and track the Base failure separately. (evidence: PR body (gh_pr_view, PR #4870): `Fixes #4851.` and `| `test_voice_001_nonstream_concurrent_no_async_chunk[no_async_chunk]` | `31070fa7` | Before fix | Failed | ... request_num=3.` — only one CustomVoice concurrent case is validated. Issue #4851 body (gh_issue_view): `Affected tests from the attached logs: - `tests/e2e/online_serving/test_qwen3_tts_base_expansion.py::test_response_format_001[no_async_chunk]` - `tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py::test_voice_002[no_async_chunk]`` — two failures reported. Anchor (changed by this diff): vllm_omni/worker/gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`. Unchanged by this diff, present in PR-time tree: tests/e2e/online_serving/test_qwen3_tts_base_expansion.py:95 `def test_response_format_001(omni_server, openai_client) -> None:` whose body calls `openai_client.send_audio_speech_request(request_config)` without `request_num`.) + +`tests/entrypoints/openai_api/test_serving_speech.py:~3471` [minor] — This regression test mocks `coerce_param_message_types` (line 3463) and asserts only that it received `False` (line 3471), so it proves the branch routing but not the FINAL_ONLY contract its name claims. Because the helper is replaced with a mock returning a fixed value, the test would still pass if `maybe_coerce_to_message_type` stopped mapping `is_streaming=False` to FINAL_ONLY. Extend the test to assert `engine_client.generate.call_args.kwargs["sampling_params_list"][0].output_kind == RequestOutputKind.FINAL_ONLY` (or patch `maybe_coerce_to_message_type` instead of the whole helper) so FINAL_ONLY is the asserted outcome. (evidence: tests/entrypoints/openai_api/test_serving_speech.py:3463 `mock_coerce = mocker.patch("vllm_omni.entrypoints.openai.serving_speech.coerce_param_message_types", return_value=qwen3_tts_server.engine_client.default_sampling_params_list)` — added by this diff; replaces the real helper with a fixed-return mock. tests/entrypoints/openai_api/test_serving_speech.py:3471 `mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, False)` — added by this diff; asserts only the routed boolean, not the resulting output_kind. Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/utils.py:759 `def coerce_param_message_types(params: list[OmniSamplingParams], is_streaming: bool):` whose callee `maybe_coerce_to_message_type` sets `target_type = RequestOutputKind.DELTA if is_streaming else RequestOutputKind.FINAL_ONLY` — the mapping this mocked test would fail to catch if it regressed.) + +`vllm_omni/utils/mm_outputs.py:131` [minor] — The slicing gate in `to_payload_element` (mm_outputs.py:131) is still an exact `element.shape[0] == seq_len` match, and this PR now passes `seq_len=total_num_scheduled_tokens`. A request-invariant/batch-level tensor whose dim0 coincidentally equals that step's total would still be silently sliced per request, with no marker distinguishing per-request per-token tensors from request-invariant ones. Minor: consider adding an explicit marker/guard in `to_payload_element`/`_build_omni_mm_payload` or filing a tracking issue for this residual collision hazard. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/utils/mm_outputs.py:131 `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len:` — any tensor whose dim0 equals seq_len is sliced via `element[start:end]` with no marker distinguishing per-request per-token tensors from request-invariant tensors. The PR diff makes the gate value the step total: gpu_ar_model_runner.py `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` and `seq_len=mm_seq_len`, so a request-invariant tensor of dim0 == total_num_scheduled_tokens would be mis-sliced per request.) + +`tests/entrypoints/openai_api/test_serving_speech.py:3473` [minor] — Confirmed: the voxtral test at line 3473 asserts coerce(..., True), but for voxtral_tts the qwen3_full_payload expression short-circuits at the model-type check, so this test passes regardless of how async_chunk is read and provides no guard for the changed accessor. Note the sibling qwen3 test `test_prepare_speech_generation_no_async_chunk_stream_uses_final_only` already sets engine_client.model_config.async_chunk=False (non-default) and asserts coerce(..., False), so the accessor is covered there; the voxtral test remains useful only for proving the scoping to qwen3_tts. (evidence: serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` — changed in this diff; serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload` — changed in this diff; tests/entrypoints/openai_api/test_serving_speech.py:3493 `mock_coerce.assert_called_once_with(voxtral_server.engine_client.default_sampling_params_list, True)` — added in this diff. For the voxtral fixture, `_tts_model_type == "qwen3_tts"` is False, so Python short-circuits and `qwen3_full_payload` is False independent of `async_chunk`; hence `is_streaming_request` is True and `coerce_param_message_types` is always called with True, making the test unable to detect any broken async_chunk read.) + +**Additional observations (beyond the comment budget):** +- `tests/worker/test_gpu_ar_model_runner.py:478` [minor] In `_async_chunk=False` the builder ships the same full payload on both channels (`gpu_ar_model_runner.py:1803 pooler_inter, pooler_client = pooler_output, pooler_output`), and the new test at `tests/worker/test_gpu_ar_m + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4870.trace.json.gz new file mode 100644 index 000000000..fcb5f5798 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4923.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4923.cost.json new file mode 100644 index 000000000..daae77d01 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1603.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4923/run-20260815-155952-027fc9", + "recorded_at": "2026-08-15T16:26:32", + "attempts": [ + { + "invocation_id": "inv-de7c84ac", + "outcome": "run:run-20260815-155952-027fc9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-155952-027fc9", + "llm_calls": 130, + "input_tokens": 1469919, + "output_tokens": 243211, + "usd": 1.022797, + "source": "spans" + } + ], + "attempt_usd_total": 1.022797, + "llm_calls": 130, + "input_tokens": 1469919, + "output_tokens": 243211, + "usd": 1.022797, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-155952-027fc9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0228, + "minutes": 26.67, + "ci_minutes": 0, + "input_tokens": 1469919, + "output_tokens": 243211, + "tool_calls": 270, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.0429, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 130, + "by_role": { + "agent": { + "usd": 0.978321, + "input_tokens": 1440961, + "output_tokens": 209952, + "calls": 127 + }, + "reducer": { + "usd": 0.044476, + "input_tokens": 28958, + "output_tokens": 33259, + "calls": 3 + } + }, + "cache_read_tokens": 13273600, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.37, + "gate": 2.26, + "review": 1594.55, + "report": 0.07 + }, + "time_to_first_result_s": 1600.23 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 43431 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4923.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4923.md new file mode 100644 index 000000000..0dcf3d785 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4923.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 6 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] concern 'model reads cudagraph_mode (runner concern)' is acknowledged in-code as TODO(#4923) and gated identically to the runner, so no model/runner mismatch at this head. +- [resolved] nested-graph bug on GPU: use_cuda_graphs now False on GPU while the runner owns the FULL outer wrap — residual NOT covered: NPU with user-set FULL mode still nests (talker_mtp_graph_safe=True unconditional + inner NPU graphs at qwen3_code_predictor.py:532); only the yaml PIECEWISE pin guards it, same posture as qwen3_omni_moe.yaml:87 ('code_predictor MaskedSelect / inner NPU graph replay') +- [claim-verified] benchmark arithmetic internally consistent: 179.44s audio / 4.56s = 39.35 audio-duration/s (calc). +- [claim-verified] #4970 scope: no `seed:` key remains in any qwen3_tts*.yaml (grep across deploy dir), so the default-seed removal is indeed out of this diff; other model yamls still ship seed: 42 (out of scope). +- [claim-verified] 'align with Qwen3-Omni': qwen3_omni_moe_code_predictor_mtp.py:21 already uses 'use_cuda_graphs=current_omni_platform.is_npu()', and qwen3_omni.py:162-164 sets the same three async-output flags +- [claim-verified] yaml prefix-caching flip is required for the PR's async path: gpu_ar_model_runner.py:1539 '_should_use_async_omni_output' returns False when 'self.omni_prefix_cache is not None' (and requires async_chunk=true, which qwen3_tts.yaml sets) +- [claim-verified] PR claim 'NPU keeps internal graphs': qwen3_code_predictor.py:532 `if current_omni_platform.is_npu() and self._wrapper_config.use_cuda_graphs` still captures NPU graphs; GPU path now has use_cuda_graphs=False. +- [claim-verified] PR claim 'runner wraps whole talker_mtp in FULL CUDAGraphWrapper': gpu_ar_model_runner.py `_capture_talker_mtp_graphs` captures sizes from compilation_config.cudagraph_capture_sizes when wrapped. +- [claim-verified] Async-output reachability: gpu_ar_model_runner.py:1537-1557 `_should_use_async_omni_output` requires async_chunk true, no omni_prefix_cache, no spec decode; qwen3_tts.yaml has async_chunk: true and enable_prefix_caching: false, so the new flags take effect. +- [claim-verified] PR RTF arithmetic is internally consistent: 1049.97ms / (179.44/32 * 1000ms) = 0.187 ≈ reported RTF 0.19. +- [claim-verified] talker_mtp_graph_safe=True is consumed at vllm_omni/worker/gpu_model_runner.py:208-209 ('if cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe)') — outer wrap claim holds. +- [claim-refuted] PR body says prefix-cache rationale 'see follow-up below' but no follow-up section exists in the body; Test Plan / vLLM Version / vLLM-Omni Commit fields are empty and the 'PLEASE FILL IN' placeholder remains. +- [claim-refuted] PR body lists 'omni_pooler_payload_include_hidden = False' as a change — qwen3_tts_talker.py:317 already set it at base with its own rationale comment; the diff's line 353 is a pure duplicate (probable merge artifact of merge commit 3b2d87c9) +- [claim-refuted] in-diff comment 'mtp seeding is silently dropped when the decode batch > 1 under full cudagraphs' — gpu_model_runner.py:1851-1856 instead routes seeded batches to the scalar per-row fallback when talker_mtp_accepts_per_row_generators is False (see review comment 1) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~350` [major] — Confirmed: with full cudagraphs, talker_mtp_accepts_per_row_generators is False, so any seeded decode batch >1 in _talker_mtp_forward recurses per row (gpu_model_runner.py:1870) while talker_mtp is graph-wrapped — N graph replays per step and no reproducible seeding. The new code comment claiming seeding is 'silently dropped' is wrong about the mechanism; correct it and either keep the batched path or warn loudly that tts_local_seed is ignored under full cudagraphs. (evidence: qwen3_tts_talker.py:350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped` (added by this diff). gpu_model_runner.py:1851-1854 (unchanged by this diff, present in PR-time tree): `if ( decode_batch_size > 1 and any(generator is not None for generator in row_generators) and not getattr(self.model, "talker_mtp_accepts_per_row_generators", False) ):` — this triggers the scalar path. gpu_model_runner.py:1870 (unchanged by this diff): `self._talker_mtp_forward([req_id], inputs_embeds, row_offsets)` — N recursive calls per step. gpu_model_runner.py `_init_talker_mtp` (unchanged by this diff): `if cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe): self.talker_mtp = graph_wrapper_cls(talker_mtp, self.vllm_config, runtime_mode=CUDAGraphMode.FULL)` — so the recursive scalar calls replay the graph wrapper rather than forcing eager.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~350` [major] — Confirmed: line 350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped` is new producer-side gating with no unit coverage. The only test that references this flag, tests/worker/test_omni_gpu_model_runner.py::test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models, sets `talker_mtp_accepts_per_row_generators=True` on a SimpleNamespace and never constructs Qwen3TTSTalkerForConditionalGeneration, so a regression in the `getattr(vllm_config.compilation_config, "cudagraph_mode", None)` default or `has_full_cudagraphs()` would stay green. Add the suggested test with mocked FULL vs NONE/PIECEWISE cudagraph_mode. (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped` (added by this PR diff). Unchanged by this diff, present in the PR-time tree: tests/worker/test_omni_gpu_model_runner.py:374 `talker_mtp_accepts_per_row_generators=True,` — set on `SimpleNamespace` in `test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models`, not on a constructed Qwen3TTSTalker. `grep talker_mtp_accepts_per_row_generators` in tests/ returns only that one test line, confirming no producer-side test exists.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:~39` [major] — qwen3_tts_code_predictor_vllm.py:39 now sets `use_cuda_graphs=current_omni_platform.is_npu()`, so on GPU `use_cuda_graphs=False`. In `CodePredictorWrapper.__init__` (qwen3_code_predictor.py:477) `_prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu`, so GPU prefix graphs are always disabled. qwen3_tts_high_concurrency.yaml:25-29 still sets `code_predictor_prefix_graphs: true` with buckets [64] and seq_lens 2-8 under the comment 'the c64 hot path', which is now a silent no-op that only logs info_once. Update the sibling profile in this PR: remove the now-dead `code_predictor_prefix_graphs*` keys and rewrite the comment, or make the enablement follow the new capture model. (evidence: Changed by this diff — vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu(),` (False on GPU, disabling inner CUDA graphs). Unchanged by this diff, present in the PR-time tree — vllm_omni/model_executor/models/common/qwen3_code_predictor.py:477 `self._prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu`. Unchanged by this diff, present in the PR-time tree — vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:27 `code_predictor_prefix_graphs: true` and line 25 `# Stage0 code-predictor prefix CUDA graphs for the c64 hot path.`) + +`docs/user_guide/examples/online_serving/text_to_speech.md:~334` [major] — Confirmed against the PR-time tree: vllm_omni/deploy/qwen3_tts.yaml now ships max_num_seqs: 64 for both stage 0 (line 38) and stage 1 (line 70), while docs/user_guide/examples/online_serving/text_to_speech.md:334 still says the bundled YAML ships stage 1 at max_num_seqs: 10 and asserts the Base-cloning tradeoff for the bundled config. docs/serving/speech_api.md:667 similarly still tells users to benchmark before reducing stage 1 below 10. Update both docs to the new 64/64 defaults or document the retune rationale. (evidence: Changed by this diff — vllm_omni/deploy/qwen3_tts.yaml:38 ` max_num_seqs: 64` (stage_id 0) and vllm_omni/deploy/qwen3_tts.yaml:70 ` max_num_seqs: 64` (stage_id 1); the pre-PR diff lines removed `max_num_seqs: 10` for both stages. Unchanged by this diff, present in the PR-time tree — docs/user_guide/examples/online_serving/text_to_speech.md:334 `The bundled `qwen3_tts.yaml` ships stage 1 (Code2Wav) at `max_num_seqs: 10`, tuned for Base voice cloning: stage-1 lifetimes are long (~3 s/req), so admitting up to 10 concurrent codec sequences lets requests progress in parallel in the scheduler — ~2× TTFA p95 at c=4 / c=8 (1× H100, 1.7B-Base, seed-tts) at an 8–12 % audio-throughput cost.` and docs/serving/speech_api.md:667 `benchmark before reducing it below `10`.`) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~353` [minor] — Confirmed. `self.omni_pooler_payload_include_hidden = False` already exists at line 317 with the 'Stage 1 (code2wav) consumes runtime audio codes, not hidden states' explanation, and the PR adds an identical, undocumented assignment at line 353. The PR body also lists this flag as newly enabled. Remove the duplicate at line 353 and keep only the earlier documented assignment. (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:317 `self.omni_pooler_payload_include_hidden = False` — pre-existing, preceded by the comment 'Stage 1 (code2wav) consumes runtime audio codes, not hidden states, so the inter-stage pooler payload never needs a CPU hidden-states view.'; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:353 `self.omni_pooler_payload_include_hidden = False` — added by this diff (unchanged line 317 is outside the diff; line 353 is the diff-added duplicate).) + +`vllm_omni/deploy/qwen3_tts.yaml:~41` [minor] — Stale YAML comments contradict this PR's values: vllm_omni/deploy/qwen3_tts.yaml:41-45 still says 'Prefix caching on by default … Disable explicitly if memory headroom is tight' while line 46 is now `enable_prefix_caching: false`, and lines 6-8 still document `max_num_batched_tokens=512` while line 48 is now `32768`. The prefix-caching comment is actively misleading: re-enabling it sets `cache_config.enable_prefix_caching`, which makes the runner construct `omni_prefix_cache` (gpu_model_runner.py:169-170), and `_should_use_async_omni_output()` then returns False (gpu_ar_model_runner.py:1540-1541), silently falling back to the synchronous output path with no warning. Rewrite both comment blocks to state the new values and that async omni output and omni prefix caching are mutually exclusive on this stage. (evidence: vllm_omni/deploy/qwen3_tts.yaml:6 `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk`; vllm_omni/deploy/qwen3_tts.yaml:41 `# Prefix caching on by default. The HS opt-out (talker only consumes the`; vllm_omni/deploy/qwen3_tts.yaml:46 ` enable_prefix_caching: false`; vllm_omni/deploy/qwen3_tts.yaml:48 ` max_num_batched_tokens: 32768`. Decisive code OUTSIDE the diff (unchanged by this PR, present in PR-time tree): vllm_omni/worker/gpu_model_runner.py:170 `self.omni_prefix_cache = OmniTensorPrefixCache(` (guarded by `if self.cache_config.enable_prefix_caching:` at line 169); vllm_omni/worker/gpu_ar_model_runner.py:1540-1541 `if self.omni_prefix_cache is not None:` / `return False` inside `_should_use_async_omni_output`.) + +`vllm_omni/deploy/qwen3_tts.yaml:~48` [minor] — Stage 0 still documents `max_num_batched_tokens=512` for async-chunk first-chunk latency (lines 5-8), but line 48 is now 32768 and both stages' `max_num_seqs` are 64 at unchanged `gpu_memory_utilization: 0.3` (line 42). The only quoted profile evidence is single-request `talker_mtp` 39→9 ms; it does not cover concurrent async-chunk TTFA/first-chunk latency or KV fit at 64×4096. Please add old-vs-new TTFA/first-chunk numbers at c=1 and c≥8 with the same main-vs-PR harness, or move the capacity retune to a follow-up. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/deploy/qwen3_tts.yaml:6-7 `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk` / `# latency tuning (not correctness) — small per-step batches keep` / `# first-chunk latency low.`; changed by this diff: vllm_omni/deploy/qwen3_tts.yaml:48 `max_num_batched_tokens: 32768`, vllm_omni/deploy/qwen3_tts.yaml:38 `max_num_seqs: 64`, vllm_omni/deploy/qwen3_tts.yaml:42 `gpu_memory_utilization: 0.3`, vllm_omni/deploy/qwen3_tts.yaml:50 `max_model_len: 4096`; unchanged stage-1 comment at vllm_omni/deploy/qwen3_tts.yaml:68-69 `# Tuned for Base voice clone; CustomVoice / VoiceDesign are TTFA-optimal at 1.` above vllm_omni/deploy/qwen3_tts.yaml:70 `max_num_seqs: 64`. PR body (gh_pr_view 4923): `Profile impact (single request, GPU): talker_mtp 39 ms → 9 ms per step.`) + +`vllm_omni/deploy/qwen3_tts.yaml:~102` [minor] — `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are both failing at this head, and this diff's surface is squarely inside both lanes' coverage: the AMD merge lane runs `test_qwen3_tts_customvoice.py` (.buildkite/test-amd-merge.yml:177) and `test_qwen3_tts_base.py` (.buildkite/test-amd-merge.yml:192), the NPU ready lane runs `test_qwen3_tts_customvoice.py` (.buildkite/test-npu-ready.yaml:21), and the merge gate lists `vllm_omni/deploy/qwen3_tts.yaml` and `vllm_omni/model_executor/models/qwen3_tts/` as rerun dependencies (.buildkite/test-merge.yml:170-174, 197-201). The NPU override `cudagraph_mode: PIECEWISE` is at qwen3_tts.yaml:99/102 (inside the `platforms.npu` block) and directly changes NPU startup graph behavior. Please confirm on main whether both failures are pre-existing or introduced here — a perf PR whose NPU fix lands while the NPU lane is red needs an explicit attribution note. (evidence: vllm_omni/deploy/qwen3_tts.yaml:99 `cudagraph_mode: PIECEWISE` (stage 0) and :102 `cudagraph_mode: PIECEWISE` (stage 1) — inside the `platforms.npu` override block added by this diff; .buildkite/test-amd-merge.yml:177 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"`; .buildkite/test-amd-merge.yml:192 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'`; .buildkite/test-npu-ready.yaml:21 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; .buildkite/test-merge.yml:174 `- vllm_omni/deploy/qwen3_tts.yaml` and :172 `- vllm_omni/model_executor/models/qwen3_tts/` (also :201/:199 for the Base test); CI read: buildkite/vllm-omni-amd-ci state FAILURE, buildkite/vllm-omni-npu-ci state FAILURE.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39` [minor] `use_cuda_graphs` is now gated on `current_omni_platform.is_npu()` instead of the actual outer wrap decision. On GPU this disables the code predictor's inner CUDA graphs in every mode — including eager and PIECEWISE, whe +- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39` [minor] This platform gate now runs `use_cuda_graphs=current_omni_platform.is_npu()`, so on ROCm/XPU (where `supports_torch_inductor()` is False) the code predictor falls back to pure eager with no graphs (`_setup_compile` warns +- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:342` [minor] Confirmed: the boundary in the in-diff comment is wrong. Under full cudagraphs `talker_mtp_accepts_per_row_generators` is False (qwen3_tts_talker.py:350), but the runner's scalar fallback only triggers for `decode_batch_ +- `docs/design/qwen3_omni_tts_performance_optimization.md:487` [minor] docs/design/qwen3_omni_tts_performance_optimization.md:487 (and the key-knobs snippet at :507) still says `max_batch_size: 10`, while vllm_omni/deploy/qwen3_tts.yaml now ships `max_num_seqs: 64` on both stages (yaml:38 a + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4923.trace.json.gz new file mode 100644 index 000000000..db898f61c Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4926.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4926.cost.json new file mode 100644 index 000000000..cc30f70e9 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1620.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4926/run-20260815-162726-3a67e9", + "recorded_at": "2026-08-15T16:54:24", + "attempts": [ + { + "invocation_id": "inv-6bdfe66f", + "outcome": "run:run-20260815-162726-3a67e9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-162726-3a67e9", + "llm_calls": 141, + "input_tokens": 666863, + "output_tokens": 236076, + "usd": 0.660157, + "source": "spans" + } + ], + "attempt_usd_total": 0.660157, + "llm_calls": 141, + "input_tokens": 666863, + "output_tokens": 236076, + "usd": 0.660157, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-162726-3a67e9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6602, + "minutes": 26.97, + "ci_minutes": 0, + "input_tokens": 666863, + "output_tokens": 236076, + "tool_calls": 293, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9131, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 141, + "by_role": { + "agent": { + "usd": 0.610566, + "input_tokens": 630084, + "output_tokens": 200087, + "calls": 138 + }, + "reducer": { + "usd": 0.049591, + "input_tokens": 36779, + "output_tokens": 35989, + "calls": 3 + } + }, + "cache_read_tokens": 8163712, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.03, + "gate": 1.6, + "review": 1612.64, + "report": 0.06 + }, + "time_to_first_result_s": 1618.32 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 48670 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4926.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4926.md new file mode 100644 index 000000000..dd5daafe0 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4926.md @@ -0,0 +1,71 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] piecewise-with-varlen-only concern (commit 332da8f2 'Fall back to varlen kernel in Hub piecewise attention'): _flash_wrapper routes to _run_varlen_dense via `causal=kwargs.get("causal", self.causal)`, matching piecewise_attn.py:92-97 `causal=(mode == "causal")`; residual: dense-homogeneous batches only, same as local sibling +- [resolved] piecewise varlen fallback landed at head (`_flash_wrapper` routes attn_func=None to varlen); residual is that `_forward_varlen_masked`/`_forward_varlen_dense` still assume varlen is not None. +- [claim-verified] 'falls back to local FLASH_ATTN when kernels missing': cuda/platform.py:124-137 probes `importlib.import_module('kernels')` and falls back; rocm/musa/npu/xpu all warn+fallback to FLASH_ATTN. +- [claim-verified] 'FLASH_ATTN_3_HUB falls back to FLASH_ATTN_HUB below Hopper': cuda/platform.py:140-147 implements major<9 fallback. +- [claim-verified] PR claim 'On non-CUDA platforms, or when kernels is not installed, logs a warning and falls back gracefully' — true for all 4 non-CUDA platforms and for missing-kernels on CUDA (fall-through to flash_attn_supported→TORCH_SDPA chain confirmed in situ) +- [claim-verified] docs 'falls back to FLASH_ATTN_HUB on older GPUs' matches cuda/platform.py:140-147 +- [claim-verified] rocm/musa/npu/xpu platform files all map HUB -> local FLASH_ATTN with a warning, matching the PR's non-CUDA fallback claim. +- [claim-refuted] 'fall back gracefully' covers only missing package: flash_attn_hub.py:103 then 109 has an unguarded final get_kernel + RuntimeError, so a package-installed-but-Hub-fetch-failure crashes at model build (comment 7). +- [claim-refuted] FA3 gate `>= 9` (cuda/platform.py:141) admits Blackwell major 10/12 (same file line 75 `_known_blackwell_sms = {(10,0),(10,3),(12,0),(12,1)}`) while docs say Hopper-only (comment 3). +- [claim-refuted] PR body 'perfect numerical consistency' is undermined by unpinned `version=1` branch-head resolution (comment 2). +- [claim-refuted] PR body 'E2E z-image FA3' is a manual screenshot; the automated FA3 half skips on L4 (test line 92-93) and no CI lane runs it. +- [claim-refuted] test_kernels_hub_execution is not selected by any .buildkite command — test-ready.yml:145 targets only tests/diffusion/*.py (not tests/diffusion/attention) with cuda; the cpu lane (test-ready.yml:13) only selects the fallback test (comment 5). +- [claim-refuted] 'falls back gracefully' does NOT cover kernels-installed-but-Hub-unreachable: FlashAttentionHubImpl.__init__ raises out of the final unversioned get_kernel and model init crashes with no local fallback (comment 4) +- [claim-refuted] docs 'perfect numerical consistency' is not guaranteed by the code: no immutable kernel revision is pinned and explicit HUB requests can silently resolve to local FLASH_ATTN. + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~92` [major] — `FlashAttentionHubImpl.__init__` (and the FA3 twin) calls `get_kernel(...)` with no in-file memoization, and `Attention.__init__` constructs a fresh impl per layer (layer.py:94-95), so a multi-layer DiT invokes `get_kernel` once per layer per worker at model init. This risks slow startup and HF rate-limit/offline failure; whether each call literally re-downloads depends on `kernels.get_kernel`'s own caching, but the repeated per-layer invocation is confirmed. Move the load into a module-level cached helper keyed by repo id so each kernel is fetched once per process. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92 ` from kernels import get_kernel` (inside `FlashAttentionHubImpl.__init__`); vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96 ` hub_module = get_kernel("kernels-community/flash-attn2", version=1)`; unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/attention/layer.py:94 ` self.attn_impl_cls = self.attn_backend.get_impl_cls()` and vllm_omni/diffusion/attention/layer.py:95 ` self.attention = self.attn_impl_cls(`. No caching exists in the changed file: grep shows the only `get_kernel` references are the FA2 calls at lines 92/96/100/103 and the FA3 twin at 271/275/279/282, with only `from functools import partial` imported.) + +`vllm_omni/platforms/cuda/platform.py:~141` [major] — At vllm_omni/platforms/cuda/platform.py:141 the FA3_HUB gate is `compute_capability.major >= 9`, so Blackwell major 10/11/12 also passes, contradicting the new docs row (`FLASH_ATTN_3_HUB` — 'CUDA Hopper (sm_90+) only'). FLASH_ATTN_HUB has no compute-capability gate: the only FLASH_ATTN guard is `if backend_upper == "FLASH_ATTN" and not flash_attn_supported` (line 149), so on sm_75 or Blackwell a surviving FLASH_ATTN_HUB reaches `DiffusionAttentionBackendEnum[backend_upper]` and loads/launches the kernel. Use `major == 9` for FA3_HUB and run both HUB names through the same `compute_supported`/Blackwell checks as local FLASH_ATTN. (evidence: vllm_omni/platforms/cuda/platform.py:141 `fa3_hub_supported = compute_capability is not None and compute_capability.major >= 9` (added by this diff) — major 10/11/12 pass, though docs/user_guide/diffusion/attention_backends.md:24 (added) says `FLASH_ATTN_3_HUB` is 'CUDA Hopper (sm_90+) only; falls back to FLASH_ATTN_HUB on older GPUs.' The sole local-FLASH capability gate, unchanged by this diff, is vllm_omni/platforms/cuda/platform.py:80 `compute_supported = capability >= 80` applied only at platform.py:149 `if backend_upper == "FLASH_ATTN" and not flash_attn_supported:`; when `backend_upper` remains `FLASH_ATTN_HUB`, control skips that branch and the SAGE_ATTN_3 branch and falls straight to `backend = DiffusionAttentionBackendEnum[backend_upper]` / `return backend.get_path()`.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~96` [major] — The Hub kernel is loaded inside FlashAttentionHubImpl.__init__ (constructed per Attention layer) via an `except Exception` chain that silently escalates version=1 → version=2 → unversioned default at logger.info; pyproject.toml pins only `kernels<0.15.0` with no lower bound. Given the PR's goal of eliminating train↔rollout numerical drift, this makes the resolved kernel version nondeterministic under transient Hub failures and can leave different layers running different builds. Resolve the module once per process with one pinned version, and fail loudly or fall back to local FLASH_ATTN instead of escalating across versions. (evidence: flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)`; flash_attn_hub.py:97-103 `except Exception as e: + try: + logger.info("Failed to load version 1, attempting version 2: %s", e) + hub_module = get_kernel("kernels-community/flash-attn2", version=2) + except Exception as e2: + logger.info("Failed to load version 2, attempting default: %s", e2) + hub_module = get_kernel("kernels-community/flash-attn2")` — the version escalates and falls back to an unversioned default on any exception. Unchanged by this diff, present in the PR-time tree: layer.py:94-95 `self.attn_impl_cls = self.attn_backend.get_impl_cls()` / `self.attention = self.attn_impl_cls(` — the impl and its __init__ are constructed per Attention layer. Unchanged by this diff, present in the PR-time tree: pyproject.toml:70 `"kernels<0.15.0", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)` — no lower bound on the kernels version.) + +`vllm_omni/platforms/cuda/platform.py:~133` [major] — Explicit `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` requests silently degrade to local `FLASH_ATTN` (or `TORCH_SDPA`) when `kernels` is unavailable — warning-only, with no hard failure and no machine-readable signal. Since the documented purpose of the HUB backends is to eliminate train/rollout numerical drift (docs/user_guide/diffusion/attention_backends.md:217), this fallback runs exactly the divergent local kernel the user opted out of. Fail hard on explicit HUB selection (or expose the effective backend and provide a strict opt-out). (evidence: vllm_omni/platforms/cuda/platform.py:124 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):` — vllm_omni/platforms/cuda/platform.py:129-134 `except ImportError:` / `if backend_upper == "FLASH_ATTN_HUB":` / `logger.warning(` / `"HuggingFace `kernels` library is not available. Falling back to local FLASH_ATTN."` / `)` / `backend_upper = "FLASH_ATTN"` — vllm_omni/platforms/cuda/platform.py:135-138 same warning-only rewrite for FLASH_ATTN_3_HUB ending in `backend_upper = "FLASH_ATTN"` — docs/user_guide/diffusion/attention_backends.md:217 `This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout.`) + +`pyproject.toml:70` [major] — Add a lower bound on `kernels` in the dev extra matching the docs pin (e.g. `kernels>=0.14.1,<0.15.0`), and make the fallback verify `get_kernel` availability rather than only `importlib.import_module("kernels")`. `from kernels import get_kernel` at flash_attn_hub.py:92/271 is unguarded, so an importable old kernels build lacking `get_kernel` passes the platform fallback and then raises ImportError in `FlashAttentionHubImpl.__init__`/`FlashAttention3HubImpl.__init__` instead of falling back to local FLASH_ATTN. (evidence: pyproject.toml:70 `"kernels<0.15.0", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)` — unchanged by this diff, present in the PR-time tree; no lower bound. vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92 ` from kernels import get_kernel` and :271 ` from kernels import get_kernel` — imported before the `try:` that wraps only the `get_kernel(...)` calls, so a missing `get_kernel` raises an uncaught ImportError. vllm_omni/platforms/cuda/platform.py:124-128 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):` / `importlib.import_module("kernels")` / `except ImportError:` — the fallback only tests that the kernels module imports, not that `get_kernel` exists, so an importable old kernels package passes the guard and then crashes at impl construction.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~103` [minor] — The graceful fallback this PR adds only probes `importlib.import_module("kernels")` at selection time (cuda/platform.py:126). Once a hub backend is selected, `Attention.__init__` constructs the impl directly (layer.py:103-108) and the final unversioned `hub_module = get_kernel("kernels-community/flash-attn2")` at flash_attn_hub.py:103 has no enclosing except, so a Hub fetch failure (HF_HUB_OFFLINE cache miss, rate limiting, deleted repo) raises out of `__init__` and crashes model init with no fallback to local FLASH_ATTN. The docs' 'fall back gracefully' sentence covers only the package-not-installed case. Either catch construction failure and fall back to the local FlashAttentionImpl (mirroring the selector), or document the pre-download / HF_HUB_OFFLINE requirement. (evidence: flash_attn_hub.py:103 `hub_module = get_kernel("kernels-community/flash-attn2")` — final retry with no enclosing `except`; a failure here raises from `__init__`. cuda/platform.py:126 `importlib.import_module("kernels")` — the only selection-time availability probe, proving package importability but not Hub fetch success. layer.py:103 `self.attention = self.attn_impl_cls(` — unchanged by this diff but present in the PR-time tree; the constructor is invoked directly with no try/except, so `__init__` exceptions propagate to model init.) + +`tests/diffusion/attention/test_kernels_hub.py:19` [minor] — This test is `@pytest.mark.cpu` with no CUDA guard, yet it calls `CudaOmniPlatform.get_diffusion_attn_backend_cls`, which executes `torch.backends.cudnn.version() or 0` (vllm_omni/platforms/cuda/platform.py:100) before the selected-backend branch. The AMD lane (.buildkite/test-amd-ready.yaml:19,30) runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`, so this test is selected on ROCm agents. Add the same guard as test_sage_attn3.py:86/114 (`@pytest.mark.skipif(not current_platform.is_cuda(), ...)`) or mock the cudnn path. Note: `torch.backends.cudnn.version()` usually returns None on non-CUDA builds rather than raising, so confirm the actual failing line from the Buildkite log before attributing the failure to line 100. (evidence: tests/diffusion/attention/test_kernels_hub.py:18-19 `@pytest.mark.core_model` / `@pytest.mark.cpu` / `def test_kernels_hub_platform_fallback(monkeypatch: pytest.MonkeyPatch):` — unchanged by the diff? No, this file is added by the diff; the quoted test is in the PR-time tree. vllm_omni/platforms/cuda/platform.py:100 `cudnn_version = torch.backends.cudnn.version() or 0` — in the path called by the test, present in the PR-time tree and also visible in the diff context (the added HUB branch is below it). .buildkite/test-amd-ready.yaml:19 `"timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — unchanged by this diff, present in the PR-time tree. tests/diffusion/attention/test_sage_attn3.py:86 `@pytest.mark.skipif(not current_platform.is_cuda(), reason="sage_attn3 tests require CUDA platform")` — sibling guard the comment cites, present in the PR-time tree.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:250` [minor] — `FlashAttention3HubImpl` (line 250) is a near line-for-line copy of `FlashAttentionHubImpl` (line 71): `_unwrap_flash_output` (112-114 vs 291-293), `_flash_wrapper` (116-126 vs 295-305), `_forward_varlen_masked` (128-161 vs 307-340), `_forward_varlen_dense` (163-176 vs 342-355), and `forward_cuda` (178-230 vs 357-406) are identical; only `__init__`'s hub repo string and log prefixes differ. `_forward_varlen_masked` also mirrors flash_attn.py:85-127 apart from the varlen function source. Extract a `_FlashAttnHubImplBase` with a `_HUB_REPO: str` class attribute consumed by `__init__`; subclasses set `"kernels-community/flash-attn2"` / `"kernels-community/flash-attn3"` so load/fallback fixes land in both backends. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:112 ` def _unwrap_flash_output(out: torch.Tensor | tuple[torch.Tensor, ...]) -> torch.Tensor:` ... :114 ` return out[0] if isinstance(out, tuple) else out` — identical at :291-293; :128 ` def _forward_varlen_masked(` and :307 ` def _forward_varlen_masked(`; :178 ` def forward_cuda(` and :357 ` def forward_cuda(` — all four method groups are line-for-line identical between the two classes; the differing `__init__` sections use `get_kernel("kernels-community/flash-attn2", ...)` vs `get_kernel("kernels-community/flash-attn3", ...)`.) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/attention/test_kernels_hub.py:57` [minor] tests/diffusion/attention/test_kernels_hub.py:57 `test_kernels_hub_execution` only drives the dense unmasked `forward_cuda` path (`attn_mask=None`, no `full_attn_spans`). The `_forward_varlen_masked`, `_forward_varlen_de +- `tests/diffusion/attention/test_kernels_hub.py:57` [minor] This diff declares res={"cuda": "L4"} for test_kernels_hub_execution, but the FA3 half skips on major < 9; L4 is sm_89, so the FlashAttention3Hub assertions can never execute on the test's own declared resource. Compound +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:147` [minor] Both Hub impls accept a module that exports only `flash_attn_func` (`__init__` only raises when BOTH funcs are None), yet `supports_attention_mask()` returns True and `forward_cuda` dispatches any masked request to `_for +- `docs/user_guide/diffusion/attention_backends.md:23` [minor] The two hub rows and the Installation paragraph don't make non-CUDA behavior clear: `FLASH_ATTN_HUB` has no platform-scope note, and the Installation section says you must install the `kernels` library without saying tha + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4926.trace.json.gz new file mode 100644 index 000000000..1706c6f0f Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4950.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4950.cost.json new file mode 100644 index 000000000..42ebe63ac --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4950.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 495.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4950/run-20260815-164353-715f20", + "recorded_at": "2026-08-15T16:52:07", + "attempts": [ + { + "invocation_id": "inv-adfc82c9", + "outcome": "run:run-20260815-164353-715f20", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-164353-715f20", + "llm_calls": 16, + "input_tokens": 254430, + "output_tokens": 31997, + "usd": 0.125576, + "source": "spans" + } + ], + "attempt_usd_total": 0.125576, + "llm_calls": 16, + "input_tokens": 254430, + "output_tokens": 31997, + "usd": 0.125576, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-164353-715f20", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1256, + "minutes": 8.22, + "ci_minutes": 0, + "input_tokens": 254430, + "output_tokens": 31997, + "tool_calls": 33, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3254, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 16, + "by_role": { + "agent": { + "usd": 0.125576, + "input_tokens": 254430, + "output_tokens": 31997, + "calls": 16 + } + }, + "cache_read_tokens": 803072, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.58, + "gate": 1.51, + "review": 488.23, + "report": 0.06 + }, + "time_to_first_result_s": 493.36 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4950.trace.json.gz", + "trace_bytes": 10232 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4950.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4950.md new file mode 100644 index 000000000..540e4e568 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4950.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4950, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 48d4fd8a5e98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Merge commits c4cd7778/48d4fd8a are in the timeline, but diff_stat against merge-base shows only the recipe file; no dropped hunks or duplicated blocks in the full file read. +- [claim-verified] Top-level chat_template_kwargs is the field the AR path consumes: serving_chat.py:373 `chat_template_kwargs=request.chat_template_kwargs` and :381 `merged_template_kwargs = self._effective_chat_template_kwargs(request)`. +- [claim-verified] Nested extra_body.chat_template_kwargs is not consulted on the AR path: grep shows extra_body is only read in diffusion branches (e.g. serving_chat.py:441, 1083, 3257), not in the non-diffusion preprocessing path. +- [claim-verified] Response shape: chat_completion_full_generator appends one choice per final_output_type, and `_create_audio_choice` builds `ChatMessage(role=role, audio=audio_obj)` (serving_chat.py ~2571). +- [validated] 24 kHz mono claim: `_create_audio_choice` defaults `sample_rate=24000` when `mm_output["sr"]` is absent and flattens ndim>1 audio to 1D (serving_chat.py). +- [sweep] diff_stat: only recipes/OpenBMB/MiniCPM-o-4_5.md changed (27 lines, 19+/8-); no test files touched. +- [sweep] Read the full recipe in situ; hunks at new lines 102-116 (curl payload), 117-126 (SDK + response shape) and 216-225 (TTS trigger note) verified against the surrounding file. +- [sweep] Sibling docs not stale: examples/online_serving/minicpmo/README.md:58 describes the SDK extra_body form, which remains correct for the demo client. +- [sweep] Without use_tts_template the stage-input processor sets tts_token_ids only when tts_bos_idx is found (minicpmo_4_5_omni.py stage processor), so the no-span path is real. +- [sweep] Residual unverified: the wrapper returns `multimodal_outputs=None` when no waveform is produced (minicpmo_4_5_omni.py:325-327 `_, waveform = talker_result; if waveform is not None: mm_out["model_outputs"] = [waveform]`); the note's 'silent audio (not text-only)' outcome therefore depends on the author's live test plan (max amplitude 0) and was not reproduced here. +- [sweep] changed_symbol_consumers: none (docs-only; no public symbols changed). + +`recipes/OpenBMB/MiniCPM-o-4_5.md:105` [nit] — + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4950.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4950.trace.json.gz new file mode 100644 index 000000000..179d1725e Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4950.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4970.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4970.cost.json new file mode 100644 index 000000000..f0e4d0f8e --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4970.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 485.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4970/run-20260815-164146-865b23", + "recorded_at": "2026-08-15T16:49:50", + "attempts": [ + { + "invocation_id": "inv-d17d1b11", + "outcome": "run:run-20260815-164146-865b23", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-164146-865b23", + "llm_calls": 20, + "input_tokens": 190743, + "output_tokens": 45423, + "usd": 0.119085, + "source": "spans" + } + ], + "attempt_usd_total": 0.119085, + "llm_calls": 20, + "input_tokens": 190743, + "output_tokens": 45423, + "usd": 0.119085, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-164146-865b23", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1191, + "minutes": 8.06, + "ci_minutes": 0, + "input_tokens": 190743, + "output_tokens": 45423, + "tool_calls": 53, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3181, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 20, + "by_role": { + "agent": { + "usd": 0.119085, + "input_tokens": 190743, + "output_tokens": 45423, + "calls": 20 + } + }, + "cache_read_tokens": 652544, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.35, + "gate": 1.81, + "review": 478.22, + "report": 0.07 + }, + "time_to_first_result_s": 483.42 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4970.trace.json.gz", + "trace_bytes": 12649 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4970.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4970.md new file mode 100644 index 000000000..f6e7a2376 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4970.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4970, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4992e10b7367) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] fleet exposure: no other qwen3_tts deploy YAML ships `seed: 42`; residual — other non-qwen3_tts models' YAMLs still ship it, but their serving path is not `tts_local_seed`-gated so they are outside this PR's blast radius. +- [claim-verified] diff_stat shows exactly 1 file changed, 2 deletions — matches '2-line YAML deletion'. +- [claim-verified] qwen3_tts_high_concurrency.yaml has no `seed:` in either `default_sampling_params` block, matching '#4869 already did' claim. +- [claim-verified] producer: serving_speech.py:3698-3706 `default_seed = getattr(stage0_params, "seed", None)` -> `extra_args.setdefault("tts_local_seed", int(default_seed))`; with seed removed, no `tts_local_seed` is set. +- [claim-verified] consumer: gpu_model_runner.py:1826 `seed = extra_args.get("tts_local_seed")`; absent seed -> `row_generators` all None -> batched path, no scalar per-row fallback. +- [claim-verified] per-request determinism intact: serving_speech.py (request.seed block) sets `stage0_params.seed = request.seed` and `extra_args["tts_local_seed"] = request.seed`. +- [claim-verified] 'no test references the deploy-level seed': tests/entrypoints/openai_api/test_serving_speech.py:3347 test constructs `SimpleNamespace(max_tokens=2048, seed=42, extra_args=None)` in memory, not the YAML. +- [validated] vllm_omni/deploy/qwen3_tts.yaml stage-0 hunk read: `seed: 42` removed between `min_tokens: 2` and `repetition_penalty: 1.05` (diff + full file read). +- [validated] vllm_omni/deploy/qwen3_tts.yaml stage-1 hunk read: `seed: 42` removed before `repetition_penalty: 1.0`; stage-1 is temperature 0.0 (greedy), so that removal is a determinism no-op but keeps profiles consistent. +- [sweep] e2e/offline qwen3_tts tests load the YAML but no `seed` assertion found (grep of tests/e2e for seed shows no qwen3_tts matches). +- [sweep] sibling census: only qwen3_tts.yaml and qwen3_tts_high_concurrency.yaml are qwen3_tts deploy profiles; both are now seed-free. qwen3_tts_forced_aligner.yaml is the forced-aligner config, not the talker profile. +- [sweep] docs: docs/configuration/stage_configs.md seed:42 lines (274/298/325) are the legacy Qwen2.5-Omni example, not qwen3_tts; recipes/Qwen/Qwen3-TTS.md and docs/serving/speech_api.md have no `seed` examples (grep). +- [sweep] NPU twin: repo-wide grep for `tts_local_seed` hits only serving_speech.py and gpu_model_runner.py (plus tests); no platforms/npu runner consumes it. +- [sweep] gate coverage: `qwen3_tts.yaml` appears as source_file_dependency only in .buildkite/test-ready.yml:328/354 and .buildkite/test-merge.yml:174/201 (the passing vllm-omni lane); no AMD/NPU pipeline file references it. + +`vllm_omni/deploy/qwen3_tts.yaml:~61` [minor] — This diff deletes `seed: 42` from stage-0 `default_sampling_params` (the fix), but leaves no in-file record of why the reference profile must NOT carry a deploy-level seed. Add a one-line tombstone comment in this block, in the style of the existing `# Official qwen-tts generates...` comment: seed is intentionally omitted because a deploy-level seed is propagated into `extra_args["tts_local_seed"]` (vllm_omni/entrypoints/openai/serving_speech.py:3698-3706) and consumed at vllm_omni/worker/gpu_model_runner.py:1826, forcing the slow per-row talker-MTP path (~40% audio_throughput loss at c>=16 per #4963); per-request determinism remains available through the request `seed` field (vllm_omni/entrypoints/openai/serving_speech.py:3694). Without it, a future 'restore determinism' edit silently reintroduces the regression. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/serving_speech.py:3694 `stage0_params.extra_args["tts_local_seed"] = request.seed`; :3698 `default_seed = getattr(stage0_params, "seed", None)`; :3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`; vllm_omni/worker/gpu_model_runner.py:1826 `seed = extra_args.get("tts_local_seed")`. The diff itself only deletes `seed: 42` from vllm_omni/deploy/qwen3_tts.yaml stage-0 `default_sampling_params`, after which `repetition_penalty: 1.05` sits at line 61 with no seed-omission comment.) + +`.buildkite/test-ready.yml:~328` [minor] — `.buildkite/test-ready.yml:328/354` and `.buildkite/test-merge.yml:174/201` are the only `vllm_omni/deploy/qwen3_tts.yaml` source-file dependencies; the AMD/NPU pipeline files have no reference to it. At this head `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are FAILURE while `buildkite/vllm-omni` and `buildkite/vllm-omni-intel-ci` pass, and this diff only deletes two `seed: 42` lines from the YAML. That is consistent with the two red lanes being pre-existing on main; please confirm by checking the same lanes on main. (evidence: [".buildkite/test-ready.yml:328 ` - vllm_omni/deploy/qwen3_tts.yaml` (inside `source_file_dependencies` of 'TTS · Qwen3-TTS CustomVoice Test'; unchanged by this diff, present in PR-time tree)", ".buildkite/test-ready.yml:354 ` - vllm_omni/deploy/qwen3_tts.yaml` (inside 'TTS · Qwen3-TTS Base Test' source_file_dependencies; unchanged by this diff)", '.buildkite/test-merge.yml:174 ` - vllm_omni/deploy/qwen3_tts.yaml` and .buildkite/test-merge.yml:201 ` - vllm_omni/deploy/qwen3_tts.yaml` (unchanged by this diff)', 'gh_ci_read(4970): `{"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`, `{"name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}`, `{"name":"buildkite/vllm-omni","state":"SUCCESS"}`, `{"name":"buildkite/vllm-omni-intel-ci","state":"SUCCESS"}`', 'PR diff deletes only `- seed: 42` (two occurrences) from `vllm_omni/deploy/qwen3_tts.yaml`; no AMD/NPU pipeline file in `.buildkite/` contains `qwen3_tts.yaml` per repo-wide grep.']) + +`vllm_omni/deploy/qwen3_tts.yaml:~60` [minor] — Anchor at vllm_omni/deploy/qwen3_tts.yaml:60 (`min_tokens: 2`) exists and the diff removes `seed: 42` from both stages, restoring the unseeded batched-MTP path. PR #4970's body states Test Result 'Pending CI' and head 4992e10b is a merge of upstream/main, so the body's nightly numbers predate the merged head; benchmarks/tts/bench_tts.py:242 (`parser.add_argument("--wer-eval", action="store_true", ...)`) confirms quality eval is opt-in. Please run the qwen3_tts perf smoke against this head (c=16 TTFP/RTF/audio_throughput vs pre-07-04 baseline, WER off) or at least the qwen3_tts advanced_model e2e pair to confirm no functional regression. (evidence: PR-time tree vllm_omni/deploy/qwen3_tts.yaml:60 ` min_tokens: 2` (anchor, unchanged by this diff; the preceding ` seed: 42` lines deleted from stage 0 and stage 1 are absent). PR body evidence: `## Test Result\n\nPending CI.` Head evidence: commit `4992e10b736788ac5003e381f85c46a25379214c` message `Merge remote-tracking branch 'upstream/main' into fix/qwen3-tts-default-seed-perf`. Harness evidence, unchanged by this diff: benchmarks/tts/bench_tts.py:242 ` parser.add_argument("--wer-eval", action="store_true", help="Enable WER/SIM/UTMOS quality eval")`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4970.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4970.trace.json.gz new file mode 100644 index 000000000..243157075 Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4970.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4977.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr4977.cost.json new file mode 100644 index 000000000..3f4845390 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 911.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr4977/run-20260815-162634-833b07", + "recorded_at": "2026-08-15T16:41:44", + "attempts": [ + { + "invocation_id": "inv-9c820cb0", + "outcome": "run:run-20260815-162634-833b07", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-162634-833b07", + "llm_calls": 57, + "input_tokens": 182382, + "output_tokens": 125740, + "usd": 0.230038, + "source": "spans" + } + ], + "attempt_usd_total": 0.230038, + "llm_calls": 57, + "input_tokens": 182382, + "output_tokens": 125740, + "usd": 0.230038, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-162634-833b07", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.23, + "minutes": 15.16, + "ci_minutes": 0, + "input_tokens": 182382, + "output_tokens": 125740, + "tool_calls": 135, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5266, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 57, + "by_role": { + "agent": { + "usd": 0.200151, + "input_tokens": 169228, + "output_tokens": 101839, + "calls": 55 + }, + "reducer": { + "usd": 0.029888, + "input_tokens": 13154, + "output_tokens": 23901, + "calls": 2 + } + }, + "cache_read_tokens": 1573376, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.39, + "gate": 1.92, + "review": 904.09, + "report": 0.07 + }, + "time_to_first_result_s": 909.45 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 28975 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4977.md b/eval/dataset/arms/copilot_v16f_train_r1/pr4977.md new file mode 100644 index 000000000..50a03c58b --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr4977.md @@ -0,0 +1,57 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] Early revisions passed `trust_remote_code=True`, which kernels 0.13.x rejects; fixed at head by 9947f414. Residual: PR description still advertises the dropped kwarg (comment 1). +- [resolved] trust_remote_code supply-chain concern (auto-executing remote Hub code) — the final commit dropped the kwarg entirely: head reads `return get_kernel(repo_id, version=version)` with no trust flag; residual is only the stale PR body text. +- [resolved] trust_remote_code: final head matches main's `get_kernel` signature after 9947f414 — "kernels 0.13.x get_kernel() does not accept trust_remote_code, so passing it made every fallback attempt fail on older installs." Residual: the PR description still claims it was added. +- [resolved] `trust_remote_code` kwarg is absent at head (commit 9947f414 "Drop trust_remote_code from hub kernel loading"; grep of flash_attn_hub.py finds no `trust_remote_code`) — likely the compat fix; residual: the PR description still claims it is added and must be updated. +- [resolved] PR-description claim 'adding trust_remote_code=True': commit 9947f414 'Drop trust_remote_code from hub kernel loading' removed the kwarg; head code calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` with no trust_remote_code — residual is only the stale description text, already covered by a kept first-round comment. +- [claim-verified] 'loaded once per process': `_hub_modules` dict + `_get_hub_module` guard (lines 21-22, 42-46) cache on first success; layer.py:100 `self.attn_impl_cls(...)` confirms one impl per layer. +- [claim-verified] fallback v1→v2→default preserved: loop `for version in (1, 2, None)` == base's nested try/except (file_at_base lines 117-128, 276-287). +- [claim-verified] 'forward behavior and backend selection unchanged': diff only touches the two constructors; forward_cuda/backends untouched; diff_stat shows 1 file, 30 insertions/30 deletions. +- [claim-verified] Motivation is real: layer.py:96 `self.attention = self.attn_impl_cls(...)` constructs one AttentionImpl per Attention module, so at base every layer re-ran get_kernel (Hub metadata traffic per layer). +- [claim-verified] Test plan's cited test exists (tests/diffusion/attention/test_kernels_hub.py::test_kernels_hub_platform_fallback, markers core_model+cpu) but it exercises `CudaOmniPlatform.get_diffusion_attn_backend_cls` selection fallback only — it never calls `_get_hub_module` or constructs either Impl, so it does not cover the changed code. +- [claim-verified] "one backend per layer": vllm_omni/diffusion/attention/layer.py:94-101 instantiates `self.attn_impl_cls(...)` per `Attention`, so per-process caching amortizes N `get_kernel` calls to 1. +- [claim-verified] "preserving the existing version fallback behavior (v1→v2→default)" holds — see fallback equivalence line. +- [claim-verified] PR claim 'preserving the existing version fallback behavior (v1 → v2 → default)': new loop `for version in (1, 2, None)` reproduces base's nested try order exactly (base: version=1, then version=2, then default). +- [claim-verified] PR claim 'loaded once per process instead of on every FlashAttention*HubImpl construction': `_get_hub_module` inserts into module-level `_hub_modules` under lock; both constructors now call it instead of `get_kernel` directly. + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:44` [minor] — The diff adds `_hub_modules` caching and `_get_hub_module` with a v1→v2→default fallback, but no test exercises either: `test_kernels_hub_platform_fallback` only monkeypatches `sys.modules['kernels']=None` and asserts backend path strings, and `test_kernels_hub_execution` constructs `FlashAttentionHubImpl` and `FlashAttention3HubImpl` exactly once each (distinct repo ids), so the cache-hit path is never reached. Add a CPU-runnable unit test that patches `kernels.get_kernel` with a counting stub, constructs two `FlashAttentionHubImpl`s, asserts one load per repo plus v1/v2 failure fallback to default, and resets or patches `_hub_modules` to avoid cross-test leakage. (evidence: flash_attn_hub.py:22 `_hub_modules: dict[str, object] = {}`; flash_attn_hub.py:44-45 `if repo_id not in _hub_modules:` / `_hub_modules[repo_id] = _load_hub_module(repo_id)`; flash_attn_hub.py:25-34 `for version in (1, 2, None): ... return get_kernel(repo_id, version=version) ... return get_kernel(repo_id)` (fallback sequence). tests/diffusion/attention/test_kernels_hub.py:19 `def test_kernels_hub_platform_fallback(monkeypatch: pytest.MonkeyPatch):` — body (lines 20-49) only calls `CudaOmniPlatform.get_diffusion_attn_backend_cls(...)` and asserts `backend_path` strings, never constructs `FlashAttentionHubImpl`; tests/diffusion/attention/test_kernels_hub.py:81 `fa_hub_impl = FlashAttentionHubImpl(...)` and :95 `fa3_hub_impl = FlashAttention3HubImpl(...)` — each constructed exactly once, so no same-repo cache hit occurs.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~34` [minor] — vllm_omni/diffusion/attention/backends/flash_attn_hub.py:34-35 calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` with no `trust_remote_code`, and no `trust_remote_code` exists under vllm_omni/diffusion/attention at head 9947f414. The PR description still says the change is 'adding `trust_remote_code=True` for more reliable Hub access'. Please update the PR description to match the merged code. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:34 `return get_kernel(repo_id, version=version)` and :35 `return get_kernel(repo_id)` — no `trust_remote_code` kwarg; grep of vllm_omni/diffusion/attention finds no `trust_remote_code`. PR body still says: 'adding `trust_remote_code=True` for more reliable Hub access'. Commit 9947f414 message: 'Drop trust_remote_code from hub kernel loading — kernels 0.13.x get_kernel() does not accept trust_remote_code, so passing it made every fallback attempt fail on older installs.') + +`tests/diffusion/attention/test_kernels_hub.py:~58` [minor] — The changed `FlashAttentionHubImpl`/`FlashAttention3HubImpl` constructors now go through memoized `_get_hub_module`, but the only test that constructs them, `test_kernels_hub_execution` (tests/diffusion/attention/test_kernels_hub.py:58), is gated by `@pytest.mark.core_model` + `@hardware_test(res={'cuda': 'L4'}, num_cards=1)`. The ready lane at `.buildkite/test-ready.yml:13` runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`, which deselects it, and no `.buildkite` lane schedules `tests/diffusion/attention`. Add the directory to a CUDA/L4 lane or add a CPU-level unit test for the constructor/caching path. (evidence: Unchanged by this diff, present in the PR-time tree: tests/diffusion/attention/test_kernels_hub.py:56 `@pytest.mark.core_model`; tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)`; tests/diffusion/attention/test_kernels_hub.py:58 `def test_kernels_hub_execution():`; .buildkite/test-ready.yml:13 `- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; tests/helpers/mark.py:15-16 `test_platform_detail = pytest.mark.cuda` / `test_resource = pytest.mark.L4` (no cpu marker); grep `.buildkite/` for `tests/diffusion/attention` returns no matches.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:4` [minor] — The diff only changes vllm_omni/diffusion/attention/backends/flash_attn_hub.py (adding `import threading`, the `_hub_modules` cache, and `_get_hub_module`), yet `buildkite/vllm-omni-npu-ci` is failing. NPU's `get_diffusion_attn_backend_cls` warns 'HuggingFace kernels-backed FlashAttention is not supported on NPU. Falling back to local FLASH_ATTN.' (platform.py:112-115) and does not import this module, so the lane is outside this diff's runtime surface. Confirm the failure also reproduces on current main (pre-existing) before merging on top of it. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/platform.py:110-116 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):` / `logger.warning("HuggingFace kernels-backed FlashAttention is "` / `"not supported on NPU. Falling back to local "` / `"FLASH_ATTN."` / `backend_upper = "FLASH_ATTN"` — the fallback path never imports flash_attn_hub. Anchored diff line: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:4 `import threading`. CI state: gh_ci_read(4977) returns `{"bucket":"fail","name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}`; diff_stat shows 1 file changed (flash_attn_hub.py).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~119` [minor] — The diff replaces both inline load blocks with `_get_hub_module(...)`, but leaves the two `getattr` fetches plus the `no functions found` raise as byte-identical twins outside the shared helper (flash-attn2: lines 119-123; flash-attn3: lines 284-288). Because `_get_hub_module` (lines 42-46) caches on successful `_load_hub_module` return with no function-presence check, a module that loads but exports neither `flash_attn_func` nor `flash_attn_varlen_func` is cached, so each subsequent constructor re-raises with no load context. Fold resolution and validation into one helper, e.g. `_resolve_hub_attn_funcs(repo_id)` returning `(flash_attn_func, flash_attn_varlen_func)` and raising once at load time with the repo id. (evidence: flash_attn_hub.py:42-46 (added by this diff): `def _get_hub_module(repo_id: str):` / ` with _hub_lock:` / ` if repo_id not in _hub_modules:` / ` _hub_modules[repo_id] = _load_hub_module(repo_id)` / ` return _hub_modules[repo_id]` — caches the module on successful load before any function-presence check. flash_attn_hub.py:119-123: ` hub_module = _get_hub_module("kernels-community/flash-attn2")` / ` self.flash_attn_func = getattr(hub_module, "flash_attn_func", None)` / ` self.flash_attn_varlen_func = getattr(hub_module, "flash_attn_varlen_func", None)` / ` if self.flash_attn_func is None and self.flash_attn_varlen_func is None:` / ` raise RuntimeError("Failed to load flash-attn2 kernel from HuggingFace Hub: no functions found")`; flash_attn_hub.py:284-288 is the identical twin for `kernels-community/flash-attn3` with `flash-attn3` in the raise.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~37` [nit] — In `_load_hub_module`, the loop `for version in (1, 2, None):` formats the sentinel directly on failure, so the third attempt logs `version None`. Map the sentinel to a readable label (e.g. `version if version is not None else 'default'`) and consider WARNING-level fallback logs. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:31 ` for version in (1, 2, None):` and vllm_omni/diffusion/attention/backends/flash_attn_hub.py:37 ` logger.info("Failed to load %s version %s: %s", repo_id, version, exc)` — when `version` is `None` the message interpolates the literal `None`.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:45` [nit] — `_get_hub_module` caches only on success: line 45 `_hub_modules[repo_id] = _load_hub_module(repo_id)` is not reached when line 39 `raise RuntimeError(...)` fires, so every later call for the same repo_id re-runs the full v1/v2/default fallback. On a permanently failing Hub this repeats Hub traffic/attempts per attention-impl instantiation, weakening the caching added by this diff. Consider caching the failure (store and re-raise the exception, with a reset path for tests) or documenting/limiting the retry behavior. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:31 ` for version in (1, 2, None):`; :39 ` raise RuntimeError(f"Failed to load HuggingFace Hub kernel {repo_id!r}") from last_error`; :44 ` if repo_id not in _hub_modules:`; :45 ` _hub_modules[repo_id] = _load_hub_module(repo_id)` — assignment executes only after `_load_hub_module` returns, so the raise path stores nothing and the next call repeats the loop.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr4977.trace.json.gz new file mode 100644 index 000000000..e46615abe Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr5009.cost.json b/eval/dataset/arms/copilot_v16f_train_r1/pr5009.cost.json new file mode 100644 index 000000000..bc66a3f2c --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1385.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16f_train_r1/runs/pr5009/run-20260815-155951-548d4a", + "recorded_at": "2026-08-15T16:22:53", + "attempts": [ + { + "invocation_id": "inv-9bd93973", + "outcome": "run:run-20260815-155951-548d4a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-155951-548d4a", + "llm_calls": 131, + "input_tokens": 775467, + "output_tokens": 192250, + "usd": 0.64875, + "source": "spans" + } + ], + "attempt_usd_total": 0.64875, + "llm_calls": 131, + "input_tokens": 775467, + "output_tokens": 192250, + "usd": 0.64875, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-155951-548d4a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6488, + "minutes": 23.03, + "ci_minutes": 0, + "input_tokens": 775467, + "output_tokens": 192250, + "tool_calls": 271, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8488, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 131, + "by_role": { + "agent": { + "usd": 0.609896, + "input_tokens": 757264, + "output_tokens": 161437, + "calls": 129 + }, + "reducer": { + "usd": 0.038854, + "input_tokens": 18203, + "output_tokens": 30813, + "calls": 2 + } + }, + "cache_read_tokens": 8440704, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.76, + "gate": 1.61, + "review": 1375.45, + "report": 0.07 + }, + "time_to_first_result_s": 1381.87 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 41995 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr5009.md b/eval/dataset/arms/copilot_v16f_train_r1/pr5009.md new file mode 100644 index 000000000..b66ff9568 --- /dev/null +++ b/eval/dataset/arms/copilot_v16f_train_r1/pr5009.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Weight-loading concern for switching RMSNorm type: in-tree W4A16 tests exist (tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py:52/73) and PR reports PASS; residual is that I could not execute them. +- [resolved] the concern that swapping away from vLLM RMSNorm could orphan checkpoint keys is resolved by the name-based loader — decisive line: `weight_loader = getattr(param, "weight_loader", default_weight_loader)` (load_weights); residual checked: unexpected keys only hit `logger.debug('Skipping unexpected Qwen-Image transformer weight %s', ...)`, a pre-existing warn-not-raise pattern unchanged by this PR. +- [claim-verified] Part 1 provenance: search_history on `using_inductor` in platforms/cuda/platform.py shows the branch added by a560ed18 (#4709) and removed by b73e0355. +- [claim-verified] Part 2 provenance: search_history shows nn.RMSNorm introduced 35ee3c75 (#4074), reverted to vllm RMSNorm by 99234881 (#3588), restored by b5589481. +- [claim-verified] test_qwen_image_matches_diffusers uses SSIM_THRESHOLD=0.94 / PSNR_THRESHOLD=30.0 (tests/e2e/accuracy/test_qwen_image.py:188-189); PR's 26.20 dB FAIL -> 31.09 dB PASS is consistent with those thresholds. +- [claim-verified] PR-body claim 'Models that need native-only already have per-model overrides (e.g. Cosmos3)': pipeline_cosmos3.py:612 get_cosmos3_ir_op_priority_func rebuilds the config with rms_norm=["native"], fused_add_rms_norm=["native"], and _resolve_ir_op_priority applies it AFTER the platform default — Cosmos3 is insulated from the flip +- [claim-verified] PR table's '+7.2% vs v0.23' Part-1 residual: calc 2.4502/2.2847 = +7.24% — number is self-consistent, but no tracking issue is named for the acknowledged remainder +- [claim-verified] CompilationMode.NONE behavior unchanged: old branch `using_inductor` is False → vllm_c; new code always vllm_c. +- [claim-verified] #3588→#4074 history: search_history shows 99234881 (#3588) added vllm RMSNorm, 35ee3c75 (#4074) added nn.RMSNorm, b5589481 restores nn.RMSNorm. +- [claim-verified] Cosmos3 per-model native override exists: pipeline_cosmos3.py get_cosmos3_ir_op_priority_func sets priority_kwargs['rms_norm']=['native'] and ['fused_add_rms_norm']=['native']. +- [claim-verified] SSIM/PSNR FAIL→PASS consistent with thresholds: test_qwen_image.py sets SSIM_THRESHOLD=0.94, PSNR_THRESHOLD=30.0; main PSNR 26.20<30 fails, post 31.09 passes. +- [claim-refuted] PR body lists FLUX.2/Klein, Wan22S2V, HunyuanVideo15, SD3, OmniGen2, LongCat, Ernie, Ovis as affected but says 'local A/B not run unless noted' — those inherit the new default with no validation evidence. +- [claim-refuted] FLUX.1-dev Δ claimed −14.1%; calc((0.6166−0.5402)/0.6166) = 12.39%, so the PR-body percentage is arithmetically off. +- [claim-refuted] cosmos3 override only pins rms_norm/fused_add_rms_norm (pipeline_cosmos3.py:620-621) and copies all other fields from the platform default (:619) — so its non-RMSNorm ops also flip to vllm_c-first. + +`vllm_omni/platforms/cuda/platform.py:~253` [major] — Confirmed at head: platform.py:253 makes `default = ["vllm_c", "native"]` unconditional, and platform.py:262 feeds it to `IrOpPriorityConfig.with_default(...)`, so VLLM_COMPILE/STOCK_TORCH_COMPILE flip every IR-op field (not just rms_norm/fused_add_rms_norm) from the old inductor-aware `["native"]` to vllm_c-first for every CUDA diffusion model. diffusion_worker.py:175 consumes this default with no per-model override, and `_DIFFUSION_IR_OP_PRIORITY_FUNCS` (registry.py:545) only overrides Cosmos3, so the inheriting set is wide. The new test asserts only `rms_norm` and `fused_add_rms_norm`; the five Qwen-Image norm sites now use `nn.RMSNorm` (transformer.py:557, 558, 595, 596, 1009), so the headline Qwen-Image result does not exercise the platform path for those sites. Keep the default inductor-aware and override only the two RMS fields, or assert all priority fields and add per-model parity evidence for the inheriting set. (evidence: vllm_omni/platforms/cuda/platform.py:253 ` default = ["vllm_c", "native"]` (changed by this diff); vllm_omni/platforms/cuda/platform.py:262 ` return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)` (changed by this diff); vllm_omni/diffusion/worker/diffusion_worker.py:175 ` ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)` (unchanged by this diff, present in PR-time tree); vllm_omni/diffusion/registry.py:545 `_DIFFUSION_IR_OP_PRIORITY_FUNCS = {` with only entry at 548 ` "Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",` (unchanged by this diff, present in PR-time tree); tests/platforms/test_cuda_ir_op_priority.py:22-23 ` assert priority.rms_norm == ["vllm_c", "native"]` and ` assert priority.fused_add_rms_norm == ["vllm_c", "native"]` (added by this diff); vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 ` self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (changed by this diff).) + +`tests/platforms/test_cuda_ir_op_priority.py:7` [major] — tests/platforms/test_cuda_ir_op_priority.py imports `CudaOmniPlatform` at module scope (line 7). The XPU hardware lane runs `pytest -v -s -m "core_model and xpu and B60"` from repo root with no path filter (.buildkite/scripts/hardware_ci/run-xpu-test.sh:54), so this CUDA platform module is imported during collection on XPU despite the `cpu` marker; `vllm_omni/platforms/cuda/platform.py` top-level imports `torch` and exposes CUDA-only methods. In-tree precedent keeps this import inside test functions (tests/diffusion/attention/test_sage_attn3.py:93 and :120). Move the import inside the test functions or use `pytest.importorskip`. AMD is path-scoped away from tests/platforms (.buildkite/test-amd-merge.yml:44) and does not collect this file. (evidence: tests/platforms/test_cuda_ir_op_priority.py:7 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` — module-scope import in the newly added test, executed at collection time regardless of the file's `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. Unchanged by this diff, present in the PR-time tree: .buildkite/scripts/hardware_ci/run-xpu-test.sh:54 `pytest -v -s -m "core_model and xpu and B60"` — pathless repo-root pytest run, so it collects the file on XPU. Unchanged by this diff: tests/diffusion/attention/test_sage_attn3.py:93 ` from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` — precedent keeps the same import inside a test function. Unchanged by this diff: vllm_omni/platforms/cuda/platform.py:7 `import torch` and :252 `major, minor = torch.cuda.get_device_capability(device_id)` — the imported CUDA module binds torch at import and exposes CUDA-only paths.) + +`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:~620` [major] — The platform default flip at `vllm_omni/platforms/cuda/platform.py:253` also changes Cosmos3 under inductor: `get_cosmos3_ir_op_priority_func` copies every `IrOpPriorityConfig` field from the platform default and pins only `rms_norm`/`fused_add_rms_norm` to `["native"]` (`pipeline_cosmos3.py:619-621`). Pre-PR, inductor-active configs applied `default = ["native"]` to every field via `with_default`, so those copied non-RMSNorm fields were `["native"]`; at head they become `["vllm_c", "native"]`. Either force all fields to `["native"]` in the Cosmos3 override or document/validate the non-RMSNorm `vllm_c` flip for Cosmos3. (evidence: ['vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]` — changed by this PR (the removed line was `default = ["native"] if using_inductor else ["vllm_c", "native"]`).', 'vllm_omni/platforms/cuda/platform.py:262 `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)` — `with_default` applies `default` to every IR-op field, so all non-overridden fields are vllm_c-first at head.', 'vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:619 `priority_kwargs = {field.name: list(getattr(ir_op_priority, field.name)) for field in fields(ir_op_priority)}` — unchanged by this diff, present in the PR-time tree; copies every field from the platform default.', 'vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:620 `priority_kwargs["rms_norm"] = ["native"]` and :621 `priority_kwargs["fused_add_rms_norm"] = ["native"]` — unchanged by this diff; only these two fields are pinned to native, so the rest inherit the new vllm_c-first default.']) + +`vllm_omni/platforms/cuda/platform.py:~252` [minor] — The Part 1 hunk restores `vllm_c` priority, but the PR body's own E2E table shows patched latency `2.4502s` vs pre-#4709 `2.2847s` (+7.2%), leaving under 3 points of headroom against the nightly ~10% baseline gate and no named follow-up for the residual v0.24 overhead outside RMSNorm. File a tracking issue for that residual and link it from the PR body so closing #4964 does not silently re-baseline the remainder. (evidence: PR body Part-1 E2E table: `c45ac74d (v0.23) 2.2847s` vs `Part 1 only (patched) 2.4502s (+7.2% vs v0.23)`; calc: 2.4502/2.2847 = 1.0724. Issue #4964 gates on ~10% deviation vs baseline (e.g. `e2e_latency_ms 3972.6384 | baseline 2395.3 | fail`).) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:~557` [minor] — Confirmed: Part 2 converts all five Qwen-Image norm sites to nn.RMSNorm and drops the vllm RMSNorm import, so the Part 1 Qwen-Image profiler/E2E variants labeled 'Part 1 only (patched)' were collected on a tree where Qwen-Image still used vLLM RMSNorm and their kernel-dispatch numbers no longer describe the merged head's Qwen-Image path. The PR body's combined P1+P2 E2E (−26.9%) and FLUX.1-dev A/B (Part 1 on a vLLM RMSNorm model) already separate the two parts; the remaining gap is specifically a Qwen-Image profiler artifact at head. (evidence: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; :558 `self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; :595 `self.norm_added_q = nn.RMSNorm(head_dim, eps=eps)`; :596 `self.norm_added_k = nn.RMSNorm(head_dim, eps=eps)`; :1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)`. The diff removes `from vllm.model_executor.layers.layernorm import RMSNorm` (unchanged pre-PR import line) and grep of vllm_omni/diffusion/models/qwen_image finds only the five nn.RMSNorm sites, no vllm layernorm import. Part 1 is unchanged-by-Part-2 platform priority: vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]` — applies only to vLLM RMSNorm IR, not torch nn.RMSNorm. PR body evidence labels the profiler variant 'Part 1 only (patched)', collected before Part 2.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:~557` [minor] — Confirmed: this PR switches the five QK/txt norm layers (`norm_q`, `norm_k`, `norm_added_q`, `norm_added_k`, `txt_norm`) from `vllm.model_executor.layers.layernorm.RMSNorm` to `torch.nn.RMSNorm` (qwen_image_transformer.py:557,558,595,596,1009) with no in-repo CPU guard — `tests/diffusion/models/qwen_image/` contains no RMSNorm-type assertion. The only Qwen-Image tests are `full_model`-marked L4/H100 accuracy tests (`tests/e2e/accuracy/test_qwen_image.py:20`), which do not run per-PR. A cheap CPU regression asserting the layer type is worthwhile, e.g. in `tests/diffusion/models/qwen_image/test_qwen_image_max_sequence_length.py`. Severity: minor. (evidence: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; :558 `self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; :595 `self.norm_added_q = nn.RMSNorm(head_dim, eps=eps)`; :596 `self.norm_added_k = nn.RMSNorm(head_dim, eps=eps)`; :1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)` — all five layers use torch.nn.RMSNorm, and the diff removes `from vllm.model_executor.layers.layernorm import RMSNorm`. Unchanged by this diff but present in the PR-time tree: tests/e2e/accuracy/test_qwen_image.py:20 `pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]`, with both test functions decorated `@pytest.mark.benchmark` and `@hardware_test(res={"cuda": "H100"}, num_cards=1)`. A PR-time grep for `RMSNorm` under tests/diffusion/models/qwen_image/ returns no matches, so no per-PR CPU guard exists.) + +`vllm_omni/platforms/rocm/platform.py:185` [minor] — The CUDA hunk removes `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` / `default = ["native"] if using_inductor else ["vllm_c", "native"]`, but that exact branch survives unchanged in the ROCm twin (`rocm/platform.py:184-185`) and the XPU twin (`xpu/platform.py:116-117`, with `xpu_kernels`). Since the PR's own regression test says inductor-active configs must not switch to native-only (#4964), AMD/Intel inductor configs keep that native-only path. If CUDA-only scoping is intentional, state it in the PR body; otherwise apply the same fix to the siblings or file a tracking issue. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/rocm/platform.py:184 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; vllm_omni/platforms/rocm/platform.py:185 `default = ["native"] if using_inductor else ["vllm_c", "native"]`. Same pattern at vllm_omni/platforms/xpu/platform.py:116 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; vllm_omni/platforms/xpu/platform.py:117 `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`. The CUDA diff deletes the identical branch, and tests/platforms/test_cuda_ir_op_priority.py declares 'Regression for #4964: inductor-active configs must not switch to native-only.') + +`PR body:0` [minor] — Correct the FLUX.1-dev Δ in the PR body: (0.6166−0.5402)/0.6166 = 12.39% improvement relative to baseline A, not −14.1%. The −14.1% value arises from (0.6166−0.5402)/0.5402 = 14.14%, which uses the patched value B as the denominator; either use the baseline-relative 12.39% or state the denominator explicitly. (evidence: PR #5009 body: '| A — post-#4709 / current main | `["native"]` only | 0.6166s | — |' and '| B — Part 1 (this PR) | `["vllm_c", "native"]` | 0.5402s | **−14.1%** |'. Arithmetic: (0.6166−0.5402)/0.6166 = 0.1239 (12.39%); (0.6166−0.5402)/0.5402 = 0.1414 (14.14%).) + +**Additional observations (beyond the comment budget):** +- `tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py:52` [minor] The W4A16 weight-loading guarantee for the nn.RMSNorm swap is author-reported: the PR body lists test_qwen_image_autoround_w4a16_load and test_qwen_image_autoround_w4a16_generate as PASSED. Both tests exist at tests/e2e/ +- `tests/platforms/test_cuda_ir_op_priority.py:26` [minor] The new test (tests/platforms/test_cuda_ir_op_priority.py:25-27) asserts only the constructed `IrOpPriorityConfig` values, so it proves the `using_inductor` branch was removed but not #4964's actual mechanism — that the + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16f_train_r1/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v16f_train_r1/pr5009.trace.json.gz new file mode 100644 index 000000000..70ad02f3e Binary files /dev/null and b/eval/dataset/arms/copilot_v16f_train_r1/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_smoke/manifest.json b/eval/dataset/arms/copilot_v16rs_smoke/manifest.json new file mode 100644 index 000000000..16d23dc15 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_smoke/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v16rs_smoke", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_smoke/pr5009.cost.json b/eval/dataset/arms/copilot_v16rs_smoke/pr5009.cost.json new file mode 100644 index 000000000..479c1a801 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_smoke/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1237.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_smoke/runs/pr5009/run-20260815-135931-e95460", + "recorded_at": "2026-08-15T14:20:07", + "attempts": [ + { + "invocation_id": "inv-0e624a64", + "outcome": "run:run-20260815-135931-e95460", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-135931-e95460", + "llm_calls": 131, + "input_tokens": 1029327, + "output_tokens": 191949, + "usd": 0.785107, + "source": "spans" + } + ], + "attempt_usd_total": 0.785107, + "llm_calls": 131, + "input_tokens": 1029327, + "output_tokens": 191949, + "usd": 0.785107, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-135931-e95460", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7851, + "minutes": 20.6, + "ci_minutes": 0, + "input_tokens": 1029327, + "output_tokens": 191949, + "tool_calls": 379, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8596, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 131, + "by_role": { + "agent": { + "usd": 0.746778, + "input_tokens": 1013851, + "output_tokens": 160944, + "calls": 129 + }, + "reducer": { + "usd": 0.038329, + "input_tokens": 15476, + "output_tokens": 31005, + "calls": 2 + } + }, + "cache_read_tokens": 10964608, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.88, + "gate": 1.99, + "review": 1228.86, + "report": 0.06 + }, + "time_to_first_result_s": 1235.78 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 37177 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_smoke/pr5009.md b/eval/dataset/arms/copilot_v16rs_smoke/pr5009.md new file mode 100644 index 000000000..740dd055e --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_smoke/pr5009.md @@ -0,0 +1,57 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Stale inductor-aware docstring on main (platform.py base: "When inductor is active use native") — head shortens to "Prefer vllm_c CUDA kernels"; residual: vllm_config param now unused +- [resolved] vllm_c priority is inherited by all non-Cosmos3 CUDA diffusion models; PR validates only Qwen-Image. Residual: every other diffusion model on CUDA under inductor now also switches to vllm_c with no evidence in the diff. +- [resolved] torch.nn.RMSNorm vs vLLM RMSNorm: numeric difference is the claimed fix (diffusers parity), but there is no added test binding the exact swap; residual: an L4 e2e accuracy test exists (tests/e2e/accuracy/test_qwen_image.py) but the PR does not show it run in the diff. +- [resolved] failing gates buildkite/vllm-omni-amd-ci, intel-ci, npu-ci: diff touches only tests/platforms (which imports vllm_omni.platforms.cuda.platform) + cuda/platform.py + qwen_image; no AMD/Intel/NPU config or test source changed. Residual: cannot attribute red gates to this PR without live CI logs; flag as minor attribution question. +- [resolved] new test is selectable in principle via -m 'core_model and cpu' if a lane collects tests/ root; current ready/merge lanes do not collect tests/platforms, so the regression is not merged-gate-gated today. +- [resolved] unused `vllm_config` param in the CUDA method is required to match the base-class override signature, not dead code. +- [resolved] docstring updated to match new behavior ('Prefer vllm_c CUDA kernels...' vllm_omni/platforms/cuda/platform.py:251); grep confirms no leftover CompilationMode/inductor text in the cuda platform. +- [resolved] kept major on cuda/platform.py:252 global default flip — Cosmos3 override exists at registry.py:549; residual unchecked models (Flux2, SD3, etc.) still inherit new default +- [claim-verified] PR latency −26.9% claim: (1.7263−1.2613)/1.7263×100 ≈ 26.94% +- [claim-verified] PR Cosmos3 per-model override claim: only Cosmos3OmniDiffusersPipeline in _DIFFUSION_IR_OP_PRIORITY_FUNCS (registry.py:545-549) +- [claim-verified] PR Part-1-only profiler −16%: (1288−1080)/1288 ≈ 16.1% +- [claim-verified] PR body numbers recomputed with calc: -26.9% = 1-1.2613/1.7263 = 0.26936 ✓; profiler -16% = 1-1080/1288 = 0.16149 ✓; E2E -1.8% = 1-2.4502/2.4960 = 0.01835 ✓. +- [claim-verified] (1.7263-1.2613)/1.7263*100 = 26.94 ≈ PR's −26.9% latency claim. +- [claim-verified] PR perf percentages recompute consistently: -26.9%, -16.1%, -1.8%, +7.2% (calc on 1.7263/1.2613, 1288/1080, 2.4960/2.4502, 2.2847/2.4502). + +`vllm_omni/platforms/cuda/platform.py:~253` [minor] — Confirmed: platform.py:253 now returns ["vllm_c", "native"] unconditionally and diffusion_worker.py:175 applies it to every pipeline without a per-model override; _DIFFUSION_IR_OP_PRIORITY_FUNCS only overrides Cosmos3, so all other vLLM-RMSNorm diffusion transformers (e.g. flux2_transformer.py:18) change dispatch under inductor. Qwen-Image itself no longer exercises this path (all five sites are torch.nn.RMSNorm). Please either register a Qwen-Image-specific ir_op_priority_func and keep the inductor-aware default for other models, or extend the existing FLUX.1-dev A/B in the PR body with same-seed accuracy evidence for the remaining RMSNorm consumers. (evidence: vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]` (changed by this diff; the preceding inductor-aware branch is deleted). vllm_omni/diffusion/worker/diffusion_worker.py:175 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)` (unchanged by this diff; resolves the default for all pipelines before any per-model override). vllm_omni/diffusion/registry.py:545-549 `_DIFFUSION_IR_OP_PRIORITY_FUNCS = { ... "Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func", }` (unchanged by this diff; the only entry is Cosmos3). vllm_omni/diffusion/models/flux2/flux2_transformer.py:18 `from vllm.model_executor.layers.layernorm import RMSNorm` (unchanged by this diff; an example of a remaining vLLM RMSNorm consumer). vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (changed by this diff; Qwen-Image no longer imports vLLM RMSNorm).) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:~557` [minor] — This diff swaps the five Qwen-Image QK/added/text norm sites from vLLM `RMSNorm` to `torch.nn.RMSNorm` (qwen_image_transformer.py:557, 558, 595, 596, 1009), which can change numerics. The only new test in this PR (`tests/platforms/test_cuda_ir_op_priority.py`) never instantiates this model, and the existing accuracy backstop `test_qwen_image_matches_diffusers` is `full_model`/`diffusion`-marked and H100-gated (pyproject.toml:239 defines full_model as L4 nightly), so it is not a merge gate. Add a lightweight CPU regression that instantiates the Qwen-Image attention/transformer blocks, asserts the norm layers are `torch.nn.RMSNorm`, and checks a small numeric RMS output, or explicitly document that the full_model e2e test is the sole coverage and is not in the merge gate. (evidence: In this diff: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 ` self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; :558 ` self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; :595 ` self.norm_added_q = nn.RMSNorm(head_dim, eps=eps)`; :596 ` self.norm_added_k = nn.RMSNorm(head_dim, eps=eps)`; :1009 ` self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)`. New test only exercises platform priority: tests/platforms/test_cuda_ir_op_priority.py:25 ` priority = CudaOmniPlatform.get_default_ir_op_priority(_vllm_config(mode=mode))`. Unchanged by this diff, present in PR-time tree: tests/e2e/accuracy/test_qwen_image.py:20 `pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]`; :197 `def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None:`; pyproject.toml:239 ` "full_model: L4 level tests (run nightly)",`.) + +`vllm_omni/platforms/rocm/platform.py:~185` [minor] — This diff removes the `using_inductor` branch only in `CudaOmniPlatform.get_default_ir_op_priority` (`cuda/platform.py:253` now always returns `["vllm_c", "native"]`). The ROCm and XPU twins keep the same inductor-aware native fallback (`rocm/platform.py:185`: `default = ["native"] if using_inductor else ["vllm_c", "native"]`; `xpu/platform.py:117`: `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`). If inductor-active RMSNorm-heavy diffusion workloads on ROCm/XPU depend on that fallback, the fix is inconsistent; please mirror it or document why CUDA-only is intended. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/rocm/platform.py:185 `default = ["native"] if using_inductor else ["vllm_c", "native"]`. Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/xpu/platform.py:117 `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`. Changed by this diff: vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]`.) + +`tests/platforms/test_cuda_ir_op_priority.py:9` [minor] — Wire this new regression test into the core_model-and-cpu merge-gate command or move it under tests/diffusion. The ready/merge lanes run `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` and `pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor`; neither collects tests/platforms/test_cuda_ir_op_priority.py (pytest testpaths=tests), so it is deselected at merge time even though it would fail against base. (evidence: tests/platforms/test_cuda_ir_op_priority.py:9 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — new test has both markers but lives under tests/platforms. Unchanged by this diff, present in the PR-time tree: .buildkite/test-ready.yml:13 `"timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` and .buildkite/test-ready.yml:29 `"timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (identical pair at .buildkite/test-merge.yml:13 and :29); pyproject.toml:227 `testpaths = ["tests"]`. Since neither lane passes tests/platforms as a path and the second lane only ignores diffusion/model_executor, no merge-gate lane collects tests/platforms/test_cuda_ir_op_priority.py.) + +`vllm_omni/platforms/cuda/platform.py:~251` [nit] — This diff removes the only read of `vllm_config.compilation_config` in `CudaOmniPlatform.get_default_ir_op_priority()` (cuda/platform.py:251-262), leaving `vllm_config` unused while rocm/platform.py:183-184 and xpu/platform.py:115-116 still branch on it. Rename to `_vllm_config` or add a one-line note that the parameter is kept for the platform interface contract. (evidence: vllm_omni/platforms/cuda/platform.py:251-262 (changed by this diff) — `def get_default_ir_op_priority(cls, vllm_config: VllmConfig) -> IrOpPriorityConfig:` / `"""Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops."""` / `default = ["vllm_c", "native"]` / `rms_norm = default` / `if envs.VLLM_USE_OINK_OPS:` / `rms_norm = ["oink"] + default` / `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)` — no read of `vllm_config` anywhere in the body. Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/rocm/platform.py:183-184 `cc = vllm_config.compilation_config` / `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; and vllm_omni/platforms/xpu/platform.py:115-116 `cc = vllm_config.compilation_config` / `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` — the siblings still branch on the parameter.) + +`vllm_omni/platforms/cuda/platform.py:~252` [nit] — The docstring at platform.py:252 says only 'for diffusion IR ops' without recording the inherited scope or the Cosmos3 exception. This CUDA default is consumed by DiffusionWorker._resolve_ir_op_priority for every diffusion pipeline, and Cosmos3OmniDiffusersPipeline is the only model with a registered ir_op_priority_func (registry.py:549); its override (pipeline_cosmos3.py get_cosmos3_ir_op_priority_func) forces rms_norm and fused_add_rms_norm back to ["native"]. Add one sentence: the default applies to all diffusion models except those with a registered per-model ir_op_priority_func (currently Cosmos3). (evidence: Changed by this diff — vllm_omni/platforms/cuda/platform.py:251-262: `def get_default_ir_op_priority(cls, vllm_config: VllmConfig) -> IrOpPriorityConfig:` / `"""Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops."""` / `default = ["vllm_c", "native"]` / `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)`. Unchanged by this diff, present in the PR-time tree — vllm_omni/diffusion/worker/diffusion_worker.py:175-178: `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)` / `ir_op_priority_func = get_diffusion_ir_op_priority_func(od_config)` / `if ir_op_priority_func is not None: ir_op_priority = ir_op_priority_func(ir_op_priority, vllm_config=vllm_config)`; vllm_omni/diffusion/registry.py:549: `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",` (the only entry in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`); vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:615-622: `def ir_op_priority_func(ir_op_priority, vllm_config=None):` ... `priority_kwargs["rms_norm"] = ["native"]` / `priority_kwargs["fused_add_rms_norm"] = ["native"]`.) + +`tests/platforms/test_cuda_ir_op_priority.py:7` [nit] — tests/platforms/test_cuda_ir_op_priority.py:7 imports CudaOmniPlatform at module scope. pytest imports this module during collection even when `-m` deselects it, and the rootless XPU lane `.buildkite/scripts/hardware_ci/run-xpu-test.sh:54` (`pytest -v -s -m "core_model and xpu and B60"`) will therefore execute this import on XPU, while `.buildkite/test-ready.yml:13` restricts paths to tests/diffusion and tests/model_executor and does not collect tests/platforms. Please confirm the import is safe on XPU (or move it inside the test functions / skip the module on non-CUDA) and confirm whether the amd/intel/npu check failures are pre-existing on main. (evidence: tests/platforms/test_cuda_ir_op_priority.py:7 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` — module-scope import added by this diff (comment cited line 6; actual line is 7). Unchanged by this diff, present in the PR-time tree: `.buildkite/scripts/hardware_ci/run-xpu-test.sh:54` `pytest -v -s -m "core_model and xpu and B60"` — rootless pytest, so it collects tests/platforms during collection. Unchanged by this diff: `.buildkite/test-ready.yml:13` `"timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' ..."` — explicit paths that exclude tests/platforms.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_smoke/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v16rs_smoke/pr5009.trace.json.gz new file mode 100644 index 000000000..1eb8b5acf Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_smoke/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/manifest.json b/eval/dataset/arms/copilot_v16rs_train_r1/manifest.json new file mode 100644 index 000000000..e0fa400c5 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v16rs_train_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.cost.json new file mode 100644 index 000000000..c835adcab --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1625.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4804/run-20260815-142043-3d16cb", + "recorded_at": "2026-08-15T14:47:44", + "attempts": [ + { + "invocation_id": "inv-9b3e0c5f", + "outcome": "run:run-20260815-142043-3d16cb", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-142043-3d16cb", + "llm_calls": 156, + "input_tokens": 1706867, + "output_tokens": 321501, + "usd": 1.405222, + "source": "spans" + } + ], + "attempt_usd_total": 1.405222, + "llm_calls": 156, + "input_tokens": 1706867, + "output_tokens": 321501, + "usd": 1.405222, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-142043-3d16cb", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.4052, + "minutes": 27.03, + "ci_minutes": 0, + "input_tokens": 1706867, + "output_tokens": 321501, + "tool_calls": 363, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1663, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 156, + "by_role": { + "agent": { + "usd": 1.341529, + "input_tokens": 1639471, + "output_tokens": 280207, + "calls": 153 + }, + "reducer": { + "usd": 0.063693, + "input_tokens": 67396, + "output_tokens": 41294, + "calls": 3 + } + }, + "cache_read_tokens": 21878400, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.31, + "gate": 1.67, + "review": 1615.4, + "report": 0.07 + }, + "time_to_first_result_s": 1621.44 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 75164 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.md new file mode 100644 index 000000000..f5e0ef7bc --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.md @@ -0,0 +1,77 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] gpu_ar_model_runner already calls on_requests_finished — MOSS codec stage-1 uses generation runner, not AR runner; abort path depends on generation runner hook +- [resolved] Round-1 codec v2 try-first gate — still present at modeling_moss_tts_codec.py:729; legacy v1 fallback remains dead on permissive config load. +- [resolved] Round-1 NPU runner gap — gpu_generation_model_runner.py:79/799 changes not mirrored in platforms/npu/worker/npu_generation_model_runner.py. +- [claim-verified] PR stereo/WER path — patch normalizes streamed PCM to 24 kHz mono; seed_tts_eval can save WAVs via SEED_TTS_WER_SAVE_AUDIO_DIR +- [claim-verified] 'stereo support' claim: v2 codec config defaults number_channels=2 and the codec infers 2 channels for the v2 path; serving WAV header honors channel count. +- [claim-verified] PR stereo benchmark env (`VLLM_OMNI_BENCH_AUDIO_CHANNELS=2`) — producer patch.py:140-161 downmixes/resamples before `tts_output_pcm_bytes`; seed_tts_eval consumer assumes that normalized shape. +- [claim-refuted] PR body 'Fix some bugs: TODO' — bug-fix section unfinished +- [claim-refuted] PR timeline claim 'Fix UT' only updates the seed-key test; the streaming session, raw chunk processor, and codec weight-loading raise path are not covered by the touched test file. +- [validated] qwen3_tts_request_seed fully removed repo-wide; rename is internally consistent (grep: 0 old-key hits) +- [validated] modeling_moss_tts_codec.py:260-270 — strict seq_token_counts required; matches generation-runner contract +- [validated] moss_tts_local.yaml:17-20,41-42 — local deploy sets initial_codec_chunk_frames=1, codec_stream_slots=8, stage-1 max_num_seqs=8 +- [validated] gpu_generation_model_runner.py:81-82 — on_requests_finished hook wired for abort cleanup +- [validated] talker2codec_delay_async_chunk retains de-delay only; realtime/local raw path split is structurally correct +- [validated] MossTTSRealtime make_omni_output emits per-step raw frames (modeling_moss_tts_talker.py:1142-1155) compatible with talker2codec_raw_async_chunk + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:729` [blocker] — `_build_codec` (modeling_moss_tts_codec.py:727-732) unconditionally constructs `MossAudioTokenizerV2Config`/`MossAudioTokenizerV2Model` first for every codec path. `PreTrainedConfig.from_pretrained` accepts the v1 config JSON, so for `codec_model_name_or_path=OpenMOSS-Team/MOSS-Audio-Tokenizer` (Delay/Realtime/SoundEffect/VoiceGenerator/TTSD) a v2 model is built from the v1 config and the `except` fallback is never triggered by weight-name mismatches; loading happens later in `load_weights`, which raises RuntimeError on the unmapped v1 `.attn.in_proj./.ff1./.ff2.` names. Gate v1 vs v2 on the codec path (e.g. `-v2` suffix or the `codec_model_name_or_path`/`audio_tokenizer_name_or_path` split) and keep the legacy branch for v1. (evidence: modeling_moss_tts_codec.py:727 `def _build_codec(self, codec_path: str) -> tuple[Any, nn.Module]:`; :729 `codec_cfg = MossAudioTokenizerV2Config.from_pretrained(codec_path)`; :730 `codec = MossAudioTokenizerV2Model(codec_cfg)`; :732 `return codec_cfg, codec`; :733 `except Exception:` — v2-first construction runs for every codec path and the fallback wraps only these lines. In `load_weights` (above `_build_codec` at 727) the v1 names fail to map and the code raises: `if missing or skipped or shape_mismatches:` `raise RuntimeError("MOSS Audio Tokenizer weights were not fully loaded: ...")`. The v1 remap entry `(".self_attn.in_projs.0.", ".attn.in_proj.")` only rewrites names containing the v2-side src, so v1 checkpoint names `.attn.in_proj.` are returned unchanged and counted as skipped. Unchanged by this diff, present in the PR-time tree: configuration_moss_tts.py:50 and :172 `codec_model_name_or_path: str = "OpenMOSS-Team/MOSS-Audio-Tokenizer"` (v1 Delay/Realtime defaults) versus the Local config's `audio_tokenizer_name_or_path: str = "OpenMOSS-Team/MOSS-Audio-Tokenizer-v2"`.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:477` [major] — In `_decode_streaming_batch`, when `session.acquire()` returns None and the request finishes, the buffered codes are discarded and no `output_index` entry is returned; `forward` therefore leaves `audios[i]` as the `empty` placeholder and emits it in `model_outputs` with no error. `deploy/moss_tts_local.yaml` sets `codec_stream_slots: 8` and stage1 `max_num_seqs: 8` (aligned), but a leaked slot can silently exhaust slots and produce success-with-empty-audio. Prefer fail-closed (raise/queue/request-error) or enforce slots >= schedulable requests with a test. (evidence: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:243 `audios: list[torch.Tensor] = [empty] * num_req`; same file `_decode_streaming_batch`: `if slot is None: + slot = session.acquire() + if slot is None: + self._append_stream_pending(request_id, codes_nq_t) +... + if finished: + logger.warning( + "MOSS codec stream request %s finished before a codec stream slot became available; " + "dropping buffered codes instead of offline-decoding a non-delta waveform.", + request_id, + ) + self._finish_stream_request(request_id, session, None) + continue` — no `outputs[output_index]` is set on this path; then `if streaming_work: + for i, wav in self._decode_streaming_batch(streaming_work).items(): + audios[i] = wav.reshape(-1) ...` only overwrites indices present in the result, so `audios[i]` remains the zero-length `empty` tensor.) + +`vllm_omni/worker/gpu_generation_model_runner.py:799` [major] — NPU twin still missing both changes: `npu_generation_model_runner.py` `_dummy_run` never sets `model_kwargs["seq_token_counts"]` (grep finds only the execute_model path at line 299), and `_update_request_states` (line 48) never calls `self.model.on_requests_finished`. With `MossTTSCodecDecoder.forward` now raising RuntimeError on missing `seq_token_counts` for non-empty input (modeling_moss_tts_codec.py:263), the NPU profile/warmup dummy run of the moss-tts codec hard-fails, and aborted streaming requests leak codec slots on NPU. Mirror both changes in the NPU generation runner. (evidence: Added by this diff: vllm_omni/worker/gpu_generation_model_runner.py:799 `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]`. Unchanged by this diff, present in PR-time tree: vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:299 `model_kwargs["seq_token_counts"] = tokens` — grep for `seq_token_counts` in that file returns ONLY this line (real execute_model path), so `_dummy_run` does not set it. Added by this diff: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:263-265 `raise RuntimeError(` / `"MossTTS codec requires seq_token_counts; otherwise concatenated "` / `"codec tokens cannot be split per request."` after `token_counts = self._normalize_seq_token_counts(kwargs.get("seq_token_counts"))` and `if token_counts is None:`. Unchanged by this diff: vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:48-50 `def _update_request_states(self, scheduler_output: SchedulerOutput):` / ` # remove requests` / ` for req_id in scheduler_output.finished_req_ids:` — no `on_requests_finished` call; grep `on_requests_finished` under vllm_omni/platforms/npu matches only npu_ar_model_runner.py:419-420, not this generation runner.) + +`vllm_omni/worker/gpu_generation_model_runner.py:81` [minor] — The new on_requests_finished slot-cleanup hook only runs inside _update_request_states, which execute_model calls only when async_chunk and num_scheduled_tokens are both truthy (gpu_generation_model_runner.py:157-158). A finish step with zero scheduled tokens hits the early EMPTY_MODEL_RUNNER_OUTPUT return (lines 160-161) and never invokes the model hook, so codec stream slots held by a request that aborts/disconnects on an idle step can leak. Invoke on_requests_finished (or an equivalent release) on the zero-token finish path so MossTTSCodecDecoder.on_requests_finished is called for every finished_req_ids, regardless of scheduled tokens. (evidence: vllm_omni/worker/gpu_generation_model_runner.py:157 `if self.model_config.async_chunk and num_scheduled_tokens:` — gates the only call to `_update_request_states` at :158 `self._update_request_states(scheduler_output)`. The model hook added in this PR lives inside that method at :81-82 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)`, so it never runs when num_scheduled_tokens == 0; the zero-token step instead returns early at :160-161 `if not scheduler_output.total_num_scheduled_tokens:` / `return self.attach_omni_connector_output(EMPTY_MODEL_RUNNER_OUTPUT)`. Contrast, present in the PR-time tree and unchanged by this diff: vllm_omni/worker/gpu_ar_model_runner.py:942-943 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)` runs before that runner's own zero-token early return.) + +`requirements/common.txt:12` [minor] — requirements/common.txt:12 pins `transformers >= 5.5.3`. This PR's new file vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:27 imports `PreTrainedAudioTokenizerBase` from transformers.modeling_utils and line 1293 subclasses it. Please verify the 5.5.3 floor actually exports that symbol, or raise the transformers floor before merge. (evidence: requirements/common.txt:12 `transformers >= 5.5.3`; vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:27 `from transformers.modeling_utils import PreTrainedAudioTokenizerBase` (new file added by this diff); vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1293 `class MossAudioTokenizerPreTrainedModel(PreTrainedAudioTokenizerBase):`.) + +`vllm_omni/benchmarks/patch/patch.py:1249` [minor] — On normalization failure this falls back to raw streamed PCM bytes at the server's native rate/channels. `compute_seed_tts_wer_metrics` assumes 24 kHz mono int16, so WER metrics become silently wrong instead of marking the sample failed. Prefer leaving `tts_output_pcm_bytes` unset / marking output failed, matching the empty-capture warning path below. (evidence: patch.py:1244-1251 except branch; seed_tts_eval.py docstring states 24 kHz mono PCM for WER; _pcm_s16le_to_seed_tts_wer_bytes resamples/downmixes on success path only.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:225` [minor] — New `talker2codec_raw_async_chunk` (moss_tts.py:225) implements the raw MOSS streaming contract — initial vs steady chunk thresholds, the code_flat_numel=0 finish sentinel, and pad-row filtering — but has no unit test under tests/model_executor/stage_input_processors/, while sibling qwen3_tts and fish_speech async chunk processors do. Add table-driven tests covering threshold gating, the finish sentinel meta, and pad-row filtering to guard regressions when codec_chunk_frames is tuned. (evidence: vllm_omni/model_executor/stage_input_processors/moss_tts.py:225 `def talker2codec_raw_async_chunk(`; moss_tts.py:304 `code_flat_numel=0,`; moss_tts.py:270 `valid_rows = frames_cpu.ne(_MOSS_AUDIO_PAD_CODE).any(dim=1)` — all added in this diff. Absence of tests: grep `talker2codec_raw_async_chunk` across tests/ returns no matches. Sibling test files exist in the PR-time tree unchanged by this diff: tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py and tests/model_executor/stage_input_processors/test_fish_speech_async_chunk.py.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1524` [minor] — The Local talker's audio-frame sampler is retuned in the new `talker_mtp` path from the removed `temperature=0.8, top_k=30, top_p=0.6, repetition_penalty=1.1` to `1.7 / 25 / 0.8 / 1.0`, changing the generated-audio distribution for every MOSS-TTS-Local request. The new code comment asserts alignment with MOSS-TTS Local v1.5 model-card/SGLang defaults, but no measured before/after WER is attached; please state this in the PR description and include the WER delta from the Seed-TTS run relative to the old values. (evidence: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1524 `audio_temperature = 1.7`; :1525 `audio_top_k = 25`; :1526 `audio_top_p = 0.8`; :1566 `repetition_penalty=1.0,` — new Local talker sampler. Removed by this diff (old `make_omni_output` path): `temperature=0.8,` / `top_k=30,` / `top_p=0.6,` / `repetition_penalty=1.1,`.) + +**Additional observations (beyond the comment budget):** +- `tests/worker/test_omni_gpu_model_runner.py:10` [minor] The only test file this PR touches validates the seed-key rename and scalar-generator routing; none of the newly added behavior (codec stream slot acquire/release/reset, talker2codec_raw_async_chunk buffering + finish se +- `vllm_omni/benchmarks/data_modules/seed_tts_eval.py:581` [minor] Add a CPU unit test in tests/benchmarks/test_seed_tts_dataset_variants.py that sets SEED_TTS_WER_SAVE_AUDIO_DIR, feeds synthetic 24 kHz mono PCM through compute_seed_tts_wer_metrics, asserts the WAV file exists with the + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.trace.json.gz new file mode 100644 index 000000000..a6052693d Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.cost.json new file mode 100644 index 000000000..f27ba1351 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1012.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4817/run-20260815-144148-0e2960", + "recorded_at": "2026-08-15T14:58:38", + "attempts": [ + { + "invocation_id": "inv-580170f2", + "outcome": "run:run-20260815-144148-0e2960", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-144148-0e2960", + "llm_calls": 68, + "input_tokens": 321446, + "output_tokens": 154379, + "usd": 0.325672, + "source": "spans" + } + ], + "attempt_usd_total": 0.325672, + "llm_calls": 68, + "input_tokens": 321446, + "output_tokens": 154379, + "usd": 0.325672, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-144148-0e2960", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3257, + "minutes": 16.83, + "ci_minutes": 0, + "input_tokens": 321446, + "output_tokens": 154379, + "tool_calls": 187, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6035, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 68, + "by_role": { + "planner": { + "usd": 0.004375, + "input_tokens": 2563, + "output_tokens": 3348, + "calls": 1 + }, + "agent": { + "usd": 0.283256, + "input_tokens": 302823, + "output_tokens": 120453, + "calls": 64 + }, + "reducer": { + "usd": 0.038041, + "input_tokens": 16060, + "output_tokens": 30578, + "calls": 3 + } + }, + "cache_read_tokens": 2557952, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.85, + "gate": 1.52, + "review": 1004.09, + "report": 0.06 + }, + "time_to_first_result_s": 1009.5 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4817.trace.json.gz", + "trace_bytes": 27724 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.md new file mode 100644 index 000000000..d80835470 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.md @@ -0,0 +1,59 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4817, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head fc423e5a9e86) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] >=10 gate matched cc 12.x — fixed by ==10; residual: future cc 11.x (sm_110a) also excluded until gate widened +- [claim-verified] PR body — sm_120/121 default quack off; datacenter 10.x unchanged; =1/=0 overrides preserved +- [claim-verified] docs/user_guide/quantization/fp8.md:47-55 — datacenter-only auto-enable and =1 force-on documented +- [claim-verified] file_at_base — pre-change `_is_blackwell` used `>= 10`; diff matches stated root cause +- [claim-verified] PR claim 'env escape hatch preserved (env checked first)' — quack_fp8.py:37-40 reads override before `_is_quack_capable()`. +- [claim-verified] New test drives the real entry points `_is_quack_capable`/`quack_enabled` with monkeypatched torch.cuda probes, not a pre-transformed value. +- [claim-refuted] the PR describes the new file as a regression guard, but at this head it is never executed by CI: no pytestmark in the file and no lane runs tests/quantization (grep tests/quantization in .buildkite = no lane references). +- [validated] vllm_omni/quantization/quack_fp8.py:32 — `return torch.cuda.get_device_capability()[0] == 10` gates only cc 10.x +- [validated] vllm_omni/patch.py:410-419 — install_quack_fp8_patch() called at import; respects quack_enabled() +- [validated] test_quack_fp8_gate.py:37-38 — (12,0)/(12,1) expect False; (10,0)/(10,3) expect True +- [validated] test_quack_fp8_gate.py:61-67 — truthy env forces on on sm_120 +- [validated] test_quack_fp8_gate.py:70-75 — non-truthy env including '' forces off on sm_100 +- [validated] grep quack_enabled callers — only vllm_omni/quantization/quack_fp8.py:102 (install_quack_fp8_patch) and the new tests; no stale in-repo consumer of the renamed behavior. +- [validated] grep install_quack_fp8_patch callers — only vllm_omni/patch.py:412-414, invoked once at patch import; no second registration path to update. + +`tests/quantization/test_quack_fp8_gate.py:21` [major] — tests/quantization/test_quack_fp8_gate.py defines no `pytestmark`, so the PR-gating 'Simple · Other Test' step (.buildkite/test-merge.yml:29 `pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor`) collects then deselects all four tests; there is no `pytest_collection_modifyitems`/`add_marker` hook in the repo to auto-mark them. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` at module scope so the sm_120 regression guard actually runs in PR CI. (evidence: ['tests/quantization/test_quack_fp8_gate.py:21 `from vllm_omni.quantization import quack_fp8` — the cited anchor line; this new file contains no `pytestmark` (grep `pytestmark` under tests/quantization returns no matches, and the directory holds only this test file).', '.buildkite/test-merge.yml:29 `"timeout 20m pytest -sv -m \'core_model and cpu\' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — unchanged by this diff, present in the PR-time tree; the whole-tree PR-gating step selects only tests marked core_model AND cpu, so the unmarked new file is deselected.']) + +`vllm_omni/quantization/quack_fp8.py:~32` [minor] — The gate narrows to major version 10 only (`vllm_omni/quantization/quack_fp8.py:32`), while the PR body lists `sm_110a` among `tcgen05.mma` targets. A cc 11.x datacenter Blackwell would silently disable quack, and the parametrize matrix (`tests/quantization/test_quack_fp8_gate.py:34-40`) has no (11, x) case. Either widen the predicate to also admit 11.x (if sm_110 is datacenter Blackwell with tcgen05), or document and test why 11.x is deliberately excluded. (evidence: vllm_omni/quantization/quack_fp8.py:32 ` return torch.cuda.get_device_capability()[0] == 10` — gate admits major version 10 only, so cc 11.x returns False. Unchanged by this diff? No, this line IS the diff's change (`>= 10` → `== 10`). PR body (gh_pr_view 4817): "- PTX `tcgen05.mma` targets `sm_100a / sm_101a / sm_103a / sm_110a`; `ptxas` rejects `tcgen05` on `sm_120`." tests/quantization/test_quack_fp8_gate.py:36-37 ` ((10, 3), True), # sm_103 datacenter Blackwell` followed directly by ` ((12, 0), False), # sm_120 workstation Blackwell — the regression` — no (11, x) test row.) + +`vllm_omni/quantization/quack_fp8.py:32` [minor] — This gate is process-global (installed at vllm_omni/patch.py import time) and flips the default for every cc 12.x CUDA FP8 ScaledMM user to native FlashInfer — NVFP4 is a separate kernel path and is not patched here. The PR's only sm_120 end-to-end evidence is HunyuanImage3 and no CI lane runs sm_120; add an sm_120 CI lane, or state explicitly in the PR/docs that this global default change is single-model-validated, before merging. (evidence: vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10` — the changed gate (in diff). vllm_omni/quantization/quack_fp8.py:101-102 `def install_quack_fp8_patch() -> None:` / ` if not quack_enabled():` — the gate governs the global patch install. vllm_omni/quantization/quack_fp8.py:108-113 imports `FlashInferFP8ScaledMMLinearKernel` (FP8 only, so NVFP4 is out of scope). Unchanged by this diff, present in the PR-time tree: vllm_omni/patch.py:410-419 `def _patch_fp8_use_quack_fused_bias():` ... `from vllm_omni.quantization.quack_fp8 import install_quack_fp8_patch` ... `install_quack_fp8_patch()` ... `_patch_fp8_use_quack_fused_bias()` — import-time process-global install.) + +`tests/quantization/test_quack_fp8_gate.py:51` [minor] — The new test only drives `_is_quack_capable()`/`quack_enabled()` with monkeypatched `torch.cuda`, and it never imports `vllm_omni.patch` or calls `install_quack_fp8_patch()`. Since `install_quack_fp8_patch()` begins with `if not quack_enabled(): return` and is invoked at patch import time (patch.py:410-414), add a case that monkeypatches `_load_quack` and drives `install_quack_fp8_patch()` (or imports `vllm_omni.patch`) for sm_100 vs sm_120 so the actual install path is covered. (evidence: In the new test, all assertions hit the gate helpers directly: tests/quantization/test_quack_fp8_gate.py:51 `assert quack_fp8.quack_enabled() is expected`, :58 `assert quack_fp8.quack_enabled() is False`, :67 `assert quack_fp8.quack_enabled() is True`, :75 `assert quack_fp8.quack_enabled() is False`; grep finds no `install_quack_fp8_patch` in the test file. The untested consumer is unchanged by this diff, present in the PR-time tree: vllm_omni/quantization/quack_fp8.py:102 `if not quack_enabled():` inside `install_quack_fp8_patch()`, and vllm_omni/patch.py:412-414 `from vllm_omni.quantization.quack_fp8 import install_quack_fp8_patch` ... `install_quack_fp8_patch()` invoked by `_patch_fp8_use_quack_fused_bias()` at module import.) + +`vllm_omni/quantization/quack_fp8.py:136` [minor] — warmup_quack_fp8 calls _load_quack() directly and therefore bypasses the new ==10 gate; a manual warmup on sm_120 will still load quack. Route it through quack_enabled() or add a guard/comment so this manual entry cannot silently re-enable the path the PR is gating off. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/quantization/quack_fp8.py:131-136 `def warmup_quack_fp8(...):` ... ` if _load_quack() is None:`; the gate it bypasses is vllm_omni/quantization/quack_fp8.py:32 ` return torch.cuda.get_device_capability()[0] == 10`, consulted only by quack_enabled() at vllm_omni/quantization/quack_fp8.py:37-41.) + +`docs/user_guide/quantization/fp8.md:~33` [minor] — fp8.md:33 still says 'On Blackwell (SM 100+)' and points all Blackwell users at quack's fused-bias benefit, while fp8.md:47-53 correctly limits quack auto-enable to datacenter cc 10.x and states sm_120/sm_121 default to FlashInfer native FP8. Add a one-line qualifier at line 33 that quack auto-enables only on datacenter Blackwell (cc 10.x); sm_120/sm_121 stay on the FlashInfer native FP8 path. (evidence: docs/user_guide/quantization/fp8.md:33 `On Blackwell (SM 100+), vLLM runs FP8 linears through the FlashInfer kernel, which` (unchanged by this diff, present in the PR-time tree). The newly added distinction lives two paragraphs later: docs/user_guide/quantization/fp8.md:47 `It is enabled automatically once installed (no flag needed) and is **datacenter` and docs/user_guide/quantization/fp8.md:51-53 `workstation/consumer Blackwell (`sm_120` / `sm_121`, compute capability `12.x`, e.g.` ... `tcgen05` is absent — so quack is **not** auto-enabled` ... `there and FlashInfer's native FP8 path is used instead.`) + +`docs/user_guide/quantization/fp8.md:54` [minor] — Minor: line 54 documents `VLLM_OMNI_USE_QUACK_FP8=1` as 'force quack on' without caveating that on sm_120/sm_121 this re-enters the per-GEMM `try quack / except -> FlashInfer` fallback the PR itself calls catastrophic. Consider adding: '=1 is intended only once CuteDSL ships sm_120a support.' (evidence: docs/user_guide/quantization/fp8.md:54 `VLLM_OMNI_USE_QUACK_FP8=1` to force quack on, or `VLLM_OMNI_USE_QUACK_FP8=0` to force` (added by this diff, no sm_120a caveat). vllm_omni/quantization/quack_fp8.py:123 `logger.warning_once("Quack FP8 GEMM failed (%s); using FlashInfer.", exc)` (unchanged by this diff, present in the PR-time tree) — on sm_120/sm_121 a force-on env override reaches the patched apply_scaled_mm, whose quack GEMM raises and hits this per-GEMM fallback.) + +`vllm_omni/quantization/quack_fp8.py:32` [nit] — gh_ci_read reports both `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` FAILURE, and neither lane is affected by this diff: the AMD lane's Simple Test steps run `tests/diffusion tests/model_executor` (its other steps run custom_pipeline, voxtral, diffusion, engine, and TTS e2e — none reference `tests/quantization`), the NPU lane runs only `test_qwen3_tts_customvoice.py`, and grep for `tests/quantization` in `.buildkite/` returns no matches. The only behavior change, `>= 10` → `== 10` in `_is_quack_capable`, evaluates identically on AMD (capability major 9) and NPU (no CUDA): False before and after. Please confirm these failures are pre-existing on main rather than introduced here. (evidence: vllm_omni/quantization/quack_fp8.py:32 ` return torch.cuda.get_device_capability()[0] == 10`; .buildkite/test-amd-ready.yaml:19 ` - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/test-npu-ready.yaml:21 ` - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`. Grep for `tests/quantization` under `.buildkite/` (unchanged by this diff, present in the PR-time tree) returns no matches.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.trace.json.gz new file mode 100644 index 000000000..81797d5c0 Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4817.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.cost.json new file mode 100644 index 000000000..7a7c7fa72 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1529.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4859/run-20260815-144747-6b128a", + "recorded_at": "2026-08-15T15:13:14", + "attempts": [ + { + "invocation_id": "inv-d4ca6816", + "outcome": "run:run-20260815-144747-6b128a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-144747-6b128a", + "llm_calls": 128, + "input_tokens": 1049457, + "output_tokens": 254782, + "usd": 0.791011, + "source": "spans" + } + ], + "attempt_usd_total": 0.791011, + "llm_calls": 128, + "input_tokens": 1049457, + "output_tokens": 254782, + "usd": 0.791011, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-144747-6b128a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.791, + "minutes": 25.45, + "ci_minutes": 0, + "input_tokens": 1049457, + "output_tokens": 254782, + "tool_calls": 336, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9418, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 128, + "by_role": { + "agent": { + "usd": 0.742994, + "input_tokens": 1023965, + "output_tokens": 217453, + "calls": 125 + }, + "reducer": { + "usd": 0.048017, + "input_tokens": 25492, + "output_tokens": 37329, + "calls": 3 + } + }, + "cache_read_tokens": 8422144, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.51, + "gate": 2.08, + "review": 1520.22, + "report": 0.07 + }, + "time_to_first_result_s": 1526.86 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4859.trace.json.gz", + "trace_bytes": 39040 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.md new file mode 100644 index 000000000..4f319df24 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.md @@ -0,0 +1,78 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4859, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6ff4ce59ed87) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | no finding reported | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] stop-head early stop concern — +2 aligns HF; residual: no unit test for window math +- [resolved] Producer→consumer PCM contract — MixRequestFuncOutput docstring patch.py:348-350 says 24 kHz mono; capture honors it at :1215; residual gap is seed_tts_eval ignoring stored format +- [claim-verified] PR aligns Ming instruction/language handling for dense Seed-TTS — `include_language` removed from `_parse_ming_instruction` only +- [claim-verified] tests/model_executor/models/ming_tts/test_ming_shared_modules.py:27 — pytestmark=[core_model, cpu, tts]; AMD ready lane command at .buildkite/test-amd-ready.yaml:19/30 selects tests/model_executor with 'core_model and cpu'. +- [claim-verified] PR description scope (instruction handling, VAE latent path, stop-head gating, benchmark PCM parsing) all present in diff. +- [claim-verified] PR's own test plan sets VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 (pr_context) — exactly the env value that triggers the double-resample. +- [claim-verified] diff_stat confirms no test files are touched — only 6 non-test files changed. +- [claim-refuted] PR 1.83% WER at head with VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 — merge conflict makes env on WER consumer inconsistent with 24 kHz normalized capture (~0.54 duration ratio) +- [claim-refuted] PR WER 1.83% — with the test plan's VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100, eval resamples the already-24k PCM as 44.1k: calc 16000/44100=0.3628 vs correct 16000/24000=0.6667 → 1.838x speed-up. +- [claim-refuted] PR claims 'benchmark PCM sample-rate parsing' improvement, but the eval env-read double-resamples the already-24kHz-normalized tts_output_pcm_bytes (patch.py:118/1215 vs seed_tts_eval.py:200) — see comment 1. +- [claim-refuted] PR body reports WER measured on commit `0455bffc3768...`, but head is merge commit 6ff4ce59 (upstream/main merge); the 1.83% number may not describe the merged state. +- [claim-refuted] PR 1.83% WER at VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 — pcm is 24 kHz after patch.py:1215 but seed_tts_eval:616 resamples as 44.1 kHz (ratio 16000/44100 vs 16000/24000 = 1.8375×) +- [validated] audio_vae.py:349-351 encode_latent — matches HF `posterior.sample()` reparameterization (inclusionAI modeling_audio_vae.py) +- [validated] audio_vae.py:141-142 aggregator — matches HF `config.num_hidden_layers = 4` pattern (inclusionAI vae_modules.py) + +`vllm_omni/model_executor/models/ming_tts/patch_emission.py:~130` [major] — `min_stop_step + 2` raises the default floor to 5 (stop_head_min_steps=3). The unchanged duration-derived window gives max_steps = target_steps + 3, and with one decode step ≈ 14112/44100 = 0.32 s, any `Duration:` value ≤ 0.32 s yields target_steps=1 → max_steps=4, so `_validate_ming_decode_window` now raises ValueError (4 < 5). This regresses previously-passing short utterances; clamp `min_required` to the resolved `max_steps` (or floor the duration-derived `max_steps` at `min_stop_step + 2`). (evidence: ['vllm_omni/model_executor/models/ming_tts/patch_emission.py:130 (changed by this diff): `min_required = max(min_stop_step + 2, min_steps)` — with the default below this equals max(5, min_steps); line 131 `if max_steps < min_required:` then raises ValueError.', 'vllm_omni/model_executor/models/ming_tts/constants.py:51 (unchanged by this diff, PR-time tree): `STOP_HEAD_MIN_STEPS = 3`.', 'vllm_omni/model_executor/models/ming_tts/prompt_assembly.py:106-107 (unchanged by this diff, PR-time tree): `min_steps = max(1, target_steps - 3)` / `max_steps = max(min_steps, target_steps + 3)`.', 'vllm_omni/model_executor/models/ming_tts/prompt_assembly.py:98-100 (unchanged by this diff, PR-time tree): `samples_per_decode_step = int(frame_hop) * int(patch_size) * int(vae_patch_size)` ... `return max(1, int(math.ceil(required_samples / float(samples_per_decode_step))))`; constants.py:39 `PATCH_SIZE = 4`, :44 `VAE_PATCH_SIZE = 4`, :45 `SAMPLE_RATE = 44100`, :48 `AUDIO_FRAME_HOP = 882` ⇒ 14112 samples/step = 0.32 s/step, so duration ≤ 0.32 s ⇒ target_steps=1 ⇒ max_steps=4 < min_required=5.']) + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:351` [major] — audio_vae.py:351 changes inference-time encoding from returning the mean (`torch.chunk(h, 2, dim=1)[0]`) to sampling `mean + std * torch.randn_like(mean)`. Both consumers call `encode_latent` without seeding this encoder path: prompt_encoder.py:137 (`latents, _ = encoder.encode_latent(waveform, waveform_length)`) and voice_presets.py:257 (`prompt_wav_lat, _ = self._audio_vae.encode_latent(...)`), so prompt latents and voice presets are non-deterministic. If HF parity requires the sampled latent, scope it to the validated path and seed it explicitly; otherwise keep mean for inference. (evidence: vllm_omni/model_executor/models/common/ming/audio_vae.py:351 `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)` (changed by this diff). Unchanged consumers in the PR-time tree: vllm_omni/model_executor/models/ming_tts/prompt_encoder.py:137 `latents, _ = encoder.encode_latent(waveform, waveform_length)`; vllm_omni/model_executor/models/ming_flash_omni/voice_presets.py:257 `prompt_wav_lat, _ = self._audio_vae.encode_latent(`. Neither path seeds the encoder.) + +`vllm_omni/entrypoints/openai/serving_speech.py:~2651` [major] — At serving_speech.py:2647-2652 `_parse_ming_instruction` no longer passes `include_language=True`, and the `include_language` parameter plus the `instruction_dict["方言"] = request.language` mapping were removed from `_parse_ming_instruction_fields`. Because `_build_ming_dense_prompt` (line 2688) passes `instruction=self._parse_ming_instruction(request)`, a dense Ming request with a non-`Auto` `language` no longer writes `方言` into the caption (BASE_CAPTION_TEMPLATE keeps `"方言": None`). Confirm this removal is intentional and documented/tested, or restore the mapping. (evidence: PR diff removes: `- include_language=False,`, `- if include_language and request.language not in (None, "", "Auto"):`, `- instruction_dict["方言"] = request.language`, and `- include_language=True,`. PR-time tree confirms the removal: vllm_omni/entrypoints/openai/serving_speech.py:2622-2624 ` *,` / ` include_voice=False,` / ` plain_text_passthrough=False,` (no include_language param); serving_speech.py:2649-2652 ` return self._parse_ming_instruction_fields( + request, + include_voice=True, + plain_text_passthrough=True, + )`; serving_speech.py:2688 ` instruction=self._parse_ming_instruction(request),` (dense Ming consumer). Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/ming_tts/prompt_assembly.py:38 ` "方言": None,` — so without the removed mapping the caption dialect stays None.) + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:141` [major] — This hunk replaces the separate aggregator config with an in-place mutation of the same `config` object passed to `Qwen2Model` at line 131. `Qwen2Model` stores the config by reference, so `self.encoder.config.num_hidden_layers` silently becomes 4 after line 141 even though the encoder's weights were built with the backbone's real depth; any later save/reload or introspection of `self.encoder` will report 4 layers while the weights contain the original count. Build a fresh aggregator config (or a copy) instead of mutating the shared object. (evidence: vllm_omni/model_executor/models/common/ming/audio_vae.py:129 `config = _qwen2_config(encoder_args)` (unchanged by this diff, present in the PR-time tree) | vllm_omni/model_executor/models/common/ming/audio_vae.py:131 `self.encoder = Qwen2Model(config)` (unchanged by this diff, present in the PR-time tree) | vllm_omni/model_executor/models/common/ming/audio_vae.py:141 `config.num_hidden_layers = 4` (changed by this PR) | vllm_omni/model_executor/models/common/ming/audio_vae.py:142 `self.aggregator = Qwen2Model(config)` (changed by this PR) — the same `config` object is mutated after it was already given to the encoder, so encoder config depth and encoder weight depth diverge.) + +`buildkite/vllm-omni-amd-ci:0` [major] — The AMD CI lane is red at head. This diff does not touch ROCm/NPU runners or AMD-loaded configs and only collects the ming_tts unit test, so the failure must be classified before merge: state whether it is pre-existing, and fix it if introduced. Do not merge behind a red AMD gate. (evidence: [sweep] AMD CI buildkite/vllm-omni-amd-ci red — diff does not touch ROCm/NPU runners or AMD-loaded configs; [sweep] red gate buildkite/vllm-omni-amd-ci — diff touches no AMD-specific code; only the ming_tts unit test is collected; pre-existing vs introduced cannot be determined from CI read-out (comment C6).) + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:~616` [major] — seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)` passes no rate/channels, so the new None-default branch re-reads VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE/VLLM_OMNI_BENCH_AUDIO_CHANNELS. But `pcm` is already normalized to 24 kHz mono by `_pcm_s16le_to_seed_tts_wer_bytes` (patch.py:130). With 44100 set, `target_len = int(len(raw) * 16000 / pcm_sample_rate)` at seed_tts_eval.py:200 computes len(raw)*16000/44100 = 0.3628x instead of len(raw)*16000/24000 = 0.6667x, a ~1.84x temporal speed-up, so WER is measured on distorted audio. Pass pcm_sample_rate=24000, channels=1 (or record the normalized rate) and re-run the sweep. (evidence: seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)` — WER path calls with no rate/channels; seed_tts_eval.py:188 `if pcm_sample_rate is None or channels is None:` and seed_tts_eval.py:189 `env_sample_rate, env_channels = stream_pcm_format_from_env()` — new branch re-reads env (added by this PR); seed_tts_eval.py:200 `target_len = int(len(raw) * 16000 / pcm_sample_rate)` — 16000/44100 = 0.3628x vs 16000/24000 = 0.6667x (~1.84x speed-up); patch.py:130 `resampler = AudioResampler(target_sr=24000)` (inside `_pcm_s16le_to_seed_tts_wer_bytes`, unchanged by this diff, present in the PR-time tree) — the `pcm`/`tts_output_pcm_bytes` fed to the WER function is already normalized to 24 kHz mono.) + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:188` [minor] — This diff makes the WER decoder read sample_rate/channels from the environment and adds stereo→mono downmix, but no test exercises `_pcm_s16le_to_f32_16k` (grep tests finds only `_transcribe_en_f32_16k` coverage). Add a unit test driving raw int16 bytes through the real function with `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100` and `VLLM_OMNI_BENCH_AUDIO_CHANNELS=2`, asserting the resampled length equals `int((len(pcm)//4)*16000/44100)` (bytes→samples halves the count, then the stereo→mono mean halves it again); this guards the exact env/channel path behind the PR's reported 1.83% WER. (evidence: vllm_omni/benchmarks/data_modules/seed_tts_eval.py:179 `def _pcm_s16le_to_f32_16k(`; :188-193 `if pcm_sample_rate is None or channels is None: + env_sample_rate, env_channels = stream_pcm_format_from_env() + if pcm_sample_rate is None: + pcm_sample_rate = env_sample_rate + if channels is None: + channels = env_channels`; :195-199 `if channels > 1: + usable = (len(raw) // channels) * channels + if usable <= 0: + return np.zeros(0, dtype=np.float32) + raw = raw[:usable].reshape(-1, channels).mean(axis=1)`. Call site, unchanged by this diff but present in the PR-time tree: vllm_omni/benchmarks/data_modules/seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)`. Test coverage proof, also outside the diff: grep of `tests` for `_pcm_s16le_to_f32_16k` returns no matches, while `_transcribe_en_f32_16k` appears only at tests/benchmarks/test_seed_tts_dataset_variants.py:220.) + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:0` [minor] — PR body reports Seed-TTS full EN WER 1.83% as 'tested on current branch based on 0455bffc3768e1fe616fea37885fb0a84cd5985b', while the PR head is merge commit 6ff4ce59. State the measured commit explicitly or re-measure/re-report at the merged head. (evidence: PR #4859 body (not part of the diff — PR metadata): "**vLLM-Omni Commit:** this PR, tested on current branch based on `0455bffc3768e1fe616fea37885fb0a84cd5985b`". PR head commit 6ff4ce59ed87 (git show): "commit 6ff4ce59ed87f2f7afc9d795f4796ad540ba7658 / Merge: 795ae240 a8b2bf11 ... Merge remote-tracking branch 'upstream/main' into fix_ming_tts_moe_precision". The measured commit 0455bffc is not the merged head.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/ming_tts/patch_emission.py:130` [minor] The min_stop_step +1→+2 floor is consistent at patch_emission.py:130 (`min_required = max(min_stop_step + 2, min_steps)`) and :151 (`min_required_decode_steps = max(min_stop_step + 2, min_decode_steps)`), but no test und +- `examples/online_serving/text_to_speech/ming_tts/README.md:101` [minor] This PR deletes the serving-side `language → 方言` mapping (`_parse_ming_instruction_fields` no longer injects `instruction_dict["方言"]`), but examples/online_serving/text_to_speech/ming_tts/README.md:101 still advertises ` +- `vllm_omni/benchmarks/data_modules/seed_tts_eval.py:116` [minor] The new module docstring says streaming PCM is decoded using `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE` / `VLLM_OMNI_BENCH_AUDIO_CHANNELS`, but `_save_seed_tts_eval_audio` still hard-codes a 24 kHz mono WAV header. `compute_see +- `tests/model_executor/models/ming_tts/test_ming_shared_modules.py:91` [minor] This diff deletes the `include_language`/`方言` mapping from `_parse_ming_instruction_fields` and stops `_parse_ming_instruction` from passing `include_language=True`, so a non-Auto `request.language` on Ming dense is now + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.trace.json.gz new file mode 100644 index 000000000..72e5494d4 Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4859.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.cost.json new file mode 100644 index 000000000..f10754db7 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1428.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4870/run-20260815-142042-034a13", + "recorded_at": "2026-08-15T14:44:28", + "attempts": [ + { + "invocation_id": "inv-8c41c296", + "outcome": "run:run-20260815-142042-034a13", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-142042-034a13", + "llm_calls": 143, + "input_tokens": 1587608, + "output_tokens": 257494, + "usd": 1.01746, + "source": "spans" + } + ], + "attempt_usd_total": 1.01746, + "llm_calls": 143, + "input_tokens": 1587608, + "output_tokens": 257494, + "usd": 1.01746, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-142042-034a13", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0175, + "minutes": 23.77, + "ci_minutes": 0, + "input_tokens": 1587608, + "output_tokens": 257494, + "tool_calls": 325, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9944, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 143, + "by_role": { + "agent": { + "usd": 0.970209, + "input_tokens": 1566370, + "output_tokens": 219818, + "calls": 140 + }, + "reducer": { + "usd": 0.04725, + "input_tokens": 21238, + "output_tokens": 37676, + "calls": 3 + } + }, + "cache_read_tokens": 11317120, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.04, + "gate": 2.31, + "review": 1419.76, + "report": 0.06 + }, + "time_to_first_result_s": 1426.16 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 38676 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.md new file mode 100644 index 000000000..567b1c63a --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.md @@ -0,0 +1,63 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 10 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Review concern that async_chunk=False FINAL_ONLY would break other TTS models — ede3dddf scoped to qwen3_tts; test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3 asserts Voxtral still gets DELTA (residual: moss/voxcpm2/indextts2 async_chunk=False streaming not individually tested) +- [resolved] Concern: does changing seq_len break the prefix-cache path? When omni_prefix_cache is not None, combined_multimodal_outputs bypass to_payload_element entirely (_build_combined_prefix_cache_mm_payload); when it is None, mm_seq_len now feeds the slice gate. Residual: the prefix-cache passthrough path is not covered by the new test. +- [resolved] removed seq_len had no other consumer: grep shows only signature-internal seq_len at gpu_ar_model_runner.py:879/923 plus new mm_seq_len at 1790. +- [resolved] prefix-cache passthrough ask: test_build_omni_output_falls_back_to_mm_cpu_without_prefix_merge (668) already covers omni_prefix_cache codes.audio splitting +- [claim-verified] PR scopes streaming coercion to qwen3_tts — code + voxtral contrast test at tests/entrypoints/openai_api/test_serving_speech.py:3473 +- [claim-verified] PR adds worker regression for batched codes.audio with tail-only hidden — test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only +- [claim-verified] PR E2E table/description uses `stream=False` (`--no-async-chunk, stream=False, and request_num=3`), so the stream=True FINAL_ONLY serving change has no real-serving evidence. +- [claim-verified] PR numbers check out arithmetically: codes.audio (3,16) = 3 requests x 16 code groups = 48 elements; per-request slice is 16 columns; calc(3*16, 48/16, 48/3) = (48, 3.0, 16.0). +- [claim-verified] PR body says tail-only hidden states caused whole batched codes.audio accumulation — consistent with to_payload_element only slicing when dim-0 == seq_len; verified via mm_outputs.py:131. +- [claim-verified] two targeted unit tests are actually runnable and assert the exact new booleans (mock_coerce.assert_called_once_with(..., False) / (..., True)). +- [claim-verified] PR root cause: codes.audio dim-0 matched scheduled tokens but seq_len came from hidden_states.shape[0] +- [claim-refuted] PR says 'Fixes #4851' but the linked issue lists two failing tests (Base test_response_format_001[no_async_chunk] and CustomVoice test_voice_002[no_async_chunk]); the PR validation table covers only CustomVoice (test_voice_001...), leaving the Base path unverified. +- [claim-refuted] PR body says 'The output builder used hidden_states.shape[0] as the multimodal split length' — true pre-fix (file_at_base confirms seq_len = hidden_states.shape[0]), fix matches claim. +- [validated] vllm_omni/worker/gpu_ar_model_runner.py:1734 — mm_seq_len uses int(scheduler_output.total_num_scheduled_tokens), fixing tail-only hidden vs batch-aligned codes.audio mismatch + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:~1144` [major] — Mirror the GPU fix on NPU: in `NPUARModelRunner.sample_tokens`, replace `seq_len = hidden_states.shape[0]` (npu_ar_model_runner.py:905) with a scheduler-token count such as `seq_len = int(scheduler_output.total_num_scheduled_tokens)`, and keep passing that value at line 1144. Add an NPU regression test equivalent to `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` so the Qwen3 customvoice NPU lane cannot ship the same cross-request codes.audio contamination. (evidence: npu_ar_model_runner.py:905 (unchanged by this diff): `seq_len = hidden_states.shape[0]`; npu_ar_model_runner.py:1144 (unchanged by this diff): `seq_len=seq_len,` inside `to_payload_element(...)`; gpu_ar_model_runner.py:1734 (this PR): `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3509` [minor] — This diff makes stream=True + async_chunk=False qwen3_tts requests coerce to FINAL_ONLY (serving_speech.py:3508-3509), but the only new evidence is a mocked routing assertion: test_serving_speech.py's added test patches `coerce_param_message_types` and asserts it was called with `False`, so it proves the gate flips, not that the streaming response actually emits the final waveform once. The PR's remote E2E ran with `stream=False`. Run a real stream=True no_async_chunk serving path (SSE or raw-audio) against the H20 deployment, or narrow the PR claim to the mocked unit level. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; vllm_omni/entrypoints/openai/serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`; tests/entrypoints/openai_api/test_serving_speech.py:3453 `def test_prepare_speech_generation_no_async_chunk_stream_uses_final_only(` with body assertion `mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, False)` — the test patches coerce_param_message_types away and never exercises the streaming response path; PR body: `The E2E probe used real remote H20 serving with Qwen3-TTS CustomVoice, `--no-async-chunk`, `stream=False`, and `request_num=3`.`) + +`tests/worker/test_gpu_ar_model_runner.py:438` [minor] — PR #4870's validation table lists only the CustomVoice E2E case (test_voice_001_nonstream_concurrent_no_async_chunk[no_async_chunk]), while issue #4851 also covers Base test_response_format_001[no_async_chunk]. The new regression test at tests/worker/test_gpu_ar_model_runner.py:438 is model-agnostic (engine_output_type='latent'), so it plausibly exercises the Base path too, but no Base no_async_chunk E2E or targeted run is evidenced. Either add test_response_format_001[no_async_chunk] (Base) to the validation table or state why the generic regression test is sufficient. (evidence: ['tests/worker/test_gpu_ar_model_runner.py:438 `def test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only(monkeypatch):` — anchor exists exactly as cited.', 'tests/worker/test_gpu_ar_model_runner.py:439 `runner = _make_async_output_runner(engine_output_type="latent")` — the regression test is generic, not CustomVoice-specific.', 'PR #4870 body (metadata, not a repo file): `| `test_voice_001_nonstream_concurrent_no_async_chunk[no_async_chunk]` | `31070fa7` | Before fix | Failed |` — the only remote E2E validation row is CustomVoice.', 'Issue #4851 body (metadata): `tests/e2e/online_serving/test_qwen3_tts_base_expansion.py::test_response_format_001[no_async_chunk]` and `tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py::test_voice_002[no_async_chunk]` — the fix covers both failing cases, but only CustomVoice is evidenced in the PR table.']) + +`tests/worker/test_gpu_ar_model_runner.py:~438` [minor] — The new regression test proves the dim-0 slice gate is reached, but it monkeypatches _should_accumulate_full_payload_output to False and _process_additional_information_updates to None, so it does not exercise the full-payload accumulation that is the actual corrupted-output path in production. Add an assertion on the inter_stage_outputs content through _should_accumulate_full_payload_output returning True (or call the real accumulate path), since the PR body's failure was each request accumulating the whole batched codes.audio. (evidence: tests/worker/test_gpu_ar_model_runner.py:451 'monkeypatch.setattr(GPUARModelRunner, "_should_accumulate_full_payload_output", lambda self: False)'; tests/worker/test_gpu_ar_model_runner.py:478 'assert torch.equal(output.inter_stage_outputs[0]["codes.audio"], codes[0:1])'.) + +`tests/worker/test_gpu_ar_model_runner.py:438` [minor] — Add a variant of test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only that sets runner.omni_prefix_cache to a non-None stub and asserts the combined-multimodal passthrough, or document that the passthrough bypasses _build_omni_mm_payload's to_payload_element seq_len gate and is intentionally excluded. As written, _make_async_output_runner sets runner.omni_prefix_cache = None, so the new test only demonstrates the non-prefix-cache to_payload_element slice gate. (evidence: tests/worker/test_gpu_ar_model_runner.py:438 `def test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only(monkeypatch):`; tests/worker/test_gpu_ar_model_runner.py:439 `runner = _make_async_output_runner(engine_output_type="latent")`; tests/worker/test_gpu_ar_model_runner.py:216 `runner.omni_prefix_cache = None` (inside `_make_async_output_runner`, unchanged by this diff, present in the PR-time tree); vllm_omni/worker/gpu_ar_model_runner.py:1736 `if self.omni_prefix_cache is not None:` — the prefix-cache branch never taken by the new test; vllm_omni/worker/gpu_ar_model_runner.py:759 `combined_multimodal_outputs = self.omni_prefix_cache.get_merged_multimodal_states(` — the passthrough source, unchanged by this diff and present in the PR-time tree.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3508` [minor] — The qwen3_tts no-async-chunk FINAL_ONLY gate (serving_speech.py:3508-3509) also governs websocket stream sessions: OmniStreamingSpeechHandler._generate_and_send calls self._speech_service._prepare_speech_generation when config.stream_audio is True (serving_speech_stream.py:273). The PR only adds direct _prepare_speech_generation tests, so add a websocket no-async-chunk qwen3_tts validation case or explicitly scope the streaming claim to non-websocket paths. (evidence: serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`; serving_speech_stream.py:273 `request_id, generator, _ = await self._speech_service._prepare_speech_generation(request)` (unchanged by this diff, present in the PR-time tree).) + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [minor] — The mm_seq_len flip is in the shared GPU AR output builder (`_build_omni_model_runner_output_from_snapshot`), and higgs_audio_v3_talker also sets `requires_full_prefix_cached_hidden_states = False`. Please record why higgs_audio_v3's batched `codes.audio` cannot diverge from `total_num_scheduled_tokens` the way Qwen3-TTS did, or add a higgs_audio_v3 case to the worker regression so the shared-value blast radius is explicit before merge. (evidence: vllm_omni/worker/gpu_ar_model_runner.py:1734 ` mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` (added by this diff) | vllm_omni/worker/gpu_ar_model_runner.py:1790 ` seq_len=mm_seq_len,` (changed by this diff, inside the shared builder) | vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_talker.py:236 ` self.requires_full_prefix_cached_hidden_states = False` (unchanged by this diff, present in PR-time tree)) + +`tests/entrypoints/openai_api/test_serving_speech.py:~3493` [minor] — The non-qwen3 branch is only checked by mocking coerce_param_message_types and asserting the boolean (test_serving_speech.py:3493), so it proves the gate routes True but not that DELTA coercion actually produces valid streaming for async_chunk=False non-qwen3 models. A real-coerce variant or an output_kind DELTA assertion would cover the 'other async_chunk=False models keep DELTA coercion' claim. (evidence: tests/entrypoints/openai_api/test_serving_speech.py:3485 " mock_coerce = mocker.patch("; :3486 " \"vllm_omni.entrypoints.openai.serving_speech.coerce_param_message_types\","; :3487 " return_value=voxtral_server.engine_client.default_sampling_params_list,"; :3493 " mock_coerce.assert_called_once_with(voxtral_server.engine_client.default_sampling_params_list, True)" — coerce is mocked and only the boolean is asserted. Production gate (in this diff): vllm_omni/entrypoints/openai/serving_speech.py:3508 " qwen3_full_payload = self._tts_model_type == \"qwen3_tts\" and not bool(async_chunk)"; :3509 " is_streaming_request = request.is_streaming() and not qwen3_full_payload"; :3510 " sampling_params_list = coerce_param_message_types(sampling_params_list, is_streaming_request)".) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/worker/gpu_ar_model_runner.py:1875` [minor] Confirmed: `sample_tokens` no longer computes `seq_len` from `hidden_states.shape[0]`, and `_build_omni_model_runner_output_from_snapshot` now derives mm splitting from `scheduler_output.total_num_scheduled_tokens` (gpu_ +- `tests/worker/test_gpu_ar_model_runner.py:250` [minor] Confirmed gap in test_build_omni_output_uses_snapshots_and_connector_after_accumulation (tests/worker/test_gpu_ar_model_runner.py:229-268): it pairs total_num_scheduled_tokens=3 with hidden_states shape (3,1), uses a fla + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.trace.json.gz new file mode 100644 index 000000000..b6617148b Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.cost.json new file mode 100644 index 000000000..3349cf8f4 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1309.5, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4923/run-20260815-142042-200701", + "recorded_at": "2026-08-15T14:42:29", + "attempts": [ + { + "invocation_id": "inv-7c22b5a6", + "outcome": "run:run-20260815-142042-200701", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-142042-200701", + "llm_calls": 101, + "input_tokens": 1083376, + "output_tokens": 245599, + "usd": 0.708718, + "source": "spans" + } + ], + "attempt_usd_total": 0.708718, + "llm_calls": 101, + "input_tokens": 1083376, + "output_tokens": 245599, + "usd": 0.708718, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-142042-200701", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7087, + "minutes": 21.77, + "ci_minutes": 0, + "input_tokens": 1083376, + "output_tokens": 245599, + "tool_calls": 282, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8516, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 101, + "by_role": { + "agent": { + "usd": 0.656467, + "input_tokens": 1059320, + "output_tokens": 204069, + "calls": 98 + }, + "reducer": { + "usd": 0.052251, + "input_tokens": 24056, + "output_tokens": 41530, + "calls": 3 + } + }, + "cache_read_tokens": 5409152, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.68, + "gate": 1.7, + "review": 1300.97, + "report": 0.06 + }, + "time_to_first_result_s": 1306.4 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 36191 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.md new file mode 100644 index 000000000..ad1d99e13 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.md @@ -0,0 +1,65 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] async+prefix mutual exclusion — enable_prefix_caching flipped true→false at qwen3_tts.yaml:46; residual: users who override prefix caching on lose async silently +- [resolved] GPU nested-graph risk: use_cuda_graphs now False on GPU and outer wrap only under has_full_cudagraphs(); residual: per-request tts_local_seed silently dropped under full cudagraphs with batch>1 (PR acknowledges via TODO follow-up). +- [claim-verified] PR nested-graph root cause — GPU inner graphs only when use_cuda_graphs=True (base); PR removes that on GPU +- [claim-verified] PR 39ms→9ms talker_mtp claim not re-run here; mechanism (single batched outer graph) matches code +- [claim-verified] 'seed:42 removal was merged in #4970, not this PR': qwen3_tts.yaml default_sampling_params contains no seed key (full file read). +- [claim-verified] 'Qwen3-Omni already opts out of hidden-state pooler payload + async output': qwen3_omni.py:156-164 sets use_async_omni_output=True, eager_omni_postprocess_before_async_output=True, omni_pooler_payload_include_hidden=False; matches qwen3_tts_talker.py:351-353. +- [claim-verified] 'GPU code predictor uses torch.compile only': qwen3_code_predictor.py _setup_compile builds torch.compile on Inductor platforms and calls _capture_cuda_graphs() only when use_cuda_graphs; is_npu()=False on GPU => no inner graphs. +- [claim-verified] 'NPU keeps internal graphs': qwen3_code_predictor.py:532-535 `if current_omni_platform.is_npu() and self._wrapper_config.use_cuda_graphs: ... _capture_npu_graphs()`. +- [claim-verified] RTF 0.19 is internally consistent: mean E2EL 1049.97ms / (179.44s/32*1000) = 0.187; also 21.88%*32 = 7.00 requests. +- [claim-verified] qwen3_tts.yaml at head has no `seed:` in default_sampling_params; serving_speech.py:3698-3706 derives the default seed from `stage0_params.seed`, so default requests carry no `tts_local_seed` and the batched-MTP win holds (residual: explicit `request.seed` still hits the scalar fallback). +- [claim-verified] `use_cuda_graphs=current_omni_platform.is_npu()` keeps NPU inner graphs via qwen3_code_predictor.py:532 and drops GPU to torch.compile-only via qwen3_code_predictor.py:552. +- [claim-refuted] The PR summary's 'cut per-step overhead from hidden-state D2H' predates this PR: `omni_pooler_payload_include_hidden = False` already exists at qwen3_tts_talker.py:317; the added line 353 is a duplicate. +- [claim-refuted] PR lists `omni_pooler_payload_include_hidden = False` as a new behavior change, but base talker.py:317 already set it False — the added line 353 is a no-op duplicate. +- [validated] vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 use_cuda_graphs=current_omni_platform.is_npu() stops GPU inner _capture_cuda_graphs (qwen3_code_predictor.py:552-556) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:~39` [major] — This hunk changes the Qwen3-TTS code predictor from use_cuda_graphs=True to use_cuda_graphs=current_omni_platform.is_npu(), which on GPU also disables code_predictor_prefix_graphs: CodePredictorWrapper.__init__ computes _prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu. The sibling profile vllm_omni/deploy/qwen3_tts_high_concurrency.yaml still sets code_predictor_prefix_graphs: true and documents it as the 'c64 hot path', so after this PR that advertised optimization is silently inert on GPU. Update that profile in this PR, or keep prefix graphs available on the eager path. (evidence: Changed by this PR: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu(),`. Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/common/qwen3_code_predictor.py:477 `self._prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu`; vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:25 `# Stage0 code-predictor prefix CUDA graphs for the c64 hot path.`; vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:27 `code_predictor_prefix_graphs: true` (diff_stat confirms qwen3_tts_high_concurrency.yaml is not in the PR's changed-file list).) + +`vllm_omni/deploy/qwen3_tts.yaml:48` [minor] — Confirmed against the PR-time tree: stage 0 in the default qwen3_tts.yaml is now max_num_seqs: 64 and max_num_batched_tokens: 32768, while the file's own header (lines 6-7) still documents 512 as the deliberate async-chunk latency tuning value, and qwen3_tts_high_concurrency.yaml:54 remains the separate c64/PROMPTS=512 profile with max_num_batched_tokens: 512. Revert the default capacity knobs to the pre-PR values (or move the new values into qwen3_tts_high_concurrency.yaml) and update the header comment accordingly, so the graph/async code fix does not silently retune first-chunk latency and memory for every consumer that loads qwen3_tts.yaml via get_deploy_config_path. (evidence: vllm_omni/deploy/qwen3_tts.yaml:38 ` max_num_seqs: 64`; vllm_omni/deploy/qwen3_tts.yaml:48 ` max_num_batched_tokens: 32768`; unchanged by this diff, present in the PR-time tree — vllm_omni/deploy/qwen3_tts.yaml:6-7 `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk` / `# latency tuning (not correctness) — small per-step batches keep`; vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:3 `# This is intentionally separate from qwen3_tts.yaml. Use it only when running`; vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:54 ` max_num_batched_tokens: 512`; consumer outside the diff — tests/e2e/online_serving/test_qwen3_tts_base.py:49 ` stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),`) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:351` [minor] — Confirmed: the PR adds use_async_omni_output/eager_omni_postprocess_before_async_output (qwen3_tts_talker.py:351-352) and new talker_mtp_accepts_per_row_generators gating (line 350), but no Qwen3-TTS-specific test exercises these; the only coverage is the generic runner guard in tests/worker/test_gpu_ar_model_runner.py:376-392, which is unchanged by this diff. Add a Qwen3-TTS unit test (stub the talker with a VllmConfig carrying full CUDAGraph mode; assert talker_mtp_accepts_per_row_generators is False and _should_use_async_omni_output is True with prefix cache off), or run tests/e2e/online_serving/test_qwen3_tts_base.py against the updated default deploy yaml and record results. (evidence: ['vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped` — new cudagraph-mode gating added by this diff.', "vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:351 `self.use_async_omni_output = True` — flag added by this diff, matching the comment's anchor.", 'vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:352 `self.eager_omni_postprocess_before_async_output = True` — flag added by this diff.', 'tests/worker/test_gpu_ar_model_runner.py:380 `runner.model.use_async_omni_output = True` and tests/worker/test_gpu_ar_model_runner.py:391 `runner.model.eager_omni_postprocess_before_async_output = True` — generic runner-guard coverage; this test file is unchanged by this diff (diff_stat shows only 3 changed files, none under tests/).', 'Grep over tests/model_executor/models/qwen3_tts found no matches for cudagraph_mode or talker_mtp_accepts_per_row_generators, confirming the absence of Qwen3-TTS-specific coverage.']) + +`vllm_omni/deploy/qwen3_tts.yaml:~43` [minor] — At qwen3_tts.yaml:46 enable_prefix_caching is false, but lines 41-45 still describe it as on-by-default and a net throughput win. The runner mutual exclusion is real: gpu_ar_model_runner.py:1540-1541 returns False from _should_use_async_omni_output when omni_prefix_cache is set, and this PR's qwen3_tts_talker.py sets use_async_omni_output=True. Also update the stale header (lines 6-8 still say Stage 0 sets max_num_batched_tokens=512; the stage now sets 32768 at line 48) and the stage-1 note (line 69 says TTFA-optimal at 1; max_num_seqs is now 64 at line 70). (evidence: vllm_omni/deploy/qwen3_tts.yaml:41 ` # Prefix caching on by default. The HS opt-out (talker only consumes the` ... vllm_omni/deploy/qwen3_tts.yaml:45 ` # Disable explicitly if memory headroom is tight on a single-GPU deploy.` then vllm_omni/deploy/qwen3_tts.yaml:46 ` enable_prefix_caching: false`. Stale header: vllm_omni/deploy/qwen3_tts.yaml:6 `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk` while vllm_omni/deploy/qwen3_tts.yaml:48 ` max_num_batched_tokens: 32768`. Stale stage-1 note: vllm_omni/deploy/qwen3_tts.yaml:69 ` # Tuned for Base voice clone; CustomVoice / VoiceDesign are TTFA-optimal at 1.` while vllm_omni/deploy/qwen3_tts.yaml:70 ` max_num_seqs: 64`. Mutual exclusion, unchanged by this diff and present in the PR-time tree: vllm_omni/worker/gpu_ar_model_runner.py:1540 ` if self.omni_prefix_cache is not None:` and vllm_omni/worker/gpu_ar_model_runner.py:1541 ` return False` inside _should_use_async_omni_output.) + +`vllm_omni/deploy/qwen3_tts.yaml:~48` [minor] — Confirmed: both stages raise max_num_seqs 10→64 and stage-0 max_num_batched_tokens 512→32768 (qwen3_tts.yaml:38,48,70). The unchanged header (lines 6-8) documents 512 as a first-chunk-latency tuning choice, yet the only reported benchmark caps concurrency at 8 and reports 21.88% streaming continuity OK with no pre-change continuity or TTFP baseline. Please add before/after TTFP + streaming continuity at concurrency 16/32 or scope the retune to the validated range. (evidence: PR-time tree, unchanged header — vllm_omni/deploy/qwen3_tts.yaml:6-8 `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk` / `# latency tuning (not correctness) — small per-step batches keep` / `# first-chunk latency low.`; changed by this diff — vllm_omni/deploy/qwen3_tts.yaml:38 ` max_num_seqs: 64`; vllm_omni/deploy/qwen3_tts.yaml:48 ` max_num_batched_tokens: 32768`; vllm_omni/deploy/qwen3_tts.yaml:70 ` max_num_seqs: 64`; PR body (not in the diff): `Maximum request concurrency: 8`, `Streaming continuity OK rate: 21.88%`, `Mean AUDIO_TTFP (ms): 272.73`.) + +`vllm_omni/deploy/qwen3_tts.yaml:~48` [minor] — Stale docs: vllm_omni/deploy/qwen3_tts.yaml:6 still says Stage 0 sets `max_num_batched_tokens=512` while line 48 now sets 32768, and docs/user_guide/examples/online_serving/text_to_speech.md:334 still says the bundled YAML ships stage 1 at `max_num_seqs: 10` while both stages now set 64. Since `max_num_seqs` (64) already caps decode batch size below 512, the 512→32768 raise affects prefill chunking/latency, not decode batch size; update both stale texts and give the prefill-latency/capacity rationale. (evidence: docs/user_guide/examples/online_serving/text_to_speech.md:334 `The bundled \`qwen3_tts.yaml\` ships stage 1 (Code2Wav) at \`max_num_seqs: 10\`, tuned for Base voice cloning: stage-1 lifetimes are long (~3 s/req), so admitting up to 10 concurrent codec sequences lets requests progress in parallel in the scheduler — ~2× TTFA p95 at c=4 / c=8 (1× H100, 1.7B-Base, seed-tts) at an 8–12 % audio-throughput cost.` — unchanged by this diff, present in the PR-time tree; vllm_omni/deploy/qwen3_tts.yaml:6 `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk` — unchanged header comment; vllm_omni/deploy/qwen3_tts.yaml:48 `max_num_batched_tokens: 32768` (diff raises it from 512).) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — This hunk (qwen3_tts_talker.py:350) makes talker_mtp_accepts_per_row_generators False whenever full cudagraphs are active, so per-request tts_local_seed reproducibility is silently dropped for decode batches > 1 — the diff's own comment (lines 343-344) acknowledges the drop. The seed is still set from request.seed with no warning (serving_speech.py:3694). Add a logger.warning_once when a seeded batch hits the graph-wrapped path and/or note the determinism change in the speech API docs, and link the runner-side cudagraph-safe-generator follow-up instead of leaving it as a bare TODO(#4923). (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350 (changed by this PR): `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`; lines 343-344 (changed by this PR): `# is silently dropped when the decode batch > 1 under full cudagraphs;` / `# be cudagraph-safe so seeded batches stay reproducible.`; unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/serving_speech.py:3694 `stage0_params.extra_args["tts_local_seed"] = request.seed`) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — This hunk introduces new model-level behavior — `talker_mtp_accepts_per_row_generators` is now derived from `cudagraph_mode.has_full_cudagraphs()` at vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:346-350 — but no test file is touched. The only existing coverage (tests/worker/test_omni_gpu_model_runner.py:374) hard-codes `talker_mtp_accepts_per_row_generators=True` and asserts runner consumption, not this derivation. Add a focused test for FULL vs NONE/PIECEWISE (or extract the decision into a helper and test it) so a future cudagraph-mode default change is caught. Also fill the empty 'Test Plan'/'vLLM Version'/'vLLM-Omni Commit' fields in the PR body. (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:346 `cudagraph_mode = getattr(vllm_config.compilation_config, "cudagraph_mode", None)`; :347-349 `talker_mtp_graph_wrapped = (self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs())`; :350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`. tests/worker/test_omni_gpu_model_runner.py:374 (unchanged by this diff, present in PR-time tree) `talker_mtp_accepts_per_row_generators=True,` inside `test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models`; the PR diff (`git diff --stat`) lists only `vllm_omni/deploy/qwen3_tts.yaml`, `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py`, and `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py` — no test files.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/deploy/qwen3_tts.yaml:98` [minor] Anchor and technical premises verified: this diff adds platforms.npu cudagraph_mode: PIECEWISE for both stages (qwen3_tts.yaml:98-102), buildkite/vllm-omni-npu-ci is failing, PIECEWISE does not take the has_full_cudagrap +- `vllm_omni/deploy/qwen3_tts.yaml:99` [minor] `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are FAILURE at this head. This diff intersects both lanes: NPU via the `is_npu()` code-predictor branch and the new NPU `cudagraph_mode: PIECEWISE` overrides, +- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39` [minor] `use_cuda_graphs` was previously `True` on every platform and is now `current_omni_platform.is_npu()`, so on AMD/ROCm (`interface.py:23` `ROCM = "rocm"`, `interface.py:42-43` is_npu==NPU only) it flips True→False; the sa +- `vllm_omni/deploy/qwen3_tts.yaml:46` [minor] At vllm_omni/worker/gpu_ar_model_runner.py:1540, `_should_use_async_omni_output` returns False when an Omni prefix cache is active with no log/warning. Since qwen3_tts_talker.py:351 now sets `use_async_omni_output = True + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.trace.json.gz new file mode 100644 index 000000000..1bb56b370 Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.cost.json new file mode 100644 index 000000000..f8116662c --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1397.6, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4926/run-20260815-144431-16e6dd", + "recorded_at": "2026-08-15T15:07:46", + "attempts": [ + { + "invocation_id": "inv-02f68d6e", + "outcome": "run:run-20260815-144431-16e6dd", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-144431-16e6dd", + "llm_calls": 136, + "input_tokens": 782301, + "output_tokens": 245384, + "usd": 0.722041, + "source": "spans" + } + ], + "attempt_usd_total": 0.722041, + "llm_calls": 136, + "input_tokens": 782301, + "output_tokens": 245384, + "usd": 0.722041, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-144431-16e6dd", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.722, + "minutes": 23.25, + "ci_minutes": 0, + "input_tokens": 782301, + "output_tokens": 245384, + "tool_calls": 336, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.881, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 136, + "by_role": { + "agent": { + "usd": 0.673998, + "input_tokens": 751146, + "output_tokens": 209422, + "calls": 133 + }, + "reducer": { + "usd": 0.048043, + "input_tokens": 31155, + "output_tokens": 35962, + "calls": 3 + } + }, + "cache_read_tokens": 8922112, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.77, + "gate": 1.67, + "review": 1389.28, + "report": 0.06 + }, + "time_to_first_result_s": 1394.76 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 43313 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.md new file mode 100644 index 000000000..2897137ff --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.md @@ -0,0 +1,64 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] piecewise FA3 hub varlen — _flash_wrapper uses varlen when flash_attn_func is None; checked piecewise_attn.py caller contract +- [resolved] pre-Hopper FA3_HUB platform gate — cuda/platform.py:140-147 falls back to FLASH_ATTN_HUB before enum resolve +- [resolved] Concern: FA3 output tuple unwrap — _unwrap_flash_output handles (out, lse); residue: FA3 varlen path (if present) is not covered by any test. +- [resolved] Concern: module-scope import breaking non-CUDA collection — imports are inside functions; residue: the fallback test still forces the CUDA platform path even when running on AMD CPU lane. +- [resolved] masked varlen hole — flash_attn_hub.py:108-109 only raises when BOTH funcs missing; supports_attention_mask True + _forward_varlen_masked still calls varlen unconditionally (residual: masked/piecewise paths untested) +- [resolved] kernel pin — flash_attn_hub.py:96-104 version=1→2→default per layer __init__; no module-level cache (residual: Hub revision drift vs train) +- [resolved] cu_seqlens — flash_attn_hub.py:32 same arange pattern as flash_attn.py:141-142; batch>1 endpoint missing in both (residual: multi-seq varlen wrong offsets) +- [claim-verified] PR unit tests pass claim — test file has core_model+cpu and hardware_test L4 markers collected by test-ready.yml +- [claim-verified] PR purpose (two backends + registry + platform fallbacks + docs + tests) is delivered in the diff. +- [claim-verified] Tests match the PR's test plan (fallback routing + dense execution vs SDPA with 1e-2 threshold); file contents confirm the assertions. +- [claim-verified] PR unit tests pass per pr_context; execution test uses hardware_test L4 where FA3 branch skips at major<9 +- [claim-refuted] PR 'graceful fallback when kernels unavailable' — only ImportError of kernels package triggers fallback; Hub load failure still selects hub backend +- [claim-refuted] docs 'perfect numerical consistency' — tests compare to SDPA at max_diff < 1e-2, not diffusers Hub parity +- [claim-refuted] PR body 'All unit tests pass' shows 'FlashAttention3Hub PASSED', but at this head tests/diffusion/attention/test_kernels_hub.py:92 `if major < 9: pytest.skip` makes the FA3 block self-skip on the only marked hardware (L4, sm_89) — the pasted result does not describe head. + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~96` [major] — Both `FlashAttentionHubImpl.__init__` (flash_attn_hub.py:96/100/103) and `FlashAttention3HubImpl.__init__` (flash_attn_hub.py:275/279/282) call `get_kernel(...)` at construction time with no pinned revision and no module-level memoization, so the loaded kernel can change between serving runs and is re-fetched per instance. Since attention_backends.md:217 promises 'perfect numerical consistency' between training and serving/rollout, pin a concrete git revision per kernel and cache the resolved module (`lru_cache`) at module level, or fail closed when the pin is unavailable. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)`; :100 `hub_module = get_kernel("kernels-community/flash-attn2", version=2)`; :103 `hub_module = get_kernel("kernels-community/flash-attn2")` — all inside `FlashAttentionHubImpl.__init__` (def at line ~83), no `lru_cache`/revision pin anywhere in the file; duplicate unpinned FA3 resolution at :275 `hub_module = get_kernel("kernels-community/flash-attn3", version=1)`, :279 `...version=2`, :282 `...flash-attn3")`. docs/user_guide/diffusion/attention_backends.md:217 `To achieve perfect numerical consistency between **training** ... and **serving/rollout** ... you can use the Hub-based attention backends. This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout.`) + +`tests/diffusion/attention/test_kernels_hub.py:92` [major] — The new test_kernels_hub_execution backing the PR's core behavioral claim is never collected by ready/merge CI: .buildkite/test-ready.yml reaches tests/diffusion/attention/ only through `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` (which deselects the cuda-only test), and the cuda lane's `pytest -sv tests/diffusion/*.py` glob does not descend into tests/diffusion/attention/. On the only cuda lane hardware there (L4, cc 8.9) the `if major < 9` branch self-skips FA3, so CI green proves nothing about either FA2 or FA3 hub execution. Add a cuda/H100 lane that selects this file (or move the execution test under an already-selected path) and run it on sm90. Note: the @hardware_test decorator is at line 57, not 59. (evidence: Unchanged by this diff, present in the PR-time tree: .buildkite/test-ready.yml:13 `"timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; .buildkite/test-ready.yml:145 `- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"`. From the diff: tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)`; tests/diffusion/attention/test_kernels_hub.py:92 `if major < 9:`.) + +`vllm_omni/platforms/cuda/platform.py:126` [major] — Platform selection only checks `import kernels` (cuda/platform.py:126); if the package imports but Hub kernel fetch/load fails (offline, rate limit, bad cache), backend_upper stays FLASH_ATTN_HUB and layer init raises — either the propagated `get_kernel` exception or the RuntimeError at flash_attn_hub.py:109 when no functions are found. Docs (attention_backends.md:231) promise graceful fallback only when the `kernels` package is missing. Probe `get_kernel` in the platform selector and fall back to FLASH_ATTN on failure, or document that Hub backends require a pre-cached Hub artifact. (evidence: vllm_omni/platforms/cuda/platform.py:126 `importlib.import_module("kernels")` and :127 `logger.info("Using HuggingFace kernels-backed attention backend '%s'", backend_upper)` — selector probes import only (changed by this diff). vllm_omni/diffusion/attention/backends/flash_attn_hub.py:103 `hub_module = get_kernel("kernels-community/flash-attn2")` and :109 `raise RuntimeError("Failed to load flash-attn2 kernel from HuggingFace Hub: no functions found")` — no graceful fallback at init (changed by this diff). docs/user_guide/diffusion/attention_backends.md:231 `If the `kernels` library is not available in the environment, vLLM-Omni will log a warning and fall back gracefully to the corresponding local backend implementations (`FLASH_ATTN`).` — fallback is documented only for missing package, not failed fetch.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~147` [major] — flash_attn_hub.py:106-109 (FA3 copy at 285-288) only raises when both `flash_attn_func` and `flash_attn_varlen_func` are None, yet `supports_attention_mask()` returns True and `forward_cuda` (205/384) sends any masked input to `_forward_varlen_masked`, which unconditionally calls `self.flash_attn_varlen_func` (147/326). A Hub module exposing only `flash_attn_func` passes the guard and then raises `TypeError: 'NoneType' object is not callable` on masked attention. Reject missing varlen in __init__ when mask support is advertised, derive mask support from varlen availability, or add a `flash_attn_func`+padding fallback. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:106 `self.flash_attn_varlen_func = getattr(hub_module, "flash_attn_varlen_func", None)`; :108 `if self.flash_attn_func is None and self.flash_attn_varlen_func is None:`; :109 `raise RuntimeError("Failed to load flash-attn2 kernel from HuggingFace Hub: no functions found")`; :55-56 `def supports_attention_mask(cls) -> bool: return True`; :205 `if attention_mask is not None and torch.any(~attention_mask):`; :147 `out_unpad = self.flash_attn_varlen_func(`. Same hole in the FA3 copy: :287 `if self.flash_attn_func is None and self.flash_attn_varlen_func is None:`; :384 `if attention_mask is not None and torch.any(~attention_mask):`; :326 `out_unpad = self.flash_attn_varlen_func(`.) + +`tests/diffusion/attention/test_kernels_hub.py:92` [minor] — Add regression tests for masked attention (_forward_varlen_masked) and piecewise full_attn_spans. test_kernels_hub_execution only covers dense unmasked forward (attn_metadata_sdpa = AttentionMetadata(attn_mask=None)); on L4 (sm_89) the FA3 hub block is skipped at line 92, so FLASH_ATTN_3_HUB has no automated forward coverage in the default L4 lane. (evidence: tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` — default lane is L4 (sm_89). tests/diffusion/attention/test_kernels_hub.py:75 ` attn_metadata_sdpa = AttentionMetadata(attn_mask=None)` — only unmasked metadata is constructed. tests/diffusion/attention/test_kernels_hub.py:84 ` output_fa_hub = fa_hub_impl.forward(q.clone(), k.clone(), v.clone(), attn_metadata_sdpa)` — dense unmasked forward only. tests/diffusion/attention/test_kernels_hub.py:92 ` if major < 9:` and tests/diffusion/attention/test_kernels_hub.py:93 ` pytest.skip("FLASH_ATTN_3_HUB execution requires Hopper-class GPU (compute capability >= 9.0)")` — FA3 block skipped on L4. Grep for `_forward_varlen_masked`, `full_attn_spans`, and `piecewise_attn` under tests/diffusion/attention returned no matches, so those newly added backend paths are untested.) + +`vllm_omni/platforms/cuda/platform.py:~138` [minor] — When `kernels` is unavailable, FLASH_ATTN_3_HUB falls back to local FLASH_ATTN (FA2), not a local FA3 path; the registry has no FLASH_ATTN_3 enum member, so a local-FA3 fallback cannot currently be targeted. The missing-kernels fallback to FLASH_ATTN is already documented; the actionable follow-up would be registering a local FLASH_ATTN_3 backend if one is ever added. (evidence: vllm_omni/platforms/cuda/platform.py:134-138 (this diff) `elif backend_upper == "FLASH_ATTN_3_HUB": ... backend_upper = "FLASH_ATTN"` — the ImportError branch rewires FA3-Hub to local FA2. vllm_omni/diffusion/attention/backends/registry.py:66 (this diff) `FLASH_ATTN_3_HUB = "vllm_omni.diffusion.attention.backends.flash_attn_hub.FlashAttention3HubBackend"` — no local `FLASH_ATTN_3` enum member exists.) + +`vllm_omni/platforms/cuda/platform.py:~126` [minor] — The fallback contract says 'fall back gracefully' when kernels is unavailable, but `vllm_omni/platforms/cuda/platform.py:126-128` catches only `ImportError` around `importlib.import_module("kernels")`. The flashinfer probe directly above (lines 116-120) documents that partially-installed/ABI-mismatched wheels raise `OSError`/`RuntimeError`, not just `ImportError`; a broken kernels install would therefore crash backend selection instead of falling back. Mirror the flashinfer probe's broad `except Exception` handling with a debug log. (evidence: vllm_omni/platforms/cuda/platform.py:126 `importlib.import_module("kernels")` (added by this PR); vllm_omni/platforms/cuda/platform.py:128 `except ImportError:` (added by this PR); unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/cuda/platform.py:116 `# A partially installed / ABI-mismatched wheel can raise OSError or` and :117 `# RuntimeError from extension loading, not just ImportError. This`) + +`pyproject.toml:70` [minor] — pyproject.toml:70 `kernels<0.15.0` is only in the `dev` extra, so a normal install never includes the package that `vllm_omni/diffusion/attention/backends/flash_attn_hub.py` imports at runtime; the CUDA platform then silently falls back to local FLASH_ATTN on ImportError. Add `kernels` to a runtime/optional extra pinned to the documented 0.14.1 (or mark the Hub backends dev-extra-only), and align the pyproject range with the `get_kernel(version=...)` contract. (evidence: pyproject.toml:70 `"kernels<0.15.0", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)` — inside [project.optional-dependencies].dev; this file is unchanged by the diff but present in the PR-time tree, and grep shows no `kernels` entry in setup.py or requirements/. New in this diff: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92 `from kernels import get_kernel`; vllm_omni/platforms/cuda/platform.py:125 `importlib.import_module("kernels")` and :130 `backend_upper = "FLASH_ATTN"` inside `except ImportError`.) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/attention/test_kernels_hub.py:19` [minor] The only new platform-routing test imports and drives CudaOmniPlatform exclusively (tests/diffusion/attention/test_kernels_hub.py:24,42,46,52), and repo grep shows no test touching MUSA/NPU/ROCm/XPU selectors. The hub→FL +- `vllm_omni/platforms/rocm/platform.py:94` [minor] At head, buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are FAILURE (gh_ci_read 4926). This diff adds fallback hunks to vllm_omni/platforms/rocm/platform.py:94 and vllm_omni/platforms/npu/platform.py:108 (both +- `docs/user_guide/diffusion/attention_backends.md:23` [minor] docs/user_guide/diffusion/attention_backends.md:23-24 — The two new Backend Options rows omit the qualifiers carried by sibling optional-backend rows. Please state for each row: Requires `kernels` (pip install kernels==0 + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.trace.json.gz new file mode 100644 index 000000000..8e31b4e90 Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.cost.json new file mode 100644 index 000000000..cb9b7c132 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 316.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4950/run-20260815-145840-63ee78", + "recorded_at": "2026-08-15T15:03:54", + "attempts": [ + { + "invocation_id": "inv-e64c0cd2", + "outcome": "run:run-20260815-145840-63ee78", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-145840-63ee78", + "llm_calls": 20, + "input_tokens": 172655, + "output_tokens": 28432, + "usd": 0.094989, + "source": "spans" + } + ], + "attempt_usd_total": 0.094989, + "llm_calls": 20, + "input_tokens": 172655, + "output_tokens": 28432, + "usd": 0.094989, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-145840-63ee78", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.095, + "minutes": 5.23, + "ci_minutes": 0, + "input_tokens": 172655, + "output_tokens": 28432, + "tool_calls": 35, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2294, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 20, + "by_role": { + "agent": { + "usd": 0.094989, + "input_tokens": 172655, + "output_tokens": 28432, + "calls": 20 + } + }, + "cache_read_tokens": 633216, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.75, + "gate": 1.73, + "review": 308.39, + "report": 0.06 + }, + "time_to_first_result_s": 313.91 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4950.trace.json.gz", + "trace_bytes": 10359 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.md new file mode 100644 index 000000000..737289432 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.md @@ -0,0 +1,37 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4950, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 48d4fd8a5e98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] Top-level chat_template_kwargs is what drives rendering: serving_chat.py:373-374 validates `request.chat_template_kwargs` and serving_chat.py:381 passes `self._effective_chat_template_kwargs(request)` as default_template_kwargs. +- [claim-verified] curl does not flatten nested extra_body; serving_chat.py:438-441 comment: 'When sending raw request with curl, no flattening happens. Directly read the `extra_body` dict.' — matches the recipe's new caveat. +- [validated] Audio-in-a-separate-choice shape is plausible: serving_chat.py:2571 builds `message=ChatMessage(role=role, audio=audio_obj)`; full two-choice assembly not traced end-to-end. +- [sweep] Diff is docs-only: single changed file recipes/OpenBMB/MiniCPM-o-4_5.md (per pr_diff and sweep_targets); no tests touched. +- [sweep] Repo-wide grep for chat_template_kwargs found a stale sibling: examples/online_serving/minicpmo/README.md:58 still recommends `extra_body.chat_template_kwargs.use_tts_template=True`; gradio_demo.py:223 uses the key (transport context not read). +- [sweep] No test files added/moved/removed in this diff, so no added-test epistemics apply; the behavior claim rests on the author's live POST and the code path verified above. + +`recipes/OpenBMB/MiniCPM-o-4_5.md:106` [minor] — This PR changes only recipes/OpenBMB/MiniCPM-o-4_5.md (docs-only; 19 insertions, 8 deletions). The markdown edit cannot exercise the NPU CI lane's code/test/config, so the failing buildkite/vllm-omni-npu-ci check is pre-existing/unrelated to this diff; please confirm before merging rather than treating the red lane as caused by this PR. (evidence: recipes/OpenBMB/MiniCPM-o-4_5.md:106 `use_tts_template=true` via the **top-level** `chat_template_kwargs` — the cited anchor exists verbatim at the cited line (unchanged by this diff's context, part of the PR-time tree). Diff scope (from `git diff --stat merge-base..HEAD`): `recipes/OpenBMB/MiniCPM-o-4_5.md | 27 +++++++++++++++++++--------` / `1 file changed, 19 insertions(+), 8 deletions(-)` — docs-only, no code/test/config touched. CI (gh_ci_read for PR 4950): `{"bucket":"fail","name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}` while `buildkite/vllm-omni`, `buildkite/vllm-omni-amd-ci`, `buildkite/vllm-omni-intel-ci`, `pre-commit`, `DCO`, and `docs/readthedocs.org:vllm-omni` are SUCCESS.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.trace.json.gz new file mode 100644 index 000000000..9a18794cb Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4950.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.cost.json new file mode 100644 index 000000000..76932c3e9 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 486.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4970/run-20260815-145647-2fa767", + "recorded_at": "2026-08-15T15:04:51", + "attempts": [ + { + "invocation_id": "inv-924cd240", + "outcome": "run:run-20260815-145647-2fa767", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-145647-2fa767", + "llm_calls": 23, + "input_tokens": 248802, + "output_tokens": 60039, + "usd": 0.160297, + "source": "spans" + } + ], + "attempt_usd_total": 0.160297, + "llm_calls": 23, + "input_tokens": 248802, + "output_tokens": 60039, + "usd": 0.160297, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-145647-2fa767", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1603, + "minutes": 8.06, + "ci_minutes": 0, + "input_tokens": 248802, + "output_tokens": 60039, + "tool_calls": 58, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3379, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 23, + "by_role": { + "agent": { + "usd": 0.160297, + "input_tokens": 248802, + "output_tokens": 60039, + "calls": 23 + } + }, + "cache_read_tokens": 1002880, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.31, + "gate": 1.53, + "review": 478.53, + "report": 0.06 + }, + "time_to_first_result_s": 483.73 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4970.trace.json.gz", + "trace_bytes": 13554 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.md new file mode 100644 index 000000000..8a317e2ef --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4970, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4992e10b7367) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Concern 'a red lane loads this yaml' is confirmed as intersecting, not resolved; the fix does NOT cover amd/npu hardware correctness — attribution of those red checks remains unproven and is raised as a question, not a blocker. +- [claim-verified] serving_speech.py:3696-3706 gates deploy-default-seed propagation to `_tts_model_type == "qwen3_tts"` via `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`; with seed absent from the yaml, no request carries tts_local_seed. +- [claim-verified] Per-request determinism preserved: serving_speech.py `if request.seed is not None: stage0_params.seed = request.seed; stage0_params.extra_args["tts_local_seed"] = request.seed` remains intact. +- [claim-verified] "no test references the deploy-level seed": grep found no `seed` in tests/e2e qwen3_tts customvoice/base files nor tests/test_config_factory.py; the only propagation test (test_serving_speech.py:3347) mocks `SimpleNamespace(max_tokens=2048, seed=42, extra_args=None)` directly rather than loading the yaml. +- [claim-verified] Nightly perf launches the default profile: benchmarks/tts/README.md:20 'The server auto-loads its Deploy YAML from vllm_omni/deploy/qwen3_tts.yaml'; quick-start `vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni`; quality eval `enabled: false` in CI. +- [claim-verified] Merge commit 4992e10b: conflict only in qwen3_tts.yaml; resolution keeps upstream `min_tokens: 2` and drops branch `seed: 42`; show_commit + diff_stat confirm net diff is still the 2 deletions. +- [validated] diff_stat: only vllm_omni/deploy/qwen3_tts.yaml changed; net 2 deletions (merge-base da15a4cb1cfe). +- [validated] Hunk 1 in situ: stage 0 default_sampling_params is now `min_tokens: 2` -> `repetition_penalty: 1.05` (qwen3_tts.yaml:60-61); Hunk 2: stage 1 ends `max_tokens: 65536` -> `repetition_penalty: 1.0` (lines 90-91). +- [validated] gpu_model_runner.py `_talker_mtp_forward`: `_explicit_talker_seed` reads `extra_args.get("tts_local_seed")`; all-None row_generators -> no generator kwarg -> single batched multinomial; scalar per-row fallback only fires when a seed is present and `talker_mtp_accepts_per_row_generators` is false. +- [sweep] Sibling contrast: qwen3_tts_high_concurrency.yaml and qwen3_tts_forced_aligner.yaml contain no `seed: 42` (grep), so no qwen3_tts deploy sibling remains exposed. +- [sweep] The ~70 other deploy yamls shipping `seed: 42` are non-qwen3 models; tts_local_seed propagation is qwen3_tts-gated, so they use standard vLLM RANDOM_SEED and do not hit the per-row multinomial cliff. +- [sweep] docs/configuration/stage_configs.md `seed: 42` examples (274/298/325) sit inside the legacy Qwen2.5-Omni stage_args example; the `default_sampling_params.seed` section (494-496) remains generally true — no doc is made stale by this PR. +- [sweep] Stage 1 seed removal is safe: stage 1 has temperature 0.0 (greedy), so the removed seed had no sampling effect. +- [sweep] PR's quoted regression numbers are internally consistent: TTFP 658->1134 (1.72x), RTF 0.365->0.635 (1.74x), audio_throughput 40.3->24.2 (0.60x, ~1/1.67). + +`.buildkite/test-npu-ready.yaml:~21` [minor] — This diff touches only vllm_omni/deploy/qwen3_tts.yaml, and both failing lanes load that yaml: the NPU ready lane runs exactly this test (tests/e2e/online_serving/test_qwen3_tts_customvoice.py:23 resolves get_deploy_config_path("qwen3_tts.yaml")) and the AMD merge lane runs the base+customvoice pair. The main buildkite/vllm-omni lane is green, so please confirm whether buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are red on main too (pre-existing) or whether the seed removal exposed a lane-specific issue. (evidence: ['diff_stat: `vllm_omni/deploy/qwen3_tts.yaml | 2 --` / `1 file changed, 2 deletions(-)` — the only changed file.', ".buildkite/test-npu-ready.yaml:21 `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'` (unchanged by this diff, present in PR-time tree)", 'tests/e2e/online_serving/test_qwen3_tts_customvoice.py:23 ` get_deploy_config_path("qwen3_tts.yaml"),` (unchanged by this diff, present in PR-time tree)', '.buildkite/test-amd-merge.yml:177 ` pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"` (unchanged by this diff, present in PR-time tree)', ".buildkite/test-amd-merge.yml:192 ` pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'` (unchanged by this diff, present in PR-time tree)", 'gh_ci_read(4970): `buildkite/vllm-omni-amd-ci` state FAILURE, `buildkite/vllm-omni-npu-ci` state FAILURE, `buildkite/vllm-omni` state SUCCESS.']) + +`vllm_omni/deploy/qwen3_tts.yaml:~60` [minor] — Confirmed. This diff removes `seed: 42` from both `default_sampling_params` blocks, but no comment records that the seed must stay unset. A future 'align defaults' sweep or determinism backport could silently re-add it: `serving_speech.py` propagates any default `seed` into `extra_args['tts_local_seed']`, which `gpu_model_runner.py` converts into a per-row `torch.Generator` and scalarizes the batched talker-MTP decode path (see #4963). Add one `# no default seed: ... (see #4963)` tombstone comment in each `default_sampling_params` block, matching the file's existing `# See #...` style. (evidence: ['vllm_omni/deploy/qwen3_tts.yaml:60 `min_tokens: 2` — the only nearby `# See #...` comment is line 59 `# audio (HTTP 400). See #4962.`, which documents `min_tokens`; no `seed:` and no tombstone comment follows line 60 in either `default_sampling_params` block (line 61 is `repetition_penalty: 1.05`).', 'vllm_omni/entrypoints/openai/serving_speech.py:3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))` — unchanged by this diff, present in the PR-time tree; this is where a deploy-level default `seed` is propagated into every unseeded request\'s `tts_local_seed` for qwen3_tts.', 'vllm_omni/worker/gpu_model_runner.py:1826 `seed = extra_args.get("tts_local_seed")` — unchanged by this diff; `_explicit_talker_seed` feeds `_row_generator`, which builds a per-row `torch.Generator`, and the batched path falls back to scalar when `any(generator is not None for generator in row_generators)`.', 'vllm_omni/deploy/moss_tts.yaml:36 `seed: 42` — representative sibling deploy yaml still shipping `seed: 42` (grep confirms dozens of matches under vllm_omni/deploy).']) + +`tests/entrypoints/openai_api/test_serving_speech.py:~3347` [minor] — This diff makes the bundled qwen3_tts.yaml seedless, and correctness depends on the invariant that a seedless qwen3_tts default does not set tts_local_seed. serving_speech.py only propagates the deploy default via `setdefault("tts_local_seed", int(default_seed))` when `stage0_params.seed` is not None, and the only serving-side test (test_serving_speech.py:3347, asserting `extra_args["tts_local_seed"] == 42` at line 3365) covers the opposite direction with a mocked seed=42. tests/test_config_factory.py:1270 loads qwen3_tts.yaml without any seed assertion. Add a sibling guard that loads the bundled deploy config (or drives _prepare_speech_generation with a seedless default) and asserts stage0_params.seed is None / "tts_local_seed" is absent, so re-introducing `seed: 42` fails CI. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/serving_speech.py:3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))` — this qwen3_tts default-seed path only runs when `default_seed = getattr(stage0_params, "seed", None)` is not None, so a seedless deploy default leaves `tts_local_seed` unset. tests/entrypoints/openai_api/test_serving_speech.py:3365 `assert stage0_params.extra_args["tts_local_seed"] == 42` — the cited test asserts the opposite (seed present -> tts_local_seed set). vllm_omni/deploy/qwen3_tts.yaml: grep for 'seed' returns no matches after the diff removed both `- seed: 42` lines.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.trace.json.gz new file mode 100644 index 000000000..543928b4e Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4970.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.cost.json new file mode 100644 index 000000000..dade2f929 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 856.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr4977/run-20260815-144231-61c006", + "recorded_at": "2026-08-15T14:56:45", + "attempts": [ + { + "invocation_id": "inv-691ecf62", + "outcome": "run:run-20260815-144231-61c006", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-144231-61c006", + "llm_calls": 61, + "input_tokens": 244549, + "output_tokens": 129160, + "usd": 0.25767, + "source": "spans" + } + ], + "attempt_usd_total": 0.25767, + "llm_calls": 61, + "input_tokens": 244549, + "output_tokens": 129160, + "usd": 0.25767, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-144231-61c006", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2577, + "minutes": 14.24, + "ci_minutes": 0, + "input_tokens": 244549, + "output_tokens": 129160, + "tool_calls": 171, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5223, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 61, + "by_role": { + "agent": { + "usd": 0.233605, + "input_tokens": 233585, + "output_tokens": 110015, + "calls": 59 + }, + "reducer": { + "usd": 0.024065, + "input_tokens": 10964, + "output_tokens": 19145, + "calls": 2 + } + }, + "cache_read_tokens": 1835776, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.28, + "gate": 1.51, + "review": 849.26, + "report": 0.07 + }, + "time_to_first_result_s": 854.09 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 26035 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.md new file mode 100644 index 000000000..ea2973086 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] trust_remote_code concern — commit 9947f414 dropped it (`kernels 0.13.x get_kernel() does not accept trust_remote_code`); head matches pyproject `kernels<0.15.0` range. +- [resolved] functools.@cache thundering-herd — commit 0f2516f2 replaced with `threading.Lock` + dict, consistent with `_MODEL_CACHE_LOCK` in rife_interpolator.py:29-30. +- [resolved] trust_remote_code dropped at head (commit 9947f414 'kernels 0.13.x get_kernel() does not accept trust_remote_code'); final call is get_kernel(repo_id, version=version)/get_kernel(repo_id) — no new external kwarg. Residual: PR description still advertises it (comment 1). +- [resolved] trust_remote_code removal matches kernels 0.13.x API (head commit: 'kernels 0.13.x get_kernel() does not accept trust_remote_code'); no new external API introduced at head — same get_kernel(repo_id, version=...) calls as base. +- [resolved] Thread-safety: earlier revision 528804e8 used functools.cache; 0f2516f2 replaced it with `_hub_lock`+dict (`with _hub_lock:` serializes load). Residual: the lock is held during the network load, blocking other repo_id loads (only two keys, negligible). +- [claim-verified] PR claim: load once per process per repo — `_hub_modules` keyed by repo_id; FA2 and FA3 are separate entries. +- [claim-verified] PR claim: forward behavior unchanged — `forward_cuda` and helper methods untouched in diff. +- [claim-verified] PR test plan entry `test_kernels_hub_platform_fallback` exists with `@pytest.mark.core_model` + `cpu` (runs in `.buildkite/test-ready.yml` CPU lane). +- [claim-verified] 'reduces redundant Hub traffic': per-layer construction previously re-ran the load chain; the cache now loads once per repo_id per process. +- [claim-verified] 'version fallback preserved': loop (1, 2, None) matches old nested try/except order v1 -> v2 -> default. +- [claim-verified] 'forward/backend selection unchanged': diff only touches the two __init__ load sites; forward_cuda paths untouched. +- [claim-verified] fallback order preserved: head loop `for version in (1, 2, None)` matches base's v1→v2→default (file_at_base :88-99 / :267-277). +- [claim-verified] Fallback order v1→v2→default preserved: old nested try/except vs new `for version in (1, 2, None)`. +- [claim-verified] Load-once reduces per-layer Hub traffic: old __init__ called get_kernel per construction; new `_get_hub_module` caches by repo_id. + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~45` [major] — New `_get_hub_module` caching/retry path (flash_attn_hub.py:20-46) is not exercised by any CI-collected test: the cited `test_kernels_hub_platform_fallback` only calls `CudaOmniPlatform.get_diffusion_attn_backend_cls(...)` and never constructs `FlashAttentionHubImpl`; `test_kernels_hub_execution` is `@hardware_test(res={'cuda':'L4'})` and is not collected by any lane (CPU lanes use `-m 'core_model and cpu'`, deselecting it; cuda lanes cover offloader/cache/batching/distributed/models/`tests/diffusion/*.py` only), and its FA3 branch skips on L4 (`major < 9`). Add a CPU unit test that monkeypatches `kernels.get_kernel` (clearing `_hub_modules` in setup), asserts one load per repo_id, per-repo keying, retry after a raising load, and the v1→v2→default fallback order — or wire the execution test into a lane and show it ran. (evidence: ['vllm_omni/diffusion/attention/backends/flash_attn_hub.py:44-45 `if repo_id not in _hub_modules:\n _hub_modules[repo_id] = _load_hub_module(repo_id)` — new caching path introduced by this diff, exactly at the cited anchor.', 'vllm_omni/diffusion/attention/backends/flash_attn_hub.py:119 `hub_module = _get_hub_module("kernels-community/flash-attn2")` and :284 `hub_module = _get_hub_module("kernels-community/flash-attn3")` — both impls now use the new cached path (changed by this diff).', 'tests/diffusion/attention/test_kernels_hub.py:42 `backend_path = CudaOmniPlatform.get_diffusion_attn_backend_cls("FLASH_ATTN_HUB", head_size=64)` — the cited fallback test only exercises platform selection; no `flash_attn_hub` import or construction appears in that test (unchanged by this diff, present in PR-time tree).', 'tests/diffusion/attention/test_kernels_hub.py:58 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` — the only test that constructs `FlashAttentionHubImpl`/`FlashAttention3HubImpl` is L4/cuda-gated (unchanged by this diff).', 'tests/diffusion/attention/test_kernels_hub.py:89-91 `major, _minor = torch.cuda.get_device_capability()` / `if major < 9:` / `pytest.skip("FLASH_ATTN_3_HUB execution requires Hopper-class GPU (compute capability >= 9.0)")` — FA3 branch skips on L4 (unchanged by this diff).', ".buildkite/test-ready.yml:13 `timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` — CPU lane deselects the L4 execution test; cuda lanes list offloader (:64), cache (:80), batching (:96), distributed (:112), models (:129), and `tests/diffusion/*.py` (:145), none of which collect `tests/diffusion/attention` (unchanged by this diff)."]) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~45` [minor] — At `_get_hub_module`, `_hub_modules[repo_id] = _load_hub_module(repo_id)` (line 45) caches the returned module before the constructors check for `flash_attn_func`/`flash_attn_varlen_func` (lines 123 and 288). A module that imports successfully but exposes neither function would raise the first time, yet remain in `_hub_modules`, so every subsequent layer reuses the bad module. Move the function-presence check into `_load_hub_module` (raise before returning) or validate immediately before assigning `_hub_modules[repo_id]`. (evidence: flash_attn_hub.py:45 `_hub_modules[repo_id] = _load_hub_module(repo_id)` (unchanged by this diff? No — introduced by this diff) caches before validation; flash_attn_hub.py:123 `if self.flash_attn_func is None and self.flash_attn_varlen_func is None: raise RuntimeError(...)` (same for flash-attn3 at line 288). Since `_get_hub_module` returns the cached entry on subsequent calls (`if repo_id not in _hub_modules: ...; return _hub_modules[repo_id]`, lines 42-46), a bad-but-importable module is permanently cached.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~34` [minor] — PR 4977's body still states it adds `trust_remote_code=True` for more reliable Hub access, but the merged head code at flash_attn_hub.py:34-35 calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` with no trust_remote_code. Commit 9947f414 intentionally removed that kwarg because kernels 0.13.x rejects it. Update the PR description to drop the trust_remote_code claim. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:34 `return get_kernel(repo_id, version=version)` and :35 `return get_kernel(repo_id)` — no trust_remote_code kwarg. PR 4977 body: "...adding `trust_remote_code=True` for more reliable Hub access." Commit 9947f414 message: "kernels 0.13.x get_kernel() does not accept trust_remote_code, so passing it made every fallback attempt fail on older installs.") + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:45` [minor] — PR body claims the change 'speeds up worker initialization', but neither the diff nor the Test Plan includes before/after timing; the listed E2E screenshot is not an initialization-time measurement. Add a `time.perf_counter()` measurement around the first `_get_hub_module` load versus per-layer construction, or trim the description to the verifiable effect: one `get_kernel` load per `repo_id` per process instead of per constructed layer. (evidence: PR 4977 body (unchanged by diff, from PR metadata): 'Diffusion models create one attention backend per layer, so this aligns rollout startup with diffusers' single-load hub kernel registry pattern, **reduces redundant HuggingFace Hub traffic**, and speeds up worker initialization while preserving the existing version fallback behavior (`v1` → `v2` → default)'. PR-time file vllm_omni/diffusion/attention/backends/flash_attn_hub.py:45 `_hub_modules[repo_id] = _load_hub_module(repo_id)` — only cache population, no timing; grep of the same file for `perf_counter|time\.time|timeit|elapsed|duration` returns no matches, and the Test Plan lists only a unit test and an E2E screenshot.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:22` [minor] — This diff introduces process-global `_hub_modules` and `_hub_lock` in `flash_attn_hub.py`, while the only failing check at head is `buildkite/vllm-omni-npu-ci`. The NPU ready lane runs only `tests/e2e/online_serving/test_qwen3_tts_customvoice.py`, which has no diffusion import, and `NPUOmniPlatform.get_diffusion_attn_backend_cls` routes FLASH_ATTN_HUB / FLASH_ATTN_3_HUB to local FLASH_ATTN before this backend is used. Please confirm the NPU failure is pre-existing on main rather than introduced by this diff; do not merge red without that check. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:22 `_hub_modules: dict[str, object] = {}` (added by this diff, with `_hub_lock = threading.Lock()` at line 23). Unchanged by this diff, present in the PR-time tree: .buildkite/test-npu-ready.yaml `commands: - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; vllm_omni/platforms/npu/platform.py:110-116 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"): logger.warning("HuggingFace kernels-backed FlashAttention is " "not supported on NPU. Falling back to local " "FLASH_ATTN.")` then `backend_upper = "FLASH_ATTN"`; grep of tests/e2e/online_serving/test_qwen3_tts_customvoice.py for `diffusion` returns no matches. CI: gh_ci_read PR 4977 shows `buildkite/vllm-omni-npu-ci` state FAILURE and all other checks SUCCESS.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.trace.json.gz new file mode 100644 index 000000000..5930f3df6 Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.cost.json b/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.cost.json new file mode 100644 index 000000000..7c6760702 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1266.6, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v16rs_train_r1/runs/pr5009/run-20260815-142041-3fc066", + "recorded_at": "2026-08-15T14:41:46", + "attempts": [ + { + "invocation_id": "inv-77ce47e9", + "outcome": "run:run-20260815-142041-3fc066", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260815-142041-3fc066", + "llm_calls": 93, + "input_tokens": 661405, + "output_tokens": 218828, + "usd": 0.591624, + "source": "spans" + } + ], + "attempt_usd_total": 0.591624, + "llm_calls": 93, + "input_tokens": 661405, + "output_tokens": 218828, + "usd": 0.591624, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260815-142041-3fc066", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5916, + "minutes": 21.08, + "ci_minutes": 0, + "input_tokens": 661405, + "output_tokens": 218828, + "tool_calls": 310, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7937, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 93, + "by_role": { + "agent": { + "usd": 0.537939, + "input_tokens": 638090, + "output_tokens": 175816, + "calls": 90 + }, + "reducer": { + "usd": 0.053684, + "input_tokens": 23315, + "output_tokens": 43012, + "calls": 3 + } + }, + "cache_read_tokens": 6382720, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.09, + "gate": 1.65, + "review": 1258.7, + "report": 0.1 + }, + "time_to_first_result_s": 1264.49 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 36792 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.md b/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.md new file mode 100644 index 000000000..b35713d07 --- /dev/null +++ b/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Weight-loading names unchanged: vLLM RMSNorm and nn.RMSNorm both expose `.weight` under norm_q/norm_k/norm_added_q/norm_added_k/txt_norm; QK norms are per-head over head_dim, so TP sharding is safe. +- [resolved] Dependency floor: torch arrives via vLLM 0.24 (>=2.6) which includes nn.RMSNorm; no new external API/kwarg to floor-check. +- [resolved] Import removal completeness — concern that bare RMSNorm refs might remain; grep confirms none at head. +- [resolved] Weight-loading regression for txt_norm — load_weights :1252-1259 handles txt_norm.weight via default_weight_loader; param names unchanged from base. +- [claim-verified] history via search_history: 35ee3c75 (#4074), 99234881 (#3588), b5589481 (this PR) — the '#3588 reverted #4074' narrative is accurate. +- [claim-verified] only in-repo caller of get_default_ir_op_priority is diffusion_worker.py:175 (_resolve_ir_op_priority); per-model override table has only Cosmos3 (registry.py:549). +- [claim-verified] PR percentages recomputed with calc: -26.9/-16.1/-1.8/+9.2/+7.2 match the body's -26.9/-16/-1.8/+9.2/+7.2. +- [claim-verified] PR numbers re-derived with calc: latency -26.94%, profiler -16.15%, E2E +9.25%/-1.83%/+7.24%, SSIM +0.0309, PSNR +4.89 dB — all match the rounded claims. +- [claim-verified] PR claim 'only in-repo caller is diffusion_worker._resolve_ir_op_priority': get_default_ir_op_priority is defined in cuda/rocm/xpu platforms, invoked at diffusion_worker.py:175, and mocked in tests; no other callers. +- [claim-verified] Cosmos3 per-model override forces native: pipeline_cosmos3.py:619-620 sets rms_norm/fused_add_rms_norm to ["native"], so Cosmos3 is unaffected by the flip. +- [claim-verified] PR arithmetic: -26.9%=(1.7263-1.2613)/1.7263; -16%=(1288-1080)/1288; -1.8%=(2.4960-2.4502)/2.4960; +7.2%=(2.4502-2.2847)/2.2847 — all consistent. +- [claim-verified] using_inductor was added by a560ed18 (#4709 rebase) and removed by b73e0355 (this PR) — search_history confirms the described lineage. +- [claim-verified] #3588 (99234881 AutoRound) reverted #4074 (35ee3c75 torch RMSNorm) in qwen_image_transformer.py — search_history confirms; this diff restores it. +- [claim-verified] Cosmos3 is the only per-model IR-priority override: vllm_omni/diffusion/registry.py:545 _DIFFUSION_IR_OP_PRIORITY_FUNCS contains only Cosmos3OmniDiffusersPipeline; pipeline_cosmos3.py:621-622 forces rms_norm/fused_add_rms_norm=['native']. + +`vllm_omni/platforms/cuda/platform.py:253` [major] — Confirmed against PR-time head. platform.py:253 now hardcodes `default = ["vllm_c", "native"]`, and qwen_image_transformer.py uses `nn.RMSNorm` at all five sites (557/558/595/596/1009) with the vLLM RMSNorm import removed, so the rms_norm/fused_add_rms_norm priorities no longer touch Qwen-Image's transformer RMSNorm layers at head. The default-bucket flip is global: registry.py:549 shows Cosmos3 is the only per-model IR-op-priority override, while 14 diffusion model files still import vLLM RMSNorm. The Part-1-only measurements therefore were taken on a tree that still had vLLM RMSNorm in Qwen-Image and cannot describe Part 1's contribution to the merged state. (evidence: vllm_omni/platforms/cuda/platform.py:253 ` default = ["vllm_c", "native"]` — changed by this diff; the prior inductor-aware branch is deleted. vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 ` self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` — changed by this diff; four more nn.RMSNorm sites at :558/:595/:596/:1009 and the `vllm.model_executor.layers.layernorm.RMSNorm` import is removed. vllm_omni/diffusion/registry.py:549 ` "Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",` — unchanged by this diff; it is the only entry in _DIFFUSION_IR_OP_PRIORITY_FUNCS, so all other CUDA diffusion models inherit the global default. vllm_omni/diffusion/models/flux/flux_transformer.py:19 `from vllm.model_executor.layers.layernorm import RMSNorm` — unchanged by this diff; one of 14 vllm_omni/diffusion/models files still importing vLLM RMSNorm.) + +`tests/platforms/test_cuda_ir_op_priority.py:7` [minor] — The module-scope import executes during whole-tree XPU collection: run-xpu-test.sh:54 runs `pytest -v -s -m "core_model and xpu and B60"` with no path, and pytest imports test modules before applying `-m` deselection. Sibling tests already import `CudaOmniPlatform` inside test bodies (test_kernels_hub.py:24; test_sage_attn3.py:93 and :120). Move the import in tests/platforms/test_cuda_ir_op_priority.py into the two test functions to match that pattern and keep non-CUDA collection safe. (evidence: In this diff, tests/platforms/test_cuda_ir_op_priority.py:7 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` is at module scope. Unchanged by this diff, present in the PR-time tree: tests/diffusion/attention/test_kernels_hub.py:24 ` from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (indented inside the test body); tests/diffusion/attention/test_sage_attn3.py:93 and :120 ` from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (indented inside test bodies); .buildkite/scripts/hardware_ci/run-xpu-test.sh:54 ` pytest -v -s -m "core_model and xpu and B60"` (no path argument, so the whole test tree is collected).) + +`vllm_omni/platforms/rocm/platform.py:~185` [minor] — This diff removes the inductor-aware branch only from CudaOmniPlatform.get_default_ir_op_priority (vllm_omni/platforms/cuda/platform.py:252-253 now returns `["vllm_c", "native"]` unconditionally). The ROCm and XPU twins still carry the identical `using_inductor` logic, so under inductor they still resolve the RMSNorm default to `["native"]`: vllm_omni/platforms/rocm/platform.py:184-185 (`using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; `default = ["native"] if using_inductor else ["vllm_c", "native"]`) and vllm_omni/platforms/xpu/platform.py:116-117 (`using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`). If Qwen-Image or other diffusion models run on ROCm/XPU under inductor they retain the slow path this PR fixes for CUDA. Confirm the asymmetry is intentional (ROCm has the aiter path and a TODO to change) or port the fix / open a tracking issue. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/rocm/platform.py:184 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; vllm_omni/platforms/rocm/platform.py:185 `default = ["native"] if using_inductor else ["vllm_c", "native"]`; vllm_omni/platforms/xpu/platform.py:116 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; vllm_omni/platforms/xpu/platform.py:117 `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`. The changed CUDA platform now reads vllm_omni/platforms/cuda/platform.py:252-253: `"""Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops."""` / `default = ["vllm_c", "native"]`.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:~557` [minor] — Swapping vllm.model_executor.layers.layernorm.RMSNorm for torch.nn.RMSNorm at lines 557, 558, 595, 596, and 1009 changes Qwen-Image numerics at fixed seed — intended as diffusers-parity, but the only diff-internal regression test asserts the priority *list*, not RMSNorm output or dispatch, and test_qwen_image_matches_diffusers is @pytest.mark.benchmark + @hardware_test(res={'cuda':'H100'}) so it does not run per-PR. Please run pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m benchmark --run-level full_model on H100 and paste SSIM/PSNR, or explicitly mark the nightly accuracy lane as the backstop for this numerics change. (evidence: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (changed by this diff from vllm RMSNorm); tests/platforms/test_cuda_ir_op_priority.py:26 `assert priority.rms_norm == ["vllm_c", "native"]` and :27 `assert priority.fused_add_rms_norm == ["vllm_c", "native"]` (new test asserts only list values, not RMSNorm output/dispatch); tests/e2e/accuracy/test_qwen_image.py:195 `@pytest.mark.benchmark`, :196 `@hardware_test(res={"cuda": "H100"}, num_cards=1)`, :197 `def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None:` (unchanged by this diff, present in the PR-time tree — the accuracy test is H100/benchmark-gated and not a per-PR gate).) + +`tests/platforms/test_cuda_ir_op_priority.py:25` [minor] — This regression test only validates the return value of get_default_ir_op_priority; it never exercises the dispatch path in diffusion_worker._resolve_ir_op_priority (or the real RMSNorm kernel selection), so a future change that keeps the config list but stops honoring it would still pass. Keep the unit test, but bind the dispatch-level regression to the profiler/trace check that actually captures the aten::mean count — tests/dfx/perf/tests/test_qwen_image_vllm_omni.json:48 (test_qwen_image_single_device_step_execution) records QPS/latency/memory baselines but does not itself assert aten::mean counts. (evidence: tests/platforms/test_cuda_ir_op_priority.py:25-26 ` priority = CudaOmniPlatform.get_default_ir_op_priority(_vllm_config(mode=mode))` / ` assert priority.rms_norm == ["vllm_c", "native"]` — the test asserts only the priority-list value. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/worker/diffusion_worker.py:174-178 `def _resolve_ir_op_priority(od_config: OmniDiffusionConfig, vllm_config: VllmConfig) -> Any:` / ` ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)` / ` ir_op_priority_func = get_diffusion_ir_op_priority_func(od_config)` / ` if ir_op_priority_func is not None:` / ` ir_op_priority = ir_op_priority_func(ir_op_priority, vllm_config=vllm_config)` — this dispatch/override step is not exercised by the cited unit test.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:1009` [minor] — This hunk swaps the entry `txt_norm` to `nn.RMSNorm`, but `forward` applies it only once before `txt_in` (line 1140), whereas the four attention-block RMSNorm sites in `QwenImageCrossAttention` — `norm_q` (557), `norm_k` (558), `norm_added_q` (595), `norm_added_k` (593) — run per transformer block per denoise step. #4964's profiler (960 `rms_norm` kernels over 4 steps = 240/step = 4 sites × 60 layers) targets that hot attention-block path, not the single entry norm. Please split PR attribution: mark `txt_norm` as accuracy/diffusers-parity and tie the latency recovery claim to the attention-block swaps, or re-profile at merged head with both parts applied. (evidence: qwen_image_transformer.py:1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)` (changed by this diff); qwen_image_transformer.py:1140 `encoder_hidden_states = self.txt_norm(encoder_hidden_states)` — the sole `txt_norm` call site in `forward`, immediately before `txt_in`; qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` and qwen_image_transformer.py:595 `self.norm_added_q = nn.RMSNorm(head_dim, eps=eps)` sit inside `QwenImageCrossAttention`, which each `QwenImageTransformerBlock` instantiates once in a `nn.ModuleList` over `num_layers`, so these execute per layer per denoise step while `txt_norm` runs once. Issue #4964 (unchanged by this diff) reports 960 `rms_norm` kernel calls over 4 denoise steps = 240/step = 4 sites × 60 layers, matching the attention-block path rather than the entry norm.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.trace.json.gz new file mode 100644 index 000000000..355591d96 Binary files /dev/null and b/eval/dataset/arms/copilot_v16rs_train_r1/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_smoke/manifest.json b/eval/dataset/arms/copilot_v17_smoke/manifest.json new file mode 100644 index 000000000..10988df2e --- /dev/null +++ b/eval/dataset/arms/copilot_v17_smoke/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v17_smoke", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "(default)", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_smoke/pr4816.cost.json b/eval/dataset/arms/copilot_v17_smoke/pr4816.cost.json new file mode 100644 index 000000000..7a3ae0022 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_smoke/pr4816.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1073.5, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_smoke/runs/pr4816/run-20260816-195605-e34bed", + "recorded_at": "2026-08-16T20:13:56", + "attempts": [ + { + "invocation_id": "inv-a21b7e19", + "outcome": "run:run-20260816-195605-e34bed", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-195605-e34bed", + "llm_calls": 102, + "input_tokens": 975093, + "output_tokens": 166280, + "usd": 0.672582, + "source": "spans" + } + ], + "attempt_usd_total": 0.672582, + "llm_calls": 102, + "input_tokens": 975093, + "output_tokens": 166280, + "usd": 0.672582, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-195605-e34bed", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6726, + "minutes": 17.84, + "ci_minutes": 0, + "input_tokens": 975093, + "output_tokens": 166280, + "tool_calls": 190, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7674, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 102, + "by_role": { + "planner": { + "usd": 0.001371, + "input_tokens": 2007, + "output_tokens": 754, + "calls": 1 + }, + "agent": { + "usd": 0.654597, + "input_tokens": 960152, + "output_tokens": 153635, + "calls": 99 + }, + "reducer": { + "usd": 0.016614, + "input_tokens": 12934, + "output_tokens": 11891, + "calls": 2 + } + }, + "cache_read_tokens": 8385152, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.79, + "gate": 2.06, + "review": 1064.56, + "report": 0.07 + }, + "time_to_first_result_s": 1070.47 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 28575 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_smoke/pr4816.md b/eval/dataset/arms/copilot_v17_smoke/pr4816.md new file mode 100644 index 000000000..cb9831d50 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_smoke/pr4816.md @@ -0,0 +1,79 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Concern "changed tests are never selected in CI": test_serving_speech.py:48 has `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` and test-ready.yml:209 runs `pytest -sv tests/entrypoints/ -m 'core_model and cuda'` (cuda auto-added), so the file is selected on the CUDA lane. Residual: AMD/Intel/NPU lanes do not reference this file (grep .buildkite `test_serving` = no matches). +- [claim-verified] PR claim that upstream `base` moved to serve.instrumentator.basic and reads `app.state.serving_tokenization` is consistent with api_server.py:67 `from vllm.entrypoints.serve.instrumentator.basic import base` and the diff's key rename. +- [claim-verified] Motivating regression chain real: api_server.py:376 `err = base(raw_request).create_error_response(...)`; before the fix only `openai_serving_tokenization` was set, so `base` raised AttributeError -> 500 on caught speech errors. +- [claim-verified] PR body's upstream claim is consistent: the file already imports `base` from `vllm.entrypoints.serve.instrumentator.basic` and `ServingTokenization` from `vllm.entrypoints.serve.tokenize.serving` (new layout), matching the quoted `tokenization()` reading `request.app.state.serving_tokenization`. +- [claim-verified] PR description's premise (upstream moved base to read `request.app.state.serving_tokenization`) is consistent with api_server.py:67 `from vllm.entrypoints.serve.instrumentator.basic import base`; old-name references are absent repo-wide at this head. +- [claim-verified] PR claim that `_create_speech_error_json_response` calls `base(raw_request).create_error_response(...)` is true: api_server.py:376 does exactly that. +- [claim-refuted] PR claim "keeps things from turning into a 500" holds only for the LLM/multi-stage branch (api_server.py:1017 assigns a real `ServingTokenization`); pure-diffusion assigns None (754) and `_create_speech_error_json_response` (376) dereferences `base()` unguarded -> still 500 for upload_voice/delete_voice errors. +- [validated] Test updates are semantically necessary: test_serving_speech.py:2533/2546/2615 set `state.serving_tokenization` so the renamed `getattr` fallback returns fake_base; with the old key these tests would raise HTTPException instead of the asserted JSON 404. +- [validated] omni_init_app_state consumers (tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py:337, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py:251) do not reference the old key - no stale assertions. +- [validated] create_chat_completion/create_speech/create_speech_batch/create_audio_generate all read the renamed key via `getattr(..., default=None)`; the TTS-only (chat handler None) LLM fallback still returns a ServingTokenization-based error response. +- [validated] grep `serving_tokenization` shows only api_server.py (7 hits: 754,1017,1157,1248,1286,1329) and test_serving_speech.py (3 hits: 2533,2546,2615); no other repo consumer of the renamed state attribute exists. +- [validated] `omni_init_app_state` is the sole setter of the attribute; `omni_run_server_worker` calls `build_openai_app` then `omni_init_app_state`, so upstream init does not separately populate `serving_tokenization`. +- [validated] No new external library call/kwarg is introduced; the rename hard-requires the already-present new-layout vllm (the file already imports `serve.*` modules), so no new dependency lower bound is needed. +- [validated] grep `openai_serving_tokenization` over the whole repo at head returns 0 matches — no stale in-repo consumer of the old name remains. + +`vllm_omni/entrypoints/openai/api_server.py:754` [major] — This hunk leaves pure-diffusion `state.serving_tokenization = None` (api_server.py:754) while the LLM branch assigns a real `ServingTokenization` (api_server.py:1014). The unchanged `_create_speech_error_json_response` still calls `base(raw_request).create_error_response(...)` at api_server.py:376; upstream `base()` now returns `request.app.state.serving_tokenization`, so in pure-diffusion mode `upload_voice`/`delete_voice` error paths (`_create_speech_error_json_response(raw_request, str(e))`, lines 1483/1539) call `.create_error_response` on `None`, raising AttributeError and returning 500 — the exact failure this PR aims to eliminate. Since `state.openai_serving_speech = OmniOpenAIServingSpeech.for_diffusion(...)` (api_server.py:784) makes those voice endpoints reachable, guard the None case in `_create_speech_error_json_response` (or provision a real tokenization error handler for pure diffusion) and confirm a sad `/v1/audio/voices` request on a pure-diffusion model returns 4xx, not 500. (evidence: api_server.py:376 `err = base(raw_request).create_error_response(`; api_server.py:754 `state.serving_tokenization = None`; api_server.py:1483 `return _create_speech_error_json_response(raw_request, str(e))` (upload_voice); api_server.py:1248 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)` followed by `if base_server is None: raise HTTPException(...404...)`.) + +```suggestion +base_server = base(raw_request) + if base_server is None: + return JSONResponse( + content={"error": {"message": message, "type": err_type, "code": int(status_code)}}, + status_code=int(status_code), + ) + err = base_server.create_error_response( + message=message, + err_type=err_type, + status_code=status_code, + ) + return _error_response_to_json_response(err, status_code=status_code) +``` + +`vllm_omni/entrypoints/openai/api_server.py:~1163` [major] — Confirmed. `create_chat_completion`'s handler-None branch still returns a bare `ErrorResponse` at api_server.py:1163 even after the rename at api_server.py:1157. In a TTS-only server `state.openai_serving_chat` is None (api_server.py:928, gated by `"generate" in supported_tasks`) while `state.serving_tokenization` is set unconditionally (api_server.py:1017), so POST /v1/chat/completions returns HTTP 200 with a BadRequestError/400 body instead of the 404 error JSON the endpoint's `responses` contract and sibling handlers (e.g. api_server.py:1259) promise. Wrap the chat error through `_error_response_to_json_response` with NotFoundError/404 and add the regression test. Note: the 200-vs-404 behavior predates this rename-only diff, but it is a real user-visible defect in the branch the diff touches. (evidence: vllm_omni/entrypoints/openai/api_server.py:1157 ` base_server = getattr(raw_request.app.state, "serving_tokenization", None)` (changed by this diff); vllm_omni/entrypoints/openai/api_server.py:1163 ` return base_server.create_error_response(message="The model does not support Chat Completions API")` (unchanged by this diff, present in PR-time tree) — bare ErrorResponse return; vllm_omni/entrypoints/openai/api_server.py:1017 ` state.serving_tokenization = ServingTokenization(` (changed by this diff; unconditionally set); vllm_omni/entrypoints/openai/api_server.py:928 ` state.openai_serving_chat = (` with the `if "generate" in supported_tasks` gate (unchanged; None for TTS-only); vllm_omni/entrypoints/openai/api_server.py:1259 ` return _error_response_to_json_response(err, status_code=HTTPStatus.NOT_FOUND)` (unchanged; sibling speech branch serializes 404 JSON).) + +```suggestion +err = base_server.create_error_response( + message="The model does not support Chat Completions API", + err_type="NotFoundError", + status_code=HTTPStatus.NOT_FOUND, + ) + return _error_response_to_json_response(err, status_code=HTTPStatus.NOT_FOUND) +``` + +`tests/entrypoints/openai_api/test_serving_speech.py:~2533` [minor] — The diff updates the three 404 tests to `raw_request.app.state.serving_tokenization = fake_base` (test_serving_speech.py:2533) but each still goes through `_patch_api_server_base(mocker)`, replacing `api_server_module.base` with a MagicMock; tests that reach `_create_speech_error_json_response` also patch `api_server_module.base` (line 2352). Therefore no test drives the real upstream `base(raw_request)` -> `request.app.state.serving_tokenization` dispatch this PR fixes, and the 404 tests would stay green even if the write still used the old `openai_serving_tokenization` name. Add a regression test that sets `app.state.serving_tokenization` to a real/stub `ServingTokenization` and drives the PR's ValueError reproduction through `upload_voice` (or `delete_voice`) without patching `api_server_module.base`, asserting a 4xx JSON error response rather than AttributeError/500. (evidence: test_serving_speech.py:2352 `mocker.patch.object(api_server_module, "base", return_value=fake_base)`; api_server.py:1248 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)` with 404 fallback; repo grep `_create_speech_error_json_response` finds it only in api_server.py (0 test references).) + +`test_serving_speech.py:48` [minor] — Concern "changed tests are never selected in CI" is resolved: test_serving_speech.py:48 declares `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` and .buildkite/test-ready.yml:209 runs `pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'`, so the file is selected on the CUDA entrypoints lane. Residual: a .buildkite grep for `test_serving` returns no matches, so no AMD/Intel/NPU lane references this file directly. (evidence: tests/entrypoints/openai_api/test_serving_speech.py:48 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — present in the PR-time tree and unchanged by this diff. .buildkite/test-ready.yml:209 `pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'` — present in the PR-time tree and outside the diff. Both lines are decisive for the comment's CI-selection claim.) + +`vllm_omni/entrypoints/openai/api_server.py:66` [minor] — Stale comment at api_server.py:66: it claims 'Keep a fallback for older/newer upstream layouts during rebase windows,' but api_server.py:67 is a single unconditional import of `base` from `vllm.entrypoints.serve.instrumentator.basic` with no fallback branch. Update or remove the comment. (evidence: api_server.py:66 `# Keep a fallback for older/newer upstream layouts during rebase windows.` followed by api_server.py:67 `from vllm.entrypoints.serve.instrumentator.basic import base` — the import is unconditional and no try/except or alternate import path exists (grep returns only line 67). These lines are unchanged by this diff but present in the PR-time tree.) + +`vllm_omni/entrypoints/openai/api_server.py:1017` [nit] — CI evidence is accurate: buildkite/vllm-omni-amd-ci, -intel-ci, and -npu-ci fail while buildkite/vllm-omni passes; grep shows no `.buildkite` lane references `test_serving` (AMD runs `tests/entrypoints/test_omni_sleep_mode.py`, and test-ready runs `pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'`). Confirm the three failures are pre-existing on main before merging. (evidence: ['vllm_omni/entrypoints/openai/api_server.py:1017 ` state.serving_tokenization = ServingTokenization(` — changed by this diff (rename from `openai_serving_tokenization`).', ".buildkite/test-ready.yml:209 ` pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'` — unchanged by this diff, present in the PR-time tree.", '.buildkite/test-amd.yaml:118 ` - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"` — unchanged by this diff, present in the PR-time tree.', 'grep `.buildkite` for literal `test_serving`: no matches — unchanged CI files, present in the PR-time tree.']) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_smoke/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v17_smoke/pr4816.trace.json.gz new file mode 100644 index 000000000..82d8f552a Binary files /dev/null and b/eval/dataset/arms/copilot_v17_smoke/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/README.md b/eval/dataset/arms/copilot_v17_train/README.md new file mode 100644 index 000000000..ef9b7050d --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/README.md @@ -0,0 +1,48 @@ +# RELABELLED — v17-moa (unintended vendor mixture), not v17 core-only + +This directory is named `v17` but does NOT hold the arm that name implies. + +`MOA_WHEN=off` was set explicitly on every arm from v13 through v16. The +launch that produced this data omitted it, so `moa_when` fell through to its +code default of `"full"` (config.py:345) and mixture-of-agents ran on every +full-depth PR review — 12 of the 15 items here. Round-1 lenses were +dispatched to three vendors instead of one: investigator + behavior to +`mimo-v2.5`, adversary + verification to `qwen3.6-plus`, reducer on tier. + +## How much of the work was actually not DeepSeek + +Attributed by the model recorded on each `agent_dispatch`, counting only +dispatches that produced usable output: + +| model | attempts | productive outputs | output tokens | +|---|---|---|---| +| `deepseek-v4-pro` | rest + all fallbacks | **155 (89.1%)** | 2,298,061 (89.3%) | +| `mimo-v2.5` | 30 | 16 (9.2%) | 183,047 (7.1%) | +| `qwen3.6-plus` | 27 | 3 (1.7%) | 94,497 (3.7%) | + +**DeepSeek still wrote ~90% of the review work**, and the two member figures +are UPPER BOUNDS. `BudgetedLLM.create` (moa.py:215-227) reserves an estimated +cost per LLM call and, when the reservation is refused, silently reruns that +call on the tier model while the lens keeps its original dispatch label. With +`moa_max_usd = $1.50` against per-item spend of $0.85–1.66, that budget trips +partway through most items, so an unknown share of the "member" rows above +was in fact DeepSeek. The trace cannot resolve it: only the dispatch carries a +model, and the per-call fallback emits none. + +(An earlier version of this file claimed 24%, inferred from seat names. That +inference ignored both `/retry` seats and the per-call budget fallback, and +was wrong by roughly 2.6x.) + +`qwen3.6-plus` returned `403 AccessDenied.Unpurchased` on 24 of its 27 +attempts — that subscription is not active. 104 minutes of this run went to +member attempts that were then discarded and redone. + +Kept rather than deleted: the data is real and the MoA behaviour it captures +is worth having. The directory names are also left alone on purpose — every +verdict's `_roles` block records the arm directory name, so renaming would +leave that provenance pointing at a directory holding different data. + +The corrected measurement, same code (`ae1b6f1`) and same items with +`MOA_WHEN=off` verified through resolved settings, lives in +`copilot_v17ds_*` / `goal_v17ds_*_sonnet`. See +`goal-eval/PROBE-v17-conversion.md` for the pre-registered amendment. diff --git a/eval/dataset/arms/copilot_v17_train/manifest.json b/eval/dataset/arms/copilot_v17_train/manifest.json new file mode 100644 index 000000000..9423aee92 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v17_train", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "(default)", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4804.cost.json b/eval/dataset/arms/copilot_v17_train/pr4804.cost.json new file mode 100644 index 000000000..22dac9594 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4804.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 2151.5, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4804/run-20260816-201527-a8c937", + "recorded_at": "2026-08-16T20:51:16", + "attempts": [ + { + "invocation_id": "inv-54e96893", + "outcome": "run:run-20260816-201527-a8c937", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-201527-a8c937", + "llm_calls": 213, + "input_tokens": 2329054, + "output_tokens": 326629, + "usd": 1.663021, + "source": "spans" + } + ], + "attempt_usd_total": 1.663021, + "llm_calls": 213, + "input_tokens": 2329054, + "output_tokens": 326629, + "usd": 1.663021, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-201527-a8c937", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.663, + "minutes": 35.82, + "ci_minutes": 0, + "input_tokens": 2329054, + "output_tokens": 326629, + "tool_calls": 383, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3676, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 213, + "by_role": { + "moa_member": { + "usd": 0.185332, + "input_tokens": 283393, + "output_tokens": 11895, + "calls": 22 + }, + "agent": { + "usd": 1.101717, + "input_tokens": 1729685, + "output_tokens": 217974, + "calls": 140 + }, + "moa_member_fallback": { + "usd": 0.328821, + "input_tokens": 253730, + "output_tokens": 69215, + "calls": 49 + }, + "reducer": { + "usd": 0.047151, + "input_tokens": 62246, + "output_tokens": 27545, + "calls": 2 + } + }, + "cache_read_tokens": 23435584, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.29, + "gate": 1.5, + "review": 2143.33, + "report": 0.07 + }, + "time_to_first_result_s": 2149.17 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 82486 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4804.md b/eval/dataset/arms/copilot_v17_train/pr4804.md new file mode 100644 index 000000000..7645fc8ef --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4804.md @@ -0,0 +1,99 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | no finding reported | +| Behavior / compatibility | 4 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] serving_speech.py:3711 now always deep-copies sampling_params_list, so the previous conditional copy that could mutate engine_client.default_sampling_params_list is gone; no remaining in-repo consumer of qwen3_tts_request_seed. +- [resolved] GPU `_dummy_run` at gpu_generation_model_runner.py:796 injects `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]`, and `_update_request_states` calls `on_requests_finished` at gpu_generation_model_runner.py:79 — GPU paths are covered; residual NPU twin gap is the round-1 kept comment. +- [claim-verified] PR Test Plan covers only Local-v1.5 (codec path OpenMOSS-Team/MOSS-Audio-Tokenizer-v2); no Delay/Realtime benchmark in the body. +- [claim-verified] commit 'Fix leak issues when aborting' delivered only the GPU hook; the NPU generation runner still leaks MossTTS stream slots on abort. +- [validated] serving_speech.py unconditional deepcopy: old code only deepcopied shared default list; new always deepcopies. Correct behavior but unconditional copy for user-provided params is a minor performance regression. +- [validated] _dummy_run seq_token_counts injection: confirmed at gpu_generation_model_runner.py:799 — uses [int(num_tokens_padded)] as synthetic segment. +- [validated] pipeline.py route changes: delay→talker2codec_delay_async_chunk, local/realtime→talker2codec_raw_async_chunk — both function names exist in stage_input_processors/moss_tts.py __all__. +- [validated] yaml changes: initial_codec_chunk_frames=1 and codec_stream_slots=8 added to connector config, max_num_seqs bumped to 8 for stage-1 concurrent decoding. +- [validated] weight loading RuntimeError on partial load: modeling_moss_tts_codec.py now raises on missing/skipped/shape_mismatch — prevents silent garbage output from unloaded weights. +- [validated] tests/worker/test_omni_gpu_model_runner.py: pytestmark=[core_model,cpu] so the touched tests are selected; CaptureTalkerMTP records generator and the two touched tests assert generator None vs not-None, so they exercise _explicit_talker_seed, not just the key string. +- [validated] PCM chain is correct on the happy path: patch.py normalizes captured 48k stereo -> 24k mono, and seed_tts_eval._pcm_s16le_to_f32_16k default pcm_sample_rate=24000 resamples 24k->16k for WER. +- [validated] qwen3_tts_request_seed fully removed — grep returns 0 matches across entire repo +- [validated] tts_local_seed has exactly 3 consumers: serving_speech.py:3715 (producer), gpu_model_runner.py:1826 (consumer), tests (3 lines) +- [validated] seq_token_counts provided in _dummy_run at gpu_generation_model_runner.py:799 and in execute_model at line 316 + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:~733` [major] — modeling_moss_tts_codec.py:729-735 tries MossAudioTokenizerV2Config.from_pretrained / MossAudioTokenizerV2Model inside a bare `except Exception` and then falls back to the legacy vendored codec after logging. A real v2 failure (corrupt checkpoint, incompatible transformers, config mismatch) is masked as a v1 fallback; the root cause is lost and the wrong codec path may be selected. Narrow the catch to the expected fallback triggers (ValueError, KeyError, OSError) or gate v2 vs v1 explicitly. Note the broad catch predates this PR, but the new try body now puts the vendored v2 path under it. (evidence: modeling_moss_tts_codec.py:733 ` except Exception:` — broad catch wrapping the new v2 instantiation (line 729 `codec_cfg = MossAudioTokenizerV2Config.from_pretrained(codec_path)`, line 730 `codec = MossAudioTokenizerV2Model(codec_cfg)`); then model_executor/models/moss_tts/modeling_moss_tts_codec.py:738 ` codec_cfg = MossAudioTokenizerConfig.from_pretrained(codec_path)` — unconditional fallback to the legacy codec after the catch.) + +```suggestion +except (ValueError, KeyError, OSError) as e: + logger.warning( + "Vendored MOSS Audio Tokenizer v2 instantiation failed (%s); falling back to legacy codec.", + e, + ) + + codec_cfg = MossAudioTokenizerConfig.from_pretrained(codec_path) +``` + +`tests/model_executor/models/moss_tts/test_per_request_codes.py:34` [major] — tests/model_executor/models/moss_tts/test_per_request_codes.py has no `pytestmark`, so the merge/test-ready `-m 'core_model and cpu'` filter deselects every test in it and the file never runs in PR CI. It only imports/covers MossTTSDelayTalkerForGeneration.make_omni_output (line 34); the PR's rewritten Realtime/Local make_omni_output and the new Local talker_mtp have zero test coverage. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` and extend the file to the Realtime/Local paths. (evidence: tests/model_executor/models/moss_tts/test_per_request_codes.py:34-35 `from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import (` / ` MossTTSDelayTalkerForGeneration,` — only the Delay talker is imported, and grep for `pytestmark` in this file returns no matches (no pytestmark exists). Unchanged by this diff, present in the PR-time tree: .buildkite/test-merge.yml:13 `- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — this marker filter deselects the unmarked test file. Grep for `MossTTSLocalTalkerForGeneration` and `MossTTSRealtimeTalkerForGeneration` under tests/ returns no matches, so the changed Local/Realtime paths have no test coverage.) + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:~299` [major] — The PR adds `seq_token_counts` and `on_requests_finished` contracts in GPUGenerationModelRunner but leaves NPUGenerationModelRunner behind. NPU `_dummy_run` builds model_kwargs with `self._init_model_kwargs()` and calls `self.model(..., **model_kwargs)` without ever setting `model_kwargs["seq_token_counts"]`, so a dummy/profile run of MossTTSCodecDecoder on NPU reaches the new `raise RuntimeError("MossTTS codec requires seq_token_counts")`. NPU `_update_request_states` also removes finished requests without calling `self.model.on_requests_finished(...)`, leaking codec stream slots on abort (the NPU AR runner already has this call at npu_ar_model_runner.py:419-420). Add `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]` in NPU `_dummy_run` and the `on_requests_finished` hook in NPU `_update_request_states`. (evidence: ['vllm_omni/worker/gpu_generation_model_runner.py:81-82 (this diff): `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)`', 'vllm_omni/worker/gpu_generation_model_runner.py:799 (this diff): `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]`', 'vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:261-264 (this diff): `token_counts = self._normalize_seq_token_counts(kwargs.get("seq_token_counts"))` / `if token_counts is None:` / `raise RuntimeError("MossTTS codec requires seq_token_counts; otherwise concatenated codec tokens cannot be split per request.")`', 'vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:299 (unchanged by this diff): `model_kwargs["seq_token_counts"] = tokens` — present only in execute_model; NPU `_dummy_run` calls `self.model(input_ids=input_ids, positions=positions, intermediate_tensors=intermediate_tensors, inputs_embeds=inputs_embeds, **model_kwargs)` with model_kwargs from `self._init_model_kwargs()` and no seq_token_counts assignment.', 'vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:48 (unchanged by this diff): `def _update_request_states(self, scheduler_output: SchedulerOutput):` — its body removes finished requests without any `on_requests_finished` call; contrast vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:419-420: `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)`.']) + +```suggestion +In NPUGenerationModelRunner._dummy_run, right before 'outputs = self.model(...)', add: + model_kwargs["seq_token_counts"] = [int(num_tokens_padded)] +And in NPUGenerationModelRunner._update_request_states, before the first remove_request loop, add: + if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): + self.model.on_requests_finished(scheduler_output.finished_req_ids) +``` + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:~480` [major] — In `_decode_streaming_batch` (vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:466-481), when `session.acquire()` returns None and the request is finished, the codes buffered in `_stream_pending_codes` are discarded by `_finish_stream_request(request_id, session, None)` with only a warning, so a successful response can be served with its tail silently truncated. Fail loudly for that request or offline-decode the buffered codes into a final delta. (evidence: modeling_moss_tts_codec.py:466-481: `slot = session.acquire() / if slot is None: / self._append_stream_pending(request_id, codes_nq_t) / ... / if finished: / logger.warning("MOSS codec stream request %s finished before a codec stream slot became available; " "dropping buffered codes instead of offline-decoding a non-delta waveform.", request_id) / self._finish_stream_request(request_id, session, None)`; and modeling_moss_tts_codec.py:554-563: `def _finish_stream_request(...) -> None: / if slot is not None: / session.release(slot) / self._stream_req_slots.pop(request_id, None) / self._stream_pending_codes.pop(request_id, None) / self._stream_starved_reqs.discard(request_id)` — the buffered codes are popped with no decode, while the caller at lines 504-507 writes the (possibly unchanged/empty) output for that request and returns normally.) + +```suggestion +Replace the warn-and-drop with a raise so truncated output is never served silently: + if finished: + raise RuntimeError( + f"MOSS codec stream request {request_id} finished with no free stream slot; " + "buffered codes would be silently dropped." + ) +``` + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:225` [major] — `talker2codec_raw_async_chunk` and `_MossCodecStreamSession` are new in this PR and carry the streaming state machine (initial/steady thresholds, buffering, `code_flat_numel=0` finish sentinel), but the only test touched is tests/worker/test_omni_gpu_model_runner.py, which swaps an extra_args key. Add a CPU unit test that drives `talker2codec_raw_async_chunk` through a 1-frame initial chunk, a 15-frame steady chunk, and the finished flush, asserting emitted chunk shapes and the terminal `code_flat_numel=0` payload. (evidence: grep tests/ for `raw_async_chunk` → no matches; tests/model_executor/models/moss_tts/test_per_request_codes.py:27 imports only `MossTTSDelayTalkerForGeneration`) + +`vllm_omni/entrypoints/openai/serving_speech.py:3710` [minor] — serving_speech.py now unconditionally deepcopies sampling_params_list whenever request.seed is set; the pre-PR code only deep-copied when the list was the shared engine default. Guard the copy with `if sampling_params_list is self.engine_client.default_sampling_params_list:` to avoid a per-request deepcopy on the speech serving hot path. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3710 `sampling_params_list = copy.deepcopy(sampling_params_list)` — unconditional deepcopy in the changed code. The diff removed the old guard: `- if sampling_params_list is self.engine_client.default_sampling_params_list:` `- import copy` `- sampling_params_list = copy.deepcopy(sampling_params_list)`.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [minor] — New files vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py and configuration_moss_audio_tokenizer_v2.py are only exercised via MossTTSCodecDecoder._build_codec; no test in tests/ imports either module. The only moss_tts codec test (test_codec_cudagraph.py) targets the legacy audio_tokenizer module. Add a CPU load/decode smoke test for the v2 path. (evidence: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:730 `codec = MossAudioTokenizerV2Model(codec_cfg)` — the new v2 codec is instantiated only here in production code (unchanged-by-diff context aside, this import/instantiation is part of the PR). tests/model_executor/models/moss_tts/test_codec_cudagraph.py:12 `from vllm_omni.model_executor.models.moss_tts.audio_tokenizer import (` — the only moss_tts codec test imports the legacy v1 tokenizer. Repo-wide grep: 'MossAudioTokenizerV2Model' under tests/ -> no matches; 'audio_tokenizer_v2' under tests/ -> no matches.) + +`vllm_omni/benchmarks/patch/patch.py:~1251` [minor] — patch.py:1246-1251: on `_pcm_s16le_to_seed_tts_wer_bytes` failure the code falls back to `bytes(pcm_capture)`, which under 48 kHz stereo capture is then written by `_save_seed_tts_eval_audio` as a 24 kHz mono WAV and fed to `_pcm_s16le_to_f32_16k` with its 24000 default — producing a mislabeled WAV and silently wrong WER/SIM/UTMOS. Set `output.tts_output_pcm_bytes = None` in the except branch so the row is counted as `no_pcm`. (evidence: vllm_omni/benchmarks/patch/patch.py:1250-1251 (in this diff): `logger.warning("Seed-TTS WER PCM normalization failed: %s", ex)` / `output.tts_output_pcm_bytes = bytes(pcm_capture)` — fallback keeps raw non-normalized PCM. vllm_omni/benchmarks/data_modules/seed_tts_eval.py:112 (added by this diff): `path.write_bytes(pcm_s16le_mono_to_wav_bytes(pcm, sample_rate=24000))` — writes the buffer as a 24 kHz mono WAV. vllm_omni/benchmarks/data_modules/seed_tts_eval.py:176 (unchanged by this diff, present in PR-time tree): `def _pcm_s16le_to_f32_16k(pcm: bytes, pcm_sample_rate: int = 24000) -> np.ndarray:` with `target_len = int(len(raw) * 16000 / pcm_sample_rate)` — treats the buffer as 24 kHz mono.) + +```suggestion +except Exception as ex: + logger.warning("Seed-TTS WER PCM normalization failed: %s", ex) + output.tts_output_pcm_bytes = None +``` + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:701` [minor] modeling_moss_tts_codec.py:701 uses `inferred_channels = 2 if "v2" in codec_path.lower() else 1` as the final fallback when `codec_cfg.number_channels`/`num_channels` is missing or null. Real v1 checkpoints store `number_channels` as an explicit JSON `null` (see audio_tokenizer.py:559-560), so a v1 checkpoint loaded […] +- `vllm_omni/entrypoints/openai/serving_speech.py:3715` [minor] serving_speech.py:3708-3715 now always deepcopies the default sampling params and writes `extra_args["tts_local_seed"] = request.seed` for every TTS model whenever `request.seed` is set (the old path gated `qwen3_tts_request_seed` on `_tts_model_type == "qwen3_tts"`). The worker test only covers the runner-side read […] +- `tests/worker/test_omni_gpu_model_runner.py:159` [minor] This diff renames the seed marker to `tts_local_seed`, but `test_talker_mtp_forward_passes_qwen3_tts_subtalker_sampling_params_to_talker` (tests/worker/test_omni_gpu_model_runner.py:275) still carries the stale `qwen3_tts` prefix (the other touched test name is already generic). More importantly, neither touched test […] +- `vllm_omni/worker/gpu_generation_model_runner.py:81` [minor] This PR adds the `on_requests_finished` hook at gpu_generation_model_runner.py:81 and a hard `seq_token_counts` requirement in MossTTSCodecDecoder.forward (modeling_moss_tts_codec.py:260-264), and buildkite/vllm-omni-npu-ci is FAILURE. diff_stat shows no NPU file in this diff. NPU's real execute path already injects […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4804.trace.json.gz new file mode 100644 index 000000000..de8e04df3 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr4817.cost.json b/eval/dataset/arms/copilot_v17_train/pr4817.cost.json new file mode 100644 index 000000000..3c79587d1 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4817.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 954.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4817/run-20260816-204519-1a98dd", + "recorded_at": "2026-08-16T21:01:12", + "attempts": [ + { + "invocation_id": "inv-847369e2", + "outcome": "run:run-20260816-204519-1a98dd", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-204519-1a98dd", + "llm_calls": 76, + "input_tokens": 310119, + "output_tokens": 171674, + "usd": 0.351135, + "source": "spans" + } + ], + "attempt_usd_total": 0.351135, + "llm_calls": 76, + "input_tokens": 310119, + "output_tokens": 171674, + "usd": 0.351135, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-204519-1a98dd", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3511, + "minutes": 15.87, + "ci_minutes": 0, + "input_tokens": 310119, + "output_tokens": 171674, + "tool_calls": 151, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5975, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 76, + "by_role": { + "planner": { + "usd": 0.003357, + "input_tokens": 3, + "output_tokens": 2988, + "calls": 1 + }, + "agent": { + "usd": 0.320572, + "input_tokens": 297574, + "output_tokens": 147072, + "calls": 73 + }, + "reducer": { + "usd": 0.027207, + "input_tokens": 12542, + "output_tokens": 21614, + "calls": 2 + } + }, + "cache_read_tokens": 2909696, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.37, + "gate": 2.51, + "review": 946.24, + "report": 0.06 + }, + "time_to_first_result_s": 952.18 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4817.trace.json.gz", + "trace_bytes": 28515 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4817.md b/eval/dataset/arms/copilot_v17_train/pr4817.md new file mode 100644 index 000000000..bf2a4e632 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4817.md @@ -0,0 +1,66 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4817, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head fc423e5a9e86) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] naming: `_is_blackwell`→`_is_quack_capable` is more precise and matches the file's private-helper convention; no placement violation in the rename itself (new test dir `tests/quantization/` mirrors `vllm_omni/quantization/`, but is CI-orphaned — see comment). +- [resolved] grep for `_is_blackwell` repo-wide returns no remaining references, so the rename to `_is_quack_capable` left no stale callers. +- [claim-verified] 'hardware-free tests' — the test monkeypatches torch.cuda.is_available/get_device_capability, no GPU required. +- [claim-verified] old default was `>= 10` (diff hunk) and new `== 10` flips the default for cc 12.x (the fix) AND cc 11.x (a silent divergence the PR does not discuss). +- [claim-verified] env escape hatch preserved and precedence unchanged: quack_enabled() reads VLLM_OMNI_USE_QUACK_FP8 before calling _is_quack_capable (quack_fp8.py:37-41); tests exercise both directions. +- [claim-verified] sm_120/121 default now native FlashInfer: quack_enabled() False -> install_quack_fp8_patch() returns at line 102-103 before patching FlashInferFP8ScaledMMLinearKernel.apply_scaled_mm. +- [claim-verified] new unit tests drive the real entry points _is_quack_capable()/quack_enabled() with monkeypatched torch.cuda probes rather than asserting an injected value. +- [claim-refuted] PR body root-cause says `tcgen05.mma` targets `sm_100a / sm_101a / sm_103a / sm_110a`, but the new gate is `== 10` and docstring/docs list only sm_100/101/103 — cc 11.x is silently excluded (finding C2). +- [claim-refuted] the guard is not wired into CI: the test file has no `pytestmark`; `.buildkite/test-ready.yml:29` selects with `-m 'core_model and cpu'`, which deselects unmarked tests; grep shows no `.buildkite` step references `tests/quantization` or `test_quack_fp8_gate`. +- [claim-refuted] PR body states 'PTX tcgen05.mma targets sm_100a / sm_101a / sm_103a / sm_110a', but the new gate `== 10` silently disables cc 11.x and the new doc/docstring asserts tcgen05 exists 'only on' 10.x parts — the two claims contradict each other. +- [claim-refuted] fp8.md:54's '`=1` to force quack on' is false on sm_120/121: quack_enabled() returns True, install_quack_fp8_patch() proceeds, and apply_scaled_mm catches the CuteDSL arch error and falls back to FlashInfer per GEMM (quack_fp8.py:106-115) — see review comment. +- [validated] vllm_omni/quantization/quack_fp8.py:32 `== 10` — 12.x major is 12 (False), 10.x is 10 (True), so the sm_120/121 regression is fixed. +- [validated] file_at_base quack_fp8.py:18-24 shows pre-change `_is_blackwell` returned `torch.cuda.get_device_capability()[0] >= 10`; PR claim 'currently turns on for every Blackwell' verified. +- [validated] quack_fp8.py:38-41 env override is checked before capability, so `=1`/`=0` still wins; tests exercise truthy and falsy values. + +`tests/quantization/test_quack_fp8_gate.py:21` [major] — This PR adds `tests/quantization/test_quack_fp8_gate.py` as the regression guard for the `>= 10` → `== 10` gate flip, but the file declares no `pytestmark` (its only decorators are `@pytest.mark.parametrize`, not selection markers). The unit lane `.buildkite/test-ready.yml` runs `pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor`, `core_model` is an explicit marker in `pyproject.toml:236`, and `tests/conftest.py` has no `pytest_collection_modifyitems`/auto-marking hook, so the file is collected then deselected — the sm_120/sm_121 guard never executes in CI. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` right after the imports. (evidence: tests/quantization/test_quack_fp8_gate.py:17-21 (changed by this diff) `import pytest` / `import torch` / (blank) / `from vllm_omni.quantization import quack_fp8` — the file contains no `pytestmark` assignment and its only decorators are `@pytest.mark.parametrize`; unchanged by this diff, present in the PR-time tree: `.buildkite/test-ready.yml:29` `- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` and `pyproject.toml:236` ` "core_model: L1&L2 tests (run in each PR)",`; tests/conftest.py defines only `pytest_terminal_summary` with no collection-modification/auto-marking hook.) + +```suggestion +from vllm_omni.quantization import quack_fp8 + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +``` + +`vllm_omni/quantization/quack_fp8.py:~32` [minor] — Gate is `return torch.cuda.get_device_capability()[0] == 10` at vllm_omni/quantization/quack_fp8.py:32, which excludes cc 11.x (sm_110a is listed in the PR body as a tcgen05.mma target) while auto-enabling all cc 10.x including 10.1/10.3 (the PR body quotes BlockScaledMmaOp hardcoding admissible_archs = [Arch.sm_100a]). Please confirm the admitted arch set for quack-kernels>=0.3.11 and either gate on that set or document the exclusions; also pin the boundary in the test (add ((10,1), True) if sm_101 remains in scope, and an (11,0) expectation matching the gate). (evidence: vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10` — includes all cc 10.x, excludes cc 11.x. tests/quantization/test_quack_fp8_gate.py:33-38 parametrizes `((10, 0), True)`, `((10, 3), True)`, `((12, 0), False)`, `((12, 1), False)`, `((9, 0), False)`, `((8, 9), False)` — no (10,1) and no (11,0). PR 4817 body (via gh_pr_view): "PTX `tcgen05.mma` targets `sm_100a / sm_101a / sm_103a / sm_110a`" and "the CuteDSL `BlockScaledMmaOp` hardcodes `admissible_archs = [Arch.sm_100a]`".) + +`?:18` [minor] — naming: `_is_blackwell` → `_is_quack_capable` (vllm_omni/quantization/quack_fp8.py:18) is more precise — the gate is compute capability == 10 (datacenter Blackwell only) — and matches the file's underscore-prefixed private-helper convention. Rename is complete: the sole caller at line 41 uses the new name and no `_is_blackwell` references remain. (evidence: vllm_omni/quantization/quack_fp8.py:18 `def _is_quack_capable() -> bool:` — renamed helper present in PR-time tree. vllm_omni/quantization/quack_fp8.py:41 ` return _is_quack_capable()` — caller updated. Repo-wide grep for `_is_blackwell` returns no matches, confirming the rename is complete.) + +`vllm_omni/patch.py:414` [minor] — Confirmed global blast radius: `vllm_omni/__init__.py:22` imports `patch` in every process, `vllm_omni/patch.py:412-414` calls `install_quack_fp8_patch()`, and that function globally replaces `FlashInferFP8ScaledMMLinearKernel.apply_scaled_mm` (quack_fp8.py:128) whenever `quack_enabled()` is true. The `>= 10` → `== 10` change in `vllm_omni/quantization/quack_fp8.py:33` therefore flips the default for every FP8 ScaledMM path that uses that kernel (sm_100/101/103 stay on; sm_120/121 turn off). Please state this global default change in the PR description/commit; if only the sm_120/121 fix was intended, scope the decision at the `install_quack_fp8_patch` call site instead. (evidence: ['vllm_omni/__init__.py:22 (unchanged by this diff, present in PR-time tree): ` from . import patch # noqa: F401` — every import of vllm_omni executes patch.py.', 'vllm_omni/patch.py:412-414 (unchanged by this diff, present in PR-time tree): ` from vllm_omni.quantization.quack_fp8 import install_quack_fp8_patch` / ` install_quack_fp8_patch()` — the gate is invoked from the always-imported patch module.', 'vllm_omni/quantization/quack_fp8.py:33 (changed in this diff): ` return torch.cuda.get_device_capability()[0] == 10` — the shared predicate that now excludes cc 12.x.', 'vllm_omni/quantization/quack_fp8.py:42: ` return _is_quack_capable()` — quack_enabled() defaults to the shared predicate.', 'vllm_omni/quantization/quack_fp8.py:128: ` FlashInferFP8ScaledMMLinearKernel.apply_scaled_mm = apply_scaled_mm` — global class-level patch, so all FP8 ScaledMM paths using this kernel inherit the new default.']) + +`docs/user_guide/quantization/fp8.md:54` [minor] — At docs/user_guide/quantization/fp8.md:53, reword to: `VLLM_OMNI_USE_QUACK_FP8=1` forces the quack attempt (on `sm_120`/`sm_121` it still falls back per GEMM — use only once CuteDSL ships `sm_120a` support), or `VLLM_OMNI_USE_QUACK_FP8=0` to force the FlashInfer path. (evidence: docs/user_guide/quantization/fp8.md:53 `VLLM_OMNI_USE_QUACK_FP8=1` to force quack on, or `VLLM_OMNI_USE_QUACK_FP8=0` to force — changed by this PR. vllm_omni/quantization/quack_fp8.py:38 `override = os.environ.get("VLLM_OMNI_USE_QUACK_FP8")`; :39 `if override is not None:`; :40 `return override.lower() in _TRUTHY` — the override branch bypasses the `_is_quack_capable()` cc==10 gate at :32. Unchanged by this diff but present in the PR-time tree: quack_fp8.py:102 `if not quack_enabled():`, :104 `if _load_quack() is None:`, :118 `try:`, :119 `out = quack_scaled_fp8_mm(A, B, As, Bs, out_dtype, bias)`, :122 `except Exception as exc:`, :123 `logger.warning_once("Quack FP8 GEMM failed (%s); using FlashInfer.", exc)`, :124 `return original(self, A=A, B=B, out_dtype=out_dtype, As=As, Bs=Bs, bias=bias, output_shape=output_shape)`.) + +```suggestion +`VLLM_OMNI_USE_QUACK_FP8=1` forces the quack attempt (on `sm_120`/`sm_121` it still falls back per GEMM — use only once CuteDSL ships `sm_120a` support), or `VLLM_OMNI_USE_QUACK_FP8=0` to force the FlashInfer path. +``` + +`vllm_omni/quantization/quack_fp8.py:~30` [nit] — Confirmed: the change cannot plausibly introduce the AMD/NPU failures. quack_fp8.py:32 now returns True only for cc 10.x (`get_device_capability()[0] == 10`), so neither an AMD (ROCm) nor NPU lane can newly enable quack; the module is still imported via vllm_omni/patch.py:412-414 but `install_quack_fp8_patch()` no-ops there. Neither .buildkite/test-amd-ready.yaml (tests/diffusion + tests/model_executor) nor .buildkite/test-npu-ready.yaml (test_qwen3_tts_customvoice.py only) collects tests/quantization/test_quack_fp8_gate.py. Minor correction: on ROCm AMD, torch.cuda.is_available() is True, so the `if not torch.cuda.is_available()` guard at line 30 is not the short-circuit on the AMD lane; the capability check is. Pre-existing-on-main still needs a main-branch CI comparison, but this diff is not a plausible cause. (evidence: vllm_omni/quantization/quack_fp8.py:30 `if not torch.cuda.is_available():` ; vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10` ; vllm_omni/patch.py:412 `from vllm_omni.quantization.quack_fp8 import install_quack_fp8_patch` ; vllm_omni/patch.py:414 `install_quack_fp8_patch()` ; unchanged by this diff, present in the PR-time tree: .buildkite/test-amd-ready.yaml:19 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` ; unchanged by this diff, present in the PR-time tree: .buildkite/test-npu-ready.yaml:21 `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4817.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4817.trace.json.gz new file mode 100644 index 000000000..e835d78e6 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4817.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr4859.cost.json b/eval/dataset/arms/copilot_v17_train/pr4859.cost.json new file mode 100644 index 000000000..714a02e71 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4859.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1873.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4859/run-20260816-210223-ca55b8", + "recorded_at": "2026-08-16T21:33:34", + "attempts": [ + { + "invocation_id": "inv-17382659", + "outcome": "run:run-20260816-210223-ca55b8", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-210223-ca55b8", + "llm_calls": 164, + "input_tokens": 1427651, + "output_tokens": 282998, + "usd": 1.089929, + "source": "spans" + } + ], + "attempt_usd_total": 1.089929, + "llm_calls": 164, + "input_tokens": 1427651, + "output_tokens": 282998, + "usd": 1.089929, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-210223-ca55b8", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0899, + "minutes": 31.18, + "ci_minutes": 0, + "input_tokens": 1427651, + "output_tokens": 282998, + "tool_calls": 286, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1316, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 164, + "by_role": { + "moa_member": { + "usd": 0.147399, + "input_tokens": 235491, + "output_tokens": 11836, + "calls": 20 + }, + "agent": { + "usd": 0.910121, + "input_tokens": 1175194, + "output_tokens": 245904, + "calls": 141 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "reducer": { + "usd": 0.03241, + "input_tokens": 16966, + "output_tokens": 25258, + "calls": 2 + } + }, + "cache_read_tokens": 13322624, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.63, + "gate": 1.77, + "review": 1865.12, + "report": 0.06 + }, + "time_to_first_result_s": 1870.57 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4859.trace.json.gz", + "trace_bytes": 45047 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4859.md b/eval/dataset/arms/copilot_v17_train/pr4859.md new file mode 100644 index 000000000..b85e5f0d7 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4859.md @@ -0,0 +1,101 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4859, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6ff4ce59ed87) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 10 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [claim-verified] diff_stat: exactly 6 files touched, no test files — tests/model_executor/models/ming_tts not updated despite changed behavior. +- [claim-verified] gh_ci_read(4859): buildkite/vllm-omni-amd-ci = FAILURE; pre-commit, build 3.11/3.12, DCO, buildkite/vllm-omni, intel, npu, docs all SUCCESS. +- [claim-verified] Diff touches no test files — confirmed by evidence 'TEST FILES TOUCHED IN THIS DIFF: NONE'; the missing regression coverage is already in the kept comments. +- [claim-refuted] PR-body WER 1.83% (env VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100) runs through a double-resample path: producer normalizes to 24k (patch.py:118), eval re-derives 44100 from env (seed_tts_eval.py:188-193); calc target_len ratio 16000/44100=0.3628 vs correct 16000/24000=0.6667. +- [claim-refuted] PR test plan exports VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100, but the captured PCM is already 24 kHz mono; 44100/24000 = 1.8375 (calc), so the 1.83% WER is computed on ~1.84x time-compressed audio — not apples-to-apples with the 2.02% official figure. +- [claim-refuted] The PR's stated purpose 'benchmark PCM sample-rate parsing' applies the override on both capture (correct) and eval (incorrect) sides, creating the double resample the repo checklist warns about. +- [claim-refuted] PR test plan sets VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 while the WER pass resamples the already-24 kHz capture by 24000/44100 ≈ 0.5442x (calc) — the 1.83% WER is measured on time-compressed audio (kept major at seed_tts_eval.py:185). +- [validated] flash path of _parse_ming_instruction_fields is unchanged (include_language defaulted False before and after); existing test still passes but does not pin the dense language-removal behavior. +- [validated] process_one_official unchanged; compute_denoise_step_latency consumer stage_pool.py:567 unaffected by metrics additions. +- [validated] diff_stat: 6 files changed, no test files touched; TEST FILES TOUCHED IN THIS DIFF: NONE consistent. +- [validated] patch.py:127 `if int(sample_rate) != 24000 and pcm_f32.size:` and MixRequestFuncOutput docstring line 427 `Raw PCM s16le mono at 24 kHz for Seed-TTS WER` prove tts_output_pcm_bytes is already normalized on the success path. +- [validated] test_ming_instruction_parser_preserves_dense_and_flash_defaults at tests/.../test_ming_shared_modules.py:96 already expects post-change behavior: flash_fields == {'风格': 'calm'} (no '方言'). Test passes at this head. +- [validated] grep for include_language across entire repo: zero matches. The removed parameter has no remaining callers. +- [validated] '方言' key is None by default in BASE_CAPTION_TEMPLATE in both ming_flash_omni/prompt_utils.py:93 and ming_tts/prompt_assembly.py:38. Not providing it keeps None, which create_instruction merges correctly. + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:141` [major] — Encoder.__init__ builds self.encoder = Qwen2Model(config) (line 131), then for patch_size != -1 mutates that same config object in place (config.num_hidden_layers = 4, line 141) before self.aggregator = Qwen2Model(config) (line 142). HF Qwen2Model stores the passed config as its own config, so self.encoder.config.num_hidden_layers now reads 4 while the encoder's layers were actually built with the backbone's original layer count. Build the aggregator from a separate config as the base code did: aggregator_config = _qwen2_config({**encoder_args, 'num_hidden_layers': 4}) and self.aggregator = Qwen2Model(aggregator_config). (evidence: audio_vae.py:131 (unchanged by this diff, present in PR-time tree): ` self.encoder = Qwen2Model(config)`; audio_vae.py:141: ` config.num_hidden_layers = 4`; audio_vae.py:142: ` self.aggregator = Qwen2Model(config)`. All three statements reference the same `config` local created at the top of Encoder.__init__ via `config = _qwen2_config(encoder_args)`, so mutating it after `self.encoder = Qwen2Model(config)` changes `self.encoder.config.num_hidden_layers` to 4.) + +```suggestion +aggregator_config = _qwen2_config({**encoder_args, "num_hidden_layers": 4}) + self.aggregator = Qwen2Model(aggregator_config) +``` + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:~351` [major] — The diff changes `encode_latent` to sample `mean + std * torch.randn_like(mean)` (audio_vae.py:351) under `@torch.inference_mode()` with no seed or generator threaded through, so every call produces different latents. `prompt_encoder.py:137` and `voice_presets.py:257` both call `encode_latent`, so per-request reference encoding and registered voice presets become nondeterministic and Seed-TTS WER becomes a single stochastic draw. If HF parity requires sampling, thread an explicit seed/generator and add a determinism test; otherwise use the posterior mean at inference to preserve the previous deterministic behavior. (evidence: vllm_omni/model_executor/models/common/ming/audio_vae.py:351 `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)`; vllm_omni/model_executor/models/common/ming/audio_vae.py:329 `def encode_latent(self, waveform, waveform_length):` preceded by `@torch.inference_mode()` (line 328); vllm_omni/model_executor/models/ming_tts/prompt_encoder.py:137 `latents, _ = encoder.encode_latent(waveform, waveform_length)` (unchanged by this diff, present in PR-time tree); vllm_omni/model_executor/models/ming_flash_omni/voice_presets.py:257 `prompt_wav_lat, _ = self._audio_vae.encode_latent(` (unchanged by this diff, present in PR-time tree).) + +```suggestion +if self.training: + latent = (mean + std * torch.randn_like(mean)).transpose(1, 2) + else: + latent = mean.transpose(1, 2) +``` + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:~188` [major] — `_pcm_s16le_to_f32_16k` (seed_tts_eval.py:188) now falls back to `stream_pcm_format_from_env()` for both `pcm_sample_rate` and `channels`. Its only caller (seed_tts_eval.py:616) passes `out.tts_output_pcm_bytes`, which `_pcm_s16le_to_seed_tts_wer_bytes` already normalizes to 24 kHz mono (patch.py:130 `AudioResampler(target_sr=24000)`). With a non-24000 env override (e.g. 44100), seed_tts_eval.py:200 computes `target_len = len(raw) * 16000 / pcm_sample_rate`, time-compressing the 24 kHz capture by 24000/pcm_sample_rate (0.544x at 44100) before Whisper. Restore the 24000/1 defaults here or pass them explicitly at the call site; the env override must not be applied to the already-normalized WER buffer. (evidence: ['seed_tts_eval.py:188 (in-diff) ` if pcm_sample_rate is None or channels is None:` — the anchor snippet is at line 188, not 185.', 'seed_tts_eval.py:189 (in-diff) ` env_sample_rate, env_channels = stream_pcm_format_from_env()`', 'seed_tts_eval.py:200 (in-diff) ` target_len = int(len(raw) * 16000 / pcm_sample_rate)`', 'seed_tts_eval.py:616 (unchanged by diff) ` wav_16k = _pcm_s16le_to_f32_16k(pcm)` — the only call site, where `pcm` is `getattr(out, "tts_output_pcm_bytes", None)`.', 'patch.py:118 (unchanged by diff) ` """Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER."""`', 'patch.py:130 (unchanged by diff) ` resampler = AudioResampler(target_sr=24000)`', 'patch.py:1215 (unchanged by diff) ` output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(`']) + +```suggestion +if pcm_sample_rate is None: + pcm_sample_rate = 24000 + if channels is None: + channels = 1 +``` + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:42` [major] — The docstring is wrong, and the change is a functional bug: `_pcm_s16le_to_f32_16k` now reads `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`/`VLLM_OMNI_BENCH_AUDIO_CHANNELS` for its only input, `tts_output_pcm_bytes`, which the capture side has already normalized to 24 kHz mono (`_pcm_s16le_to_seed_tts_wer_bytes` and the chat path resample/downmix to 24000 mono). With a non-default env (e.g. 44100) the WER/UTMOS buffer is resampled as if 44.1 kHz, yielding ~0.544x the correct length. Keep the env override on the capture side and restore the 24000/1 default here (or pass explicit 24000/1 at the call site). (evidence: ['vllm_omni/benchmarks/data_modules/seed_tts_eval.py:42-43 "Streaming PCM is decoded using ``VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`` /" "``VLLM_OMNI_BENCH_AUDIO_CHANNELS`` (default: 24 kHz mono)."', 'vllm_omni/benchmarks/data_modules/seed_tts_eval.py:586 `pcm = getattr(out, "tts_output_pcm_bytes", None)` and seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)` — the only call site, passing the captured field.', 'vllm_omni/benchmarks/data_modules/seed_tts_eval.py:188-193 (changed by this diff) `if pcm_sample_rate is None or channels is None:` / `env_sample_rate, env_channels = stream_pcm_format_from_env()` / `if pcm_sample_rate is None:` / `pcm_sample_rate = env_sample_rate` / `if channels is None:` / `channels = env_channels`.', 'vllm_omni/benchmarks/patch/patch.py:118 (unchanged by this diff, present in the PR-time tree) `"""Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER."""`.', 'vllm_omni/benchmarks/patch/patch.py:1215 (unchanged by this diff, present in the PR-time tree) `output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(`.']) + +```suggestion +``tts_output_pcm_bytes`` is already normalized to 24 kHz mono by the capture side; +do not set ``VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`` / ``VLLM_OMNI_BENCH_AUDIO_CHANNELS`` here. +``` + +`vllm_omni/metrics/definitions.py:337` [minor] — `stream_pcm_format_from_env` only warns on `ValueError` and then returns `max(sample_rate, 1), max(channels, 1)` (definitions.py:326-337), so a parseable but non-positive env value such as `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=0` is silently converted to 1 Hz instead of falling back to the documented defaults. The benchmark consumer then computes `audio_duration = total_samples / sample_rate` (patch.py:1194), producing a huge duration and ~0 RTF from invalid input. Treat zero/negative parsed values as invalid, warn with the same style used for `ValueError`, and fall back to the documented defaults. (evidence: vllm_omni/metrics/definitions.py:326-337 `raw_sr = os.environ.get(AUDIO_SAMPLE_RATE_ENV)` ... `return max(sample_rate, 1), max(channels, 1)` — only `ValueError` triggers a warning; valid-int zero/negative values are clamped to 1. Unchanged by this diff? No, this helper is added by the PR. Consumer, also in this diff: vllm_omni/benchmarks/patch/patch.py:1194 `output.audio_duration = total_samples / sample_rate` (with line 1193 `total_samples = total_pcm_bytes // (sample_width * channels)`).) + +```suggestion +if sample_rate <= 0: + logger.warning("Invalid %s=%r; using default %d", AUDIO_SAMPLE_RATE_ENV, raw_sr, default_sample_rate) + sample_rate = default_sample_rate + if channels <= 0: + logger.warning("Invalid %s=%r; using default %d", AUDIO_CHANNELS_ENV, raw_ch, default_channels) + channels = default_channels + return sample_rate, channels +``` + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:351` [minor] — buildkite/vllm-omni-amd-ci is FAILURE on this PR head while all other checks are SUCCESS, and the six-file diff touches no AMD/ROCm-specific code, so the failure is likely pre-existing/environmental. Note PR 4859 is already MERGED: rerun the AMD gate on the head and record the outcome; do not leave a known red required gate unacknowledged. (evidence: ['CI API output (outside the diff, decisive for a CI-gate claim): gh_ci_read(4859) → {"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}; every other returned check has state SUCCESS (pre-commit, build (3.11), build (3.12), DCO, buildkite/vllm-omni, buildkite/vllm-omni-intel-ci, buildkite/vllm-omni-npu-ci, docs/readthedocs.org:vllm-omni).', 'diff_stat(4859) (merge-base a8b2bf11d2f2) changed files: vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/metrics/definitions.py, vllm_omni/model_executor/models/common/ming/audio_vae.py, vllm_omni/model_executor/models/ming_tts/patch_emission.py — no AMD/ROCm-specific file.']) + +`tests/model_executor/models/ming_tts/test_ming_shared_modules.py:96` [minor] — Add regression tests for the changed Ming behaviors in this PR: the diff touches no test files while changing (1) shared AudioVAE encode_latent to mean + softplus(scale)*randn sampling (vllm_omni/model_executor/models/common/ming/audio_vae.py:349-351), (2) the decode-window boundary to max(min_stop_step + 2, min_steps) (vllm_omni/model_executor/models/ming_tts/patch_emission.py:130 and :151), (3) removal of the include_language=True dense instruction path (vllm_omni/entrypoints/openai/serving_speech.py), and (4) PCM sample-rate/channel defaults moved to defs.stream_pcm_format_from_env (vllm_omni/metrics/definitions.py). The existing test_ming_instruction_parser_preserves_dense_and_flash_defaults only checks language='Auto' dense and include_voice flash, so it does not cover the removed non-Auto dense language path. (evidence: ['diff_stat (merge-base a8b2bf11d2f2) lists exactly 6 changed files: vllm_omni/benchmarks/data_modules/seed_tts_eval.py, vllm_omni/benchmarks/patch/patch.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/metrics/definitions.py, vllm_omni/model_executor/models/common/ming/audio_vae.py, vllm_omni/model_executor/models/ming_tts/patch_emission.py — none under tests/.', 'vllm_omni/model_executor/models/common/ming/audio_vae.py:349-351 `mean, scale = torch.chunk(h, 2, dim=1)` / `std = F.softplus(scale) + 1e-4` / `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)` (changed by this diff).', 'vllm_omni/model_executor/models/ming_tts/patch_emission.py:130 `min_required = max(min_stop_step + 2, min_steps)` and :151 `min_required_decode_steps = max(min_stop_step + 2, min_decode_steps)` (changed by this diff).', 'tests/model_executor/models/ming_tts/test_ming_shared_modules.py:96 `flash_fields = serving._parse_ming_instruction_fields(` — present in the PR-time tree; the surrounding test does not exercise the removed dense include_language=True path.']) + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:141` [minor] — Add a unit test that instantiates AudioVAE (or its Encoder) with patch_size != -1, e.g. via AudioVAEConfig(patch_size=4), and runs the encoder forward / encode_latent path. The PR changes the aggregator construction (config.num_hidden_layers = 4; Qwen2Model(config)) and encode_latent's mean/scale sampling, but tests/ only constructs AudioVAEConfig with patch_size=-1 or uses a voxcpm2 _FakeAudioVAE, so neither changed path is executed. (evidence: Changed by this diff: vllm_omni/model_executor/models/common/ming/audio_vae.py:140 `if patch_size != -1:`; :141 `config.num_hidden_layers = 4`; :142 `self.aggregator = Qwen2Model(config)`; and :349-351 `mean, scale = torch.chunk(h, 2, dim=1)` / `std = F.softplus(scale) + 1e-4` / `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)`. Unchanged by this diff, present in the PR-time tree: tests/model_executor/models/ming_tts/test_ming_shared_modules.py:32 `cfg = AudioVAEConfig(sample_rate=16000, patch_size=-1)` (config-only; no AudioVAE instantiation); tests/model_executor/models/ming_tts/test_ming_shared_modules.py:57-63 uses `AudioVAEConfig(... patch_size=VAE_PATCH_SIZE ...)` only inside validate_ming_tts_config; tests/model_executor/models/voxcpm2/test_talker_state_eviction.py:396 `class _FakeAudioVAE:` and :401 `audio_vae = _FakeAudioVAE()`. A grep for `AudioVAE(` under tests/ matches only `_FakeAudioVAE`, i.e. no real AudioVAE is ever constructed with patch_size != -1.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/entrypoints/openai/serving_speech.py:2651` [minor] The diff removes the `include_language` parameter and the `instruction_dict['方言'] = request.language` branch from `_parse_ming_instruction_fields`/`_parse_ming_instruction` (serving_speech.py:2619-2653), so `request.language` no longer reaches Ming generation as the 方言 instruction. The parser test […] +- `vllm_omni/model_executor/models/ming_tts/patch_emission.py:151` [minor] The diff changes the stop-head boundary from `min_stop_step + 1` to `min_stop_step + 2` in both `_validate_ming_decode_window` (patch_emission.py:130) and `_resolve_ming_stop_decision` (patch_emission.py:151), delaying stop gating by one decode step. No tests reference either helper, and this change is bundled with […] +- `vllm_omni/metrics/definitions.py:21` [minor] vllm_omni/metrics/definitions.py:21 introduces `logger = logging.getLogger(__name__)`, while vllm_omni/metrics/stats.py:18 and vllm_omni/benchmarks/patch/patch.py:52 use `logger = init_logger(__name__)`. The two new warnings in `stream_pcm_format_from_env` (definitions.py:330 and :336) therefore use the stdlib logger. […] +- `vllm_omni/metrics/definitions.py:286` [minor] `DEFAULT_AUDIO_CHANNELS`, the two `VLLM_OMNI_BENCH_AUDIO_*` env constants, and `stream_pcm_format_from_env` are bench-CLI env parsing added to `vllm_omni/metrics/definitions.py`, whose module docstring scopes it to 'Prometheus + bench CLI metric naming'. The new `import os`/`import logging` are therefore inherited by […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. +- **vllm-omni-pr-review-hub-kernel-loading**: Review pattern: optional HuggingFace kernels-backed attention backends — per-layer unpinned get_kernel loads, selector fallback gaps, and CI lane wiring. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4859.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4859.trace.json.gz new file mode 100644 index 000000000..8dac1ce28 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4859.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr4870.cost.json b/eval/dataset/arms/copilot_v17_train/pr4870.cost.json new file mode 100644 index 000000000..2bf028522 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4870.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1759.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4870/run-20260816-203303-a19e14", + "recorded_at": "2026-08-16T21:02:21", + "attempts": [ + { + "invocation_id": "inv-fc1f3b1a", + "outcome": "run:run-20260816-203303-a19e14", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-203303-a19e14", + "llm_calls": 204, + "input_tokens": 2192343, + "output_tokens": 272692, + "usd": 1.475222, + "source": "spans" + } + ], + "attempt_usd_total": 1.475222, + "llm_calls": 204, + "input_tokens": 2192343, + "output_tokens": 272692, + "usd": 1.475222, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-203303-a19e14", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.4752, + "minutes": 29.3, + "ci_minutes": 0, + "input_tokens": 2192343, + "output_tokens": 272692, + "tool_calls": 293, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2219, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 204, + "by_role": { + "moa_member": { + "usd": 0.154089, + "input_tokens": 255987, + "output_tokens": 5174, + "calls": 23 + }, + "agent": { + "usd": 1.293741, + "input_tokens": 1921395, + "output_tokens": 246329, + "calls": 177 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 2 + }, + "reducer": { + "usd": 0.027392, + "input_tokens": 14961, + "output_tokens": 21189, + "calls": 2 + } + }, + "cache_read_tokens": 20304128, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.44, + "gate": 1.56, + "review": 1751.77, + "report": 0.06 + }, + "time_to_first_result_s": 1757.82 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 42403 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4870.md b/eval/dataset/arms/copilot_v17_train/pr4870.md new file mode 100644 index 000000000..38a57866b --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4870.md @@ -0,0 +1,90 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] earlier revision read `self.engine_client.model_config`; df8137e8 changed it to `self.model_config`, consistent with other `self.model_config` uses in the same file (serving_speech.py:2466, 3283, 3600). +- [resolved] non-qwen3 async_chunk=False streaming keeps DELTA coercion: new test `...keeps_delta_for_non_qwen3` asserts coerce called with True. +- [resolved] GPU producer census — qwen3_tts_talker.py:591-605 cats per-request audio code rows into codes.audio aligned to scheduled tokens; no conflicting GPU AR producer of codes.audio found. +- [claim-verified] codes.audio is batch-aligned to scheduled token count: qwen3_tts_talker.py make_omni_output uses `audio_codes = torch.cat(audio_codes_list, dim=0)`. +- [claim-verified] tail-only hidden states are the Qwen3-TTS path: qwen3_tts_talker.py sets `requires_full_prefix_cached_hidden_states = False`; the new test passes hidden shape (1,) vs total 3. +- [claim-verified] diff_stat confirms exactly 4 files changed (2 prod, 2 tests); the probe file is not in the diff, matching the PR's claim. +- [claim-verified] PR's named tests exist: test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only and test_prepare_speech_generation_no_async_chunk_stream_uses_final_only. +- [claim-verified] PR body says ruff format passes on four changed files — the diff contains exactly four files (2 src + 2 test); ruff itself was not re-run in this read-only pass. +- [claim-refuted] PR test plan claims E2E coverage for 'stream=True plus async_chunk=False response mode', but the E2E probe in the PR body used 'stream=False'; only the coercion argument is unit-tested, not the actual FINAL_ONLY streaming response. +- [validated] gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` feeds `_build_omni_pooler_payload(seq_len=mm_seq_len)`; `to_payload_element` slices `element[start:end]` only when `element.shape[0]==seq_len` (mm_outputs.py:129-130). +- [validated] prefix-cache counterpart already computes `total_scheduled_tokens = sum(...)` in `_coerce_to_payload_dict` (core/prefix_cache.py:597-605); the new GPU code aligns the no-prefix-cache path. +- [validated] new worker test drives the real `_build_omni_model_runner_output_from_snapshot` with batched `codes.audio` (3x16) and per-request `num_scheduled_tokens [1,1,1]`; it would fail under the old `hidden_states.shape[0]` split length. +- [validated] removed seq_len arg from _build_omni_model_runner_output_from_snapshot; grep finds no remaining callers outside gpu_ar_model_runner.py:2027 (output_builder) and tests/worker/test_gpu_ar_model_runner.py. +- [validated] all 5 test call sites dropped seq_len= and no 'seq_len=' remains in tests/worker/test_gpu_ar_model_runner.py (grep). + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:~905` [major] — This diff fixes the GPU payload-split bug by slicing multimodal tensors on scheduler layout (mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) at vllm_omni/worker/gpu_ar_model_runner.py:1734, passed as seq_len=mm_seq_len at :1790). The NPU AR-runner twin (unchanged by this diff) still derives seq_len = hidden_states.shape[0] at vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905 and passes it at :1144, so the same tail-only cross-request contamination remains on NPU: with tail hidden states seq_len==1 while codes.audio is [batch,16], to_payload_element's element.shape[0] == seq_len gate (vllm_omni/utils/mm_outputs.py:131) falls through and clones the whole batched payload. Apply the scheduler-layout length here too (mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)) and pass it at line 1144. (evidence: ['vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905 `seq_len = hidden_states.shape[0]` — unchanged by this diff, present in the PR-time tree.', 'vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:1144 `seq_len=seq_len,` — unchanged by this diff, present in the PR-time tree.', 'vllm_omni/utils/mm_outputs.py:131 `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len:` — when the gate fails, the function falls through to `return element.clone()` for tensors, cloning the whole batched payload.', 'vllm_omni/worker/gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` and :1790 `seq_len=mm_seq_len,` — the fix this PR applied on GPU only.']) + +```suggestion +mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) +``` + +`vllm_omni/entrypoints/openai/serving_speech.py:~3508` [minor] — vllm_omni/entrypoints/openai/serving_speech.py:3508-3509 makes Qwen3-TTS streaming requests with async_chunk=False coerce as FINAL_ONLY (coerce_param_message_types(..., False)), but the new tests only assert the coercion argument and never assert the emitted streaming response, so the one-final-waveform path is unverified. Add an E2E probe for stream=True/stream_format='audio' with async_chunk=False that asserts one complete waveform is received. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` and vllm_omni/entrypoints/openai/serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload` — for qwen3_tts with async_chunk=False any streaming request yields is_streaming_request=False, so coerce_param_message_types gets False (FINAL_ONLY). The PR's added tests (diff hunk tests/entrypoints/openai_api/test_serving_speech.py @@ -3450,6 +3450,48 @@) assert only `mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, False)`, not any emitted streaming chunk.) + +`vllm_omni/entrypoints/openai/serving_speech.py:~3507` [minor] — Align the fallback to False: async_chunk defaults to False in arg_utils and all runner consumers, so a missing model_config attribute should resolve as async_chunk disabled (Qwen3-TTS full-payload FINAL_ONLY), not DELTA coercion. (evidence: Diff line (PR-time tree): vllm_omni/entrypoints/openai/serving_speech.py:3507 `async_chunk = getattr(self.model_config, "async_chunk", True)`; 3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; 3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`. With the True fallback, a missing async_chunk makes async_chunk=True, so qwen3_full_payload=False and streaming Qwen3 requests coerce DELTA. Unchanged by this diff, present in the PR-time tree: vllm_omni/worker/gpu_ar_model_runner.py:293 `self._async_chunk = getattr(self.model_config, "async_chunk", False)`; vllm_omni/worker/omni_connector_model_runner_mixin.py:104 `self._async_chunk: bool = getattr(model_config, "async_chunk", False)`; vllm_omni/engine/arg_utils.py:161 `async_chunk: bool = False`.) + +```suggestion +async_chunk = getattr(self.model_config, "async_chunk", False) +``` + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [minor] — mm_seq_len at line 1734 flows into _build_omni_pooler_payload(seq_len=...) for every GPU AR model with a pooler payload (needs_pooler_payload = len(downstream_req_ids) > 0), not just Qwen3-TTS. The new worker test covers the tail-only (hidden opt-out) path but not a full-prefix model under the new total_num_scheduled_tokens semantics; either add a full-prefix assertion or scope mm_seq_len to the tail-only path. Full-prefix models are unaffected only because their hidden_states.shape[0] equals total_num_scheduled_tokens there. (evidence: vllm_omni/worker/gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` — added by this diff inside `if needs_pooler_payload:`. vllm_omni/worker/gpu_ar_model_runner.py:1778 `seq_len=mm_seq_len,` — inside the `self._build_omni_pooler_payload(...)` call, so the new length flows to every GPU AR model that enters the pooler-payload branch.) + +`docs/user_guide/examples/online_serving/text_to_speech.md:~422` [minor] — This diff keeps Qwen3-TTS `async_chunk=False` speech requests in FINAL_ONLY mode (`qwen3_full_payload = self._tts_model_type == 'qwen3_tts' and not bool(async_chunk)` in `vllm_omni/entrypoints/openai/serving_speech.py`), so `stream_format='audio'` no longer requires async chunks. `docs/user_guide/examples/online_serving/text_to_speech.md:422` still says raw PCM streaming requires `async_chunk: true`, which is now stale. Clarify that `async_chunk=False` yields final-only streaming for qwen3_tts, or keep the requirement but document the new behavior. (evidence: Changed by this diff: vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` and serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`. Unchanged by this diff, present in the PR-time tree: docs/user_guide/examples/online_serving/text_to_speech.md:422 `Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.` The diff's new test `test_prepare_speech_generation_no_async_chunk_stream_uses_final_only` (tests/entrypoints/openai_api/test_serving_speech.py) builds a raw-PCM request (`stream_format="audio", response_format="pcm"`) with `async_chunk=False` and asserts `mock_coerce.assert_called_once_with(..., False)`, proving the async_chunk=False streaming path is intentional.) + +```suggestion +Raw PCM streaming requires `stream_format="audio"` and `response_format="pcm"`. With `async_chunk: true` (default in `qwen3_tts.yaml`) audio is emitted incrementally; with `async_chunk: false` the server streams a single final-only waveform. `speed` is not supported when streaming. +``` + +`.buildkite/test-amd-merge.yml:~177` [minor] — This diff changes the GPU AR-runner multimodal split length (`vllm_omni/worker/gpu_ar_model_runner.py`) and serving-speech FINAL_ONLY behavior (`vllm_omni/entrypoints/openai/serving_speech.py`), and `buildkite/vllm-omni-amd-ci` is FAILURE at head. `.buildkite/test-amd-merge.yml:177` (Qwen3-TTS CustomVoice, `grade: Blocking`) and `:192` (Qwen3-TTS Base, `grade: Blocking`) run E2E tests that exercise both changed code paths via the serving/worker stack. Since this PR fixes #4851 (Qwen3-TTS no_async_chunk serving correctness), pull the AMD job log and confirm whether the failure is this regression or pre-existing on main before merging. (evidence: Unchanged by this diff, present in the PR-time tree: `.buildkite/test-amd-merge.yml:177` `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"`; `.buildkite/test-amd-merge.yml:192` `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'`. `gh_ci_read(4870)` returns `{"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` (API result, not a file line).) + +`tests/entrypoints/openai_api/test_serving_speech.py:~3453` [minor] — The diff makes `is_streaming_request = request.is_streaming() and not qwen3_full_payload` (vllm_omni/entrypoints/openai/serving_speech.py:3509), so every `is_streaming()` form — `stream_format='audio'`, `stream=True` (SSE), and `stream_format='sse'` — is coerced to FINAL_ONLY for Qwen3-TTS when `async_chunk=False`. The new test only constructs `OpenAICreateSpeechRequest(input='hello', stream_format='audio', response_format='pcm')` and only asserts `coerce_param_message_types(..., False)`; the `stream=True` and `stream_format='sse'` forms under `async_chunk=False` are not covered. Parametrize the test to cover all three streaming forms and, where observable, assert a single final waveform. (evidence: ['vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`', 'vllm_omni/entrypoints/openai/serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`', 'tests/entrypoints/openai_api/test_serving_speech.py:3453 `def test_prepare_speech_generation_no_async_chunk_stream_uses_final_only(`', 'tests/entrypoints/openai_api/test_serving_speech.py:3467 `request = OpenAICreateSpeechRequest(input="hello", stream_format="audio", response_format="pcm")`', 'tests/entrypoints/openai_api/test_serving_speech.py:3471 `mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, False)`', 'unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/protocol/audio.py:302 `def is_streaming(self) -> bool: return self.is_raw_audio_stream() or self.is_sse_stream()` (comment-cited line; method content verified, line number not independently re-grepped).']) + +```suggestion +@pytest.mark.parametrize( + "request", + [ + OpenAICreateSpeechRequest(input="hello", stream_format="audio", response_format="pcm"), + OpenAICreateSpeechRequest(input="hello", stream=True, response_format="pcm"), + OpenAICreateSpeechRequest(input="hello", stream_format="sse"), + ], + ) + def test_prepare_speech_generation_no_async_chunk_stream_uses_final_only( + self, qwen3_tts_server, mocker: MockerFixture, request + ): +``` + +`tests/worker/test_gpu_ar_model_runner.py:478` [minor] — This new test correctly forces the seq_len switch (hidden_states has 1 row vs total_num_scheduled_tokens=3 and codes shape [3,16], so the old seq_len=hidden_states.shape[0] would return the full codes tensor for every request), but every request schedules exactly 1 token, so start/end all collapse to idx:idx+1 and per-request slicing with differing counts is not covered. Consider adding a mixed-layout case, e.g. num_scheduled_tokens={'r1':1,'r2':2,'r3':1}, total_num_scheduled_tokens=4, codes shape [4,16], hidden_states shape [1], query_start_loc_cpu=[0,1,3], asserting codes[0:1], codes[1:3], codes[3:4]. (evidence: tests/worker/test_gpu_ar_model_runner.py:458 ` num_scheduled_tokens={"r1": 1, "r2": 1, "r3": 1},` +tests/worker/test_gpu_ar_model_runner.py:473 ` num_scheduled_tokens_np=np.array([1, 1, 1], dtype=np.int32),` +tests/worker/test_gpu_ar_model_runner.py:474 ` query_start_loc_cpu=torch.tensor([0, 1, 2], dtype=torch.long),` +tests/worker/test_gpu_ar_model_runner.py:478 ` assert torch.equal(output.inter_stage_outputs[0]["codes.audio"], codes[0:1])` +vllm_omni/worker/gpu_ar_model_runner.py:1734 (diff-added) ` mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4870.trace.json.gz new file mode 100644 index 000000000..c7febab60 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr4923.cost.json b/eval/dataset/arms/copilot_v17_train/pr4923.cost.json new file mode 100644 index 000000000..814a3fda8 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4923.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1056.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4923/run-20260816-201527-0faa63", + "recorded_at": "2026-08-16T20:33:01", + "attempts": [ + { + "invocation_id": "inv-10643315", + "outcome": "run:run-20260816-201527-0faa63", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-201527-0faa63", + "llm_calls": 123, + "input_tokens": 1242069, + "output_tokens": 162555, + "usd": 0.836324, + "source": "spans" + } + ], + "attempt_usd_total": 0.836324, + "llm_calls": 123, + "input_tokens": 1242069, + "output_tokens": 162555, + "usd": 0.836324, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-201527-0faa63", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8363, + "minutes": 17.56, + "ci_minutes": 0, + "input_tokens": 1242069, + "output_tokens": 162555, + "tool_calls": 213, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8205, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 123, + "by_role": { + "moa_member": { + "usd": 0.142488, + "input_tokens": 198650, + "output_tokens": 9358, + "calls": 29 + }, + "agent": { + "usd": 0.652789, + "input_tokens": 1026461, + "output_tokens": 120113, + "calls": 91 + }, + "reducer": { + "usd": 0.041047, + "input_tokens": 16958, + "output_tokens": 33084, + "calls": 3 + } + }, + "cache_read_tokens": 10732352, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.77, + "gate": 1.61, + "review": 1048.12, + "report": 0.07 + }, + "time_to_first_result_s": 1053.56 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 33188 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4923.md b/eval/dataset/arms/copilot_v17_train/pr4923.md new file mode 100644 index 000000000..22445179e --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4923.md @@ -0,0 +1,80 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [claim-verified] PR body claim 'talker_mtp_graph_safe = True so the runner wraps talker_mtp in a FULL CUDAGraphWrapper' — verified at gpu_model_runner.py:209: `if cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe)` +- [claim-verified] PR body claim 'GPU: torch.compile only; no inner device graphs' — verified: use_cuda_graphs=is_npu() → False on GPU, CodePredictorWrapper skips device graph capture +- [claim-verified] PR description '39ms→9ms per step' — benchmark not runnable here, but the mechanism (outer FULL graph capture replacing per-step eager MTP calls) is consistent with the claim. +- [claim-verified] benchmark arithmetic: 32/4.56=7.02 req/s and 179.44/4.56=39.35 audio-duration/s match the reported table. +- [claim-refuted] PR description says the talker newly opts out of hidden-state pooler payload, but at this head `self.omni_pooler_payload_include_hidden = False` already exists ~line 310 outside the diff; the hunk's assignment is a duplicate no-op. +- [claim-refuted] 'no test change' is true but inadequate: the changed full-graph MTP + async-output path has no automated coverage; tests/worker/test_omni_gpu_model_runner.py:374 hardcodes talker_mtp_accepts_per_row_generators=True on a fake model and never drives the __init__ gate. +- [claim-refuted] PR says 'GPU code predictor uses torch.compile only' — true for CUDA, but the same value disables inner graphs on ROCm too, and the only evidence is 1x H100 (CUDA); AMD lane is red and intersects this exact line. +- [validated] qwen3_tts_talker.py: model reads cudagraph_mode from compilation_config — TODO(#4923) acknowledges this is a runner concern; temporary coupling is acceptable with tracking issue +- [validated] qwen3_tts.yaml: max_num_seqs 10→64, max_num_batched_tokens 512→32768 — significant capacity increase, justified by PR profile data showing 39ms→9ms per-step improvement +- [validated] gpu_model_runner.py:209 wraps talker_mtp only when `cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe)`; Qwen3-TTS has no `.talker` and now sets `talker_mtp_graph_safe=True`, so outer wrap is consistent. +- [validated] gpu_ar_model_runner.py:1537-1564 `_should_use_async_omni_output` requires async_chunk, no omni_prefix_cache, no spec decode, and model flags; qwen3_tts.yaml has async_chunk=true and now enable_prefix_caching=false, so the new async flags actually take effect. +- [validated] qwen3_code_predictor.py:477 `_prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu`; GPU `use_cuda_graphs=False` disables inner graphs as intended while NPU keeps them. +- [validated] fish_speech_slow_ar.py:198 already uses the same `talker_mtp_graph_safe = True` naming/placement, so the new flag follows the sibling convention. +- [validated] YAML max_num_seqs 10→64 and max_num_batched_tokens 512→32768 — these are stage-0 capacity increases matching the PR's throughput goal. Stage-1 also 10→64. No consumer outside this YAML is affected (deploy configs are leaf-level). + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~350` [major] — Confirmed: under full cudagraphs `talker_mtp` is graph-wrapped (gpu_model_runner.py:208-210) while `talker_mtp_accepts_per_row_generators` is set to False (qwen3_tts_talker.py:350). The runner's scalar path still passes the per-request `generator` into the wrapped module (gpu_model_runner.py:1885-1886), so seeded requests are not reproducible and nothing warns the API user. Add `logger.warning_once` when a seeded request hits the graph-wrapped path, or surface the determinism loss in the speech API docs. (evidence: Changed by this diff — vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:334-335 `# Consequence: under full cudagraphs, per-request ``tts_local_seed`` is / # not reproducible (matching Qwen3-Omni, which has no per-row seeding);`. Unchanged by this diff, present in the PR-time tree — vllm_omni/worker/gpu_model_runner.py:209-210 `if cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe): ... self.talker_mtp = graph_wrapper_cls(talker_mtp, self.vllm_config, runtime_mode=CUDAGraphMode.FULL)`; vllm_omni/worker/gpu_model_runner.py:1885-1886 `if row_generators[0] is not None: talker_kwargs["generator"] = row_generators[0]`.) + +```suggestion +if ( + isinstance(self.talker_mtp, current_omni_platform.get_graph_wrapper_cls()) + and any(g is not None for g in row_generators) + ): + logger.warning_once( + "tts_local_seed is ignored under full-cudagraph talker_mtp replay; " + "seeded requests are not reproducible." + ) +``` + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~348` [major] — Confirmed at qwen3_tts_talker.py:348. No test constructs the talker with a VllmConfig whose compilation_config.cudagraph_mode.has_full_cudagraphs() is True, so the new gating branch never executes in CI. The only talker_mtp coverage hardcodes talker_mtp_accepts_per_row_generators=True on a fake model (tests/worker/test_omni_gpu_model_runner.py:374), and the qwen3_tts e2e disables cudagraphs via enforce_eager=True (tests/e2e/offline_inference/test_qwen3_tts_base.py:42). Add the requested CPU unit test asserting both flags under a full-cudagraphs VllmConfig, plus a GPU run of the shipped yaml through _capture_talker_mtp_graphs. (evidence: Changed by this diff — vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:346 `cudagraph_mode = getattr(vllm_config.compilation_config, "cudagraph_mode", None)`; :348 `self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs()`; :350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`. Unchanged by this diff, present in the PR-time tree — tests/worker/test_omni_gpu_model_runner.py:374 `talker_mtp_accepts_per_row_generators=True,` (hardcoded on a fake SimpleNamespace, so the real model gate never runs); tests/e2e/offline_inference/test_qwen3_tts_base.py:42 `"enforce_eager": True,` (cudagraphs disabled in the only qwen3_tts e2e).) + +`vllm_omni/deploy/qwen3_tts.yaml:95` [major] — Both buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are FAILURE on this PR (verified via CI checks), and the diff touches platform-specific behavior in both lanes: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu()` and vllm_omni/deploy/qwen3_tts.yaml:99/102 `cudagraph_mode: PIECEWISE` under `platforms.npu`. Local evidence cannot attribute the failures as pre-existing vs introduced; require green re-runs or a main-side comparison for both lanes before merging rather than assuming pre-existing failures. (evidence: CI checks (PR 4923): `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` and `{"bucket":"fail","name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}` (external CI data, not a repo file). Diff-changed lines: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu(),`; vllm_omni/deploy/qwen3_tts.yaml:99 ` cudagraph_mode: PIECEWISE`; vllm_omni/deploy/qwen3_tts.yaml:102 ` cudagraph_mode: PIECEWISE`.) + +`vllm_omni/deploy/qwen3_tts.yaml:~46` [minor] — The comment block at lines 41-45 still says 'Prefix caching on by default ... Disable explicitly if memory headroom is tight', but line 46 is `enable_prefix_caching: false`. The top-of-file note at lines 6-8 still says 'Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk latency tuning', but line 48 is `max_num_batched_tokens: 32768`. Update both comments to match the new values. (evidence: vllm_omni/deploy/qwen3_tts.yaml:41-46: `# Prefix caching on by default. The HS opt-out (talker only consumes the` / `# Disable explicitly if memory headroom is tight on a single-GPU deploy.` directly precedes `enable_prefix_caching: false`. vllm_omni/deploy/qwen3_tts.yaml:6-8: `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk` / `# latency tuning (not correctness) — small per-step batches keep` / `# first-chunk latency low.` while vllm_omni/deploy/qwen3_tts.yaml:48 is `max_num_batched_tokens: 32768`.) + +```suggestion +# Prefix caching is disabled on stage 0: async omni output and omni + # prefix caching are mutually exclusive, and this deploy opts into the + # async-output fast-path. + enable_prefix_caching: false +``` + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39` [minor] — Narrow the PR body's 'GPU: torch.compile only' claim to CUDA (or explicitly cover ROCm/XPU/MUSA): `use_cuda_graphs=current_omni_platform.is_npu()` is False on every non-NPU platform, so this hunk also disables inner device graphs on ROCm/XPU/MUSA without evidence, and the AMD CI lane covering this line is red. The 1x H100 result supports CUDA only. (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu(),` — set from the platform check, not a CUDA-only gate. Unchanged by this diff but present in the PR-time tree, proving the platform semantics: vllm_omni/platforms/interface.py:42 `def is_npu(self) -> bool:` and vllm_omni/platforms/interface.py:43 `return self._omni_enum == OmniPlatformEnum.NPU` — so `is_npu()` is False on ROCm/XPU/MUSA/CUDA and the hunk disables inner device graphs there too.) + +`vllm_omni/deploy/qwen3_tts.yaml:38` [minor] — The `talker_mtp 39ms→9ms` per-step claim is not backed by any benchmark artifact or harness in the diff (only qwen3_tts.yaml and two model files change), and benchmarks/tts/README.md's concurrency-cliff table is pre-existing and contains RTF/TTFP only. Mark the number as author-local profiling or check in the artifact/harness. Note: the capacity-bump values are at qwen3_tts.yaml:38/48, not line 7. (evidence: ['vllm_omni/deploy/qwen3_tts.yaml:38 `max_num_seqs: 64` and vllm_omni/deploy/qwen3_tts.yaml:48 `max_num_batched_tokens: 32768` — the capacity bump values the comment refers to, present at these lines (not line 7).', 'diff_stat for PR #4923: only three files changed — `vllm_omni/deploy/qwen3_tts.yaml`, `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py`, `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py`; no benchmark harness or result artifact is added.', 'benchmarks/tts/README.md:218 `Observed on H20-3e, Qwen3-TTS-1.7B (measured pre-merge on this branch):` — unchanged by this diff (README not in diff_stat) and the table lists RTF/TTFP, never a `talker_mtp 39ms→9ms` figure.', 'Repo grep for the claim finds only docs/design/qwen3_omni_tts_performance_optimization.md matches (a Qwen3-Omni doc, different model), not a Qwen3-TTS 39ms→9ms measurement.']) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:352` [nit] — `self.omni_pooler_payload_include_hidden = False` is already assigned at line 317 (unchanged by this diff); this hunk re-adds the identical assignment at line 353. Drop the duplicate line so the diff reflects only the two newly-enabled async flags. (evidence: qwen3_tts_talker.py:317 `self.omni_pooler_payload_include_hidden = False` (pre-existing, unchanged by this diff) and qwen3_tts_talker.py:353 `self.omni_pooler_payload_include_hidden = False` (newly added duplicate in the diff, between `self.eager_omni_postprocess_before_async_output = True` and `self.model = Qwen3Model(...)`).) + +```suggestion +self.use_async_omni_output = True + self.eager_omni_postprocess_before_async_output = True +``` + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4923.trace.json.gz new file mode 100644 index 000000000..d93a664bf Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr4926.cost.json b/eval/dataset/arms/copilot_v17_train/pr4926.cost.json new file mode 100644 index 000000000..9d6842872 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4926.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 2001.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4926/run-20260816-210114-e7c792", + "recorded_at": "2026-08-16T21:34:33", + "attempts": [ + { + "invocation_id": "inv-36bc3f82", + "outcome": "run:run-20260816-210114-e7c792", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-210114-e7c792", + "llm_calls": 198, + "input_tokens": 1048036, + "output_tokens": 318136, + "usd": 1.013634, + "source": "spans" + } + ], + "attempt_usd_total": 1.013634, + "llm_calls": 198, + "input_tokens": 1048036, + "output_tokens": 318136, + "usd": 1.013634, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-210114-e7c792", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0136, + "minutes": 33.32, + "ci_minutes": 0, + "input_tokens": 1048036, + "output_tokens": 318136, + "tool_calls": 283, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1342, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 198, + "by_role": { + "moa_member": { + "usd": 0.101898, + "input_tokens": 107020, + "output_tokens": 8586, + "calls": 33 + }, + "agent": { + "usd": 0.849073, + "input_tokens": 901635, + "output_tokens": 262319, + "calls": 160 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 2 + }, + "reducer": { + "usd": 0.062663, + "input_tokens": 39381, + "output_tokens": 47231, + "calls": 3 + } + }, + "cache_read_tokens": 13244288, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.56, + "gate": 1.55, + "review": 1993.98, + "report": 0.07 + }, + "time_to_first_result_s": 1999.13 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 52338 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4926.md b/eval/dataset/arms/copilot_v17_train/pr4926.md new file mode 100644 index 000000000..8beb24374 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4926.md @@ -0,0 +1,130 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] piecewise varlen fallback landed mid-PR (commit 332da8f2): `_flash_wrapper` with attn_func=None routes to `_run_varlen_dense` (flash_attn_hub.py:117-125) — residual: the piecewise path itself is still untested by the shipped test. +- [claim-verified] registry.py:65-66 adds FLASH_ATTN_HUB and FLASH_ATTN_3_HUB members to DiffusionAttentionBackendEnum; both enum lookups resolve. +- [claim-verified] Test-result number check: max_diff 0.00390625 == 1/256, consistent with bf16 ULP rounding at softmax scale 1/8 for head_dim 64. +- [claim-verified] HEAD tree state: commit 332da8f2's piecewise fallback to varlen and 35aa741e's strict assertions are present in the checked-out head. +- [claim-verified] PR claim 'non-CUDA or kernels-missing → local FLASH_ATTN fallback' is present in all five platform files (cuda/rocm/musa/npu/xpu). +- [claim-verified] PR 'All unit tests pass' log is a local run (kernels fetched from Hub); CI cannot reproduce because no lane collects the execution test. +- [claim-refuted] PR-body "falls back gracefully to local FLASH_ATTN" is incomplete: cuda/platform.py:124-127 probes only `importlib.import_module("kernels")`, bypassing the `compute_supported` gate that local FLASH_ATTN still checks at cuda/platform.py:149 — sm<80 with kernels installed returns the HUB backend, not SDPA. +- [claim-refuted] Docs' "perfect numerical consistency" (attention_backends.md:217) is unsupported: the PR's own test accepts max_diff < 1e-2, and the two HUB kernels are numerically distinct from SDPA and each other. +- [claim-refuted] Docs pin `pip install kernels==0.14.1` but pyproject.toml:70 declares only `kernels<0.15.0` (no lower bound); older versions in range need not provide the `get_kernel` API the new module imports. +- [claim-refuted] PR body 'All unit tests pass' locally shows FA3 PASSED, but FA3 requires sm>=9 and the test is marked res={"cuda":"L4"} (sm_89); on the marked hardware the FA3 branch (test:92-93 `if major < 9: pytest.skip`) always skips. +- [claim-refuted] docs claim 'perfect numerical consistency ... eliminates numerical drift' (attention_backends.md:217) vs unpinned get_kernel version=1/2/default refs and a 1e-2-vs-SDPA smoke only — no Diffusers parity test in the diff. +- [validated] test_kernels_hub_platform_fallback does exercise the real `CudaOmniPlatform.get_diffusion_attn_backend_cls` entry point with monkeypatched capability/package checkers (tests/diffusion/attention/test_kernels_hub.py:19-53). +- [validated] _run_varlen_dense cu_seqlens construction (flash_attn_hub.py:33-36) is correct for dense batches (arange step q_len/k_len). +- [validated] _upad_input call matches fa.py signature (fa.py:297: query,key,value,attention_mask,query_length,unpad_input_func) and 6-tuple return. + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92` [major] — This constructor is run once per attention layer (vllm_omni/diffusion/attention/layer.py:94-95 `self.attn_impl_cls = self.attn_backend.get_impl_cls()` then `self.attention = self.attn_impl_cls(...)`), yet every instantiation re-runs `from kernels import get_kernel` and retries `get_kernel(... version=1)`, then `version=2`, then default — an unpinned, unmemoized moving target. That contradicts the PR's stated goal of making rollout execute the same kernel version as training and multiplies disk/network/parse work by the layer count. Load once at module scope (or via an `lru_cache`) and pin an explicit revision instead of the `version=1/2/default` chain; the same defect is duplicated in `FlashAttention3HubImpl.__init__` at flash_attn_hub.py:271-282. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92 ` from kernels import get_kernel` and :96 ` hub_module = get_kernel("kernels-community/flash-attn2", version=1)` and :100 ` hub_module = get_kernel("kernels-community/flash-attn2", version=2)` and :103 ` hub_module = get_kernel("kernels-community/flash-attn2")` — unpinned, unmemoized per-constructor fetch. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/attention/layer.py:94-95 ` self.attn_impl_cls = self.attn_backend.get_impl_cls() + self.attention = self.attn_impl_cls( +` — each Attention layer instantiates the impl, so the fetch runs per layer. Same pattern duplicated in vllm_omni/diffusion/attention/backends/flash_attn_hub.py:271-282 for FlashAttention3HubImpl.) + +```suggestion +from functools import lru_cache + +@lru_cache(maxsize=None) +def _load_hub_kernel(repo_id: str) -> object: + from kernels import get_kernel + + return get_kernel(repo_id) # pin an explicit revision via kernels' rev kwarg + +... + hub_module = _load_hub_kernel("kernels-community/flash-attn2") +``` + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~147` [major] — `__init__` only raises when BOTH `flash_attn_func` and `flash_attn_varlen_func` are missing (line 108), but `_forward_varlen_masked` (line 147) and `_forward_varlen_dense` (line 163) call `self.flash_attn_varlen_func` unconditionally. A Hub module exposing only the dense `flash_attn_func` passes construction, then the advertised mask path crashes with `TypeError: 'NoneType' object is not callable`. Require `flash_attn_varlen_func` at construction or route masks through the dense function. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:108 ` if self.flash_attn_func is None and self.flash_attn_varlen_func is None:` — guard raises only when BOTH are None; :147 ` out_unpad = self.flash_attn_varlen_func(` — unconditional call in `_forward_varlen_masked`; :55-56 ` def supports_attention_mask(cls) -> bool:` / ` return True` — backend advertises mask support. All three lines are in the new file added by this diff.) + +```suggestion +if self.flash_attn_varlen_func is None: + raise RuntimeError( + "FlashAttentionHubImpl requires flash_attn_varlen_func: the Hub module " + "exposed only flash_attn_func, but masked/varlen paths need the varlen kernel." + ) +``` + +`vllm_omni/platforms/cuda/platform.py:124` [major] — FLASH_ATTN_HUB/FLASH_ATTN_3_HUB selection only checks that `kernels` is importable (lines 124-127) and never applies the `compute_supported` gate that local FLASH_ATTN applies at line 149. With `kernels` installed on a pre-Ampere GPU (capability < 8.0), the FLASH_ATTN_HUB backend is returned at line 175, running the Hub FA2 kernel on unsupported hardware instead of the TORCH_SDPA fallback used by the local path. Gate both HUB names on `compute_supported` and fall back to TORCH_SDPA before importing `kernels`. (evidence: ['vllm_omni/platforms/cuda/platform.py:80 (unchanged by this diff, present in PR-time tree) `compute_supported = capability >= 80`', 'vllm_omni/platforms/cuda/platform.py:124-126 (added by this diff) `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):` / `try:` / `importlib.import_module("kernels")`', 'vllm_omni/platforms/cuda/platform.py:149 (unchanged by this diff, present in PR-time tree) `if backend_upper == "FLASH_ATTN" and not flash_attn_supported:`', 'vllm_omni/platforms/cuda/platform.py:175-177 (unchanged by this diff, present in PR-time tree) `backend = DiffusionAttentionBackendEnum[backend_upper]` / `logger.debug("Using diffusion attention backend \'%s\'", backend_upper)` / `return backend.get_path()`']) + +```suggestion +if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"): + if not compute_supported: + logger.warning( + "Flash Attention requires GPU with compute capability >= 8.0. " + "Falling back to TORCH_SDPA backend." + ) + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + try: + importlib.import_module("kernels") +``` + +`vllm_omni/platforms/cuda/platform.py:133` [major] — Confirmed at vllm_omni/platforms/cuda/platform.py:124-156. An explicit FLASH_ATTN_HUB/FLASH_ATTN_3_HUB request is downgraded to local FLASH_ATTN on `ImportError` for `kernels`, and the existing `flash_attn_supported` gate can further route to TORCH_SDPA. The only user-visible signal is a warning that does not state outputs will no longer match HF-Diffusers training, contradicting the 'perfect numerical consistency' claim in docs/user_guide/diffusion/attention_backends.md:217. Hard-fail (or an explicit opt-out) is a reasonable fix. (evidence: vllm_omni/platforms/cuda/platform.py:126-138 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"): try: importlib.import_module("kernels") ... except ImportError: if backend_upper == "FLASH_ATTN_HUB": logger.warning("HuggingFace `kernels` library is not available. Falling back to local FLASH_ATTN.") backend_upper = "FLASH_ATTN" elif backend_upper == "FLASH_ATTN_3_HUB": logger.warning("HuggingFace `kernels` library is not available. Falling back to local FLASH_ATTN.") backend_upper = "FLASH_ATTN"`; vllm_omni/platforms/cuda/platform.py:149-156 `if backend_upper == "FLASH_ATTN" and not flash_attn_supported: ... return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path()`; docs/user_guide/diffusion/attention_backends.md:217 `To achieve perfect numerical consistency between **training** ... and **serving/rollout** ... This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout.`) + +```suggestion +if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"): + try: + importlib.import_module("kernels") + logger.info("Using HuggingFace kernels-backed attention backend '%s'", backend_upper) + except ImportError as e: + raise ValueError( + f"'{selected_backend}' requires the HuggingFace `kernels` package " + "(pip install kernels==0.14.1). Refusing to silently substitute a " + "different attention kernel; select FLASH_ATTN explicitly to opt out." + ) from e +``` + +`vllm_omni/platforms/cuda/platform.py:141` [major] — The gate at vllm_omni/platforms/cuda/platform.py:141 accepts FLASH_ATTN_3_HUB for every compute capability major >= 9, including Blackwell sm_100/sm_120, contradicting the PR's own documentation that FLASH_ATTN_3_HUB is Hopper-only and falls back to FLASH_ATTN_HUB on older GPUs. tests/diffusion/attention/test_kernels_hub.py:92 mirrors the same over-broad `major < 9` skip and would attempt FA3 on Blackwell. Restrict the gate and the test skip to Hopper (major == 9), or correct the docs to say '>= 9.0' if Blackwell is actually supported. (evidence: vllm_omni/platforms/cuda/platform.py:141: `fa3_hub_supported = compute_capability is not None and compute_capability.major >= 9`; vllm_omni/platforms/cuda/platform.py:144: `"FLASH_ATTN_3_HUB requires a Hopper-class GPU with compute capability >= 9.0. "`; tests/diffusion/attention/test_kernels_hub.py:92: `if major < 9:`; tests/diffusion/attention/test_kernels_hub.py:93: `pytest.skip("FLASH_ATTN_3_HUB execution requires Hopper-class GPU (compute capability >= 9.0)")`; docs/user_guide/diffusion/attention_backends.md:24: `| FLASH_ATTN_3_HUB | FlashAttention 3 from HuggingFace kernels library. CUDA Hopper (sm_90+) only; falls back to FLASH_ATTN_HUB on older GPUs. |`) + +```suggestion +fa3_hub_supported = compute_capability is not None and compute_capability.major == 9 +``` + +`.buildkite/test-ready.yml:~145` [major] — The only execution test for the new hub kernels (`test_kernels_hub_execution`) is never collected: `test-ready.yml:145` runs `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'`, which does not descend into `tests/diffusion/attention/`; the recursive run at `test-merge.yml:13` selects `'core_model and cpu'`, while `@hardware_test(res={"cuda": "L4"})` adds only `cuda`/`L4` marks. On L4 the FA3 half also always skips (`test_kernels_hub.py:92`), so the new FA2/FA3 forward code has zero CI execution coverage. Add the `attention/` directory to a collected CUDA lane (covers FA2) and split/gate the FA3 section onto an H100-marked test. (evidence: Unchanged by this diff, present in the PR-time tree: .buildkite/test-ready.yml:145 ` - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"` — non-recursive glob, does not collect tests/diffusion/attention/. Unchanged by this diff, present in the PR-time tree: .buildkite/test-merge.yml:13 ` - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — cpu selection only. Added by this diff: tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` — adds cuda/L4 marks, no cpu mark. Added by this diff: tests/diffusion/attention/test_kernels_hub.py:92 ` if major < 9:` followed by ` pytest.skip("FLASH_ATTN_3_HUB execution requires Hopper-class GPU (compute capability >= 9.0)")` — FA3 half always skips on L4 (sm_89).) + +```suggestion +- timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/attention -m 'core_model and cuda' --run-level "core_model" +``` + +`vllm_omni/platforms/cuda/platform.py:126` [minor] — The fallback probe at vllm_omni/platforms/cuda/platform.py:126 only imports the `kernels` package, but `FlashAttentionHubImpl.__init__` and `FlashAttention3HubImpl.__init__` (vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92 and :271) execute `from kernels import get_kernel` before any try/except. A `kernels` install that imports but lacks `get_kernel` therefore passes selection and then crashes in the constructor instead of falling back, so the documented graceful fallback does not cover too-old or broken packages. Probe `get_kernel` (and/or add a lower version bound) so the fallback decision matches what the constructor actually requires. (evidence: vllm_omni/platforms/cuda/platform.py:126 ` importlib.import_module("kernels")` — the probe only checks package import. vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92 ` from kernels import get_kernel` — this import sits before the try/except, so a package that imports but lacks `get_kernel` raises an uncaught ImportError in the impl constructor. pyproject.toml:70 ` "kernels<0.15.0", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)` — no lower bound excludes such too-old/broken installs.) + +```suggestion +try: + from kernels import get_kernel # noqa: F401 + importlib.import_module("kernels") + logger.info("Using HuggingFace kernels-backed attention backend '%s'", backend_upper) + except ImportError: +``` + +`flash_attn_hub.py:116` [minor] — `_flash_wrapper` (flash_attn_hub.py:116-125) routes attn_func=None to `_run_varlen_dense`, and the piecewise branch in forward_cuda (lines 189-201) can hit that fallback when `flash_attn_func` is None. The shipped test_kernels_hub.py only builds `AttentionMetadata(attn_mask=None)` (line 75) and never sets `full_attn_spans`, so the hub backend's piecewise path remains untested. (evidence: flash_attn_hub.py:116-125 ` def _flash_wrapper(self, q, k, v, *, attn_func, **kwargs):` / ` if attn_func is not None:` / ` return self._unwrap_flash_output(attn_func(q, k, v, **kwargs))` / ` return _run_varlen_dense(` / ` self.flash_attn_varlen_func,` / ` q,` / ` k,` / ` v,` / ` causal=kwargs.get("causal", self.causal),` / ` softmax_scale=kwargs.get("softmax_scale", self.softmax_scale),` / ` )`; flash_attn_hub.py:189-201 ` if full_attn_spans is not None:` / ` logger.debug("Using piecewise Flash Attention for mixed causal/full mask")` / ` attn_func = partial(` / ` self._flash_wrapper,` / ` attn_func=self.flash_attn_func,` / ` )` / ` return piecewise_attn(` / ` query,` / ` key,` / ` value,` / ` full_attn_spans,` / ` self.softmax_scale,` / ` attn_func,` / ` )`; tests/diffusion/attention/test_kernels_hub.py:75 ` attn_metadata_sdpa = AttentionMetadata(attn_mask=None)` — grep confirms no occurrence of `piecewise` or `full_attn_spans` in test_kernels_hub.py.) + +**Additional observations (beyond the comment budget):** +- `docs/user_guide/diffusion/attention_backends.md:217` [minor] 'Perfect numerical consistency' overstates the guarantee: the Hub backends are still distinct kernels (flash-attn2 vs flash-attn3), and the PR's own test compares them against an SDPA reference with `max_diff < 1e-2` (tests/diffusion/attention/test_kernels_hub.py:88 and :104), not bit-exact equality. Tone the doc down […] +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:230` [minor] `FlashAttentionHubImpl` (flash_attn_hub.py:71) and `FlashAttention3HubImpl` (flash_attn_hub.py:250) are near-identical copies — differing only in the `get_kernel` repo id (`kernels-community/flash-attn2` vs `kernels-community/flash-attn3`), log strings and the warning class name — and both re-implement the […] +- `docs/user_guide/diffusion/attention_backends.md:23` [minor] docs/user_guide/diffusion/attention_backends.md:23-24 — the two HUB rows omit the dependency note carried by the sibling `SAGE_ATTN` row (line 21: 'Requires `sageattention`.'). Both backends are CUDA-only and require `kernels`; state that in the table. Qualify the `FLASH_ATTN_3_HUB` fallback: it falls back to […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. +- **vllm-omni-pr-review-hub-kernel-loading**: Review pattern: optional HuggingFace kernels-backed attention backends — per-layer unpinned get_kernel loads, selector fallback gaps, and CI lane wiring. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4926.trace.json.gz new file mode 100644 index 000000000..c5e76da59 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr4950.cost.json b/eval/dataset/arms/copilot_v17_train/pr4950.cost.json new file mode 100644 index 000000000..cbe93856d --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4950.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 352.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4950/run-20260816-211500-486338", + "recorded_at": "2026-08-16T21:20:51", + "attempts": [ + { + "invocation_id": "inv-99b68df7", + "outcome": "run:run-20260816-211500-486338", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-211500-486338", + "llm_calls": 19, + "input_tokens": 170621, + "output_tokens": 27671, + "usd": 0.091657, + "source": "spans" + } + ], + "attempt_usd_total": 0.091657, + "llm_calls": 19, + "input_tokens": 170621, + "output_tokens": 27671, + "usd": 0.091657, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-211500-486338", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0917, + "minutes": 5.84, + "ci_minutes": 0, + "input_tokens": 170621, + "output_tokens": 27671, + "tool_calls": 28, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2454, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 19, + "by_role": { + "agent": { + "usd": 0.091657, + "input_tokens": 170621, + "output_tokens": 27671, + "calls": 19 + } + }, + "cache_read_tokens": 561152, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.85, + "gate": 1.54, + "review": 344.8, + "report": 0.07 + }, + "time_to_first_result_s": 350.24 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4950.trace.json.gz", + "trace_bytes": 11088 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4950.md b/eval/dataset/arms/copilot_v17_train/pr4950.md new file mode 100644 index 000000000..8fb67fce2 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4950.md @@ -0,0 +1,41 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4950, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 48d4fd8a5e98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] 'curl does not flatten nested extra_body' matches the repo NOTE at serving_chat.py image branch: 'When sending raw request with curl, no flattening happens.' +- [claim-verified] 151703 is the 4.5 TTS BOS: vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:90-94 uses 151703/151704; when absent `tts_token_ids` is not attached (lines 97-110). +- [validated] Top-level `chat_template_kwargs` reaches the template: vllm_omni/entrypoints/openai/serving_chat.py:379-381 validates `request.chat_template_kwargs` and folds it via `_effective_chat_template_kwargs(request)` into `merged_template_kwargs`. +- [validated] Nested `extra_body.chat_template_kwargs` is ignored for the LLM template path: in `_create_chat_completion` `extra_body` is only read inside the diffusion `image` branch (serving_chat.py ~400+); repo grep for `chat_template_kwargs` shows no `extra_body`-sourced consumer. +- [sweep] diff surface is a single file recipes/OpenBMB/MiniCPM-o-4_5.md; no tests/configs/yamls touched (per pr_diff/diff_stat). +- [sweep] SDK-based consumers remain correct after this change: examples/online_serving/minicpmo/README.md:58 and gradio_demo.py:223 send `extra_body` through the OpenAI client (flattened to root), so they are not stale. + +`recipes/OpenBMB/MiniCPM-o-4_5.md:124` [minor] — This diff changes the documented response shape from a single choice (old text: text in `choices[0].message.content` and WAV in `choices[0].message.audio.data`) to 'text in one choice ... audio in another choice ... two `choices` entries'. I could not confirm in `vllm_omni/entrypoints/openai/serving_chat.py:chat_completion_full_generator` that the omni AR path emits one choice per modality, and the PR test plan records only WAV amplitude ('~12s WAV', 'max amplitude = 0'), not the choices array. Can you paste the `choices` array from the corrected curl response so the recipe does not teach a shape the server does not actually return? (evidence: Diff adds 'Response carries text in one choice's `message.content` and base64 WAV ... you typically get two `choices` entries (one text, one audio)' while removing the old 'text in `choices[0].message.content` and base64 WAV in `choices[0].message.audio.data`'; PR test plan lists only 'non-zero audio amplitude (~12s WAV)' and 'silent audio (max amplitude = 0)'. Client-side `examples/online_serving/minicpmo/gradio_demo.py` iterates `for choice in completion.choices`, which proves only that the client tolerates multiple choices, not the server's emitted shape.) + +`recipes/OpenBMB/MiniCPM-o-4_5.md:~114` [minor] — The `buildkite/vllm-omni-npu-ci` check is FAILURE at this head, while the diff changes only `recipes/OpenBMB/MiniCPM-o-4_5.md` (no code/config/test paths), so this red gate is almost certainly pre-existing or flaky relative to this doc-only branch. Still confirm the same check on main before merging. (Anchor correction: the quoted snippet is at line 116, not 114.) (evidence: ['recipes/OpenBMB/MiniCPM-o-4_5.md:116 ` "chat_template_kwargs": {"use_tts_template": true}` — present in PR-time tree; the comment\'s cited line 114 should be 116.', 'gh_ci_read(4950): `{"bucket":"fail","name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}` — check is red at this head.', 'diff_stat: `recipes/OpenBMB/MiniCPM-o-4_5.md | 27 +++++++++++++++++++--------` / `1 file changed, 19 insertions(+), 8 deletions(-)` — no .buildkite, deploy YAML, or test file in the changed set.']) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. +- **vllm-omni-pr-review-hub-kernel-loading**: Review pattern: optional HuggingFace kernels-backed attention backends — per-layer unpinned get_kernel loads, selector fallback gaps, and CI lane wiring. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4950.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4950.trace.json.gz new file mode 100644 index 000000000..e580656d0 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4950.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr4970.cost.json b/eval/dataset/arms/copilot_v17_train/pr4970.cost.json new file mode 100644 index 000000000..371eb94a2 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4970.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 427.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4970/run-20260816-210753-661ebf", + "recorded_at": "2026-08-16T21:14:58", + "attempts": [ + { + "invocation_id": "inv-643b871f", + "outcome": "run:run-20260816-210753-661ebf", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-210753-661ebf", + "llm_calls": 24, + "input_tokens": 289524, + "output_tokens": 47876, + "usd": 0.148951, + "source": "spans" + } + ], + "attempt_usd_total": 0.148951, + "llm_calls": 24, + "input_tokens": 289524, + "output_tokens": 47876, + "usd": 0.148951, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-210753-661ebf", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.149, + "minutes": 7.08, + "ci_minutes": 0, + "input_tokens": 289524, + "output_tokens": 47876, + "tool_calls": 46, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3067, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 24, + "by_role": { + "agent": { + "usd": 0.148951, + "input_tokens": 289524, + "output_tokens": 47876, + "calls": 24 + } + }, + "cache_read_tokens": 670976, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.28, + "gate": 2.05, + "review": 419.17, + "report": 0.07 + }, + "time_to_first_result_s": 424.55 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4970.trace.json.gz", + "trace_bytes": 12467 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4970.md b/eval/dataset/arms/copilot_v17_train/pr4970.md new file mode 100644 index 000000000..c87371d49 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4970.md @@ -0,0 +1,57 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4970, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4992e10b7367) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [claim-verified] per-request seed preserved: serving_speech.py:3694 `stage0_params.extra_args["tts_local_seed"] = request.seed`. +- [claim-verified] default-seed propagation exists: serving_speech.py:3698 `default_seed = getattr(stage0_params, "seed", None)` and :3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`. +- [claim-verified] 'matching qwen3_tts_high_concurrency.yaml': grep `seed` in that file and in qwen3_tts_forced_aligner.yaml → no matches. +- [claim-verified] 'no test references the deploy-level seed': grep `seed` in tests/test_config_factory.py and tests/e2e/offline_inference/test_qwen3_tts_base.py → no matches. +- [claim-verified] PR numbers internally consistent: audio_throughput 40.3→24.2 = -40.0%; RTF 0.365→0.635 = +74%; TTFP 658→1134 = +72% (author's '~2x TTFP/RTF' is loose but not self-refuting). +- [validated] fast path restored: gpu_model_runner.py `_explicit_talker_seed` returns None when `tts_local_seed` absent → `row_generators` all None → no per-row loop and no `generator(s)` kwarg. +- [validated] seeded requests stay batched: qwen3_tts_talker.py:326 `self.talker_mtp_accepts_per_row_generators = True` keeps #4889 per-row generator path for explicit seeds. +- [sweep] diff_stat: only `vllm_omni/deploy/qwen3_tts.yaml` changed — 2 deletions, no test files. +- [sweep] Read qwen3_tts.yaml in situ: both stages' default_sampling_params no longer contain `seed`; npu platform override only sets `enforce_eager`. +- [sweep] grep `tts_local_seed`: producer serving_speech.py:3694/3706, consumer gpu_model_runner.py:1826. +- [sweep] grep `seed: 42` in vllm_omni/deploy: ~60 other YAMLs still carry it, but propagation is gated on `_tts_model_type == "qwen3_tts"`, so no qwen3_tts sibling remains exposed and non-qwen3_tts seeds go through normal SamplingParams.seed. +- [sweep] docs sweep: grep `seed: 42` in docs → only legacy Qwen2.5-Omni stage_configs example and pd_disaggregation; no qwen3_tts doc asserts the removed default. +- [sweep] .buildkite/test-merge.yml lists qwen3_tts.yaml as source_file_dependency for functional TTS e2e jobs, so CI will load/parse the changed config. +- [sweep] no in-file comment/docstring in the touched file asserts a seed default, so nothing reads as stale except the missing tombstone itself. + +`vllm_omni/deploy/qwen3_tts.yaml:~61` [minor] — Confirmed: `vllm_omni/deploy/qwen3_tts.yaml` no longer sets a stage-0 default `seed`, and `serving_speech.py:3706` propagates a stage-default seed into every request's `tts_local_seed` when the request has no seed of its own (`serving_speech.py:3694`). Please add the proposed tombstone above `min_tokens` (line 60) noting that `seed` is intentionally omitted from the default profile and that per-request determinism is available via the request seed field. (evidence: vllm_omni/deploy/qwen3_tts.yaml:60 `min_tokens: 2` and :61 `repetition_penalty: 1.05` with no `seed` key between them (diff removed `- seed: 42`). Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/serving_speech.py:3694 `stage0_params.extra_args["tts_local_seed"] = request.seed`; vllm_omni/entrypoints/openai/serving_speech.py:3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`; vllm_omni/worker/gpu_model_runner.py:1826 `seed = extra_args.get("tts_local_seed")`.) + +```suggestion +# No default `seed`: a deploy-level seed propagates into every + # request's `tts_local_seed` and forces the slow per-row talker-MTP + # multinomial path (#4963). Use the per-request `seed` field instead. + min_tokens: 2 +``` + +`vllm_omni/deploy/qwen3_tts.yaml:~60` [minor] — This PR removes only the two `seed: 42` default sampling params from `vllm_omni/deploy/qwen3_tts.yaml` — a performance-only change with no test or benchmark files in the diff. The merge CI steps that reference this YAML are functional e2e pytest jobs only (`.buildkite/test-merge.yml` Qwen3-TTS CustomVoice and Base steps), not perf lanes, and the PR's Test Result is `Pending CI.` Please attach the nightly `qwen3_tts_base` and `qwen3_tts_customvoice` cells (audio_throughput/TTFP/RTF vs pre-07-04 baselines) or an equivalent `benchmarks/tts` comparison at concurrency >= 16 to prove the restored fast path actually lands. (evidence: PR #4970 diff: only `vllm_omni/deploy/qwen3_tts.yaml` changed, `1 file changed, 2 deletions(-)`, both deletions `- seed: 42`. PR body: `"Test Result": "Pending CI."`. Unchanged by this diff but present in the PR-time tree: `.buildkite/test-merge.yml:174` `- vllm_omni/deploy/qwen3_tts.yaml` and `.buildkite/test-merge.yml:201` `- vllm_omni/deploy/qwen3_tts.yaml` appear only under `source_file_dependencies` of `TTS · Qwen3-TTS CustomVoice Test` / `TTS · Qwen3-TTS Base Test`, whose commands are `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'` and the analogous base test command — functional e2e, not a perf/benchmark lane.) + +`vllm_omni/deploy/qwen3_tts.yaml:~91` [minor] — Confirmed from the PR-time tree: the diff is config-only (2 deletions), and vllm_omni/deploy/qwen3_tts.yaml:93-95 still carries an `npu` platform override (unchanged by this diff). gh_ci_read(4970) shows both buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci as FAILURE. Whether those failures are pre-existing on main or introduced by this change cannot be determined from CI status metadata alone — please check the lane logs. (evidence: vllm_omni/deploy/qwen3_tts.yaml:91 ` repetition_penalty: 1.0` — cited anchor exists verbatim. vllm_omni/deploy/qwen3_tts.yaml:93-95 (unchanged by this diff, present in PR-time tree): `platforms:` / ` npu:` / ` stages:`. diff_stat: `vllm_omni/deploy/qwen3_tts.yaml | 2 --` (only changed file). gh_ci_read(4970): `buildkite/vllm-omni-amd-ci` state FAILURE and `buildkite/vllm-omni-npu-ci` state FAILURE.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4970.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4970.trace.json.gz new file mode 100644 index 000000000..42005907a Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4970.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr4977.cost.json b/eval/dataset/arms/copilot_v17_train/pr4977.cost.json new file mode 100644 index 000000000..2d63fd8b4 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4977.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 994.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr4977/run-20260816-205118-3df7ac", + "recorded_at": "2026-08-16T21:07:51", + "attempts": [ + { + "invocation_id": "inv-a4084ca4", + "outcome": "run:run-20260816-205118-3df7ac", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-205118-3df7ac", + "llm_calls": 82, + "input_tokens": 255972, + "output_tokens": 128899, + "usd": 0.281282, + "source": "spans" + } + ], + "attempt_usd_total": 0.281282, + "llm_calls": 82, + "input_tokens": 255972, + "output_tokens": 128899, + "usd": 0.281282, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-205118-3df7ac", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2813, + "minutes": 16.54, + "ci_minutes": 0, + "input_tokens": 255972, + "output_tokens": 128899, + "tool_calls": 142, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5771, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 82, + "by_role": { + "moa_member": { + "usd": 0.065178, + "input_tokens": 72528, + "output_tokens": 12065, + "calls": 27 + }, + "agent": { + "usd": 0.19311, + "input_tokens": 172710, + "output_tokens": 98605, + "calls": 53 + }, + "reducer": { + "usd": 0.022995, + "input_tokens": 10734, + "output_tokens": 18229, + "calls": 2 + } + }, + "cache_read_tokens": 2063872, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.35, + "gate": 1.77, + "review": 986.87, + "report": 0.07 + }, + "time_to_first_result_s": 992.03 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 27450 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr4977.md b/eval/dataset/arms/copilot_v17_train/pr4977.md new file mode 100644 index 000000000..bb2542d61 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr4977.md @@ -0,0 +1,83 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Failed-load cache poisoning: flash_attn_hub.py:45 `_hub_modules[repo_id] = _load_hub_module(repo_id)` executes only after the loader returns, so a raised RuntimeError leaves no dict entry and a later construction retries; residual: retries have no backoff/cap, but each retry is a new construction. +- [claim-verified] diff_stat shows only `vllm_omni/diffusion/attention/backends/flash_attn_hub.py` changed (30 insertions/30 deletions); no test files touched, matching sweep_targets. +- [claim-verified] PR body claims `trust_remote_code=True`; head file has no such kwarg (read flash_attn_hub.py:1-50) and commit timeline shows `9947f414 Drop trust_remote_code` — description is stale (already a kept comment). +- [claim-refuted] PR body says 'adding trust_remote_code=True for more reliable Hub access' but commit 9947f414 explicitly dropped it (diff shows no trust_remote_code). +- [claim-refuted] PR body says the diff adds `trust_remote_code=True`; head commit 9947f414 dropped it — commit message: 'kernels 0.13.x get_kernel() does not accept trust_remote_code, so passing it made every fallback attempt fail on older installs.' +- [claim-refuted] PR body says 'adding `trust_remote_code=True` for more reliable Hub access', but at head `grep trust_remote_code` over flash_attn_hub.py returns nothing; commit `9947f414 Drop trust_remote_code from hub kernel loading` confirms it was intentionally dropped — only the PR description is stale. +- [claim-refuted] PR Test Plan `test_kernels_hub_platform_fallback` does not exercise the changed path: tests/diffusion/attention/test_kernels_hub.py:24 sets `sys.modules["kernels"]=None` and the test only asserts `get_diffusion_attn_backend_cls` routing, so `_load_hub_module`/`_get_hub_module` never execute. +- [validated] flash_attn_hub.py:42-46 — _get_hub_module caches on success, does NOT cache on exception, so retries work correctly. +- [validated] flash_attn_hub.py:27 — _load_hub_module does `from kernels import get_kernel` lazily inside the function, so import-time side effects unchanged. +- [validated] flash_attn_hub.py:31-39 — version fallback loop (1, 2, None) matches old inline try/except behavior; last_error chained via `raise RuntimeError from last_error`. +- [validated] flash_attn_hub.py:119,284 — both FlashAttentionHubImpl and FlashAttention3HubImpl.__init__ now call _get_hub_module; no inline get_kernel remains in the file. +- [validated] registry.py:65-66 — FLASH_ATTN_HUB and FLASH_ATTN_3_HUB enum entries reference FlashAttentionHubBackend/FlashAttention3HubBackend, unchanged. +- [validated] Per-layer impl construction confirmed at vllm_omni/diffusion/attention/layer.py:94-100 (`self.attention = self.attn_impl_cls(...)`), so the new process cache removes N-1 repeated `get_kernel` calls per worker. +- [validated] Version fallback preserved: flash_attn_hub.py:29-36 loops `for version in (1, 2, None)` and calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` — the same v1→v2→default order as the removed nested try/except. + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [major] — Both `FlashAttentionHubImpl.__init__` (line 119) and `FlashAttention3HubImpl.__init__` (line 284) now call `_get_hub_module()`, which memoizes per-repo module loads in `_hub_modules` and performs the v1→v2→default fallback. No test covers this path: `test_kernels_hub_platform_fallback` only monkeypatches `sys.modules['kernels']` (line 27) for platform routing and never constructs an impl, while `test_kernels_hub_execution` constructs an impl once (FA2, and FA3 only on Hopper), leaving cache-hit, fallback-order, and failed-load-retry behavior unverified. Add a CPU/mock test that monkeypatches `kernels.get_kernel` with a counting fake, constructs two impls, asserts one load per repo, asserts fallback order when version=1 raises and version=2 succeeds, and clears `_hub_modules` (or adds a reset hook) between tests. (evidence: PR-time tree (test file unchanged by this diff, diff touches only vllm_omni/diffusion/attention/backends/flash_attn_hub.py): flash_attn_hub.py:22-23 `_hub_modules: dict[str, object] = {}` / `_hub_lock = threading.Lock()`; flash_attn_hub.py:42-45 `def _get_hub_module(repo_id: str):` ` with _hub_lock:` ` if repo_id not in _hub_modules:` ` _hub_modules[repo_id] = _load_hub_module(repo_id)`; flash_attn_hub.py:119 `hub_module = _get_hub_module("kernels-community/flash-attn2")`; flash_attn_hub.py:284 `hub_module = _get_hub_module("kernels-community/flash-attn3")`. Unchanged by diff, present in PR-time tree: tests/diffusion/attention/test_kernels_hub.py:27 `monkeypatch.setitem(sys.modules, "kernels", None)` (platform routing only, no impl construction) and tests/diffusion/attention/test_kernels_hub.py:79-82 `from vllm_omni.diffusion.attention.backends.flash_attn_hub import FlashAttentionHubImpl` / `fa_hub_impl = FlashAttentionHubImpl(...)` (single construction; no get_kernel monkeypatch or cache-hit/fallback-retry assertions anywhere in tests).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:26` [minor] — PR body still promises `trust_remote_code=True`, but the merged head (commit 9947f414 'Drop trust_remote_code from hub kernel loading') calls `get_kernel(repo_id)` / `get_kernel(repo_id, version=version)` without that kwarg because kernels 0.13.x rejects it. Update the PR description to drop the stale `trust_remote_code=True` claim (code is at vllm_omni/diffusion/attention/backends/flash_attn_hub.py:33-35). (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:33-35 ` if version is not None: + return get_kernel(repo_id, version=version) + return get_kernel(repo_id)` — no `trust_remote_code` kwarg (present in PR-time tree, part of the diff). Commit 9947f41430aa subject: `Drop trust_remote_code from hub kernel loading`; body: `kernels 0.13.x get_kernel() does not accept trust_remote_code, so passing it made every fallback attempt fail on older installs. Match main's call signature; caching remains unchanged.` PR #4977 body (verbatim): `...and adding `trust_remote_code=True` for more reliable Hub access.`) + +`flash_attn_hub.py:45` [minor] — flash_attn_hub.py:45 `_hub_modules[repo_id] = _load_hub_module(repo_id)` runs only after `_load_hub_module` returns, so a RuntimeError raised at line 39 leaves no cache entry and each later `_get_hub_module` call retries the full 1→2→default load sequence with no backoff/cap. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:39 ` raise RuntimeError(f"Failed to load HuggingFace Hub kernel {repo_id!r}") from last_error`; vllm_omni/diffusion/attention/backends/flash_attn_hub.py:44-46 ` if repo_id not in _hub_modules:` / ` _hub_modules[repo_id] = _load_hub_module(repo_id)` / ` return _hub_modules[repo_id]` — the assignment is on the RHS call's return path, so if line 39 raises the key is never inserted and the next `_get_hub_module` call re-enters `_load_hub_module`.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:4` [minor] — This diff adds `import threading` and a process-wide `_hub_modules`/`_hub_lock` cache in `flash_attn_hub.py`. The failing `buildkite/vllm-omni-npu-ci` gate does not obviously touch this file: `NPUOmniPlatform.get_diffusion_attn_backend_cls` maps `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` to local `FLASH_ATTN` before this module is imported, and `.buildkite/test-npu-ready.yaml` only runs `tests/e2e/online_serving/test_qwen3_tts_customvoice.py`. Confirm whether this NPU failure is pre-existing on main and unrelated, or point to the NPU step that imports this backend. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/platform.py:110-111 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):`; vllm_omni/platforms/npu/platform.py:115 `backend_upper = "FLASH_ATTN"`; .buildkite/test-npu-ready.yaml:21 `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; changed by this diff, present in the PR-time tree: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:4 `import threading`) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:119` [minor] — This diff centralizes kernel loading into `_get_hub_module()`, but the attribute extraction and no-functions validation remain duplicated verbatim in `FlashAttentionHubImpl` (lines 120-123) and `FlashAttention3HubImpl` (lines 285-288), differing only by repo_id and error text. Fold them into `_get_hub_module` (return `(flash_attn_func, flash_attn_varlen_func)` and raise the no-functions error there) so the two backends cannot drift on required function names or error text. (evidence: flash_attn_hub.py:119 ` hub_module = _get_hub_module("kernels-community/flash-attn2")` (line added by this PR's diff); flash_attn_hub.py:120 ` self.flash_attn_func = getattr(hub_module, "flash_attn_func", None)`; flash_attn_hub.py:121 ` self.flash_attn_varlen_func = getattr(hub_module, "flash_attn_varlen_func", None)`; flash_attn_hub.py:122-123 ` if self.flash_attn_func is None and self.flash_attn_varlen_func is None: + raise RuntimeError("Failed to load flash-attn2 kernel from HuggingFace Hub: no functions found")`; flash_attn_hub.py:284 ` hub_module = _get_hub_module("kernels-community/flash-attn3")`; flash_attn_hub.py:285 ` self.flash_attn_func = getattr(hub_module, "flash_attn_func", None)`; flash_attn_hub.py:286 ` self.flash_attn_varlen_func = getattr(hub_module, "flash_attn_varlen_func", None)`; flash_attn_hub.py:287-288 ` if self.flash_attn_func is None and self.flash_attn_varlen_func is None: + raise RuntimeError("Failed to load flash-attn3 kernel from HuggingFace Hub: no functions found")`. The duplicated extraction/validation lines are unchanged context present in the PR-time tree.) + +```suggestion +self.flash_attn_func, self.flash_attn_varlen_func = _get_hub_module( + "kernels-community/flash-attn2" + ) +``` + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:43` [nit] — `_get_hub_module` calls `_load_hub_module` while holding the single global `_hub_lock` (line 43), and `_load_hub_module` performs the Hub kernel load (line 27: `from kernels import get_kernel`). Since both flash-attn2 (line 119) and flash-attn3 (line 284) go through this one lock, a slow load of one repo blocks the other. Consider per-repo locks guarded by a separate dict lock so each repo still loads exactly once without cross-repo serialization. (evidence: flash_attn_hub.py:22-23 `_hub_modules: dict[str, object] = {}` / `_hub_lock = threading.Lock()`; flash_attn_hub.py:27 `from kernels import get_kernel` (inside `_load_hub_module`, which performs the Hub load); flash_attn_hub.py:43-46 `with _hub_lock: + if repo_id not in _hub_modules: + _hub_modules[repo_id] = _load_hub_module(repo_id) + return _hub_modules[repo_id]`; flash_attn_hub.py:119 `hub_module = _get_hub_module("kernels-community/flash-attn2")`; flash_attn_hub.py:284 `hub_module = _get_hub_module("kernels-community/flash-attn3")`.) + +```suggestion +_hub_locks: dict[str, threading.Lock] = {} +_hub_locks_guard = threading.Lock() + + +def _get_hub_module(repo_id: str): + with _hub_locks_guard: + lock = _hub_locks.setdefault(repo_id, threading.Lock()) + with lock: + if repo_id not in _hub_modules: + _hub_modules[repo_id] = _load_hub_module(repo_id) + return _hub_modules[repo_id] +``` + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr4977.trace.json.gz new file mode 100644 index 000000000..0e028bb27 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_train/pr5009.cost.json b/eval/dataset/arms/copilot_v17_train/pr5009.cost.json new file mode 100644 index 000000000..f52fb734a --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr5009.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1792.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_train/runs/pr5009/run-20260816-201527-cce0cb", + "recorded_at": "2026-08-16T20:45:17", + "attempts": [ + { + "invocation_id": "inv-4de634dd", + "outcome": "run:run-20260816-201527-cce0cb", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-201527-cce0cb", + "llm_calls": 162, + "input_tokens": 1054494, + "output_tokens": 252610, + "usd": 0.896854, + "source": "spans" + } + ], + "attempt_usd_total": 0.896854, + "llm_calls": 162, + "input_tokens": 1054494, + "output_tokens": 252610, + "usd": 0.896854, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-201527-cce0cb", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8969, + "minutes": 29.82, + "ci_minutes": 0, + "input_tokens": 1054494, + "output_tokens": 252610, + "tool_calls": 282, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.045, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 162, + "by_role": { + "moa_member": { + "usd": 0.139989, + "input_tokens": 157456, + "output_tokens": 10476, + "calls": 34 + }, + "agent": { + "usd": 0.712613, + "input_tokens": 874431, + "output_tokens": 207523, + "calls": 124 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 1 + }, + "reducer": { + "usd": 0.044252, + "input_tokens": 22607, + "output_tokens": 34611, + "calls": 3 + } + }, + "cache_read_tokens": 11203008, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.89, + "gate": 1.66, + "review": 1783.45, + "report": 0.06 + }, + "time_to_first_result_s": 1789.05 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 41081 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_train/pr5009.md b/eval/dataset/arms/copilot_v17_train/pr5009.md new file mode 100644 index 000000000..3d9ca78c7 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_train/pr5009.md @@ -0,0 +1,108 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Test is actually selected: .buildkite/test-merge.yml:29 runs `pytest -m 'core_model and cpu'` over non-diffusion/non-executor paths (includes tests/platforms); CUDA lane is green. +- [resolved] Module-scope CudaOmniPlatform import is CPU-safe: tests/diffusion/attention/test_kernels_hub.py:24 imports it inside an existing core_model+cpu test. +- [resolved] nn.RMSNorm per-head normalization (over head_dim) is TP-safe for Qwen-Image; unlike ltx2's rms_norm_across_heads it needs no all-reduce path. +- [claim-verified] PR claim 'only in-repo caller is diffusion_worker._resolve_ir_op_priority' holds: get_default_ir_op_priority referenced only at diffusion_worker.py:175. +- [claim-verified] 'Cosmos3 forces native via per-model override' — pipeline_cosmos3.py:612-624 sets rms_norm/fused_add_rms_norm=['native'] but copies every other field from the new default, so Cosmos3's non-RMSNorm ops still inherit the flip. +- [claim-verified] '#3588 accidentally reverted #4074' — search_history: 35ee3c75 (#4074) introduced nn.RMSNorm; later 99234881 (#3588) changed the file; this PR's diff reverts vllm RMSNorm back to nn.RMSNorm. +- [claim-verified] PR's own numbers recompute: (1.7263-1.2613)/1.7263=0.2694, (1288-1080)/1288=0.1615, (2.4960-2.4502)/2.4960=0.0183, 2.4502/2.2847-1=0.0724. +- [claim-verified] Only in-repo caller of get_default_ir_op_priority is diffusion_worker.py:175 (_resolve_ir_op_priority); no AR path reaches it. +- [claim-verified] Cosmos3 is the sole _DIFFUSION_IR_OP_PRIORITY_FUNCS entry (registry.py:545-550) and its override pins rms_norm/fused_add_rms_norm to ["native"] (pipeline_cosmos3.py:620-621). +- [claim-verified] PR description claim 'Part 1 scope: CUDA platform default for all diffusion models using vLLM RMSNorm under inductor' is accurate — the diff removes the inductor branch for all CUDA diffusion models. +- [claim-verified] PR description claim 'Models that need native-only already have per-model overrides (e.g. Cosmos3)' — confirmed via registry.py _DIFFUSION_IR_OP_PRIORITY_FUNCS. +- [claim-verified] PR numbers recomputed with calc: (1.2613-1.7263)/1.7263 = -0.2694 (-26.9%) and (1.2613-1.4037)/1.4037 = -0.1014 (-10.1%); both match the body. +- [claim-verified] diff_stat confirms net change is only the 3 named files (45 insertions, 16 deletions) with merge-base cbd3939c3d50 — no dropped hunks or extra artifacts. +- [claim-verified] W4A16 tests the body cites exist (test_qwen_image_autoround_w4a16_load/_generate) and exercise the changed module's load/generate path, but under eager and only for non-blank output. + +`vllm_omni/platforms/cuda/platform.py:~253` [major] — Confirmed: `default` at platform.py:253 is passed as the base of `IrOpPriorityConfig.with_default(default, rms_norm=..., fused_add_rms_norm=...)` at platform.py:262, so removing the inductor-aware branch changes the priority for every non-overridden IR op (silu_and_mul, rotary, etc.) under inductor from native-first to vllm_c-first. Scope the flip to rms_norm/fused_add_rms_norm or add per-model ir_op_priority_func coverage. (evidence: vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]`; vllm_omni/platforms/cuda/platform.py:262 `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)` — `default` is the base for every non-overridden IR op. Removed by this diff (not present in the PR-time file): `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` and `default = ["native"] if using_inductor else ["vllm_c", "native"]`, so inductor-active configs previously got native-first for all default ops and now get vllm_c-first.) + +```suggestion +@classmethod + def get_default_ir_op_priority(cls, vllm_config: VllmConfig) -> IrOpPriorityConfig: + """Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops.""" + from vllm.config.compilation import CompilationMode + + cc = vllm_config.compilation_config + using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE + default = ["native"] if using_inductor else ["vllm_c", "native"] + rms_norm = ["vllm_c", "native"] if using_inductor else default + if envs.VLLM_USE_OINK_OPS: + rms_norm = ["oink"] + rms_norm + + return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm) +``` + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:~557` [minor] — Confirmed at this head: qwen_image_transformer.py:557/558/595/596/1009 are all `nn.RMSNorm` (vllm RMSNorm import removed), and pipeline_qwen_image.py:22 loads HF `Qwen2_5_VLForConditionalGeneration`, so Qwen-Image's denoise loop has no vllm `rms_norm`/`fused_add_rms_norm` IR sites left for Part 1's `IrOpPriorityConfig` to affect. The PR body's Qwen-Image profiler table is labeled 'Part 1 only (patched)' and was measured on a tree that still had vllm RMSNorm in the DiT, so it does not describe this merged head. Note the PR body does include a supplementary Part 2 isolation table (1.4037s -> 1.2613s, -10.1%), so a split exists, but it keeps vllm.RMSNorm for the Part 1 leg and therefore does not represent the merged state; at this head Part 1's Qwen-Image RMSNorm contribution is zero. Please re-measure Part 1 at this head for Qwen-Image or explicitly restate its Qwen-Image scope as nil and use the FLUX.1-dev A/B as the platform-default evidence. (evidence: ['vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (changed by this diff)', 'vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:558 `self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` (changed by this diff)', 'vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:595 `self.norm_added_q = nn.RMSNorm(head_dim, eps=eps)` (changed by this diff)', 'vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:596 `self.norm_added_k = nn.RMSNorm(head_dim, eps=eps)` (changed by this diff)', 'vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)` (changed by this diff)', 'vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py:22 `from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer` (unchanged by this diff, present in the PR-time tree)', 'vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]` and platform.py:264 `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)` (Part 1 only sets RMSNorm IR priorities, which nn.RMSNorm bypasses)']) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — The RMSNorm→nn.RMSNorm swap at qwen_image_transformer.py:557 (and :558, :595-596, :1009) is shared by all five `qwen_image` pipelines — QwenImagePipeline, QwenImageEditPipeline, QwenImageEditPlusPipeline, QwenImageLayeredPipeline, and QwenImageDMD2Pipeline (registry.py:24-44, 209-213) — but the PR adds only a CPU IR-op-priority test and no pixelwise parity evidence for the edit/layered/DMD2 variants. Please name the test that guards those variants or run their pixelwise-against-diffusers tests. (evidence: qwen_image_transformer.py:557 ` self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()` — changed by this PR from vLLM RMSNorm to nn.RMSNorm. Unchanged by this diff, present in the PR-time tree: registry.py:24-44 ` "QwenImagePipeline": ( + "qwen_image", + "pipeline_qwen_image", + "QwenImagePipeline", + ), + "QwenImageEditPipeline": ( + "qwen_image", + "pipeline_qwen_image_edit", + "QwenImageEditPipeline", + ), + "QwenImageEditPlusPipeline": ( + "qwen_image", + "pipeline_qwen_image_edit_plus", + "QwenImageEditPlusPipeline", + ), + "QwenImageLayeredPipeline": ( + "qwen_image", + "pipeline_qwen_image_layered", + "QwenImageLayeredPipeline", + ),` and registry.py:209-213 ` "QwenImageDMD2Pipeline": ( + "qwen_image", + "pipeline_qwen_image", + "QwenImageDMD2Pipeline", + ),`; grep also shows pipeline_qwen_image.py:334, pipeline_qwen_image_edit.py:285, pipeline_qwen_image_edit_plus.py:253, and pipeline_qwen_image_layered.py:278 each instantiate QwenImageTransformer2DModel imported from qwen_image_transformer.) + +`vllm_omni/platforms/rocm/platform.py:~184` [minor] — The CUDA hunk removes the inductor-aware native-only branch, but the ROCm and XPU twins still keep it: ROCm rms_norm stays ['native'] under inductor unless both VLLM_ROCM_USE_AITER and VLLM_ROCM_USE_AITER_RMSNORM are set, and XPU stays ['native'] under inductor. If the #4964 mechanism (inductor → native-only RMSNorm decomposition) is RMSNorm-heavy-diffusion-generic, it likely survives on those platforms. Please state intent: is this CUDA-only by design, or do ROCm/XPU diffusion models need the same fix/override? (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/rocm/platform.py:184 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; vllm_omni/platforms/rocm/platform.py:185 `default = ["native"] if using_inductor else ["vllm_c", "native"]`; vllm_omni/platforms/xpu/platform.py:116 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE`; vllm_omni/platforms/xpu/platform.py:117 `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`. Changed by this diff: vllm_omni/platforms/cuda/platform.py:252 `default = ["vllm_c", "native"]` with the inductor-aware branch removed.) + +`vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:619` [minor] — Cosmos3's override does not fully protect it from the platform-default flip: get_cosmos3_ir_op_priority_func (pipeline_cosmos3.py:612-624) pins only rms_norm/fused_add_rms_norm=['native'] and copies every other field from the incoming default via fields(ir_op_priority). With platform.py now always returning default=['vllm_c','native'] (the old code used ['native'] when inductor was active), Cosmos3's non-RMSNorm ops under inductor inherit the vllm_c-first change. Pin the non-RMSNorm fields to ['native'] explicitly in this override (the alternative of keeping default inductor-aware conflicts with this PR's own new test), so native-only Cosmos3 stays native-only. (evidence: platform.py:253 (changed by this PR) `default = ["vllm_c", "native"]` — the old diff line it replaces was `default = ["native"] if using_inductor else ["vllm_c", "native"]`; pipeline_cosmos3.py:619-621 (unchanged by this diff, present in PR-time tree) `priority_kwargs = {field.name: list(getattr(ir_op_priority, field.name)) for field in fields(ir_op_priority)}` then only `priority_kwargs["rms_norm"] = ["native"]` and `priority_kwargs["fused_add_rms_norm"] = ["native"]` — every other field is copied from the incoming default, so under inductor those fields now inherit vllm_c-first.) + +`tests/platforms/test_cuda_ir_op_priority.py:1` [minor] — Placement breaks the platforms test hierarchy: tests/platforms contains test_cuda_ir_op_priority.py directly with no cuda/ subdir, while the NPU platform test lives at tests/platforms/npu/test_310p_patches.py. Move this test to tests/platforms/cuda/test_ir_op_priority.py for consistent per-platform grouping. (evidence: tests/platforms/test_cuda_ir_op_priority.py:1 '# SPDX-License-Identifier: Apache-2.0' — file exists at tests/platforms/test_cuda_ir_op_priority.py, i.e. directly under tests/platforms with no tests/platforms/cuda/ subdir. Directory listing of tests/platforms shows only: __pycache__/, npu/, test_cuda_ir_op_priority.py. Directory listing of tests/platforms/npu shows: quant/, test_310p_patches.py (unchanged by this diff, present in the PR-time tree).) + +`tests/e2e/accuracy/test_qwen_image.py:~197` [minor] — Anchor verified: tests/e2e/accuracy/test_qwen_image.py:197 is the Qwen-Image diffusers parity check and is nightly-gated (module pytestmark full_model+diffusion, H100). The W4A16 expansion test runs enforce_eager=True and asserts only arr.std()>1.0, so it exercises neither the inductor path nor diffusers parity. Note that tests/e2e/accuracy/test_diffusers_backend_similarity.py also contains Qwen-Image vs diffusers helpers, so avoid the word 'only' unless that file is also full_model/nightly. If SSIM/PSNR parity must be protected per-PR, add a merge-gate compiled-path parity smoke or state explicitly that parity remains nightly-only. (evidence: The decisive files are unchanged by this diff but present in the PR-time tree: +tests/e2e/accuracy/test_qwen_image.py:20 `pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]` +tests/e2e/accuracy/test_qwen_image.py:196 `@hardware_test(res={"cuda": "H100"}, num_cards=1)` +tests/e2e/accuracy/test_qwen_image.py:197 `def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None:` +tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py:58 `with OmniRunner(QUANTIZED_MODEL, enforce_eager=True) as runner:` +tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py:68 `assert arr.std() > 1.0, "Generated image appears blank (std ≈ 0)"`) + +`tests/platforms/test_cuda_ir_op_priority.py:12` [nit] — `backend` is never varied (all three call sites rely on the default) and `get_default_ir_op_priority` no longer reads `vllm_config` at all, so the parameter is dead and slightly overstates what the test exercises. Drop it. (evidence: tests/platforms/test_cuda_ir_op_priority.py:12 `def _vllm_config(*, backend: str = "inductor", mode: CompilationMode) -> VllmConfig:`; tests/platforms/test_cuda_ir_op_priority.py:25 `priority = CudaOmniPlatform.get_default_ir_op_priority(_vllm_config(mode=mode))`; tests/platforms/test_cuda_ir_op_priority.py:35 `_vllm_config(mode=CompilationMode.VLLM_COMPILE),`; vllm_omni/platforms/cuda/platform.py:251-259 `def get_default_ir_op_priority(cls, vllm_config: VllmConfig) -> IrOpPriorityConfig:` body reads `default = ["vllm_c", "native"]` and never references `vllm_config`.) + +```suggestion +def _vllm_config(*, mode: CompilationMode) -> VllmConfig: +``` + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_train/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v17_train/pr5009.trace.json.gz new file mode 100644 index 000000000..3d469e193 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_train/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_val/README.md b/eval/dataset/arms/copilot_v17_val/README.md new file mode 100644 index 000000000..ef9b7050d --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/README.md @@ -0,0 +1,48 @@ +# RELABELLED — v17-moa (unintended vendor mixture), not v17 core-only + +This directory is named `v17` but does NOT hold the arm that name implies. + +`MOA_WHEN=off` was set explicitly on every arm from v13 through v16. The +launch that produced this data omitted it, so `moa_when` fell through to its +code default of `"full"` (config.py:345) and mixture-of-agents ran on every +full-depth PR review — 12 of the 15 items here. Round-1 lenses were +dispatched to three vendors instead of one: investigator + behavior to +`mimo-v2.5`, adversary + verification to `qwen3.6-plus`, reducer on tier. + +## How much of the work was actually not DeepSeek + +Attributed by the model recorded on each `agent_dispatch`, counting only +dispatches that produced usable output: + +| model | attempts | productive outputs | output tokens | +|---|---|---|---| +| `deepseek-v4-pro` | rest + all fallbacks | **155 (89.1%)** | 2,298,061 (89.3%) | +| `mimo-v2.5` | 30 | 16 (9.2%) | 183,047 (7.1%) | +| `qwen3.6-plus` | 27 | 3 (1.7%) | 94,497 (3.7%) | + +**DeepSeek still wrote ~90% of the review work**, and the two member figures +are UPPER BOUNDS. `BudgetedLLM.create` (moa.py:215-227) reserves an estimated +cost per LLM call and, when the reservation is refused, silently reruns that +call on the tier model while the lens keeps its original dispatch label. With +`moa_max_usd = $1.50` against per-item spend of $0.85–1.66, that budget trips +partway through most items, so an unknown share of the "member" rows above +was in fact DeepSeek. The trace cannot resolve it: only the dispatch carries a +model, and the per-call fallback emits none. + +(An earlier version of this file claimed 24%, inferred from seat names. That +inference ignored both `/retry` seats and the per-call budget fallback, and +was wrong by roughly 2.6x.) + +`qwen3.6-plus` returned `403 AccessDenied.Unpurchased` on 24 of its 27 +attempts — that subscription is not active. 104 minutes of this run went to +member attempts that were then discarded and redone. + +Kept rather than deleted: the data is real and the MoA behaviour it captures +is worth having. The directory names are also left alone on purpose — every +verdict's `_roles` block records the arm directory name, so renaming would +leave that provenance pointing at a directory holding different data. + +The corrected measurement, same code (`ae1b6f1`) and same items with +`MOA_WHEN=off` verified through resolved settings, lives in +`copilot_v17ds_*` / `goal_v17ds_*_sonnet`. See +`goal-eval/PROBE-v17-conversion.md` for the pre-registered amendment. diff --git a/eval/dataset/arms/copilot_v17_val/manifest.json b/eval/dataset/arms/copilot_v17_val/manifest.json new file mode 100644 index 000000000..2776bf717 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v17_val", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 5, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "(default)", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_val/pr4810.cost.json b/eval/dataset/arms/copilot_v17_val/pr4810.cost.json new file mode 100644 index 000000000..c61f07b57 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4810.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 2893.7, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_val/runs/pr4810/run-20260816-201524-c6140d", + "recorded_at": "2026-08-16T21:03:36", + "attempts": [ + { + "invocation_id": "inv-625eaf4a", + "outcome": "run:run-20260816-201524-c6140d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-201524-c6140d", + "llm_calls": 180, + "input_tokens": 1606888, + "output_tokens": 323039, + "usd": 1.275802, + "source": "spans" + } + ], + "attempt_usd_total": 1.275802, + "llm_calls": 180, + "input_tokens": 1606888, + "output_tokens": 323039, + "usd": 1.275802, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-201524-c6140d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2758, + "minutes": 48.19, + "ci_minutes": 0, + "input_tokens": 1606888, + "output_tokens": 323039, + "tool_calls": 279, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3951, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 180, + "by_role": { + "moa_member": { + "usd": 0.137247, + "input_tokens": 220659, + "output_tokens": 7912, + "calls": 23 + }, + "agent": { + "usd": 0.842072, + "input_tokens": 1067378, + "output_tokens": 233670, + "calls": 122 + }, + "moa_member_fallback": { + "usd": 0.261938, + "input_tokens": 303817, + "output_tokens": 53783, + "calls": 33 + }, + "reducer": { + "usd": 0.034546, + "input_tokens": 15034, + "output_tokens": 27674, + "calls": 2 + } + }, + "cache_read_tokens": 16866176, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.85, + "gate": 1.79, + "review": 2885.4, + "report": 0.07 + }, + "time_to_first_result_s": 2891.09 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 36439 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_val/pr4810.md b/eval/dataset/arms/copilot_v17_val/pr4810.md new file mode 100644 index 000000000..4cd4f4031 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4810.md @@ -0,0 +1,82 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [claim-verified] mimo_audio_llm.py has no AutoWeightsLoader usage (grep), so its top-level custom load_weights genuinely bypasses the outer mapper and needs the new call. +- [claim-verified] each of the 4 changed non-test files has a new test covering it (delegated: Hunyuan/Qwen2; direct: Mammoth/MiMo). +- [claim-verified] tests/model_executor/models/test_kv_cache_scale_mapper.py:15 _STALE_API_FILES has exactly four paths; verified by reading the diff and grep. +- [claim-verified] tests/model_executor/models/test_kv_cache_scale_mapper.py carries pytestmark = [pytest.mark.core_model, pytest.mark.cpu], so it will be collected/run by the `.buildkite/test-amd-merge.yml:32` `pytest -sv tests/model_executor -m 'core_model and cpu'` lane. +- [claim-refuted] PR description says 'Fix custom model loaders that still call the removed vLLM get_cache_scale API'. Repo-wide grep shows `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` STILL calls `self.quant_config.get_cache_scale(name)` — same removed API, same bug, unfixed. +- [claim-refuted] PR claims to fix custom loaders calling removed get_cache_scale, but repo grep finds vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 still calls self.quant_config.get_cache_scale(name). +- [claim-refuted] 'Fix custom model loaders that still call get_cache_scale' is incomplete: diffusion/hunyuan_image3/hunyuan_image3_transformer.py:2238 still calls it and is the only remaining in-repo caller. +- [claim-refuted] PR's own pre-fix run listed the diffusion transformer as FAILING; the shipped `_STALE_API_FILES` drops that path, narrowing the test to hide the unfixed caller. +- [claim-refuted] PR body says only the four model_executor loaders need fixing; grep of vllm_omni for 'get_cache_scale' finds vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 still calls `self.quant_config.get_cache_scale(name)`. The PR's own current-main test run listed this file as failing, and the new _STALE_API_FILES omits it. This is covered in kept comments; residual diffusion risk remains unfixed at head. +- [validated] hunyuan_image3.py:2178 — `loader = AutoWeightsLoader(self, ...)` confirms outer loader does cache scale mapping before HunyuanModel.load_weights receives weights. +- [validated] test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale: feeds _MAPPED_SCALE_NAME to HunyuanModel.load_weights via _FakeHunyuanModel; asserts param receives value. Correctly tests the post-mapping path. +- [validated] test_direct_custom_loaders_apply_cache_scale_mapper: feeds _SOURCE_SCALE_NAME through MammothModa2/MiMoAudio load_weights; asserts mapper.called and mapped name appears in loaded set. Correctly tests the pre-mapping transform. +- [validated] test_model_loaders_do_not_call_removed_get_cache_scale: source-level string assertion `.get_cache_scale(` not in source. Brittle but valid as a regression guard; PR author may remove. +- [validated] hunyuan_image3.py:2178 uses AutoWeightsLoader and the new comment at line 226 matches the outer-mapper contract. + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:15` [major] — Confirmed. `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` still calls the removed `self.quant_config.get_cache_scale(name)` (repo-wide grep finds this as the only remaining production call), yet the new `_STALE_API_FILES` at tests/model_executor/models/test_kv_cache_scale_mapper.py:15 lists only the four model_executor loaders. Since `pipeline_hunyuan_image3.py` constructs `HunyuanImage3Model(..., quant_config=quant_config)`, this stale branch is reachable and the regression test hides the remaining AttributeError. Add the diffusion transformer path to `_STALE_API_FILES` and fix its `load_weights` (remove the manual branch or apply `get_cache_scale_mapper`) before merge. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 ` if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`; the same file has `def load_weights` at line 2125 and grep finds no `get_cache_scale_mapper` there. The new test's whitelist omits this file: tests/model_executor/models/test_kv_cache_scale_mapper.py:15-20 `_STALE_API_FILES = [ + "vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py", + "vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py", + "vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py", + "vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py", +]`. Reachability: vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py:386 ` self.model = HunyuanImage3Model(self.hf_config, quant_config=quant_config)`.) + +```suggestion +_STALE_API_FILES = [ + "vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py", + "vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py", + "vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py", + "vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py", + "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", +] +``` + +`vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1162` [major] — Pre-existing in the PR-time tree (context lines, not changed by this diff): vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1162-1163 unconditionally prepend `model.` to any name already starting with `model.`, while self.model's parameter names carry the `model.` prefix via `maybe_prefix(prefix, "model")` (line 539). The new direct-loader test passes unprefixed `_SOURCE_SCALE_NAME`, so it does not cover the prefixed path. Fix or explicitly document the expected prefix convention. (evidence: vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1162 `if name.startswith("model."):` / :1163 `name = "model." + name` (unchanged context in this diff) — an already-prefixed `model.layers...` name becomes `model.model.layers...`, which cannot match `params_dict` keys. vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:539 `prefix=maybe_prefix(prefix, "model"),` — self.model's named parameters therefore start with `model.`.) + +```suggestion +if not name.startswith("model."): + name = "model." + name +``` + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:81` [minor] — The guard at tests/model_executor/models/test_kv_cache_scale_mapper.py:81 (`assert ".get_cache_scale(" not in source`) does not match the deleted qwen2_old.py form `getattr(self.quant_config, "get_cache_scale", None)` or its call `get_cache_scale(name)` (no dot prefix), so reintroducing that fallback would leave this test green. Tighten to `assert "get_cache_scale(" not in source` rather than `assert "get_cache_scale" not in source`, which would false-positive on the new correct `get_cache_scale_mapper()` calls. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:81 ` assert ".get_cache_scale(" not in source` (new file in this diff). The deleted old form is in this same diff at vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py hunk `@@ -335,19 +335,10 @@`: `- get_cache_scale = getattr(self.quant_config, "get_cache_scale", None)` — that line contains `"get_cache_scale"` but not `.get_cache_scale(`, so the assertion cannot detect the removed-API pattern it claims to guard for qwen2_old.py.) + +```suggestion +assert "get_cache_scale" not in source +``` + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:35` [minor] — This diff adds `_RecordingCacheScaleMapper`, whose `apply` only renames `k_proj.output_scale` to `attn.k_scale` (tests/model_executor/models/test_kv_cache_scale_mapper.py:36), and `test_direct_custom_loaders_apply_cache_scale_mapper` passes a scalar `torch.ones(())`; the test therefore never exercises q_scale/v_scale renaming or the 1-D `loaded_weight[0]` path whose manual handling this diff removes. Add a q_scale/v_scale case and a 1-D scale tensor, or use the real quant-config mapper. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:36 ` yield name.replace(".k_proj.output_scale", ".attn.k_scale"), weight` — the recording mapper renames only k_scale; tests/model_executor/models/test_kv_cache_scale_mapper.py:147 ` loaded = load_weights(fake_model, [(_SOURCE_SCALE_NAME, torch.ones(()))])` and :161 ` loaded = load_weights(fake_model, [(_SOURCE_SCALE_NAME, torch.ones(()))])` — the direct-loader tests pass a scalar weight, never a 1-D scale tensor.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:112` [minor] — This test calls HunyuanModel.load_weights/Qwen2Model.load_weights directly with the already-mapped `_MAPPED_SCALE_NAME` (line 112), bypassing the outer AutoWeightsLoader at hunyuan_image3.py:2178 that actually applies `get_cache_scale_mapper()`. It proves only the fall-through branch accepts a mapped name and cannot catch a regression where the outer loader stops applying the mapper. Drive the real outer entry point with a real quant config, or rename the test to describe the fall-through contract it actually asserts. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:23 `_MAPPED_SCALE_NAME = "layers.0.self_attn.attn.k_scale"` — the name is pre-mapped by the test; tests/model_executor/models/test_kv_cache_scale_mapper.py:112 `loaded = load_weights(fake_model, [(_MAPPED_SCALE_NAME, torch.ones(()))])` — passed directly to `HunyuanModel.load_weights`/`Qwen2Model.load_weights` (parametrized class_name at lines 83/88), so the `AutoWeightsLoader` path is never exercised; the production mapper application lives outside this diff in the PR-time tree at vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py:2178 `loader = AutoWeightsLoader(`.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:12` [nit] — New test file tests/model_executor/models/test_kv_cache_scale_mapper.py:12 has `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; .buildkite/test-amd-merge.yml:32 runs `pytest -sv tests/model_executor -m 'core_model and cpu'`, so the AMD merge lane collects this test. Grep across .buildkite finds no NPU yaml containing `core_model and cpu`. Please confirm on main whether the AMD/NPU CI failures are pre-existing before merging. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — added by this diff. .buildkite/test-amd-merge.yml:32 ` - "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` — unchanged by this diff, present in the PR-time tree. Absence of an NPU lane: `grep -r "core_model and cpu" .buildkite` at head f9665cc223c8 returns matches only in test-amd-merge.yml, test-amd-ready.yaml, test-merge.yml, test-ready.yml (no npu yaml). CI read of PR 4810 returns buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci with state FAILURE.) + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:487` [nit] — The diff adds a manual `get_cache_scale_mapper()` call in `MammothModa2Qwen2ForCausalLM.load_weights`, but that class is the `language_model` of `MammothModa2ARForConditionalGeneration(Qwen2_5_VLForConditionalGeneration)` (line 540) and no parent `load_weights` override is defined in this file. If the inherited parent loader already applies `AutoWeightsLoader`, this manual call may be redundant/double-applied; please confirm the parent path and remove the manual call if it is already covered. (evidence: vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:255 `class MammothModa2Qwen2ForCausalLM(nn.Module, SupportsPP):` and :540 `class MammothModa2ARForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):`; grep shows no `load_weights` defined on MammothModa2ARForConditionalGeneration in this file, so the inherited parent loader is the entry to this hunk.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_val/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v17_val/pr4810.trace.json.gz new file mode 100644 index 000000000..825498d42 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_val/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_val/pr4816.cost.json b/eval/dataset/arms/copilot_v17_val/pr4816.cost.json new file mode 100644 index 000000000..7a3ae0022 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4816.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1073.5, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_smoke/runs/pr4816/run-20260816-195605-e34bed", + "recorded_at": "2026-08-16T20:13:56", + "attempts": [ + { + "invocation_id": "inv-a21b7e19", + "outcome": "run:run-20260816-195605-e34bed", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-195605-e34bed", + "llm_calls": 102, + "input_tokens": 975093, + "output_tokens": 166280, + "usd": 0.672582, + "source": "spans" + } + ], + "attempt_usd_total": 0.672582, + "llm_calls": 102, + "input_tokens": 975093, + "output_tokens": 166280, + "usd": 0.672582, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-195605-e34bed", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.6726, + "minutes": 17.84, + "ci_minutes": 0, + "input_tokens": 975093, + "output_tokens": 166280, + "tool_calls": 190, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7674, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 102, + "by_role": { + "planner": { + "usd": 0.001371, + "input_tokens": 2007, + "output_tokens": 754, + "calls": 1 + }, + "agent": { + "usd": 0.654597, + "input_tokens": 960152, + "output_tokens": 153635, + "calls": 99 + }, + "reducer": { + "usd": 0.016614, + "input_tokens": 12934, + "output_tokens": 11891, + "calls": 2 + } + }, + "cache_read_tokens": 8385152, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.79, + "gate": 2.06, + "review": 1064.56, + "report": 0.07 + }, + "time_to_first_result_s": 1070.47 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 28575 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_val/pr4816.md b/eval/dataset/arms/copilot_v17_val/pr4816.md new file mode 100644 index 000000000..cb9831d50 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4816.md @@ -0,0 +1,79 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Concern "changed tests are never selected in CI": test_serving_speech.py:48 has `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` and test-ready.yml:209 runs `pytest -sv tests/entrypoints/ -m 'core_model and cuda'` (cuda auto-added), so the file is selected on the CUDA lane. Residual: AMD/Intel/NPU lanes do not reference this file (grep .buildkite `test_serving` = no matches). +- [claim-verified] PR claim that upstream `base` moved to serve.instrumentator.basic and reads `app.state.serving_tokenization` is consistent with api_server.py:67 `from vllm.entrypoints.serve.instrumentator.basic import base` and the diff's key rename. +- [claim-verified] Motivating regression chain real: api_server.py:376 `err = base(raw_request).create_error_response(...)`; before the fix only `openai_serving_tokenization` was set, so `base` raised AttributeError -> 500 on caught speech errors. +- [claim-verified] PR body's upstream claim is consistent: the file already imports `base` from `vllm.entrypoints.serve.instrumentator.basic` and `ServingTokenization` from `vllm.entrypoints.serve.tokenize.serving` (new layout), matching the quoted `tokenization()` reading `request.app.state.serving_tokenization`. +- [claim-verified] PR description's premise (upstream moved base to read `request.app.state.serving_tokenization`) is consistent with api_server.py:67 `from vllm.entrypoints.serve.instrumentator.basic import base`; old-name references are absent repo-wide at this head. +- [claim-verified] PR claim that `_create_speech_error_json_response` calls `base(raw_request).create_error_response(...)` is true: api_server.py:376 does exactly that. +- [claim-refuted] PR claim "keeps things from turning into a 500" holds only for the LLM/multi-stage branch (api_server.py:1017 assigns a real `ServingTokenization`); pure-diffusion assigns None (754) and `_create_speech_error_json_response` (376) dereferences `base()` unguarded -> still 500 for upload_voice/delete_voice errors. +- [validated] Test updates are semantically necessary: test_serving_speech.py:2533/2546/2615 set `state.serving_tokenization` so the renamed `getattr` fallback returns fake_base; with the old key these tests would raise HTTPException instead of the asserted JSON 404. +- [validated] omni_init_app_state consumers (tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py:337, tests/entrypoints/openai_api/test_omni_sleep_wakeup.py:251) do not reference the old key - no stale assertions. +- [validated] create_chat_completion/create_speech/create_speech_batch/create_audio_generate all read the renamed key via `getattr(..., default=None)`; the TTS-only (chat handler None) LLM fallback still returns a ServingTokenization-based error response. +- [validated] grep `serving_tokenization` shows only api_server.py (7 hits: 754,1017,1157,1248,1286,1329) and test_serving_speech.py (3 hits: 2533,2546,2615); no other repo consumer of the renamed state attribute exists. +- [validated] `omni_init_app_state` is the sole setter of the attribute; `omni_run_server_worker` calls `build_openai_app` then `omni_init_app_state`, so upstream init does not separately populate `serving_tokenization`. +- [validated] No new external library call/kwarg is introduced; the rename hard-requires the already-present new-layout vllm (the file already imports `serve.*` modules), so no new dependency lower bound is needed. +- [validated] grep `openai_serving_tokenization` over the whole repo at head returns 0 matches — no stale in-repo consumer of the old name remains. + +`vllm_omni/entrypoints/openai/api_server.py:754` [major] — This hunk leaves pure-diffusion `state.serving_tokenization = None` (api_server.py:754) while the LLM branch assigns a real `ServingTokenization` (api_server.py:1014). The unchanged `_create_speech_error_json_response` still calls `base(raw_request).create_error_response(...)` at api_server.py:376; upstream `base()` now returns `request.app.state.serving_tokenization`, so in pure-diffusion mode `upload_voice`/`delete_voice` error paths (`_create_speech_error_json_response(raw_request, str(e))`, lines 1483/1539) call `.create_error_response` on `None`, raising AttributeError and returning 500 — the exact failure this PR aims to eliminate. Since `state.openai_serving_speech = OmniOpenAIServingSpeech.for_diffusion(...)` (api_server.py:784) makes those voice endpoints reachable, guard the None case in `_create_speech_error_json_response` (or provision a real tokenization error handler for pure diffusion) and confirm a sad `/v1/audio/voices` request on a pure-diffusion model returns 4xx, not 500. (evidence: api_server.py:376 `err = base(raw_request).create_error_response(`; api_server.py:754 `state.serving_tokenization = None`; api_server.py:1483 `return _create_speech_error_json_response(raw_request, str(e))` (upload_voice); api_server.py:1248 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)` followed by `if base_server is None: raise HTTPException(...404...)`.) + +```suggestion +base_server = base(raw_request) + if base_server is None: + return JSONResponse( + content={"error": {"message": message, "type": err_type, "code": int(status_code)}}, + status_code=int(status_code), + ) + err = base_server.create_error_response( + message=message, + err_type=err_type, + status_code=status_code, + ) + return _error_response_to_json_response(err, status_code=status_code) +``` + +`vllm_omni/entrypoints/openai/api_server.py:~1163` [major] — Confirmed. `create_chat_completion`'s handler-None branch still returns a bare `ErrorResponse` at api_server.py:1163 even after the rename at api_server.py:1157. In a TTS-only server `state.openai_serving_chat` is None (api_server.py:928, gated by `"generate" in supported_tasks`) while `state.serving_tokenization` is set unconditionally (api_server.py:1017), so POST /v1/chat/completions returns HTTP 200 with a BadRequestError/400 body instead of the 404 error JSON the endpoint's `responses` contract and sibling handlers (e.g. api_server.py:1259) promise. Wrap the chat error through `_error_response_to_json_response` with NotFoundError/404 and add the regression test. Note: the 200-vs-404 behavior predates this rename-only diff, but it is a real user-visible defect in the branch the diff touches. (evidence: vllm_omni/entrypoints/openai/api_server.py:1157 ` base_server = getattr(raw_request.app.state, "serving_tokenization", None)` (changed by this diff); vllm_omni/entrypoints/openai/api_server.py:1163 ` return base_server.create_error_response(message="The model does not support Chat Completions API")` (unchanged by this diff, present in PR-time tree) — bare ErrorResponse return; vllm_omni/entrypoints/openai/api_server.py:1017 ` state.serving_tokenization = ServingTokenization(` (changed by this diff; unconditionally set); vllm_omni/entrypoints/openai/api_server.py:928 ` state.openai_serving_chat = (` with the `if "generate" in supported_tasks` gate (unchanged; None for TTS-only); vllm_omni/entrypoints/openai/api_server.py:1259 ` return _error_response_to_json_response(err, status_code=HTTPStatus.NOT_FOUND)` (unchanged; sibling speech branch serializes 404 JSON).) + +```suggestion +err = base_server.create_error_response( + message="The model does not support Chat Completions API", + err_type="NotFoundError", + status_code=HTTPStatus.NOT_FOUND, + ) + return _error_response_to_json_response(err, status_code=HTTPStatus.NOT_FOUND) +``` + +`tests/entrypoints/openai_api/test_serving_speech.py:~2533` [minor] — The diff updates the three 404 tests to `raw_request.app.state.serving_tokenization = fake_base` (test_serving_speech.py:2533) but each still goes through `_patch_api_server_base(mocker)`, replacing `api_server_module.base` with a MagicMock; tests that reach `_create_speech_error_json_response` also patch `api_server_module.base` (line 2352). Therefore no test drives the real upstream `base(raw_request)` -> `request.app.state.serving_tokenization` dispatch this PR fixes, and the 404 tests would stay green even if the write still used the old `openai_serving_tokenization` name. Add a regression test that sets `app.state.serving_tokenization` to a real/stub `ServingTokenization` and drives the PR's ValueError reproduction through `upload_voice` (or `delete_voice`) without patching `api_server_module.base`, asserting a 4xx JSON error response rather than AttributeError/500. (evidence: test_serving_speech.py:2352 `mocker.patch.object(api_server_module, "base", return_value=fake_base)`; api_server.py:1248 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)` with 404 fallback; repo grep `_create_speech_error_json_response` finds it only in api_server.py (0 test references).) + +`test_serving_speech.py:48` [minor] — Concern "changed tests are never selected in CI" is resolved: test_serving_speech.py:48 declares `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` and .buildkite/test-ready.yml:209 runs `pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'`, so the file is selected on the CUDA entrypoints lane. Residual: a .buildkite grep for `test_serving` returns no matches, so no AMD/Intel/NPU lane references this file directly. (evidence: tests/entrypoints/openai_api/test_serving_speech.py:48 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` — present in the PR-time tree and unchanged by this diff. .buildkite/test-ready.yml:209 `pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'` — present in the PR-time tree and outside the diff. Both lines are decisive for the comment's CI-selection claim.) + +`vllm_omni/entrypoints/openai/api_server.py:66` [minor] — Stale comment at api_server.py:66: it claims 'Keep a fallback for older/newer upstream layouts during rebase windows,' but api_server.py:67 is a single unconditional import of `base` from `vllm.entrypoints.serve.instrumentator.basic` with no fallback branch. Update or remove the comment. (evidence: api_server.py:66 `# Keep a fallback for older/newer upstream layouts during rebase windows.` followed by api_server.py:67 `from vllm.entrypoints.serve.instrumentator.basic import base` — the import is unconditional and no try/except or alternate import path exists (grep returns only line 67). These lines are unchanged by this diff but present in the PR-time tree.) + +`vllm_omni/entrypoints/openai/api_server.py:1017` [nit] — CI evidence is accurate: buildkite/vllm-omni-amd-ci, -intel-ci, and -npu-ci fail while buildkite/vllm-omni passes; grep shows no `.buildkite` lane references `test_serving` (AMD runs `tests/entrypoints/test_omni_sleep_mode.py`, and test-ready runs `pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'`). Confirm the three failures are pre-existing on main before merging. (evidence: ['vllm_omni/entrypoints/openai/api_server.py:1017 ` state.serving_tokenization = ServingTokenization(` — changed by this diff (rename from `openai_serving_tokenization`).', ".buildkite/test-ready.yml:209 ` pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'` — unchanged by this diff, present in the PR-time tree.", '.buildkite/test-amd.yaml:118 ` - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"` — unchanged by this diff, present in the PR-time tree.', 'grep `.buildkite` for literal `test_serving`: no matches — unchanged CI files, present in the PR-time tree.']) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_val/pr4825.cost.json b/eval/dataset/arms/copilot_v17_val/pr4825.cost.json new file mode 100644 index 000000000..b2dcc9d39 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4825.cost.json @@ -0,0 +1,126 @@ +{ + "wall_s": 1055.9, + "split": "val", + "rc": 3, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_val/runs/pr4825/run-20260816-202436-09df0e", + "recorded_at": "2026-08-16T20:32:58", + "attempts": [ + { + "invocation_id": "inv-769ba0c3", + "outcome": "run:run-20260816-201524-2f167a", + "rc": 3, + "pre_run": null + }, + { + "invocation_id": "inv-be695108", + "outcome": "run:run-20260816-202436-09df0e", + "rc": 3, + "pre_run": null + }, + { + "run_dir": "run-20260816-201524-2f167a", + "llm_calls": 104, + "input_tokens": 532285, + "output_tokens": 85780, + "usd": 0.450982, + "source": "spans" + }, + { + "run_dir": "run-20260816-202436-09df0e", + "llm_calls": 87, + "input_tokens": 308150, + "output_tokens": 76190, + "usd": 0.328559, + "source": "spans" + } + ], + "attempt_usd_total": 0.779541, + "llm_calls": 87, + "input_tokens": 308150, + "output_tokens": 76190, + "usd": 0.328559, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-202436-09df0e", + "task_kind": "pr_review", + "status": "blocked", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3286, + "minutes": 8.35, + "ci_minutes": 0, + "input_tokens": 308150, + "output_tokens": 76190, + "tool_calls": 138, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4195, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 87, + "by_role": { + "moa_member": { + "usd": 0.132093, + "input_tokens": 163273, + "output_tokens": 11635, + "calls": 44 + }, + "agent": { + "usd": 0.196465, + "input_tokens": 144877, + "output_tokens": 64555, + "calls": 41 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 2 + } + }, + "cache_read_tokens": 4873664, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.26, + "gate": 2.35, + "review": 491.71 + }, + "time_to_first_result_s": 497.37 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 1, + "pushes": 0, + "posted": 0, + "steps_completed": 2 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 26574 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_val/pr4825.md b/eval/dataset/arms/copilot_v17_val/pr4825.md new file mode 100644 index 000000000..238a2f494 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4825.md @@ -0,0 +1,17 @@ +(no RUN_REPORT.md — rc=3) + +## stdout +→ task: pr_review PR #4825 on vllm-omni (tier L2) +→ plan: reuse pr-review@6 (active) steps=['pr.fetch_diff', 'pr.gate_check', 'agent.review_diff', 'pr.post_review', 'report.final_summary'] + · recalled pr-review@6 (active) +→ path: eco · agent-model=deepseek-v4-pro @ api.deepseek.com + metrics: usd≈0.33 8.3min S=1.00 (/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_val/runs/pr4825/run-20260816-202436-09df0e/metrics.json) + ✓ fetch: fetched PR #4825 diff (3256 chars, context 2022 chars); checkout: PR-TIME TREE (head ac3a2ce5a9b9) + ✓ gate: FAILING CHECKS (2): ['buildkite/vllm-omni-amd-ci', 'buildkite/vllm-omni-npu-ci'] — do not re-argue what CI already repor + ✗ review: unhandled error: BadRequestError: Error code: 400 - {'error': {'message': 'The `content[].thinking` in the thinking mode must be passed back to the API.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_request_error'}} +run run-20260816-202436-09df0e: blocked (/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_val/runs/pr4825/run-20260816-202436-09df0e) + ⚠ step 'review' (agent.review_diff): unhandled error: BadRequestError: Error code: 400 - {'error': {'message': 'The `content[].thinking` in the thinking mode must be passed back to the API.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_request_error'}} + see /data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_val/runs/pr4825/run-20260816-202436-09df0e/ESCALATION.md + + +## stderr diff --git a/eval/dataset/arms/copilot_v17_val/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v17_val/pr4825.trace.json.gz new file mode 100644 index 000000000..ded282c89 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_val/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_val/pr4837.cost.json b/eval/dataset/arms/copilot_v17_val/pr4837.cost.json new file mode 100644 index 000000000..6b7277bb7 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4837.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 1294.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_val/runs/pr4837/run-20260816-203300-69c9a5", + "recorded_at": "2026-08-16T20:54:32", + "attempts": [ + { + "invocation_id": "inv-83dcacb2", + "outcome": "run:run-20260816-203300-69c9a5", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-203300-69c9a5", + "llm_calls": 167, + "input_tokens": 1146307, + "output_tokens": 190423, + "usd": 0.911075, + "source": "spans" + } + ], + "attempt_usd_total": 0.911075, + "llm_calls": 167, + "input_tokens": 1146307, + "output_tokens": 190423, + "usd": 0.911075, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-203300-69c9a5", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9111, + "minutes": 21.53, + "ci_minutes": 0, + "input_tokens": 1146307, + "output_tokens": 190423, + "tool_calls": 247, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9203, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 167, + "by_role": { + "moa_member": { + "usd": 0.141565, + "input_tokens": 252015, + "output_tokens": 3119, + "calls": 23 + }, + "agent": { + "usd": 0.739538, + "input_tokens": 882265, + "output_tokens": 163050, + "calls": 140 + }, + "moa_member_fallback": { + "usd": 0.0, + "input_tokens": 0, + "output_tokens": 0, + "calls": 2 + }, + "reducer": { + "usd": 0.029972, + "input_tokens": 12027, + "output_tokens": 24254, + "calls": 2 + } + }, + "cache_read_tokens": 13326336, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.31, + "gate": 1.63, + "review": 1285.81, + "report": 0.06 + }, + "time_to_first_result_s": 1291.8 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 35436 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_val/pr4837.md b/eval/dataset/arms/copilot_v17_val/pr4837.md new file mode 100644 index 000000000..4e0377397 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4837.md @@ -0,0 +1,81 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4837, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] PR body 'unwrapping covered only when already_submitted is true' matches base code via search_history/`file_at_base` reading of the hunk and the issue traceback. +- [claim-verified] Linked issue #4839 single-request curl path now flows: thinker2imagegen returns `[{'prompt': '', 'extra': {...}}]` (ming_flash_omni.py:411) → len==1 unwrap → submit_initial receives a dict, avoiding the ValueError. +- [claim-verified] Old orchestrator condition `if already_submitted and len(diffusion_prompt) == 1:` only unwrapped on the update path; new `if len(diffusion_prompt) == 1:` unwraps on initial submit too (orchestrator.py:1290) — matches the PR description. +- [claim-verified] Both diffusion submit paths reject a list: stage_pool.py submit_initial and submit_update each raise `Diffusion list-prompt batch requests are no longer supported.` +- [claim-refuted] PR test-plan screenshot cannot hold at merged head: pipeline_ming_imagegen.py:449-461 passes prompt=None/height/.../return_dict=True while pipeline_z_image.py:413 'def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:' has no **kwargs. +- [claim-refuted] PR body Test Result image was produced at pre-merge commit 5c51782c; merge commit 78cc4dcb pulled in #1235 (show_commit lists pipeline_z_image.py -190 lines) which changed ZImagePipeline.forward — the merged head was not re-tested. +- [claim-refuted] PR body's test plan is stale at head: body cites commit 5c51782c, but head is merge 78cc4dcb which pulled main's `pipeline_z_image.py | 190 +++++----------------` refactor; the tested behavior is no longer what head executes. +- [validated] orchestrator.py:1290 `if len(diffusion_prompt) == 1:` now unwraps before BOTH submit_update (already_submitted) and submit_initial (initial); the reported #4839 ValueError path reaches submit_initial, so the fix targets the right branch. +- [validated] ZImagePipeline.forward (pipeline_z_image.py:413-750) returns a single `DiffusionOutput(output=image, stage_durations=...)`; the old `outputs[0]` in the Ming wrapper was indexing a DiffusionOutput that has no __getitem__, so the new direct return is correct. +- [validated] orchestrator.py:1290 'if len(diffusion_prompt) == 1:' now unwraps on the initial submit path too; the update path previously already unwrapped under already_submitted. +- [validated] pipeline_ming_imagegen.py:465 'raw = outputs.output' matches ZImagePipeline.forward's return 'return DiffusionOutput(output=image, stage_durations=stage_durations)' (pipeline_z_image.py:719); returning outputs preserves stage_durations. +- [validated] orchestrator.py:1290 fix is correct: StagePool.submit_initial (stage_pool.py:953) and submit_update (stage_pool.py:1024) both raise ValueError for list prompts, so unwrapping len==1 before submit_initial is required; the already-submitted path was already unwrapping. +- [validated] Multi-element list (>1) still reaches submit and raises the existing loud ValueError "Diffusion list-prompt batch requests are no longer supported" (stage_pool.py:953) — not silent, and consistent with scheduler batching. +- [validated] The pipeline return change (return `outputs` instead of a stripped DiffusionOutput(output=raw)) preserves ZImagePipeline's stage_durations/finished fields; no in-repo consumer assumes the old stripped output. + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449` [blocker] — pipeline_ming_imagegen.py:449 calls `super().forward(z_req, prompt=None, height=height, width=width, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, generator=generator, output_type="pt", return_dict=True)`, but the parent signature is `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:` (pipeline_z_image.py:413) with no `**kwargs`, so every Ming imagegen request raises `TypeError: forward() got an unexpected keyword argument 'prompt'` before `raw = outputs.output`. Change to `super().forward(z_req)`; since Ming's `__init__` sets `self.tokenizer = None`, also carry the precomputed `prompt_embeds`/`negative_prompt_embeds` through an `_encode_prompt` override or `_ZPipelineSamplingParams` fields, then re-run the serve+curl test at head. (evidence: vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449 ` outputs: DiffusionOutput = super().forward(` followed by lines 450-461: ` z_req,` ` prompt=None,` ` height=height,` ` width=width,` ` num_inference_steps=num_inference_steps,` ` guidance_scale=guidance_scale,` ` prompt_embeds=prompt_embeds,` ` negative_prompt_embeds=negative_prompt_embeds,` ` generator=generator,` ` output_type="pt",` ` return_dict=True,` — these kwargs appear in the diff hunk context. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413 ` def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:` — signature accepts only `req`, no `**kwargs`. Also present in the PR-time tree: pipeline_ming_imagegen.py __init__ ` self.text_encoder = None` and ` self.tokenizer = None`.) + +```suggestion +outputs: DiffusionOutput = super().forward(z_req) +``` + +`vllm_omni/engine/orchestrator.py:~1290` [major] — The orchestrator hunk deletes `already_submitted and` at orchestrator.py:1290, so a one-element `diffusion_prompt` list is unwrapped on the initial-submit path too, avoiding the `Diffusion list-prompt batch requests are no longer supported` ValueError at stage_pool.py:953. No automated test covers this branch: tests/engine/test_orchestrator_stage_input_bridge.py only drives a FakeStageClient with `stage_type='llm'` and `custom_process_input_func=None`. Add a regression test that gives the diffusion-stage fake a `custom_process_input_func` returning `[{'prompt': '', 'extra': {}}]`, calls `_forward_to_next_stage` on the non-submitted initial path, and asserts the submitted request is the unwrapped dict. (evidence: vllm_omni/engine/orchestrator.py:1290 ` if len(diffusion_prompt) == 1:` — the diff replaces `if already_submitted and len(diffusion_prompt) == 1:` with this line, so the following unwrap `diffusion_prompt = diffusion_prompt[0]` (line 1291) now also runs on the initial-submit path. vllm_omni/engine/stage_pool.py:953 ` "Diffusion list-prompt batch requests are no longer supported. "` — unchanged by this diff, present in the PR-time tree; this is the list-prompt ValueError the fix avoids. tests/engine/test_orchestrator_stage_input_bridge.py (FakeStageClient.__init__) ` self.stage_type = "llm"` and ` self.custom_process_input_func = None`; the file contains only `test_forward_text_prompt_uses_target_stage_input_processor`, so no diffusion single-element-list bridge path is covered.) + +```suggestion +In tests/engine/test_orchestrator_stage_input_bridge.py add a test: + +class FakeDiffusionStageClient(FakeStageClient): + def __init__(self, **kw): + super().__init__(**kw) + self.stage_type = "diffusion" + self.custom_process_input_func = lambda outputs, prompt, requires_mm, **kw: [{"prompt": "", "extra": {}}] + self.added_requests = [] + async def add_request_async(self, *args, **kwargs): + self.added_requests.append(args) + +@pytest.mark.asyncio +async def test_forward_single_element_bridge_unwraps_on_initial_submit(): + ... + await orchestrator._forward_to_next_stage("req-diff", 0, _request_output("req-diff"), req_state) + submitted = stage1.added_requests[0][2] + assert isinstance(submitted, dict) +``` + +`.buildkite/test-amd.yaml:1` [minor] — Head shows two failing lanes — buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci — while buildkite/vllm-omni is green. The diff only touches vllm_omni/engine/orchestrator.py and vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py (no AMD/NPU platform files), and .buildkite/test-amd.yaml/.buildkite/test-npu-ready.yaml do not directly load those changed paths. Establish whether these failures pre-date this PR on main; if caused by this PR, fix them, and if pre-existing, state that explicitly rather than letting the main-lane SUCCESS stand in for the full matrix. (The AMD lane does run tests/e2e/offline_inference/test_z_image.py, which may indirectly hit the changed ming pipeline.) (evidence: gh_ci_read(PR 4837): buildkite/vllm-omni-amd-ci = FAILURE; buildkite/vllm-omni-npu-ci = FAILURE; buildkite/vllm-omni = SUCCESS. diff_stat: only 2 changed files — vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py and vllm_omni/engine/orchestrator.py (neither is AMD/NPU-specific). grep across .buildkite for 'pipeline_ming_imagegen|orchestrator' returned no matches (unchanged by this diff), so the CI configs do not directly load the changed paths. .buildkite/test-amd.yaml:7 ` - pytest -s -v tests/e2e/offline_inference/test_z_image.py` and .buildkite/test-npu-ready.yaml:14 ` - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'` show the failing lanes' test surfaces.) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:~475` [minor] — The pipeline hunk replaces `return DiffusionOutput(output=raw)` with `return outputs` (pipeline_ming_imagegen.py:475), so parent `DiffusionOutput` metadata such as `stage_durations` (data.py:1212) now survives this wrapper; the old reconstruction silently defaulted it to {}. No test locks this in. Add a unit test that monkeypatches `super().forward` to return `DiffusionOutput(output=torch.randn(...), stage_durations={'x': 1.0})` and asserts `MingImagePipeline.forward` returns the same object/metadata, so a future ZImagePipeline return-contract refactor cannot silently regress this wrapper. (evidence: pipeline_ming_imagegen.py:475 ` return outputs` — changed by this PR; vllm_omni/diffusion/models/z_image/pipeline_z_image.py:719-720 (unchanged by this diff, present in PR-time tree): ` stage_durations = self.stage_durations if hasattr(self, "stage_durations") else None` / ` return DiffusionOutput(output=image, stage_durations=stage_durations)`; vllm_omni/diffusion/data.py:1212 (unchanged by this diff): ` stage_durations: dict[str, float] = field(default_factory=dict)` — the removed `return DiffusionOutput(output=raw)` would have constructed a new DiffusionOutput with the default empty stage_durations.) + +```suggestion +assert returned is outputs +assert returned.stage_durations == {"x": 1.0} +``` + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_val/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v17_val/pr4837.trace.json.gz new file mode 100644 index 000000000..01963f695 Binary files /dev/null and b/eval/dataset/arms/copilot_v17_val/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17_val/pr4893.cost.json b/eval/dataset/arms/copilot_v17_val/pr4893.cost.json new file mode 100644 index 000000000..74f5515ad --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4893.cost.json @@ -0,0 +1,119 @@ +{ + "wall_s": 2063.6, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17_val/runs/pr4893/run-20260816-201525-1f6dba", + "recorded_at": "2026-08-16T20:49:46", + "attempts": [ + { + "invocation_id": "inv-17c70b86", + "outcome": "run:run-20260816-201525-1f6dba", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260816-201525-1f6dba", + "llm_calls": 246, + "input_tokens": 1559528, + "output_tokens": 295984, + "usd": 1.166582, + "source": "spans" + } + ], + "attempt_usd_total": 1.166582, + "llm_calls": 246, + "input_tokens": 1559528, + "output_tokens": 295984, + "usd": 1.166582, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260816-201525-1f6dba", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.1666, + "minutes": 34.34, + "ci_minutes": 0, + "input_tokens": 1559528, + "output_tokens": 295984, + "tool_calls": 423, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1998, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 246, + "by_role": { + "moa_member": { + "usd": 0.124408, + "input_tokens": 139603, + "output_tokens": 13980, + "calls": 37 + }, + "agent": { + "usd": 0.728701, + "input_tokens": 1056978, + "output_tokens": 189026, + "calls": 151 + }, + "moa_member_fallback": { + "usd": 0.281961, + "input_tokens": 346375, + "output_tokens": 68439, + "calls": 56 + }, + "reducer": { + "usd": 0.031512, + "input_tokens": 16572, + "output_tokens": 24539, + "calls": 2 + } + }, + "cache_read_tokens": 14510016, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.6, + "gate": 1.59, + "review": 2054.32, + "report": 0.06 + }, + "time_to_first_result_s": 2060.56 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 46205 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17_val/pr4893.md b/eval/dataset/arms/copilot_v17_val/pr4893.md new file mode 100644 index 000000000..bf94b8f25 --- /dev/null +++ b/eval/dataset/arms/copilot_v17_val/pr4893.md @@ -0,0 +1,76 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4893, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] diff contains exactly the 4 stated files; no unrelated scope in this head. +- [claim-verified] '11 passed' arithmetic holds: test_hunyuan_fused_moe.py has 8 tests (3 num_tokens + 2 dp_metadata + 1 dispatch + 2 factory) and test_expert_parallel_layout.py has 3; total 11. +- [claim-verified] PR test-plan command targets exactly the two touched test files; both carry @pytest.mark.cpu + @pytest.mark.core_model so they collect in the cpu/core_model lane. Could not re-run the reported '11 passed' here (read-only, no shell). +- [validated] parallel_state.py:853-872 — MoE PCP correctly creates a separate vLLM coordinator with sp_group_ranks (not aliasing diffusion _SP), device_communicator and reduce_scatter available +- [validated] parallel_state.py:863-872 — TP branch: MoE path uses init_vllm_model_parallel_group; non-MoE path uses diffusion init_model_parallel_group with parallel_mode='tensor' +- [validated] parallel_state.py:873-880 — DP branch: only assigns vLLM DP when use_moe_parallel_mapping; otherwise keeps diffusion _DP aliased to vllm_parallel_state._DP +- [validated] parallel_state.py:901-910 — EP branch correctly creates vLLM EP with tp-sp-cfg-dp ranks +- [validated] parallel_state.py:917-940 — destroy_model_parallel correctly guards identity (vllm_parallel_state._DP is not _DP, _PCP is not _SP) to prevent double-destroy +- [validated] hunyuan_fused_moe.py:29-50 — _set_forward_context_dp_metadata has proper early-return guards: no context, metadata already set, single-DP group +- [validated] hunyuan_fused_moe.py:91 — _set_forward_context_dp_metadata called from _num_tokens_pre_hook alongside _set_forward_context_num_tokens +- [validated] test_expert_parallel_layout.py:62-73 — fake_init_vllm_model_parallel_group creates groups with 'vllm_' prefix, device_communicator, and reduce_scatter +- [validated] test_expert_parallel_layout.py:109 — assertion correctly changed from 'is' to 'is not' for PCP vs SP identity +- [validated] test_expert_parallel_layout.py:118-123 — asserts device_communicator and reduce_scatter on PCP, DP, EP +- [validated] test_expert_parallel_layout.py:187-189 — vllm group names correctly verified as ['vllm_pcp', 'vllm_tp', 'vllm_dp', 'vllm_ep'] + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:38` [minor] — In non-expert-parallel DP>1 runs, `_vllm_ps._DP` is the diffusion GroupCoordinator (parallel_state.py:816) and `_vllm_ps._EP` is None (only `use_moe_parallel_mapping` builds it). The pre-hook therefore invokes `_set_forward_context_dp_metadata(hidden_states.shape[0])` at hunyuan_fused_moe.py:91, which performs an `all_gather_object` over the diffusion DP cpu_group and populates `forward_context.dp_metadata` that no vLLM all2all dispatcher consumes. Add an early return when `_EP` is None to preserve non-EP behavior; minor performance/behavior concern. (evidence: hunyuan_fused_moe.py:91 `_set_forward_context_dp_metadata(hidden_states.shape[0])` — called unconditionally from the FusedMoE pre-hook. hunyuan_fused_moe.py:38 `if dp_group is None or getattr(dp_group, "world_size", 1) <= 1:` then :42-46 `torch.distributed.all_gather_object(gathered_num_tokens, int(num_tokens), group=dp_group.cpu_group)` and :50 `forward_context.dp_metadata = _vllm_fc.DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64))` — so any DP world_size>1 performs the collective and writes dp_metadata. parallel_state.py:816 (unchanged by this diff, present in the PR-time tree) `vllm_parallel_state._DP = _DP` — non-EP vLLM DP stays the diffusion coordinator. parallel_state.py:902 `if use_moe_parallel_mapping:` and :904 `vllm_parallel_state._EP = init_vllm_model_parallel_group(...)` — `_EP` only exists in the EP branch, so in non-EP runs the dp_metadata written above has no all2all consumer.) + +```suggestion +if getattr(_vllm_ps, "_EP", None) is None: + return +``` + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:~38` [minor] — The new helper's two error-path branches are untested: the `dp_group.world_size <= 1` early return at hunyuan_fused_moe.py:38 and the `any(count is None)` RuntimeError at hunyuan_fused_moe.py:48. The only new DP-metadata test uses `fake_group.world_size = 2` (test_hunyuan_fused_moe.py:62) and always populates the gather output, so add a world_size=1 no-collective test and a None-injected RuntimeError test. (evidence: PR-time tree, all present in the diff: `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:38` `if dp_group is None or getattr(dp_group, "world_size", 1) <= 1:`; `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:48` `raise RuntimeError(f"Failed to gather MoE DP token counts: {gathered_num_tokens}")`; `tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:62` `fake_group.world_size = 2`.) + +`requirements:0` [minor] — Declare a vLLM version constraint before landing: the new wrapper calls vLLM's `init_model_parallel_group(..., use_device_communicator=True)` (parallel_state.py:517) and constructs `forward_context.DPMetadata` (hunyuan_fused_moe.py:~46-50), but vllm is not declared in requirements/* or pyproject.toml (only docs pin vllm==0.24.0), so these API requirements are unenforced across supported vLLM versions. Add the minimum-version pin or a compatibility guard. (evidence: requirements/common.txt has no vllm runtime dependency (only `# transformers_keys_to_ignore_compat (vllm_omni/model_executor/models/utils.py).` at line 11 mentions vllm); pyproject.toml:11 `dynamic = ["version", "dependencies"]` and no vllm constraint appears anywhere in the file; docs/getting_started/installation/gpu/cuda.inc.md:23 `uv pip install vllm==0.24.0 --torch-backend=auto` is the only declared vllm version. Changed code outside packaging: vllm_omni/diffusion/distributed/parallel_state.py:515-522 `return vllm_parallel_state.init_model_parallel_group(..., use_device_communicator=True)`; vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:46-50 constructs `forward_context.dp_metadata = _vllm_fc.DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64))`.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:91` [minor] — The PR adds a mocked, single-process unit test (test_sets_dp_metadata_from_vllm_dp_group) asserting dp_metadata.num_tokens_across_dp_cpu == [1024, 1025], but it fabricates rank order via output[:] = [obj, obj + 1]. Please add a true multi-rank assertion (launch 2+ ranks with the real vLLM DP/cfg-dp group) verifying num_tokens_across_dp_cpu matches the actual cfg-dp rank order, since the mocked test cannot catch a rank-order/DP-count regression. (evidence: tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:66 `output[:] = [obj, obj + 1]` — the mocked all_gather_object side effect fabricates rank order in a single process. tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:76 `assert mock_ctx.dp_metadata.num_tokens_across_dp_cpu.tolist() == [1024, 1025]` — only checks the fabricated tensor, not the real cfg-dp rank order. No multi-rank test is present. Production hook anchor (unchanged by this diff): vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:91 `_set_forward_context_dp_metadata(hidden_states.shape[0])`.) + +`vllm_omni/diffusion/distributed/parallel_state.py:876` [minor] — The non-MoE TP else-branch at parallel_state.py:876-880 still assigns the diffusion GroupCoordinator to vllm_parallel_state._TP (only the MoE branch uses init_vllm_model_parallel_group with use_device_communicator=True). This branch is reachable for every non-expert-parallel diffusion run, not gated behind the EP+non-MoE raise, so add a comment/assert that these paths must not feed vLLM TP consumers that require device_communicator/reduce_scatter; a future non-MoE model calling vLLM TP APIs would silently receive the wrong coordinator type. (evidence: vllm_omni/diffusion/distributed/parallel_state.py:876 ` vllm_parallel_state._TP = init_model_parallel_group(` (else-branch, `parallel_mode="tensor"`) returns the diffusion GroupCoordinator; contrast vllm_omni/diffusion/distributed/parallel_state.py:869 ` vllm_parallel_state._TP = init_vllm_model_parallel_group(` which routes through init_vllm_model_parallel_group -> vllm_parallel_state.init_model_parallel_group(..., use_device_communicator=True) at parallel_state.py:515-525. The non-MoE branch is present in the diff (hunk @@ -838,23 +853,40 @@), unchanged from the prior unconditional assignment except now gated by `else`.) + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:~68` [minor] — The new test fabricates the all_gather result as `output[:] = [obj, obj + 1]` (test_hunyuan_fused_moe.py:68) and only asserts production re-packs it into `DPMetadata` (test_hunyuan_fused_moe.py:77); it never verifies the rank-order contract production depends on — that `num_tokens_across_dp_cpu[dp_rank]` matches the vLLM cfg-dp group's rank order. Derive expected per-rank counts from that rank layout or run a real two-rank gloo `all_gather_object` so a rank-order regression fails here. (evidence: tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:67-68 `def _all_gather_object(output, obj, group): assert group is fake_group.cpu_group; output[:] = [obj, obj + 1]`; tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:77 `assert mock_ctx.dp_metadata.num_tokens_across_dp_cpu.tolist() == [1024, 1025]`; vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:45-51 `gathered_num_tokens: list[int | None] = [None] * dp_group.world_size; torch.distributed.all_gather_object(gathered_num_tokens, int(num_tokens), group=dp_group.cpu_group); ... forward_context.dp_metadata = _vllm_fc.DPMetadata(torch.tensor(gathered_num_tokens, dtype=torch.int64))`.) + +`vllm_omni/diffusion/distributed/parallel_state.py:885` [minor] — Confirmed: the diff drops the `cfg_parallel_size > 1` guard at parallel_state.py:882, so cfg=1 MoE+EP now builds `vllm_parallel_state._DP` through `init_vllm_model_parallel_group` (parallel_state.py:885). The only EP layout test uses cfg_parallel_size=2 (test_expert_parallel_layout.py:106), leaving the cfg=1 EP path unpinned. Minor is the right severity; consider adding a cfg=1 EP case or parameterizing the test. (evidence: vllm_omni/diffusion/distributed/parallel_state.py:882 ` if use_moe_parallel_mapping:` (guard has no `and cfg_parallel_size > 1`); vllm_omni/diffusion/distributed/parallel_state.py:885 ` vllm_parallel_state._DP = init_vllm_model_parallel_group(`; tests/diffusion/distributed/test_expert_parallel_layout.py:106 ` cfg_parallel_size=2,` (only EP layout test is cfg=2).) + +`tests/diffusion/distributed/test_expert_parallel_layout.py:~121` [minor] — This test monkeypatches `parallel_state.init_vllm_model_parallel_group` (test_expert_parallel_layout.py:92), so the production wrapper (parallel_state.py:515-526) never runs; the `device_communicator` (121-123) and `reduce_scatter` (124-126) assertions validate attributes the fake injects at lines 73-74. Instead, patch only `vllm_parallel_state.init_model_parallel_group` with a recorder and assert the four calls pass `group_name in {"pcp","tp","dp","ep"}` and `use_device_communicator=True`. (evidence: tests/diffusion/distributed/test_expert_parallel_layout.py:92 `monkeypatch.setattr(parallel_state, "init_vllm_model_parallel_group", fake_init_vllm_model_parallel_group)` — monkeypatches the production wrapper, so it never runs. tests/diffusion/distributed/test_expert_parallel_layout.py:73-74 `device_communicator=object(),` / `reduce_scatter=lambda tensor, **kwargs: tensor,` — the fake injects the very attributes later asserted. tests/diffusion/distributed/test_expert_parallel_layout.py:121-126 `assert parallel_state.vllm_parallel_state._PCP.device_communicator is not None` / `assert hasattr(parallel_state.vllm_parallel_state._PCP, "reduce_scatter")` — assertions only check fake-injected attributes. vllm_omni/diffusion/distributed/parallel_state.py:521-526 `return vllm_parallel_state.init_model_parallel_group(` ... `use_device_communicator=True,` — the production contract bypassed by the monkeypatch.) + +```suggestion +recorded_kwargs = [] + def fake_upstream_init_model_parallel_group(group_ranks, local_rank, backend, **kwargs): + recorded_kwargs.append(kwargs) + return fake_init_vllm_model_parallel_group(group_ranks, local_rank, backend, kwargs.get("group_name")) + monkeypatch.setattr(parallel_state.vllm_parallel_state, "init_model_parallel_group", fake_upstream_init_model_parallel_group) +``` + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/distributed/test_expert_parallel_layout.py:190` [minor] This diff retargets the EP filter from parallel_mode == "expert" to parallel_mode == "vllm_ep" (line 189), matching the new f"vllm_{group_name}" fake naming. But line 190 is self-referential and cannot catch a rank-layout regression: fake_init_vllm_model_parallel_group both appends the group to created_groups and […] + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **pr-review-adversary-findings-moss-streaming**: Adversary-lens review notes for MOSS-TTS v1.5 streaming PR: dead v1/v2 codec fallback, NPU twin drift, stream-slot silent-drop. +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17_val/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v17_val/pr4893.trace.json.gz new file mode 100644 index 000000000..7a3b5cabe Binary files /dev/null and b/eval/dataset/arms/copilot_v17_val/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/manifest.json b/eval/dataset/arms/copilot_v17cb_train/manifest.json new file mode 100644 index 000000000..ee6c449e2 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v17cb_train", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4804.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4804.cost.json new file mode 100644 index 000000000..eb206b4e6 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4804.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 878.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4804/run-20260817-133249-69a519", + "recorded_at": "2026-08-17T13:47:27", + "attempts": [ + { + "invocation_id": "inv-34ad06eb", + "outcome": "run:run-20260817-133249-69a519", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-133249-69a519", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-133249-69a519", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.7043, + "minutes": 14.63, + "ci_minutes": 0, + "input_tokens": 2034181, + "output_tokens": 140951, + "tool_calls": 583, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7136, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.41, + "gate": 1.55, + "review": 871.47, + "report": 0.06 + }, + "time_to_first_result_s": 877.48 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 65496 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4804.md b/eval/dataset/arms/copilot_v17cb_train/pr4804.md new file mode 100644 index 000000000..c6e00b480 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4804.md @@ -0,0 +1,102 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] Abort slot leak on GPU — on_requests_finished added to MossTTSCodecDecoder and gpu_generation_model_runner; residual: NPU generation runner still missing both hooks. +- [resolved] talker_mtp code merge when codes None — gpu_model_runner guards merge with code_predictor_codes is not None (gpu_model_runner.py:1898-1900). +- [resolved] Weight load strictness: modeling_moss_tts_codec load_weights raises on missing/shape mismatch — checked residual v2 remap path. +- [resolved] Realtime raw path: pipeline already used raw processor name; concern was unified `talker2codec_async_chunk` skip_dedelay — split into delay vs raw functions at head. +- [resolved] Abort slot leak — on_requests_finished added in codec + gpu_generation_model_runner; residual unchecked on NPU generation runner and untested on GPU +- [resolved] qwen3_tts_request_seed fully removed — repo-wide grep at head returns zero matches; tests and serving_speech use tts_local_seed. +- [resolved] NPU real forward path already passes seq_token_counts at npu_generation_model_runner.py:299 — only dummy/profile path missing. +- [claim-verified] PR adds MOSS Local streaming and vendored v2 tokenizer — diff adds audio_tokenizer_v2.py, _MossCodecStreamSession, talker2codec_raw_async_chunk. +- [claim-verified] Unified local seed via tts_local_seed — grep shows only serving_speech + gpu_model_runner + tests, zero qwen3_tts_request_seed. +- [claim-verified] tts_local_seed rename: repo grep shows no remaining qwen3_tts_request_seed; gpu_model_runner reads tts_local_seed. +- [claim-verified] vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:48 — _update_request_states still lacks on_requests_finished hook present at gpu_generation_model_runner.py:81-82. +- [claim-verified] vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:780 — _dummy_run sets runtime_additional_information but not seq_token_counts; GPU adds it at gpu_generation_model_runner.py:799. +- [claim-verified] Moss codec streaming slots keyed by meta.req_id/external id (moss_tts.py:238-239, modeling_moss_tts_codec.py:424-431) but on_requests_finished receives scheduler internal IDs (modeling_moss_tts_codec.py:574, gpu_generation_model_runner.py:82). +- [claim-refuted] serving_speech seed gate removed for qwen3 only — tts_local_seed now set for any TTS when request.seed set; only talker_mtp consumes it today. + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:~48` [major] — GPU generation runner now calls model.on_requests_finished in _update_request_states before removing finished requests, but NPUGenerationModelRunner still only removes input_batch entries. Mirror the same hook so MossTTSCodecDecoder releases codec stream slots on NPU abort/disconnect (NPU AR already does this). (evidence: Changed by this PR: gpu_generation_model_runner.py:81-82 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): self.model.on_requests_finished(scheduler_output.finished_req_ids)`. Unchanged in PR-time tree but decisive for the gap: npu_generation_model_runner.py:48-51 `def _update_request_states(self, scheduler_output: SchedulerOutput):` then `for req_id in scheduler_output.finished_req_ids: self.input_batch.remove_request(req_id)` with no on_requests_finished call. NPU AR already has parity: npu_ar_model_runner.py:419-420 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): self.model.on_requests_finished(scheduler_output.finished_req_ids)`. Moss codec defines the hook this PR relies on: modeling_moss_tts_codec.py:564-571 `def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:` — "so the runner calls this hook from its finished-request path to avoid leaking stream slots and buffered codes.") + +```suggestion +def _update_request_states(self, scheduler_output: SchedulerOutput): + if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): + self.model.on_requests_finished(scheduler_output.finished_req_ids) + for req_id in scheduler_output.finished_req_ids: + self.input_batch.remove_request(req_id) +``` + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:~782` [major] — MossTTSCodecDecoder.forward now requires seq_token_counts for non-empty codec inputs. gpu_generation_model_runner._dummy_run injects model_kwargs["seq_token_counts"] = [int(num_tokens_padded)] (line 799), but npu_generation_model_runner._dummy_run (lines 780-782) still only sets runtime_additional_information. Mirror the GPU dummy-run seq_token_counts assignment so NPU warmup/profile runs for Moss code2wav stages do not raise RuntimeError. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:780-782 `if hasattr(self.model, "get_dummy_runtime_additional_information"): runtime_addi = self.model.get_dummy_runtime_additional_information(num_reqs); model_kwargs["runtime_additional_information"] = runtime_addi` (no seq_token_counts); PR diff adds gpu_generation_model_runner.py:799 `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]`; modeling_moss_tts_codec.py:260-265 `token_counts = self._normalize_seq_token_counts(kwargs.get("seq_token_counts")); if token_counts is None: raise RuntimeError("MossTTS codec requires seq_token_counts; otherwise concatenated codec tokens cannot be split per request.")`) + +```suggestion +model_kwargs["seq_token_counts"] = [int(num_tokens_padded)] +``` + +`vllm_omni/benchmarks/patch/patch.py:1251` [minor] — In patch.py async_request_openai_audio_speech, _pcm_s16le_to_seed_tts_wer_bytes normalizes streamed PCM to 24 kHz mono for Seed-TTS WER, but the except at patch.py:1249-1251 stores raw pcm_capture. compute_seed_tts_wer_metrics (seed_tts_eval.py:598) always calls _pcm_s16le_to_f32_16k(pcm) with default pcm_sample_rate=24000 and no channel handling. When normalization fails under 48 kHz stereo bench settings, WER can still run on mis-scaled/mis-channelled audio. On failure, set tts_output_pcm_bytes=None instead of bytes(pcm_capture). (evidence: patch.py:1249-1251 `except Exception as ex: logger.warning("Seed-TTS WER PCM normalization failed: %s", ex); output.tts_output_pcm_bytes = bytes(pcm_capture)` — stores unnormalized server-rate/channel PCM on failure. Unchanged consumer in PR-time tree: seed_tts_eval.py:598 `wav_16k = _pcm_s16le_to_f32_16k(pcm)` with seed_tts_eval.py:176 `def _pcm_s16le_to_f32_16k(pcm: bytes, pcm_sample_rate: int = 24000)` — assumes 24 kHz mono with no channel parameter.) + +```suggestion +except Exception as ex: + logger.warning("Seed-TTS WER PCM normalization failed: %s", ex) + output.tts_output_pcm_bytes = None + output.success = False +``` + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:483` [minor] — In _decode_streaming_batch, when session.acquire() returns None the request buffers codec frames; if finished=True before a slot is available (modeling_moss_tts_codec.py:477-483), the code warns and calls _finish_stream_request without decoding, so audios[i] stays empty. Consider failing the request or enforcing codec_stream_slots >= peak concurrent codec_streaming streams in deploy config. (evidence: modeling_moss_tts_codec.py:468 `slot = session.acquire()`; modeling_moss_tts_codec.py:469-483 `if slot is None: ... self._append_stream_pending(request_id, codes_nq_t) ... if finished: ... self._finish_stream_request(request_id, session, None)`; modeling_moss_tts_codec.py:560-561 `self._stream_pending_codes.pop(request_id, None)` (no decode); modeling_moss_tts_codec.py:243 `audios: list[torch.Tensor] = [empty] * num_req`) + +```suggestion +if finished: + logger.error( + "MOSS codec stream request %s finished before a stream slot was available; " + "rejecting request instead of silently dropping %d buffered code chunks.", + request_id, + len(self._stream_pending_codes.get(request_id, [])), + ) + self._finish_stream_request(request_id, session, None) + raise RuntimeError(f"MOSS codec stream slots exhausted for {request_id}") +``` + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:225` [minor] — This PR adds talker2codec_raw_async_chunk (moss_tts.py:225) as the Local/Realtime stage-0→codec contract (initial_codec_chunk_frames, buffering, finish sentinel with code_flat_numel=0) and wires it in pipeline.py for moss_tts_local/realtime, but tests/model_executor/stage_input_processors/ has no test_moss_tts*.py while GLM-TTS has test_glm_tts_async_chunk.py for the same async-chunk pattern. Add CPU tests for initial vs steady chunk thresholds, finish-only sentinel emission, and partial-buffer flush on is_finished. (evidence: moss_tts.py:225 `def talker2codec_raw_async_chunk(` — new raw async processor; moss_tts.py:268-269 `initial_chunk_frames = int(cfg.get("initial_codec_chunk_frames") or 0)` and `threshold = initial_chunk_frames if initial_chunk_frames > 0 and not emitted_any else chunk_frames`; moss_tts.py:286-287 `# ``code_flat_numel=0`` tells the codec this is a control-only finish packet` with `code_flat_numel=0`; pipeline.py:94 `async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_raw_async_chunk")` for moss_tts_local; unchanged precedent in PR-time tree: tests/model_executor/stage_input_processors/test_glm_tts_async_chunk.py:18 `from vllm_omni.model_executor.stage_input_processors.glm_tts import ar_to_dit_async_chunk`; repo-wide tests/ grep for talker2codec_raw_async_chunk|talker2codec_delay_async_chunk → 0 hits; tests/model_executor/stage_input_processors/ list_dir has test_glm_tts_async_chunk.py and no test_moss_tts*.py) + +```suggestion +# tests/model_executor/stage_input_processors/test_moss_tts_async_chunk.py +# def test_raw_async_chunk_emits_after_initial_threshold(...) +# def test_raw_async_chunk_finish_sentinel_has_code_flat_numel_zero(...) +``` + +`?:48` [minor] — Abort slot leak fixed on GPU: MossTTSCodecDecoder.on_requests_finished is wired from GPUGenerationModelRunner._update_request_states. Residual parity gap: NPUGenerationModelRunner still does not call on_requests_finished on finished_req_ids (NPU AR runner already does), so MOSS codec streaming slots can leak on NPU abort/disconnect. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:564 `def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:`; vllm_omni/worker/gpu_generation_model_runner.py:81-82 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)`; vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:48-51 `def _update_request_states(self, scheduler_output: SchedulerOutput):` / `# remove requests` / `for req_id in scheduler_output.finished_req_ids:` / `self.input_batch.remove_request(req_id)` with no on_requests_finished call; contrast vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:419-420 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / `self.model.on_requests_finished(scheduler_output.finished_req_ids)`) + +`?:680` [minor] — MossTTSCodecDecoder.load_weights is strict: it raises RuntimeError on missing params, skipped checkpoint keys, or shape mismatches after applying v2 in_proj/out_proj/ffn remaps to vendored MossAudioTokenizerV2Model parameter names. (evidence: modeling_moss_tts_codec.py:641-646 `# v2 checkpoint names use singular in_proj/out_proj and ffn.{0,2};` / `(".self_attn.in_proj.", ".self_attn.in_projs.0."),` / `(".ffn.0.", ".linear1."),`; modeling_moss_tts_codec.py:668-672 `if expected_shape != actual_shape:` / `shape_mismatches.append((name, tgt, actual_shape, expected_shape))`; modeling_moss_tts_codec.py:679-680 `if missing or skipped or shape_mismatches:` / `raise RuntimeError(`) + +`?:48` [minor] — Abort slot cleanup is wired for GPU generation (gpu_generation_model_runner + MossTTSCodecDecoder.on_requests_finished) but NPUGenerationModelRunner._update_request_states still skips the hook; add parity and a GPU generation-runner unit test. (evidence: gpu_generation_model_runner.py:81-82 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): self.model.on_requests_finished(scheduler_output.finished_req_ids)` — GPU generation path calls cleanup on abort/finish. modeling_moss_tts_codec.py:564-583 `def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:` … `self._finish_stream_request(request_id, session, slot)` — codec releases stream slots on finished IDs. npu_generation_model_runner.py:48-51 `def _update_request_states(self, scheduler_output: SchedulerOutput):` … `for req_id in scheduler_output.finished_req_ids: self.input_batch.remove_request(req_id)` — no on_requests_finished call (unchanged by this diff, present in PR-time tree). npu_ar_model_runner.py:419-420 has the hook, so only NPU generation runner lacks parity. tests/worker/test_gpu_generation_model_runner.py — repo-wide grep finds no on_requests_finished reference in that file.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:727` [minor] `_build_codec` still broad-falls back to legacy `audio_tokenizer.MossAudioTokenizerModel`, but `_MossCodecStreamSession` requires v2-only `streaming()`/`_decode_frame()`. If the fallback is taken, the first `codec_streaming` chunk crashes in `_ensure_stream_session()` with `AttributeError`; remove or gate the legacy […] +- `vllm_omni/deploy/moss_tts_realtime.yaml:42` [minor] Realtime now uses talker2codec_raw_async_chunk (pipeline.py) and moss_tts_local.yaml sets codec_stream_slots: 8 with stage-1 max_num_seqs: 8, but moss_tts_realtime.yaml still has stage-1 max_num_seqs: 1 and no codec_stream_slots while stage-0 allows max_num_seqs: 8. Without codec_stream_slots the codec falls back to […] +- `vllm_omni/entrypoints/openai/serving_speech.py:3715` [minor] serving_speech.py now always writes tts_local_seed into stage-0 extra_args when request.seed is set (3707-3715), with no _tts_model_type guard. Only gpu_model_runner._talker_mtp_forward reads that key (1826). Consider documenting it as a talker_mtp contract or gating the write on models that implement talker_mtp so […] +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:564` [minor] This PR adds MossTTSCodecDecoder.on_requests_finished (modeling_moss_tts_codec.py:564) and gpu_generation_model_runner._update_request_states calls it at lines 79–80 to release codec stream slots on abort/disconnect, but no Moss codec test covers that hook—only voxcpm2/test_talker_state_eviction.py exercises the same […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4804.trace.json.gz new file mode 100644 index 000000000..bc1b13448 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4817.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4817.cost.json new file mode 100644 index 000000000..22c7bc68a --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4817.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 322.6, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4817/run-20260817-134547-ac87db", + "recorded_at": "2026-08-17T13:51:10", + "attempts": [ + { + "invocation_id": "inv-e670061e", + "outcome": "run:run-20260817-134547-ac87db", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-134547-ac87db", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-134547-ac87db", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1656, + "minutes": 5.37, + "ci_minutes": 0, + "input_tokens": 405871, + "output_tokens": 50894, + "tool_calls": 157, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2656, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.88, + "gate": 2.44, + "review": 315.64, + "report": 0.07 + }, + "time_to_first_result_s": 322.01 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4817.trace.json.gz", + "trace_bytes": 20301 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4817.md b/eval/dataset/arms/copilot_v17cb_train/pr4817.md new file mode 100644 index 000000000..feaa17d11 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4817.md @@ -0,0 +1,88 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4817, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head fc423e5a9e86) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Prior `>= 10` gate matched cc 12.x — file_at_base shows `_is_blackwell()` used `>= 10`; narrowed to `== 10`. Residual: future cc 11.x Blackwell (platform.py:82-83 anticipates major 11) would also be excluded +- [resolved] sm_120 default victim — quack_enabled() False at cc 12.x stops install_quack_fp8_patch; residual: manual warmup_quack_fp8 + VLLM_OMNI_USE_QUACK_FP8=1 still reachable +- [resolved] Hopper/Ada unaffected — cc 9.x/8.x remain False under == 10 gate +- [resolved] sm_120 users needing `VLLM_OMNI_USE_QUACK_FP8=0` workaround — gate now defaults off; override `=1` still forces quack per test_env_override_forces_on_even_on_sm120 +- [claim-verified] PR claim sm_120 auto-disables quack: test parametrizes (12,0)/(12,1) → False +- [claim-verified] PR claim sm_100/103 unchanged: test parametrizes (10,0)/(10,3) → True +- [claim-verified] PR root-cause claim: base used `_is_blackwell()` with `>= 10` (file_at_base quack_fp8.py:22) +- [claim-verified] tests/quantization/test_quack_fp8_gate.py — monkeypatched cc matrix covers 10.x on, 12.x/9.x/8.x off, env force-on/off +- [claim-verified] PR repro path (HunyuanImage-3.0 FP8 on sm_120) — behavioral fix aligns with described per-GEMM FlashInfer fallback on wrong arch +- [validated] vllm_omni/quantization/quack_fp8.py:32 — `== 10` gate excludes cc 12.x (sm_120/sm_121) and keeps cc 10.x enabled; matches PR root-cause fix +- [validated] vllm_omni/patch.py:410-419 — only consumer calls `install_quack_fp8_patch()` which gates on `quack_enabled()` before patching FlashInfer +- [validated] tests/quantization/test_quack_fp8_gate.py — monkeypatched `torch.cuda` probes exercise real `_is_quack_capable()`/`quack_enabled()` entry points, not tautological fixtures +- [validated] Env override precedence preserved: `quack_enabled()` checks `VLLM_OMNI_USE_QUACK_FP8` before `_is_quack_capable()` (quack_fp8.py:37-41) +- [validated] vllm_omni/quantization/quack_fp8.py:32 — `== 10` gate blocks cc 12.x auto-enable; install path gated via quack_enabled() at quack_fp8.py:102 + +`tests/quantization/test_quack_fp8_gate.py:21` [major] — tests/quantization/test_quack_fp8_gate.py adds a regression guard for the >= 10 → == 10 gate but omits pytestmark = [pytest.mark.core_model, pytest.mark.cpu]. Buildkite PR CI "Simple · Other Test" (.buildkite/test-ready.yml:29) runs pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor, which collects tests/quantization/ but deselects unmarked tests — the guard never runs in CI. (evidence: tests/quantization/test_quack_fp8_gate.py:18-21 `import pytest` / `import torch` / `from vllm_omni.quantization import quack_fp8` — no pytestmark in this 75-line file; unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:29 `- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; unchanged by this diff, present in PR-time tree: tests/model_executor/quantization/test_modelopt_nvfp4_nan_clamp.py:33 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +```suggestion +import pytest +import torch + +from vllm_omni.quantization import quack_fp8 + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +``` + +`docs/user_guide/quantization/fp8.md:~33` [minor] — The diff updates fp8.md lines 47–53 to document datacenter-only quack auto-enable (sm_100/sm_101/sm_103, cc 10.x) and exclude workstation sm_120/sm_121, matching the == 10 gate in quack_fp8.py:32, but the subsection opener at line 33 still reads "On Blackwell (SM 100+)" as if all Blackwell gets the quack fused-bias path. Align line 33 with the datacenter-only gate so sm_120 users are not steered to install vllm-omni[quack] expecting auto-enable. (evidence: unchanged by this diff, present in the PR-time tree: docs/user_guide/quantization/fp8.md:33 `On Blackwell (SM 100+), vLLM runs FP8 linears through the FlashInfer kernel, which`; changed in this PR: docs/user_guide/quantization/fp8.md:47-52 `It is enabled automatically once installed (no flag needed) and is **datacenter Blackwell only** (`sm_100` / `sm_101` / `sm_103`, compute capability `10.x`, e.g. B200)... on workstation/consumer Blackwell (`sm_120` / `sm_121`, compute capability `12.x`, e.g. RTX PRO 6000 / RTX 50-series) `tcgen05` is absent — so quack is **not** auto-enabled`; changed in this PR: vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`) + +```suggestion +On datacenter Blackwell (`sm_100` / `sm_101` / `sm_103`, compute capability `10.x`), vLLM runs FP8 linears through the FlashInfer kernel, which +``` + +`platform.py:82` [minor] — Prior `>= 10` gate matched cc 12.x (`_is_blackwell()` at base); narrowed to `== 10` in `_is_quack_capable()`. Residual: `vllm_omni/platforms/cuda/platform.py:82-83` anticipates major 11 as Blackwell (`major in (10, 11, 12)`), but the quack gate excludes cc 11.x — future datacenter 11.x would need `VLLM_OMNI_USE_QUACK_FP8=1` or a gate update. (evidence: unchanged by this diff, present at base: quack_fp8.py `_is_blackwell()` used `return torch.cuda.get_device_capability()[0] >= 10`; PR head quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`; unchanged sibling, PR-time tree: vllm_omni/platforms/cuda/platform.py:82-83 `# Accept major in {10, 11, 12} to cover future Blackwell refreshes.` / `is_blackwell = major in (10, 11, 12)`) + +`?:102` [minor] — On sm_120/cc 12.x, default quack_enabled() is False (_is_quack_capable major==10), so install_quack_fp8_patch() no-ops at patch.py import; residual opt-in remains via VLLM_OMNI_USE_QUACK_FP8=1 (re-enables patch) and manual warmup_quack_fp8() (no capability gate, only _load_quack()). (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/quantization/quack_fp8.py:33 `return torch.cuda.get_device_capability()[0] == 10`; vllm_omni/quantization/quack_fp8.py:102-103 `if not quack_enabled():` / ` return`; vllm_omni/quantization/quack_fp8.py:37-42 `override = os.environ.get("VLLM_OMNI_USE_QUACK_FP8")` … `return override.lower() in _TRUTHY`; vllm_omni/quantization/quack_fp8.py:136-137 `if _load_quack() is None:` / ` return` (warmup_quack_fp8 skips quack_enabled); vllm_omni/patch.py:414 `install_quack_fp8_patch()`) + +`?:33` [minor] — sm_120 (cc 12.x) no longer needs `VLLM_OMNI_USE_QUACK_FP8=0` — `_is_quack_capable()` uses `== 10`, so quack defaults off on sm_120/sm_121; `VLLM_OMNI_USE_QUACK_FP8=1` still forces quack on (`test_env_override_forces_on_even_on_sm120`). (evidence: vllm_omni/quantization/quack_fp8.py:33 `return torch.cuda.get_device_capability()[0] == 10` — sm_120 (major 12) does not auto-enable quack; vllm_omni/quantization/quack_fp8.py:37-41 `override = os.environ.get("VLLM_OMNI_USE_QUACK_FP8")` / `if override is not None: return override.lower() in _TRUTHY` — explicit truthy override still forces quack on; tests/quantization/test_quack_fp8_gate.py:38-39 `((12, 0), False)` / `((12, 1), False)` — regression guard for sm_120/sm_121 default-off; tests/quantization/test_quack_fp8_gate.py:63-69 `test_env_override_forces_on_even_on_sm120` asserts `quack_enabled() is True` with truthy env on (12, 0).) + +`vllm_omni/quantization/quack_fp8.py:~136` [minor] — The diff gates auto-enable through `_is_quack_capable()` / `quack_enabled()` (including `install_quack_fp8_patch()` at lines 100–102), but `warmup_quack_fp8()` still calls `_load_quack()` at line 136 without checking `quack_enabled()`. On sm_120/sm_121 with `quack-kernels` installed, following the fp8.md prewarm snippet (lines 73–75) can still load CuteDSL and pay per-GEMM FlashInfer fallback even though serving no longer patches the hot path by default. Add a `quack_enabled()` guard to `warmup_quack_fp8()` before `_load_quack()`. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/quantization/quack_fp8.py:100-103 `if not quack_enabled(): return` / `if _load_quack() is None: return`; vllm_omni/quantization/quack_fp8.py:136 `if _load_quack() is None:` (no preceding quack_enabled() check in warmup_quack_fp8); docs/user_guide/quantization/fp8.md:73-75 `from vllm_omni.quantization.quack_fp8 import warmup_quack_fp8` / `warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])`) + +```suggestion +def warmup_quack_fp8( + shapes: list[tuple[int, int, int]], + device: str = "cuda", + out_dtype: torch.dtype = torch.bfloat16, +) -> None: + if not quack_enabled(): + return + if _load_quack() is None: + return +``` + +`docs/user_guide/quantization/fp8.md:54` [minor] — The diff documents `VLLM_OMNI_USE_QUACK_FP8=1` to force quack on at fp8.md:54, but on sm_120/sm_121 a truthy override bypasses the new datacenter-only auto gate and can re-enable the per-GEMM FlashInfer fallback this PR avoids by default. Document that `=1` is for datacenter Blackwell or future CuteDSL sm_120 support, not routine workstation use. (evidence: docs/user_guide/quantization/fp8.md:54 `VLLM_OMNI_USE_QUACK_FP8=1` to force quack on, or `VLLM_OMNI_USE_QUACK_FP8=0` to force`; vllm_omni/quantization/quack_fp8.py:38-40 `override = os.environ.get("VLLM_OMNI_USE_QUACK_FP8")` / `if override is not None:` / `return override.lower() in _TRUTHY`; vllm_omni/quantization/quack_fp8.py:24-27 `Workstation/consumer Blackwell (``sm_120`` / ``sm_121``, compute capability ``12.x`` ... lacks ``tcgen05``, so quack can never run there — CuteDSL rejects the arch and every GEMM falls back to FlashInfer one call at a time, which is catastrophically slow.`; tests/quantization/test_quack_fp8_gate.py:65-67 `_fake_cuda(monkeypatch, (12, 0))` / `monkeypatch.setenv(_ENV, value)` / `assert quack_fp8.quack_enabled() is True`) + +```suggestion +`VLLM_OMNI_USE_QUACK_FP8=1` to force quack on (datacenter Blackwell only — on sm_120/sm_121 this can reintroduce severe per-GEMM fallback), or `VLLM_OMNI_USE_QUACK_FP8=0` to force +``` + +`vllm_omni/quantization/quack_fp8.py:32` [nit] — Auto-enable now uses get_device_capability()[0] == 10 (quack_fp8.py:32), which correctly excludes cc 12.x workstation Blackwell but also excludes any future cc 11.x datacenter SKU. cuda/platform.py already routes major in (10, 11, 12) as Blackwell-class. Can you confirm cc-11.x should stay off quack auto-enable until explicitly whitelisted (e.g. major in (10, 11) once sm_110a is validated), rather than only documenting sm_100/101/103? (evidence: quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10` — auto-enable only on cc 10.x; unchanged by this diff, present in PR-time tree: vllm_omni/platforms/cuda/platform.py:83 `is_blackwell = major in (10, 11, 12)` — platform already anticipates major-11 Blackwell GPUs) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4817.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4817.trace.json.gz new file mode 100644 index 000000000..6739068a8 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4817.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4859.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4859.cost.json new file mode 100644 index 000000000..5abd3ecbe --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4859.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 845.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4859/run-20260817-135515-c38403", + "recorded_at": "2026-08-17T14:09:20", + "attempts": [ + { + "invocation_id": "inv-f08fa019", + "outcome": "run:run-20260817-135515-c38403", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-135515-c38403", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-135515-c38403", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4318, + "minutes": 14.09, + "ci_minutes": 0, + "input_tokens": 1066501, + "output_tokens": 130728, + "tool_calls": 482, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5972, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.89, + "gate": 2.2, + "review": 839.12, + "report": 0.06 + }, + "time_to_first_result_s": 845.26 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4859.trace.json.gz", + "trace_bytes": 30146 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4859.md b/eval/dataset/arms/copilot_v17cb_train/pr4859.md new file mode 100644 index 000000000..0b9c92788 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4859.md @@ -0,0 +1,134 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4859, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6ff4ce59ed87) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] Concern stop-head fired one step early at base (+1); head uses +2 aligned to HF `step > 3`; residual: custom stop_head_min_steps still hard-codes +2 offset without named constant. +- [resolved] Concern: aggregator config copy vs mutate — upstream HF mutates shared Qwen2Config; residual: encoder.config.num_hidden_layers may disagree with built layer count +- [resolved] Seed-TTS `language: English/Chinese` (seed_tts_dataset.py:238) no longer becomes `方言` in Ming instructions — likely WER fix; residual: request.language still salts prefix cache at serving_speech.py:276 but no longer affects generation. +- [resolved] Benchmark PCM refactor duplication — _audio_pcm_format removed from patch.py; concern closed, residual: new max(sample_rate,1) clamp untested +- [resolved] benchmark PCM refactor duplication — _audio_pcm_format removed; residual max(.,1) clamp untested (kept at definitions.py:337) +- [claim-verified] PR body Seed-TTS EN WER 1.83% (1088/0 failures) is stated but not reproducible from CI; only author L4 numbers. +- [claim-verified] PR WER 1.83% author-reported only; merge commit 6ff4ce59 in timeline — numbers not re-verified at this head +- [claim-verified] Diff adds zero test files; test_ming_shared_modules.py does not exercise encode_latent, stop gate, or language mapping +- [claim-verified] PR test plan sets VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 for Ming MoE EN Seed-TTS; matches harness change scope. +- [claim-verified] PR test plan names ming_tts_moe.yaml + VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 — deploy file exists at vllm_omni/deploy/ming_tts_moe.yaml +- [claim-refuted] Automated regression for reported WER — no test/benchmark job in diff; tests/e2e/online_serving/test_ming_tts_expansion.py pytest.mark.skip(#4704) +- [claim-refuted] PR WER 1.83% with VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100: patch.py:1215-1219 stores 24 kHz mono but seed_tts_eval.py:616 decodes with env 44100 → ~46% shorter 16 kHz waveform +- [validated] audio_vae.py:349-351 encode_latent uses mean+softplus+randn; matches HF VAE reparameterization vs base mean-only (file_at_base:349-350). +- [validated] patch_emission.py:130,151 min_required=max(min_stop_step+2,...); with STOP_HEAD_MIN_STEPS=3 (constants.py:51) stop fires at step>=4, matching HF modeling_bailingmm.py `step > 3`. + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:351` [major] — Shared `AudioVAE.encode_latent` (audio_vae.py:349-351) now samples `mean + std * torch.randn_like(mean)` instead of returning the mean chunk. This is not Ming-TTS-local: `prompt_encoder.py:137` uses it for reference-audio latents and `voice_presets.py:257` uses it when loading voice presets. Please confirm HF inference parity or guard/scope the change so preset stability and reproducible voice-clone runs are not regressed. (evidence: audio_vae.py:349 `mean, scale = torch.chunk(h, 2, dim=1)`; audio_vae.py:350 `std = F.softplus(scale) + 1e-4`; audio_vae.py:351 `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)` — unchanged by this diff, present in PR-time tree: prompt_encoder.py:137 `latents, _ = encoder.encode_latent(waveform, waveform_length)`; voice_presets.py:257 `prompt_wav_lat, _ = self._audio_vae.encode_latent(`; base/main (pre-PR) audio_vae.py:346-348 `mean = torch.chunk(h, 2, dim=1)[0]` / `latent = mean.transpose(1, 2)`) + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:~616` [major] — `tts_output_pcm_bytes` is always normalized to 24 kHz mono by `_pcm_s16le_to_seed_tts_wer_bytes` (patch.py:118-133, :1215-1218), but WER eval at seed_tts_eval.py:616 calls `_pcm_s16le_to_f32_16k(pcm)` without rate/channels; after this PR that reads `stream_pcm_format_from_env()` (:189-193). With `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100` (stream decode), eval treats stored 24 kHz PCM as 44.1 kHz and skews ASR/WER. Pass `pcm_sample_rate=24000, channels=1` at the call site. (evidence: unchanged by this diff, present in PR-time tree: patch.py:118 `Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER.`; patch.py:127-131 `if int(sample_rate) != 24000 and pcm_f32.size: ... resampler.resample(pcm_f32, orig_sr=int(sample_rate))`; patch.py:1215-1218 `output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(bytes(pcm_capture), sample_rate=sample_rate, channels=channels)`; seed_tts_eval.py:189-193 `if pcm_sample_rate is None or channels is None: env_sample_rate, env_channels = stream_pcm_format_from_env()`; seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)`) + +```suggestion +wav_16k = _pcm_s16le_to_f32_16k(pcm, pcm_sample_rate=24000, channels=1) +``` + +`vllm_omni/entrypoints/openai/serving_speech.py:2629` [major] — This diff removes the `request.language` → `instruction_dict["方言"]` path from `_parse_ming_instruction_fields` / `_parse_ming_instruction`, but `examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py` still sends `--dialect` as `payload["language"]` (argparse: "Maps to Ming `方言`"). `serving_speech.py` has no remaining `方言` mapping, so `--dialect` is silently ignored; dialect still works only via `instructions` JSON. Likely fixes bogus locale injection (e.g. Seed-TTS `language="English"`), but either restore dialect mapping for non-locale values (exclude `_TTS_LANGUAGES`) or update the client/README in this PR. (evidence: unchanged by this diff, present in PR-time tree: examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py:103-104 `if args.dialect:\n payload["language"] = args.dialect`; examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py:192 `parser.add_argument("--dialect", default=None, help="Maps to Ming \`方言\`")`; serving_speech.py:2627-2631 `instruction_dict: dict[str, Any] = {}\n\n voice_lower = request.voice.lower() if isinstance(request.voice, str) else None` (no `方言` mapping; grep shows zero `方言` in serving_speech.py); diff removes `if include_language and request.language not in (None, "", "Auto"): instruction_dict["方言"] = request.language` and `include_language=True` from `_parse_ming_instruction`.) + +```suggestion +def _parse_ming_instruction_fields( + self, + request, + *, + include_voice=False, + plain_text_passthrough=False, + ): + instruction_text = request.instructions.strip() if isinstance(request.instructions, str) else None + instruction_dict: dict[str, Any] = {} + + lang = request.language + if isinstance(lang, str) and lang.strip() and lang not in ("Auto",) and lang not in _TTS_LANGUAGES: + instruction_dict["方言"] = lang.strip() + + voice_lower = request.voice.lower() if isinstance(request.voice, str) else None +``` + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:189` [major] — This hunk wires stream_pcm_format_from_env() into _pcm_s16le_to_f32_16k when rate/channels are omitted. The only caller (compute_seed_tts_wer_metrics at seed_tts_eval.py:616) decodes out.tts_output_pcm_bytes, which patch.py:1215-1218 already normalizes to 24 kHz mono via _pcm_s16le_to_seed_tts_wer_bytes using the stream env rate only for ingestion. Re-reading VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE here decodes normalized bytes at the wrong rate whenever the bench env is not 24000 (e.g. 44100 for Ming). Drop the env import/branches and keep fixed 24 kHz mono defaults in this helper. (evidence: unchanged by this diff, present in PR-time tree: patch.py:118 `"""Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER."""`; patch.py:1215-1218 `output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(bytes(pcm_capture), sample_rate=sample_rate, channels=channels,)`; seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)`; seed_tts_eval.py:188 `env_sample_rate, env_channels = stream_pcm_format_from_env()`) + +```suggestion +def _pcm_s16le_to_f32_16k( + pcm: bytes, + pcm_sample_rate: int = 24000, + channels: int = 1, +) -> np.ndarray: + import scipy.signal + + if not pcm: + return np.zeros(0, dtype=np.float32) + raw = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0 + if channels > 1: + usable = (len(raw) // channels) * channels + if usable <= 0: + return np.zeros(0, dtype=np.float32) + raw = raw[:usable].reshape(-1, channels).mean(axis=1) + target_len = int(len(raw) * 16000 / pcm_sample_rate) + if target_len <= 0: + return np.zeros(0, dtype=np.float32) + return scipy.signal.resample(raw, target_len).astype(np.float32) +``` + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:349` [minor] — This diff makes `encode_latent` stochastic (`mean + softplus(scale) * randn` at audio_vae.py:349-351) instead of returning only the mean chunk. Both callers (`prompt_encoder.py:137`, `voice_presets.py:257`) inherit that behavior, but the PR adds no CPU smoke test with a fixed seed to lock shape and non-deterministic sampling. (evidence: audio_vae.py:349 `mean, scale = torch.chunk(h, 2, dim=1)`; audio_vae.py:351 `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)`; unchanged by this diff, present in PR-time tree: prompt_encoder.py:137 `latents, _ = encoder.encode_latent(waveform, waveform_length)`; unchanged by this diff, present in PR-time tree: voice_presets.py:257 `prompt_wav_lat, _ = self._audio_vae.encode_latent(`; tests/ grep for `encode_latent|audio_vae`: no matches) + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:141` [minor] — After `self.encoder = Qwen2Model(config)`, in-place `config.num_hidden_layers = 4` mutates the shared config object already held by the encoder. Encoder weights keep the original backbone depth, but `self.encoder.config.num_hidden_layers` reads 4. Restore a separate copied config for the aggregator (as in base). (evidence: audio_vae.py:131 `self.encoder = Qwen2Model(config)` — encoder instantiated before config mutation; audio_vae.py:141 `config.num_hidden_layers = 4` — shared config mutated in place; audio_vae.py:142 `self.aggregator = Qwen2Model(config)` — aggregator uses the same mutated config object. Base (unchanged by this diff, present at merge-base): audio_vae.py:141-142 `aggregator_config = _qwen2_config({**encoder_args, "num_hidden_layers": 4})` / `self.aggregator = Qwen2Model(aggregator_config)` — separate config copy for the 4-layer aggregator.) + +```suggestion +aggregator_config = _qwen2_config({**encoder_args, "num_hidden_layers": 4}) + self.aggregator = Qwen2Model(aggregator_config) +``` + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:42` [minor] — The module doc says streaming PCM decode follows VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE / VLLM_OMNI_BENCH_AUDIO_CHANNELS, but the openai-audio-speech WER path already receives 24 kHz mono in tts_output_pcm_bytes after patch.py::_pcm_s16le_to_seed_tts_wer_bytes. Clarify that those env vars configure raw stream capture and bench metrics in patch.py, while WER eval consumes normalized 24 kHz mono bytes. (evidence: seed_tts_eval.py:42 `Streaming PCM is decoded using ``VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`` /` — module doc claims env-driven decode here; unchanged by this diff, present in PR-time tree: patch.py:118 `"""Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER."""` — WER bytes are normalized before export; patch.py:1215 `output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(` — openai-audio-speech WER capture path; seed_tts_eval.py:586 `pcm = getattr(out, "tts_output_pcm_bytes", None)` — WER reads pre-exported bytes from patch output.) + +```suggestion +Streaming PCM capture uses ``VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`` / ``VLLM_OMNI_BENCH_AUDIO_CHANNELS`` in the bench patch; WER eval consumes 24 kHz mono bytes produced by ``_pcm_s16le_to_seed_tts_wer_bytes``. +``` + +`vllm_omni/model_executor/models/ming_tts/patch_emission.py:151` [minor] — The diff changes Ming stop-head gating from `min_stop_step + 1` to `+ 2` in `_validate_ming_decode_window` and `_resolve_ming_stop_decision` (patch_emission.py:130, :151). With default `stop_head_min_steps=3`, `min_required_decode_steps` becomes 5, so `should_stop_head` (line 159) cannot fire until `step>=4` instead of `step>=3`. No in-repo unit test covers this offset; consider adding a focused test on `_resolve_ming_stop_decision` asserting continue at step 3 and `stop_head` at step 4 when `stop_prob > threshold`. (evidence: patch_emission.py:130 `min_required = max(min_stop_step + 2, min_steps)`; patch_emission.py:151 `min_required_decode_steps = max(min_stop_step + 2, min_decode_steps)`; patch_emission.py:159 `should_stop_head = ((step + 1) >= min_required_decode_steps) and stop_prob > stop_threshold`; constants.py:51 `STOP_HEAD_MIN_STEPS = 3`; unchanged by this diff, present in PR-time tree: tests/model_executor/models/ming_tts/test_ming_shared_modules.py:76 `stop_head_min_steps=0` (no test imports `_resolve_ming_stop_decision`)) + +```suggestion +def test_resolve_ming_stop_decision_respects_plus_two_offset(): + _, _, _, min_req, _ = _resolve_ming_stop_decision( + step=3, stop_prob=1.0, stop_threshold=0.5, + min_stop_step=3, min_decode_steps=0, max_decode_steps=10, + audio_dummy_token_id=0, text_eos_token_id=1, + ) + assert min_req == 5 + reason, should_stop, _, _, _ = _resolve_ming_stop_decision( + step=3, stop_prob=1.0, stop_threshold=0.5, + min_stop_step=3, min_decode_steps=0, max_decode_steps=10, + audio_dummy_token_id=0, text_eos_token_id=1, + ) + assert reason == MING_STOP_REASON_CONTINUE and not should_stop +``` + +**Additional observations (beyond the comment budget):** +- `vllm_omni/benchmarks/data_modules/seed_tts_eval.py:616` [minor] WER decoding at seed_tts_eval.py:616 calls `_pcm_s16le_to_f32_16k(pcm)` with no explicit rate/channels. The helper now reads `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE` / `VLLM_OMNI_BENCH_AUDIO_CHANNELS` and downmixes multi-channel PCM (179–197) before 16 kHz resample. No test under `tests/` covers this helper; add a CPU test […] +- `?:151` [minor] Base `min_stop_step + 1` let stop-head fire at step 3; head `+ 2` in `patch_emission.py` aligns with HF `step > 3` when `stop_head_min_steps=3`. Residual nit: the `+ 2` offset is still a magic literal, not a named constant. +- `?:141` [minor] audio_vae.py:141 — mutating shared `config` after `self.encoder = Qwen2Model(config)` makes `self.encoder.config.num_hidden_layers` report 4 while the encoder stack was built with the backbone's original layer count; matches upstream HF mutate pattern, differs from pre-PR copy approach; metadata-only residual risk. +- `seed_tts_dataset.py:238` [minor] Seed-TTS still sends `language: English/Chinese` (seed_tts_dataset.py:238), but this PR stops mapping it to Ming `方言` in `_parse_ming_instruction_fields` — likely WER fix. Residual: `request.language` is still hashed in `_conditioning_cache_salt` (serving_speech.py:276), yet the Ming dense path does not set […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4859.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4859.trace.json.gz new file mode 100644 index 000000000..5b6d68a31 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4859.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4870.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4870.cost.json new file mode 100644 index 000000000..c24eff51a --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4870.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 672.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4870/run-20260817-133435-cc9009", + "recorded_at": "2026-08-17T13:45:47", + "attempts": [ + { + "invocation_id": "inv-ad96c3fd", + "outcome": "run:run-20260817-133435-cc9009", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-133435-cc9009", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-133435-cc9009", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.345, + "minutes": 11.2, + "ci_minutes": 0, + "input_tokens": 896633, + "output_tokens": 93595, + "tool_calls": 427, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4972, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.09, + "gate": 1.57, + "review": 666.49, + "report": 0.06 + }, + "time_to_first_result_s": 672.19 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 25340 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4870.md b/eval/dataset/arms/copilot_v17cb_train/pr4870.md new file mode 100644 index 000000000..9f9267d1f --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4870.md @@ -0,0 +1,93 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Reviewer scoping FINAL_ONLY to qwen3_tts only — ede3dddf + voxtral_server test; residual: other async_chunk=False TTS families still use DELTA streaming coercion by design +- [resolved] Scoped streaming coercion to qwen3_tts only (`qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`); residual: moss_tts/indextts2/higgs with `async_chunk=False` streaming still get DELTA coercion by design. +- [resolved] async_chunk=False streaming coercion scoped to qwen3 only (line 3508); residual: voxcpm2/moss_tts_nano/indextts2/etc. unchanged — checked deploy yamls + voxtral test +- [resolved] async_chunk=False streaming coercion scoped to qwen3_tts (serving_speech.py:3507-3509); voxtral test at :3473-3491 guards non-qwen3 still gets delta=True +- [resolved] mm `seq_len` source — concern that `hidden_states.shape[0]` was wrong for batched mm; fixed at :1734 using scheduler total; residual hidden tail-only slicing when `include_hidden=True` still uses `min(total, hidden.shape[0])` at :1711-1717 (Qwen3 no-async-chunk sets `omni_pooler_payload_include_hidden=False`). +- [claim-verified] PR Fixes #4851 — worker fix is in shared GPUARModelRunner path used by Qwen3 Base/CustomVoice talker stages +- [claim-verified] PR remote H20 E2E — not re-run here; unit tests in PR target the changed contracts +- [claim-verified] Issue #4851 non-streaming `no_async_chunk` failures align with worker batch-split bug; serving hunk is orthogonal to `stream=False` concurrent path. +- [claim-verified] PR remote H20 E2E claims not re-run here; unit tests cited in PR body match added test names at head. +- [claim-verified] PR #4851 failure mode — concurrent no_async_chunk assigns whole-batch codes.audio when seq_len mismatches batch axis; worker fix addresses root cause +- [claim-verified] PR remote H20 E2E table references probe not in diff; not reproducible from repo CI config alone +- [claim-verified] PR body root-cause — `to_payload_element` slices when `element.shape[0] == seq_len` (mm_outputs.py:129); old `seq_len=1` with `codes.audio` (3,16) cloned full batch per request. +- [claim-verified] PR remote E2E after fix — author table claims pass; not re-run in this read-only step. +- [claim-refuted] merge-gate Qwen3 E2E covers no_async_chunk concurrent repro: test-ready.yml:323-333 source_file_dependencies omit worker/serving; online test has async_chunk only + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:~1144` [minor] — GPU head fixes batched MM slicing by using scheduled token count for `seq_len`; NPU AR twin still passes `hidden_states.shape[0]` into `to_payload_element` at the pooler path (npu_ar_model_runner.py:905 → :1144). Mirror: `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` in the `needs_pooler_payload` block and `seq_len=mm_seq_len` at the `to_payload_element` call. (evidence: unchanged by this diff, present in PR-time tree: npu_ar_model_runner.py:905 `seq_len = hidden_states.shape[0]`; npu_ar_model_runner.py:1144 `seq_len=seq_len,` inside `to_payload_element(...)`; gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`; gpu_ar_model_runner.py:1790 `seq_len=mm_seq_len,`; mm_outputs.py:131-132 `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len:` / `return element[start:end].contiguous()`) + +```suggestion +if needs_pooler_payload: + mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) + combined_hidden_states = None + combined_multimodal_outputs = None + mm_cpu = None +... + mm_payload[mm_key] = to_payload_element( + element=mm_val, + idx=idx, + start=start, + end=end, + pass_lists_through=False, + seq_len=mm_seq_len, + ) +``` + +`?:3508` [minor] — FINAL_ONLY streaming coercion for async_chunk=False is scoped to qwen3_tts only (ede3dddf); voxtral_server test at test_serving_speech.py:3493 keeps DELTA for other TTS families by design. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3503 `# waveform once. Scoped to qwen3_tts: other async_chunk=False models`; vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; vllm_omni/entrypoints/openai/serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`; tests/entrypoints/openai_api/test_serving_speech.py:3493 `mock_coerce.assert_called_once_with(voxtral_server.engine_client.default_sampling_params_list, True)`) + +`?:3508` [minor] — FINAL_ONLY streaming coercion is scoped to qwen3_tts only (`qwen3_full_payload` at serving_speech.py:3508); moss_tts/indextts2/higgs with async_chunk=False still pass `is_streaming_request=True` into `coerce_param_message_types` and keep DELTA coercion by design (see comment at :3503-3504 and voxtral regression test). (evidence: serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`; serving_speech.py:3510 `sampling_params_list = coerce_param_message_types(sampling_params_list, is_streaming_request)`; serving_speech.py:3503-3504 `# Scoped to qwen3_tts: other async_chunk=False models / # keep the DELTA coercion they stream with today.`; unchanged by this diff, present in PR-time tree: vllm_omni/entrypoints/utils.py:759-761 `def coerce_param_message_types(...): ... convert to DELTA messages, if streaming is enabled, or FINAL_ONLY if it's disabled`) + +`?:3508` [minor] — async_chunk=False streaming coercion is scoped to qwen3_tts only (serving_speech.py:3508); voxcpm2/moss_tts_nano/indextts2 and other non-qwen3 models with async_chunk: false still use DELTA coercion when streaming — deploy YAMLs unchanged; voxtral regression test added. (evidence: serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`; unchanged by this diff, present in PR-time tree: vllm_omni/deploy/voxcpm2.yaml:6 `async_chunk: false`; vllm_omni/deploy/moss_tts_nano.yaml:14 `async_chunk: false`; vllm_omni/deploy/indextts2.yaml:1 `async_chunk: false`; tests/entrypoints/openai_api/test_serving_speech.py:3493 `mock_coerce.assert_called_once_with(voxtral_server.engine_client.default_sampling_params_list, True)`) + +`serving_speech.py:3493` [minor] — async_chunk=False streaming coercion scoped to qwen3_tts (serving_speech.py:3507-3509); voxtral test test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3 (test_serving_speech.py:3473-3493) asserts coerce_param_message_types(..., True). (evidence: serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`; test_serving_speech.py:3493 `mock_coerce.assert_called_once_with(voxtral_server.engine_client.default_sampling_params_list, True)`) + +`vllm_omni/utils/mm_outputs.py:~123` [minor] — This PR passes scheduler_output.total_num_scheduled_tokens as seq_len for batched mm tensor slicing (gpu_ar_model_runner.py:1734) when hidden states are tail-only. to_payload_element's seq_len docstring at mm_outputs.py:123 still leads with "dim 0 of hidden states"; update it to describe scheduler-aligned batch-axis length for per-request mm slicing (e.g. codes.audio), not hidden-state row count. (evidence: unchanged by this diff, present in PR-time tree: mm_outputs.py:123 `seq_len: Optional sequence length (i.e., dim 0 of hidden states).`; gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`; gpu_ar_model_runner.py:1790 `seq_len=mm_seq_len,`) + +```suggestion +seq_len: Optional batch axis length for per-request slicing. When set, a tensor whose first dimension equals seq_len is sliced per request. Use scheduler total scheduled tokens for batched mm tensors (e.g. codes.audio) even when hidden states are tail-only. +``` + +`.buildkite/test-merge.yml:~169` [minor] — This PR fixes Qwen3-TTS full-payload streaming in vllm_omni/entrypoints/openai/serving_speech.py and multimodal payload splitting in vllm_omni/worker/gpu_ar_model_runner.py, but the merge E2E step at .buildkite/test-merge.yml:169-174 only lists e2e tests and model_executor/deploy paths. upload_pipeline.py skips steps when no changed file matches source_file_dependencies, so merge-test CI may omit test_qwen3_tts_customvoice despite it covering the #4851 failure mode. Add the worker and serving files to that step's dependencies (IndexTTS2 already lists serving_speech.py at :262). (evidence: unchanged by this diff, present in PR-time tree: .buildkite/test-merge.yml:169-174 `source_file_dependencies:` / `- tests/e2e/online_serving/test_qwen3_tts_customvoice.py` / `- tests/e2e/offline_inference/test_qwen3_tts_customvoice.py` / `- vllm_omni/model_executor/models/qwen3_tts/` / `- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py` / `- vllm_omni/deploy/qwen3_tts.yaml` (no worker or serving_speech entries); .buildkite/scripts/upload_pipeline.py:10 `Drop steps or groups whose source_file_dependencies do not match changed files.`; .buildkite/scripts/upload_pipeline.py:378-383 `if deps is not None and not _matches_dependencies(changed_files, deps):` / `_log(f"skip {_step_label(step)!r} (no changes under {deps})",)` / `continue`; PR diff touches vllm_omni/worker/gpu_ar_model_runner.py and vllm_omni/entrypoints/openai/serving_speech.py only under vllm_omni/, which do not match the listed prefixes) + +```suggestion +- tests/e2e/online_serving/test_qwen3_tts_customvoice.py + - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py + - vllm_omni/worker/gpu_ar_model_runner.py + - vllm_omni/entrypoints/openai/serving_speech.py + - vllm_omni/model_executor/models/qwen3_tts/ +``` + +`tests/e2e/offline_inference/test_qwen3_tts_customvoice.py:~36` [minor] — The offline no-async-chunk CustomVoice E2E (`test_qwen3_tts_customvoice.py`) pins `max_num_seqs: 1` and calls `send_audio_speech_request` once (default `request_num=1`), so it cannot exercise multi-request worker batching. The new worker regression `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` guards 3-request `codes.audio` splitting with tail-only hidden states—the batched cross-request contamination class this PR fixes. Consider raising `max_num_seqs` and adding a concurrent multi-request offline case for that config. (evidence: unchanged by this diff, present in PR-time tree: tests/e2e/offline_inference/test_qwen3_tts_customvoice.py:36 `"max_num_seqs": 1,`; :79 `omni_runner_handler.send_audio_speech_request(request_config)`; tests/helpers/runtime.py:1731 `def send_audio_speech_request(self, request_config: dict[str, Any], request_num: int = 1)`; added in this PR: tests/worker/test_gpu_ar_model_runner.py:438 `def test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` with req_ids `["r1", "r2", "r3"]` and per-request `codes.audio` slice asserts at :481-483.) + +```suggestion +"max_num_seqs": 8, + "gpu_memory_utilization": 0.2, +``` + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4870.trace.json.gz new file mode 100644 index 000000000..ca48eb691 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4923.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4923.cost.json new file mode 100644 index 000000000..8d61ed641 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4923.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 662.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4923/run-20260817-132147-dfa6b0", + "recorded_at": "2026-08-17T13:32:49", + "attempts": [ + { + "invocation_id": "inv-3ce7adc6", + "outcome": "run:run-20260817-132147-dfa6b0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-132147-dfa6b0", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-132147-dfa6b0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3167, + "minutes": 11.02, + "ci_minutes": 0, + "input_tokens": 704296, + "output_tokens": 115027, + "tool_calls": 446, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4807, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.95, + "gate": 1.73, + "review": 655.45, + "report": 0.06 + }, + "time_to_first_result_s": 661.17 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 29671 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4923.md b/eval/dataset/arms/copilot_v17cb_train/pr4923.md new file mode 100644 index 000000000..223f141dd --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4923.md @@ -0,0 +1,81 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Nested GPU graph concern — inner CodePredictorWrapper._capture_cuda_graphs skipped when use_cuda_graphs=False on GPU (qwen3_code_predictor.py:552-556); residual: NPU still captures inner NPU graphs with use_cuda_graphs=True +- [resolved] Per-row generator gating — talker sets talker_mtp_accepts_per_row_generators from has_full_cudagraphs() (talker.py:346-350), runner scalar fallback at gpu_model_runner.py:1851-1855; residual: tts_local_seed non-reproducible for batched decode under full graphs (documented TODO) +- [resolved] prefix-cache vs async-output mutual exclusion: yaml enable_prefix_caching:false is required; runner guard at gpu_ar_model_runner.py:1540-1541 — residual: comment block at qwen3_tts.yaml:41-44 still argues for prefix caching. +- [resolved] Seeded batched eager path preserved when has_full_cudagraphs() false — talker_mtp_accepts_per_row_generators stays True; test at tests/worker/test_omni_gpu_model_runner.py:356-388. +- [resolved] Nested GPU code-predictor graphs vs outer talker_mtp graph — inner graphs disabled on GPU; residual: high_concurrency.yaml code_predictor_prefix_graphs:true no longer enables prefix graphs on GPU (qwen3_code_predictor.py:477) +- [resolved] async output vs prefix cache — gpu_ar_model_runner.py:1540-1541 returns False when omni_prefix_cache set; yaml now enable_prefix_caching:false aligns +- [resolved] Nested GPU graph concern — GPU path disables inner graphs (`use_cuda_graphs=False`) and relies on outer FULL wrap; NPU path keeps inner graphs and uses yaml `PIECEWISE` instead; residual: custom deploy yamls on NPU without `PIECEWISE` can still nest graphs now that `talker_mtp_graph_safe=True`. +- [claim-verified] PR claim that #4970 seed removal is not in this diff — no seed: key in qwen3_tts.yaml at head +- [claim-verified] PR nested-graph root cause — GPU code predictor no longer calls _capture_cuda_graphs +- [claim-verified] PR nested-graph root cause: GPU inner _capture_cuda_graphs only when use_cuda_graphs True (qwen3_code_predictor.py:552-554); outer wrap at gpu_model_runner.py:209-211. +- [claim-verified] PR 39ms→9ms talker_mtp claim not re-run here; yaml retune co-changes scheduler defaults beyond graph fix +- [claim-verified] PR perf table (39ms→9ms talker_mtp) — not reproducible in this read-only step; relies on author benchmark harness +- [claim-verified] PR states #4970 seed fix is upstream-only — diff has no seed/default_sampling_params change at this head +- [claim-verified] PR body claim to mirror Qwen3-Omni graph strategy — GPU outer FULL wrap + async output; NPU uses sibling `PIECEWISE` yaml rather than FULL outer MTP wrap. + +`vllm_omni/deploy/qwen3_tts.yaml:48` [major] — This diff retunes the bundled default deploy—stage-0 `max_num_seqs` 10→64 (line 38), `enable_prefix_caching` true→false (line 46), `max_num_batched_tokens` 512→32768 (line 48), stage-1 `max_num_seqs` 10→64 (line 69)—but the file header (lines 6–8) still documents `max_num_batched_tokens=512`, the stage-1 comment (line 68) still says CustomVoice/VoiceDesign are TTFA-optimal at 1, user docs still prescribe bundled stage-1 `max_num_seqs: 10` (`text_to_speech.md:334`, `speech_api.md:658`), and `qwen3_tts_high_concurrency.yaml` still carries the explicit 512/64 throughput profile. Consumers loading this yaml without overrides—including CI CustomVoice e2e—inherit throughput-oriented admission/batching. Either scope these knobs back to the high-concurrency profile and keep latency defaults here, or update header, inline comments, and user docs in the same PR with TTFA evidence for CustomVoice/VoiceDesign at 64. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/deploy/qwen3_tts.yaml:6 `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk`; vllm_omni/deploy/qwen3_tts.yaml:48 ` max_num_batched_tokens: 32768`; vllm_omni/deploy/qwen3_tts.yaml:69 ` max_num_seqs: 64`; docs/user_guide/examples/online_serving/text_to_speech.md:334 `The bundled \`qwen3_tts.yaml\` ships stage 1 (Code2Wav) at \`max_num_seqs: 10\``; docs/serving/speech_api.md:658 `stage 1 performed best at \`max_num_seqs: 10\``; vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:54 ` max_num_batched_tokens: 512`; tests/e2e/online_serving/test_qwen3_tts_customvoice.py:22-24 `_STAGE_CONFIG = modify_stage_config(get_deploy_config_path("qwen3_tts.yaml"), updates={"stages": {0: {"default_sampling_params.max_tokens": 500}}})`) + +```suggestion +max_num_batched_tokens: 512 + max_num_seqs: 10 +``` + +`docs/user_guide/examples/online_serving/text_to_speech.md:~334` [major] — This diff changes bundled `qwen3_tts.yaml` stage-0 and stage-1 `max_num_seqs` from 10 to 64 (`qwen3_tts.yaml:38`, `:70`), but `docs/user_guide/examples/online_serving/text_to_speech.md:334` still states the bundled Code2Wav default is `max_num_seqs: 10` with TTFA guidance derived from that value, and `docs/serving/speech_api.md:658–667` still references `max_num_seqs: 10` and reducing below 10. Update these user-facing docs in the same PR so operators and benchmarks align with the new bundled default and retain CustomVoice/VoiceDesign override guidance (`max_num_seqs: 1`). (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/deploy/qwen3_tts.yaml:38 `max_num_seqs: 64`; vllm_omni/deploy/qwen3_tts.yaml:70 `max_num_seqs: 64`; docs/user_guide/examples/online_serving/text_to_speech.md:334 `The bundled `qwen3_tts.yaml` ships stage 1 (Code2Wav) at `max_num_seqs: 10`, tuned for Base voice cloning:`; docs/serving/speech_api.md:658 `On the current Qwen3-TTS CustomVoice benchmark, stage 1 performed best at `max_num_seqs: 10`:`; docs/serving/speech_api.md:667 `benchmark before reducing it below `10`.`) + +```suggestion +The bundled `qwen3_tts.yaml` ships stage 1 (Code2Wav) at `max_num_seqs: 64` for throughput-oriented Base cloning benchmarks on 1x H100. For CustomVoice / VoiceDesign (TTFA-optimal at `max_num_seqs: 1`) or latency-sensitive Base serves, override via `--stage-overrides '{"1": {"max_num_seqs": 1}}'` and benchmark before adopting the bundled default. +``` + +`vllm_omni/deploy/qwen3_tts.yaml:46` [minor] — Stage-0 sets enable_prefix_caching: false (line 46) while the comment block above (lines 41–45) still recommends prefix caching for shared ref_audio throughput. The same PR sets use_async_omni_output = True on the talker; GPUARModelRunner._should_use_async_omni_output returns False when omni_prefix_cache is set (initialized from enable_prefix_caching), so the two features are mutually exclusive. Update the yaml comment to document that prefix caching is disabled here to allow async omni output. (evidence: vllm_omni/deploy/qwen3_tts.yaml:41-46 `# Prefix caching on by default...` / `enable_prefix_caching: false`; unchanged by this diff, present in PR-time tree: vllm_omni/worker/gpu_ar_model_runner.py:1540-1541 `if self.omni_prefix_cache is not None:` / `return False`; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:351 `self.use_async_omni_output = True`; vllm_omni/worker/gpu_model_runner.py:169 `if self.cache_config.enable_prefix_caching:`) + +```suggestion +# Prefix caching disabled so async omni output can run (mutually exclusive + # with omni_prefix_cache; see GPUARModelRunner._should_use_async_omni_output). + # Enable explicitly only if you disable async output / accept sync D2H. + enable_prefix_caching: false +``` + +`vllm_omni/deploy/qwen3_tts.yaml:~38` [minor] — Gate report shows `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` failing at PR head. This PR changes `vllm_omni/deploy/qwen3_tts.yaml` (throughput defaults at line 38 and new `platforms.npu` `compilation_config.cudagraph_mode: PIECEWISE` at lines 95–101) plus Qwen3-TTS talker/code-predictor model code—all exercised by the CustomVoice e2e job wired in `.buildkite/test-amd-ready.yaml:173` and `.buildkite/test-npu-ready.yaml:21` (`test_qwen3_tts_customvoice.py` loads `qwen3_tts.yaml`). Please confirm in CI logs whether these reds predate this PR or are regressions (OOM, NPU PIECEWISE block, or behavior change from the yaml retune) and paste the failing assertion if introduced here. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/deploy/qwen3_tts.yaml:38 ` max_num_seqs: 64`; vllm_omni/deploy/qwen3_tts.yaml:95-101 `platforms:` / `npu:` / `compilation_config:` / `cudagraph_mode: PIECEWISE`; tests/e2e/online_serving/test_qwen3_tts_customvoice.py:21-24 `_STAGE_CONFIG = modify_stage_config(get_deploy_config_path("qwen3_tts.yaml"), ...)`; tests/helpers/stage_config.py:627-631 `def get_deploy_config_path(rel_path: str) -> str:` / `return str(_DEPLOY_DIR / rel_path)`; .buildkite/test-amd-ready.yaml:173 `- timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"`; .buildkite/test-npu-ready.yaml:21 `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — This diff gates `talker_mtp_accepts_per_row_generators` on `cudagraph_mode.has_full_cudagraphs()` (`qwen3_tts_talker.py:346-350`), disabling per-row MTP generators under full cudagraphs; the in-file comment notes `tts_local_seed` is then not reproducible. HTTP serving still injects `tts_local_seed` when clients pass `seed` (`serving_speech.py:3686-3694`), and the runner falls back to scalar forwards when generators are present but the flag is false (`gpu_model_runner.py:1851-1854`). No test covers talker `__init__` gating with mocked full vs non-full `cudagraph_mode` (`test_omni_gpu_model_runner.py` only sets the flag on a `SimpleNamespace`). Consider a small CPU init test and a speech_api note that seeded concurrent requests are best-effort under full cudagraphs. (evidence: qwen3_tts_talker.py:346-350 `cudagraph_mode = getattr(vllm_config.compilation_config, "cudagraph_mode", None)` / `talker_mtp_graph_wrapped = (self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs())` / `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`; unchanged by this diff, present in PR-time tree: serving_speech.py:3686-3694 `if request.seed is not None and sampling_params_list:` / `stage0_params.extra_args["tts_local_seed"] = request.seed`; unchanged by this diff, present in PR-time tree: gpu_model_runner.py:1851-1854 `if (decode_batch_size > 1 and any(generator is not None for generator in row_generators) and not getattr(self.model, "talker_mtp_accepts_per_row_generators", False)):`; unchanged by this diff, present in PR-time tree: test_omni_gpu_model_runner.py:372-374 `runner.model = SimpleNamespace(talker_mtp_output_key=("codes", "audio"), talker_mtp_accepts_per_row_generators=True,)`) + +`qwen3_code_predictor.py:552` [minor] — Nested GPU graph concern resolved — inner CodePredictorWrapper._capture_cuda_graphs skipped on GPU because use_cuda_graphs=False (common/qwen3_code_predictor.py:552-556, set via qwen3_tts_code_predictor_vllm.py:39); residual: NPU still captures inner NPU graphs when use_cuda_graphs=True (common/qwen3_code_predictor.py:532-535). (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/common/qwen3_code_predictor.py:552-556 `if self._wrapper_config.use_cuda_graphs: self._capture_cuda_graphs()` else torch.compile only; PR diff sets GPU wrapper: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu()`; NPU residual: vllm_omni/model_executor/models/common/qwen3_code_predictor.py:532-535 `if current_omni_platform.is_npu() and self._wrapper_config.use_cuda_graphs: ... self._capture_npu_graphs()`) + +`talker.py:350` [minor] — Per-row generator gating: talker sets talker_mtp_accepts_per_row_generators from has_full_cudagraphs() (qwen3_tts_talker.py:346-350); runner scalar fallback when batched decode has explicit seeds but the flag is False (gpu_model_runner.py:1851-1858). Residual: tts_local_seed non-reproducible for batched decode under full cudagraphs (TODO #4923, qwen3_tts_talker.py:334-345). Default deploy uses PIECEWISE, so this path is latent unless cudagraph_mode is FULL. (evidence: qwen3_tts_talker.py:346 `cudagraph_mode = getattr(vllm_config.compilation_config, "cudagraph_mode", None)`; qwen3_tts_talker.py:347-349 `talker_mtp_graph_wrapped = (self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs())`; qwen3_tts_talker.py:350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`; unchanged by this diff, present in PR-time tree: gpu_model_runner.py:1851-1855 `if (decode_batch_size > 1 and any(generator is not None for generator in row_generators) and not getattr(self.model, "talker_mtp_accepts_per_row_generators", False)):`; qwen3_tts_talker.py:334 `# Consequence: under full cudagraphs, per-request ``tts_local_seed`` is not reproducible`) + +`gpu_ar_model_runner.py:1540` [minor] — prefix-cache vs async-output mutual exclusion: `enable_prefix_caching: false` in deploy YAML is required because `_should_use_async_omni_output()` bails out when `omni_prefix_cache` is set (`vllm_omni/worker/gpu_ar_model_runner.py:1540-1541`). Residual: the comment block at `qwen3_tts.yaml:41-44` still argues prefix caching is on by default and a throughput win, but the stage sets `enable_prefix_caching: false`. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/worker/gpu_ar_model_runner.py:1540-1541 `if self.omni_prefix_cache is not None:` / `return False`; PR diff sets qwen3_tts.yaml `enable_prefix_caching: false` and talker `self.use_async_omni_output = True`; unchanged comment at qwen3_tts.yaml:41-44 `# Prefix caching on by default. The HS opt-out (talker only consumes the` / `# last token's hidden state) and deferred mm-output writes added in this` / `# PR make per-step bookkeeping negligible, and any text repetition across` / `# requests (same voice or shared ref_audio) is a net throughput win.` vs line 46 `enable_prefix_caching: false`) + +**Additional observations (beyond the comment budget):** +- `qwen3_code_predictor.py:477` [minor] Nested talker_mtp CUDA graphs vs inner code-predictor graphs: this PR disables GPU code-predictor graphs (`use_cuda_graphs=current_omni_platform.is_npu()`), so `qwen3_tts_high_concurrency.yaml` `code_predictor_prefix_graphs: true` no longer enables prefix graphs on GPU […] +- `vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39` [minor] This diff gates inner CUDA graphs with `use_cuda_graphs=current_omni_platform.is_npu()` (line 39), disabling them on GPU while NPU keeps them—matching Qwen3-Omni MTP. `test_tts_config` still builds `CodePredictorWrapperConfig(use_cuda_graphs=True)` and asserts True (481–487), which no longer reflects wrapper init […] +- `tests/e2e/offline_inference/test_qwen3_tts_base.py:42` [minor] This diff targets the default cudagraph + outer `talker_mtp` graph-wrap path (`talker_mtp_graph_safe = True`, per-row generators gated on full cudagraphs), but L3 offline `test_qwen3_tts_base.py` builds a temp yaml with `enforce_eager: True` on both stages (lines 42 and 47), so offline merge-gate coverage never […] +- `vllm_omni/deploy/qwen3_tts.yaml:99` [minor] NPU `platforms` block adds `cudagraph_mode: PIECEWISE` for stages 0/1 without explaining why NPU must avoid FULL outer `talker_mtp` wrapping (`talker_mtp_graph_safe=True` + `has_full_cudagraphs()`) while inner code-predictor NPU graphs stay on (`use_cuda_graphs=is_npu()`). Mirror `qwen3_omni_moe.yaml:81,87` inline […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4923.trace.json.gz new file mode 100644 index 000000000..82da2000a Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4926.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4926.cost.json new file mode 100644 index 000000000..9135a977f --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4926.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1095.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4926/run-20260817-135110-a4e07d", + "recorded_at": "2026-08-17T14:09:25", + "attempts": [ + { + "invocation_id": "inv-bdeae926", + "outcome": "run:run-20260817-135110-a4e07d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-135110-a4e07d", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-135110-a4e07d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4024, + "minutes": 18.24, + "ci_minutes": 0, + "input_tokens": 1036760, + "output_tokens": 111354, + "tool_calls": 375, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.664, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.76, + "gate": 1.58, + "review": 1088.92, + "report": 0.09 + }, + "time_to_first_result_s": 1094.31 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 34463 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4926.md b/eval/dataset/arms/copilot_v17cb_train/pr4926.md new file mode 100644 index 000000000..041f62490 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4926.md @@ -0,0 +1,126 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] piecewise hub crash when flash_attn_func missing — fixed via _flash_wrapper varlen fallback at flash_attn_hub.py:117-126; residual: no test exercises full_attn_spans on hub backends. +- [resolved] FA3 hub Hopper gate added at cuda/platform.py:140-147; residual Blackwell major 10/12 still passes >=9 +- [resolved] piecewise varlen fallback — flash_attn_hub.py:117-126 _flash_wrapper uses varlen when flash_attn_func is None; residual: masked/piecewise untested +- [resolved] FA3_HUB Hopper gate at cuda/platform.py:141-147 — concern addressed; residual: test uses major<9 skip on L4 so FA3 forward never runs even locally on CI hardware +- [resolved] registry import surface — flash_attn_hub.py is string-referenced only; no eager import at collection; residual: no get_class() smoke test for new enum members +- [claim-verified] PR unit-test command — test_kernels_hub.py marked core_model+cpu; collected by .buildkite/test-ready.yml `tests/diffusion … -m 'core_model and cpu'`. +- [claim-verified] PR max_diff 0.00390625 satisfies assert max_diff < 1e-2 threshold +- [claim-verified] pyproject.toml:70 already lists kernels<0.15.0 in test optional-deps; docs pin kernels==0.14.1 — in range; kernels not added to serving optional-deps (same as sageattention) +- [claim-refuted] PR 'perfect numerical consistency' — test_kernels_hub_execution only asserts max_diff<1e-2 vs SDPA on dense unmasked B=1; no diffusers/hub cross-check in diff. +- [claim-refuted] PR test plan implies GPU Hub execution is CI-gated: test_kernels_hub_execution only has cuda/L4 marks, not in .buildkite test-ready.yml tests/diffusion/*.py lane +- [claim-refuted] docs attention_backends.md:231 “fall back gracefully to FLASH_ATTN” — only when `import kernels` fails; flash_attn_hub.py:96-109 RuntimeError if get_kernel fails after import +- [claim-refuted] docs “perfect numerical consistency” — get_kernel version 1/2/default chain + test uses q=k=v vs SDPA at 1e-2 +- [claim-refuted] PR 'all unit tests pass' — test_kernels_hub_execution not in merge-gate CI collection path (cuda jobs use tests/diffusion/*.py only, line 145) +- [claim-refuted] PR E2E z-image FA3 — manual screenshots only, no automated e2e/benchmark in diff + +`vllm_omni/platforms/cuda/platform.py:126` [major] — Platform selection keeps `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` once `importlib.import_module("kernels")` succeeds (`cuda/platform.py:124-128`), but only falls back on `ImportError` (`:128-138`). Later, `FlashAttentionHubImpl.__init__` loads Hub kernels via `get_kernel(...)` and raises `RuntimeError` when no functions are found (`flash_attn_hub.py:96-109`), during `Attention.__init__` (`layer.py:94-105`). Docs promise graceful fallback to local `FLASH_ATTN` when Hub kernels are unavailable (`attention_backends.md:231`). Probe Hub kernel load during platform selection (or catch init failure and re-resolve) so partial installs/offline/Hub outages degrade instead of aborting model load. (evidence: vllm_omni/platforms/cuda/platform.py:126 `importlib.import_module("kernels")` — only `except ImportError:` at :128 reroutes to `FLASH_ATTN`; unchanged by this diff, present in PR-time tree. vllm_omni/diffusion/attention/backends/flash_attn_hub.py:109 `raise RuntimeError("Failed to load flash-attn2 kernel from HuggingFace Hub: no functions found")`. vllm_omni/diffusion/attention/layer.py:95-105 `self.attention = self.attn_impl_cls(...)` instantiates Hub impl at model init. docs/user_guide/diffusion/attention_backends.md:231 `If the \`kernels\` library is not available in the environment, vLLM-Omni will log a warning and fall back gracefully to the corresponding local backend implementations (\`FLASH_ATTN\`).`) + +```suggestion +except ImportError: + backend_upper = "FLASH_ATTN" +else: + try: + from vllm_omni.diffusion.attention.backends.flash_attn_hub import _probe_hub_kernels_available + if not _probe_hub_kernels_available(): + logger.warning("HuggingFace kernels hub load probe failed; falling back to FLASH_ATTN.") + backend_upper = "FLASH_ATTN" + except Exception: + backend_upper = "FLASH_ATTN" +``` + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96` [major] — Hub FA2/FA3 loaders use a version fallback chain (`version=1` → `version=2` → unpinned default at flash_attn_hub.py:96-103 and 275-282) while docs only pin `kernels==0.14.1`. For train↔rollout parity, pin one explicit Hub revision in code and document it instead of probing multiple moving versions. (evidence: flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)`; flash_attn_hub.py:100 `hub_module = get_kernel("kernels-community/flash-attn2", version=2)`; flash_attn_hub.py:103 `hub_module = get_kernel("kernels-community/flash-attn2")`; flash_attn_hub.py:275-282 mirrors the same chain for `kernels-community/flash-attn3`; docs/user_guide/diffusion/attention_backends.md documents `pip install kernels==0.14.1` and repo names `kernels-community/flash-attn2` / `flash-attn3` with no pinned Hub revision.) + +```suggestion +HUB_FLASH_ATTN2_REVISION = 1 # or env DIFFUSION_HUB_FA2_REVISION +hub_module = get_kernel("kernels-community/flash-attn2", version=HUB_FLASH_ATTN2_REVISION) +``` + +`vllm_omni/diffusion/models/ltx2/ltx2_transformer.py:~397` [major] — This PR registers `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB`, and `FlashAttentionHubImpl._forward_varlen_masked` requires a 2D padding mask (`flash_attn_hub.py:141`), but LTX-2 still converts 4D masks to 2D only when `get_name().upper() == "FLASH_ATTN"` (`ltx2_transformer.py:397`). Extend the gate to Hub backend names (or an equivalent capability check) so masked LTX-2 steps do not pass 4D masks into Hub varlen paths. (evidence: unchanged by this diff, present in PR-time tree: ltx2_transformer.py:396-398 `attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])` / `if attn.attn.attn_backend.get_name().upper() == "FLASH_ATTN":` / `attention_mask = self._to_padding_mask(attention_mask)`; flash_attn_hub.py:141 `assert attention_mask.ndim == 2, "attention_mask must be 2D, (batch_size, seq_len)"`; ltx2_transformer.py:519 `attn_metadata = AttentionMetadata(attn_mask=attention_mask) if attention_mask is not None else None`) + +```suggestion +backend_name = attn.attn.attn_backend.get_name().upper() + if backend_name in ("FLASH_ATTN", "FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"): + attention_mask = self._to_padding_mask(attention_mask) +``` + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92` [minor] — `FlashAttentionHubImpl.__init__` calls `get_kernel('kernels-community/flash-attn2', ...)` on every instance (flash_attn_hub.py:92–104) with no module-level cache; `Attention.__init__` builds `self.attn_impl_cls(...)` per layer (layer.py:95–103), so a multi-layer DiT repeats Hub resolve/download and info logging at init. Cache the loaded hub module once per process before per-layer construction; same pattern applies to `FlashAttention3HubImpl`. (evidence: flash_attn_hub.py:92 `from kernels import get_kernel` — import and resolve happen inside per-instance `__init__`; flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)` — no `@lru_cache`/`@cache` wrapper in file (grep: no cache symbols). unchanged by this diff, present in PR-time tree: layer.py:95-96 `self.attn_impl_cls = self.attn_backend.get_impl_cls()` / `self.attention = self.attn_impl_cls(` — one impl constructed per `Attention` layer.) + +```suggestion +@functools.lru_cache(maxsize=1) +def _load_flash_attn2_hub_module(): + from kernels import get_kernel + logger.info("Loading flash-attn2 kernel from HuggingFace Hub...") + for version in (1, 2, None): + try: + return get_kernel("kernels-community/flash-attn2", version=version) if version else get_kernel("kernels-community/flash-attn2") + except Exception as e: + logger.info("flash-attn2 hub load failed (version=%s): %s", version, e) + raise RuntimeError("Failed to load flash-attn2 kernel from HuggingFace Hub") + +# in FlashAttentionHubImpl.__init__: +hub_module = _load_flash_attn2_hub_module() +``` + +`tests/diffusion/attention/test_kernels_hub.py:58` [minor] — `FlashAttentionHubImpl.forward_cuda` (flash_attn_hub.py:189-203) handles `full_attn_spans` via `_flash_wrapper`/`piecewise_attn`, but `test_kernels_hub_execution` (test_kernels_hub.py:74-99) only passes `AttentionMetadata(attn_mask=None)` with no `full_attn_spans`. Consider adding a Hub piecewise hardware test reusing span layouts from `test_piecewise_attn.py`. (evidence: tests/diffusion/attention/test_kernels_hub.py:74 `attn_metadata_sdpa = AttentionMetadata(attn_mask=None)` — execution test never sets full_attn_spans; vllm_omni/diffusion/attention/backends/flash_attn_hub.py:189 `if full_attn_spans is not None:` — piecewise branch added in this PR; vllm_omni/diffusion/attention/backends/flash_attn_hub.py:117-127 `_flash_wrapper` falls back to `_run_varlen_dense(...)` when `attn_func is None`; tests/diffusion/attention/test_piecewise_attn.py:89-96 `got = piecewise_attn(..., attn_func=_sdpa_attn_func,)` — existing piecewise coverage uses SDPA, not FlashAttentionHubImpl) + +```suggestion +@pytest.mark.core_model +@hardware_test(res={"cuda": "L4"}, num_cards=1) +def test_kernels_hub_piecewise_matches_sdpa(): + # build q,k,v and full_attn_spans like test_piecewise_attn.py, run FlashAttentionHubImpl.forward_cuda, + # assert close to SDPA piecewise reference + ... +``` + +`tests/diffusion/attention/test_kernels_hub.py:57` [minor] — This diff adds tests/diffusion/attention/test_kernels_hub.py, but merge-gate CUDA jobs run `pytest tests/diffusion/*.py -m 'core_model and cuda'` (test-ready.yml:145) while CPU lanes use recursive `tests/diffusion` with `-m 'core_model and cpu'` (line 13). The new cuda-marked `test_kernels_hub_execution` sits under tests/diffusion/attention/ and is outside the top-level glob, so hub load/forward regressions are not caught in per-PR CI. Extend the diffusion CUDA job to collect `tests/diffusion/attention`. (evidence: tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` — cuda execution test in attention subdir; unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:13 `timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` — CPU lane recurses tests/diffusion but excludes cuda tests; unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:145 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"` — CUDA lane glob does not include tests/diffusion/attention/) + +```suggestion +# In .buildkite/test-ready.yml under Diffusion · Other Test, replace or extend: +- timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/attention -m 'core_model and cuda' --run-level "core_model" +``` + +`vllm_omni/platforms/cuda/platform.py:~141` [minor] — `FLASH_ATTN_3_HUB` is enabled whenever `compute_capability.major >= 9` (`cuda/platform.py:141`), so Blackwell GPUs (major 10/11/12 per the same file) stay on the FA3 Hub path while fallback to `FLASH_ATTN_HUB` only runs for `major < 9`. Docs call this backend “Hopper (sm_90+) only” and only describe pre-Hopper fallback; there is no Blackwell coverage in `test_kernels_hub.py`. Tighten the gate to `major == 9` or document/test Blackwell FA3 Hub behavior. (evidence: vllm_omni/platforms/cuda/platform.py:141 `fa3_hub_supported = compute_capability is not None and compute_capability.major >= 9`; vllm_omni/platforms/cuda/platform.py:77-78 `is_blackwell = major in (10, 11, 12)`; docs/user_guide/diffusion/attention_backends.md:24 `FLASH_ATTN_3_HUB` | FlashAttention 3 from HuggingFace `kernels` library. CUDA Hopper (sm_90+) only; falls back to `FLASH_ATTN_HUB` on older GPUs.; docs/user_guide/diffusion/attention_backends.md:231 On CUDA GPUs below Hopper (compute capability < 9.0), `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`.) + +```suggestion +fa3_hub_supported = ( + compute_capability is not None + and compute_capability.major == 9 +) +``` + +`flash_attn_hub.py:116` [minor] — Resolved: piecewise hub no longer crashes when hub exposes only varlen (`flash_attn_func` missing)—`_flash_wrapper` at flash_attn_hub.py:116-126 falls back to `_run_varlen_dense`, and piecewise at :189-203 binds `attn_func=self.flash_attn_func` through that wrapper. Residual (minor): no test exercises `full_attn_spans` on hub backends. (evidence: flash_attn_hub.py:116-126 `def _flash_wrapper(self, q, k, v, *, attn_func, **kwargs):` / `if attn_func is not None:` / `return _run_varlen_dense(`; flash_attn_hub.py:189-203 `if full_attn_spans is not None:` / `attn_func = partial(self._flash_wrapper, attn_func=self.flash_attn_func,)` / `return piecewise_attn(...)`; tests/diffusion/attention/test_kernels_hub.py:74 `attn_metadata_sdpa = AttentionMetadata(attn_mask=None)` — no `full_attn_spans` coverage for hub backends) + +**Additional observations (beyond the comment budget):** +- `cuda/platform.py:141` [minor] FLASH_ATTN_3_HUB gate at vllm_omni/platforms/cuda/platform.py:140-147 uses compute_capability.major >= 9; Blackwell GPUs (major 10/11/12 per lines 79-81) still pass, so they stay on FLASH_ATTN_3_HUB despite the Hopper-only warning/docs. +- `flash_attn_hub.py:116` [minor] piecewise varlen fallback — flash_attn_hub.py:116-126 `_flash_wrapper` routes to `_run_varlen_dense(flash_attn_varlen_func, ...)` when `attn_func` is None; piecewise binds `attn_func=self.flash_attn_func` (189-203). Residual gap: masked/piecewise paths untested in test_kernels_hub.py. +- `cuda/platform.py:90` [minor] FA3_HUB Hopper gate at vllm_omni/platforms/cuda/platform.py:141-147 correctly requires major >= 9 and falls back to FLASH_ATTN_HUB; residual gap: test_kernels_hub_execution (L4 hardware_test) skips FA3 forward via major < 9, so FlashAttention3HubImpl is never executed on CI L4 hardware. +- `vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py:247` [minor] Docs advertise global `FLASH_ATTN_HUB` / `FLASH_ATTN_3_HUB`, but `DiffusersAdapterPipeline._set_attention_backend` only matches `FLASH_ATTN`, `SAGE_ATTN`, `ASCEND`, and `TORCH_SDPA`. Explicit Hub values hit `case _:` and fall back to diffusers `native` (SDPA), defeating train/rollout alignment on the diffusers load […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4926.trace.json.gz new file mode 100644 index 000000000..8e9723d35 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4950.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4950.cost.json new file mode 100644 index 000000000..59b018db3 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4950.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 329.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4950/run-20260817-140925-6a2a3a", + "recorded_at": "2026-08-17T14:14:54", + "attempts": [ + { + "invocation_id": "inv-3616f55e", + "outcome": "run:run-20260817-140925-6a2a3a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-140925-6a2a3a", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-140925-6a2a3a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0522, + "minutes": 5.47, + "ci_minutes": 0, + "input_tokens": 130455, + "output_tokens": 15425, + "tool_calls": 79, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2159, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.82, + "gate": 1.56, + "review": 322.85, + "report": 0.08 + }, + "time_to_first_result_s": 328.28 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4950.trace.json.gz", + "trace_bytes": 10434 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4950.md b/eval/dataset/arms/copilot_v17cb_train/pr4950.md new file mode 100644 index 000000000..85e445936 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4950.md @@ -0,0 +1,68 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4950, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 48d4fd8a5e98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Recipe TTS trigger note formerly claimed text-only without flag (`file_at_base`); head now documents silent audio — residual: intro still says “same response/one shot” while body documents two choices. +- [claim-verified] PR top-level `chat_template_kwargs` vs nested `extra_body` — consistent with server request field wiring above. +- [claim-verified] PR separate-choices response shape — consistent with `choices.extend(choices_data)` per modality. +- [claim-verified] OpenAI SDK `extra_body` flattening — `gradio_demo.py:222-331` uses nested `extra_body.chat_template_kwargs` via SDK client (valid path). +- [validated] `vllm_omni/entrypoints/openai/serving_chat.py:373` — `validate_chat_template(..., chat_template_kwargs=request.chat_template_kwargs)`; raw JSON `extra_body` nesting is not auto-merged. +- [validated] `vllm_omni/entrypoints/openai/serving_chat.py:2125-2185` — text and audio `final_output_type` each append separate `choices` entries. +- [validated] `vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:104-119` — without `<|tts_bos|>` in token_ids, `tts_token_ids`/`tts_hidden_states` are omitted from talker payload. +- [validated] `vllm_omni/engine/orchestrator.py:73-77` — terminal audio path uses `torch.zeros((0,))` when no downstream payload; supports silent-audio-not-text-only when `modalities` includes `audio`. +- [sweep] recipes/OpenBMB/MiniCPM-o-4_5.md hunk at ~102–127: curl uses top-level `chat_template_kwargs`; SDK `extra_body` note added — matches `serving_chat.py:373`. +- [sweep] Stale sibling: `examples/online_serving/minicpmo/README.md:57-61` still documents only `extra_body.chat_template_kwargs` without curl-vs-SDK distinction. +- [sweep] Blast radius grep `use_tts_template|extra_body.*chat_template` — only recipe, gradio demo (SDK), and example README; no code defaults changed. +- [sweep] Tests: no test files in diff; recipe behavior covered only by manual `/render` + curl claims in PR body (not re-run here). +- [sweep] CI `buildkite/vllm-omni-npu-ci` red — diff touches only `recipes/OpenBMB/MiniCPM-o-4_5.md`; no NPU config/yaml intersection at this head. +- [sweep] Merge commits `c4cd7778`/`48d4fd8a` on branch — docs-only net change; no code-path revalidation needed beyond head reads. + +`examples/online_serving/minicpmo/README.md:~57` [minor] — This PR fixes the recipe's curl guidance, but the linked Gradio example README (examples/online_serving/minicpmo/README.md:57-58) still describes only extra_body.chat_template_kwargs and omits that raw HTTP/curl must use top-level chat_template_kwargs. Align that sibling doc so users following the recipe link do not assume extra_body works for all clients. (evidence: unchanged by this diff, present in PR-time tree: examples/online_serving/minicpmo/README.md:57-58 `- **TTS trigger**: the demo sets` / `` `extra_body.chat_template_kwargs.use_tts_template=True`, which appends ``; PR updates recipe at recipes/OpenBMB/MiniCPM-o-4_5.md:106-107 `` `use_tts_template=true` via the **top-level** `chat_template_kwargs` `` / `` field (curl does not flatten nested `extra_body`): ``) + +```suggestion +- **TTS trigger**: the Gradio demo (OpenAI Python SDK) sets `extra_body.chat_template_kwargs.use_tts_template=True`, which the client merges into the request root. For raw `curl`, pass top-level `chat_template_kwargs.use_tts_template=true` instead (nested `extra_body` is ignored). +``` + +`recipes/OpenBMB/MiniCPM-o-4_5.md:219` [minor] — The corrected TTS trigger note explains silent audio when the flag is missing with modalities: ["text", "audio"], but does not point readers to the existing text-only curl smoke test (modalities: ["text"] at lines 92–100) for faster text-only serving. A one-line cross-reference would close the loop on the old "text-only (faster)" guidance. (evidence: unchanged by this diff, present in the PR-time tree: recipes/OpenBMB/MiniCPM-o-4_5.md:92 `**Quick smoke test (text-only output)**:`; recipes/OpenBMB/MiniCPM-o-4_5.md:100 `"modalities": ["text"]"; recipes/OpenBMB/MiniCPM-o-4_5.md:222 `receives no TTS token span and returns silent audio (not text-only).` — no mention of modalities: ["text"] or the smoke test in lines 219–225) + +```suggestion +- **TTS trigger**: speech output requires + `chat_template_kwargs.use_tts_template=true` so the chat template + appends `<|tts_bos|>` before generation. Without it, Stage-1 talker + receives no TTS token span and returns silent audio (not text-only). + For text-only output, request `modalities: ["text"]` (see smoke test above). + For **curl**, put `chat_template_kwargs` at the request root; nested + `extra_body.chat_template_kwargs` is ignored. The OpenAI Python SDK + may use `extra_body` because it flattens those fields into the root. +``` + +`recipes/OpenBMB/MiniCPM-o-4_5.md:104` [nit] — The section title and Summary still imply text and speech share one choice (`choices[0]`), but the updated body correctly notes separate `choices` entries for text vs audio. Tweak the header/intro (and Summary lines 10 and 21-22) so readers do not expect both modalities on `choices[0]`. (evidence: unchanged by this diff, present in the PR-time tree: recipes/OpenBMB/MiniCPM-o-4_5.md:10 `emits text and 24 kHz mono speech in the same response`; recipes/OpenBMB/MiniCPM-o-4_5.md:21-22 `call can return text and 24 kHz speech in one shot`; recipes/OpenBMB/MiniCPM-o-4_5.md:104 `**Text + speech in one response** (the headline 4.5 feature). The TTS`; added/changed in this PR: recipes/OpenBMB/MiniCPM-o-4_5.md:124-127 `Response carries text in one choice's message.content and base64 WAV in another choice's message.audio.data (24 kHz mono, see Notes). With modalities: ["text", "audio"] you typically get two choices entries (one text, one audio).`) + +```suggestion +**Text + speech in one HTTP response** (the headline 4.5 feature). The TTS +``` + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4950.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4950.trace.json.gz new file mode 100644 index 000000000..9f0d54830 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4950.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4970.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4970.cost.json new file mode 100644 index 000000000..d348c34ea --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4970.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 264.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4970/run-20260817-140921-1cc5a9", + "recorded_at": "2026-08-17T14:13:45", + "attempts": [ + { + "invocation_id": "inv-acbe21fe", + "outcome": "run:run-20260817-140921-1cc5a9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-140921-1cc5a9", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-140921-1cc5a9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0531, + "minutes": 4.4, + "ci_minutes": 0, + "input_tokens": 125778, + "output_tokens": 17364, + "tool_calls": 85, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.1845, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.46, + "gate": 2.06, + "review": 258.55, + "report": 0.06 + }, + "time_to_first_result_s": 264.13 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4970.trace.json.gz", + "trace_bytes": 9608 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4970.md b/eval/dataset/arms/copilot_v17cb_train/pr4970.md new file mode 100644 index 000000000..2a15f4a3d --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4970.md @@ -0,0 +1,52 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4970, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4992e10b7367) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Stage-1 seed removal — only stage-0 seed feeds `tts_local_seed`; stage-1 `temperature: 0.0` so stage-1 seed was perf-neutral +- [claim-verified] qwen3_tts_high_concurrency.yaml has no `seed:` — matches PR claim that sibling already dropped it (#4869) +- [claim-verified] grep tests/ + tests/dfx/perf for deploy seed — no test asserts `qwen3_tts.yaml` seed; nightly `test_tts.json` quality phase `enabled: false`, no request `seed` field +- [claim-verified] grep repo `qwen3_tts.yaml` — CI path filters in `.buildkite/test-ready.yml` and `test-merge.yml` list this file; e2e smoke checks audio presence not determinism +- [validated] serving_speech.py:3696-3706 — qwen3_tts deploy `stage0_params.seed` propagates to `tts_local_seed` only when non-None; omission stops default seeding +- [validated] gpu_model_runner.py:1821-1888 — `_explicit_talker_seed` reads only `extra_args["tts_local_seed"]`; all-None generators take batched unseeded `talker_mtp` path +- [validated] qwen3_tts_talker.py:326 — `talker_mtp_accepts_per_row_generators = True`; per-request seed stays batched (not scalar fallback) +- [sweep] vllm_omni/deploy/qwen3_tts.yaml:58,88 — read head; both `seed: 42` lines removed, `min_tokens: 2` and other sampling params unchanged +- [sweep] Blast radius — other deploy yamls still ship `seed: 42` (moss_tts, voxcpm2, qwen2_5_omni, etc.) but only qwen3_tts HTTP path had #4869 `tts_local_seed` propagation; VoxCPM2 #4963 remainder correctly out of scope +- [sweep] Offline path — `tts_local_seed` set only in `serving_speech.py`; offline never hit deploy-seed MTP regression +- [sweep] Rebase — merge commit in timeline; diff is 2-line yaml deletion only, no semantic merge conflict +- [sweep] CI red — `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` intersect diff via `qwen3_tts.yaml` path triggers + `test_qwen3_tts_*` e2e; attribution unverified + +`vllm_omni/deploy/qwen3_tts.yaml:60` [minor] — This diff removes deploy-level `seed: 42` from stage 0 (and stage 1). Users copy `qwen3_tts.yaml` as the reference profile; without a tombstone, someone may re-add `seed: 42` for reproducibility and silently re-trigger per-row MTP seeding (`vllm_omni/entrypoints/openai/serving_speech.py:3698-3706` propagates stage-0 deploy seed into `tts_local_seed`; see #4963). Add a brief comment matching the existing `min_tokens` style noting deploy seed must stay unset and per-request API `seed` is the supported determinism knob. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/entrypoints/openai/serving_speech.py:3698 `default_seed = getattr(stage0_params, "seed", None)`; vllm_omni/entrypoints/openai/serving_speech.py:3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`; base vllm_omni/deploy/qwen3_tts.yaml (stage 0) had `seed: 42` immediately after `min_tokens: 2`; PR head removes it) + +```suggestion +# Do not set deploy-level seed here: serving propagates stage-0 seed + # into tts_local_seed and forces slower per-row MTP sampling (#4963). + # Use per-request API seed when determinism is required. + min_tokens: 2 +``` + +`vllm_omni/deploy/qwen3_tts.yaml:61` [minor] — This diff removes deploy-default `seed: 42` from `qwen3_tts.yaml` (non-deterministic default audio). In `.buildkite/test-ready.yml`, that yaml is a `source_file_dependencies` path for `TTS · Qwen3-TTS CustomVoice Test` and `TTS · Qwen3-TTS Base Test` (lines 328, 354), so this PR will re-run those jobs. AMD/NPU ready pipelines also run `test_qwen3_tts_customvoice.py` but without `source_file_dependencies`. If `buildkite/vllm-omni-amd-ci` / `buildkite/vllm-omni-npu-ci` are red, do those failures reproduce on current `main` or are they introduced here? Smoke e2e tests do not assert on deploy seed, so functional breakage seems unlikely. (evidence: unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:328 `- vllm_omni/deploy/qwen3_tts.yaml`; .buildkite/test-ready.yml:354 `- vllm_omni/deploy/qwen3_tts.yaml`; vllm_omni/deploy/qwen3_tts.yaml:61 ` repetition_penalty: 1.05` (PR diff removes `seed: 42` above this line); unchanged by this diff: .buildkite/test-amd-ready.yaml:173 `- timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"`; .buildkite/test-npu-ready.yaml:21 `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4970.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4970.trace.json.gz new file mode 100644 index 000000000..c216076bf Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4970.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4977.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr4977.cost.json new file mode 100644 index 000000000..09f8614c4 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4977.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 467.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr4977/run-20260817-134728-a40ea1", + "recorded_at": "2026-08-17T13:55:14", + "attempts": [ + { + "invocation_id": "inv-ead5f484", + "outcome": "run:run-20260817-134728-a40ea1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-134728-a40ea1", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-134728-a40ea1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2048, + "minutes": 7.78, + "ci_minutes": 0, + "input_tokens": 499190, + "output_tokens": 63644, + "tool_calls": 224, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.351, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.31, + "gate": 1.71, + "review": 461.41, + "report": 0.07 + }, + "time_to_first_result_s": 466.47 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 18709 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4977.md b/eval/dataset/arms/copilot_v17cb_train/pr4977.md new file mode 100644 index 000000000..435d439b7 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr4977.md @@ -0,0 +1,98 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] trust_remote_code kwarg — dropped at head for kernels 0.13.x compat per commit message; residual checked: PR description not updated +- [resolved] trust_remote_code claim — commit 9947f414 dropped kwargs; flash_attn_hub.py:34-35 calls get_kernel(repo_id, version=version) / get_kernel(repo_id) only; pyproject.toml:70 pins kernels<0.15.0 +- [resolved] PR-body trust_remote_code claim — dropped at 9947f414; head calls get_kernel(repo_id, version=version) with no trust_remote_code (flash_attn_hub.py:33-35); residual: PR description still mentions it +- [resolved] trust_remote_code kwarg concern — removed at 9947f414 for kernels 0.13.x compat; residual: PR description still advertises it +- [resolved] trust_remote_code kwarg — commit 9947f414 dropped it; head get_kernel(repo_id, version=version) at flash_attn_hub.py:34 has no trust_remote_code; residual: PR body still advertises it (first-round minor) +- [claim-verified] PR goal to cut redundant Hub traffic: cache keyed by repo_id (`flash-attn2` / `flash-attn3`) loads once per worker process +- [claim-verified] version fallback v1→v2→default preserved vs file_at_base nested try/except in both Hub impl `__init__` paths +- [claim-verified] PR test plan test_kernels_hub_platform_fallback — test has @pytest.mark.core_model @pytest.mark.cpu; collected by .buildkite/test-ready.yml core_model and cpu lane for tests/diffusion +- [claim-verified] 'preserving version fallback behavior' — loop (1,2,None) matches pre-refactor try order (file_at_base:96-104, 274-282) +- [claim-verified] 'Attention forward behavior unchanged' — forward_cuda bodies untouched in diff +- [claim-verified] test_kernels_hub_platform_fallback — collected by merge-gate CPU job pytest -sv tests/diffusion -m 'core_model and cpu' (.buildkite/test-merge.yml:13), not only manual PR test plan +- [claim-refuted] PR body still claims adding `trust_remote_code=True`; at head 9947f414 it was removed (`get_kernel(repo_id, version=version)` only) +- [claim-refuted] PR body still says adding trust_remote_code=True at head; code and final commit explicitly removed it +- [claim-refuted] PR purpose line 'adding trust_remote_code=True' — absent at head after 9947f414 + +`tests/diffusion/attention/test_kernels_hub.py:~81` [minor] — This diff caches hub kernels in `_get_hub_module()` so each repo loads once per process (`flash_attn_hub.py:42-46`, used at `:119` and `:284`). Existing tests only construct one `FlashAttentionHubImpl` (`test_kernels_hub.py:81-83`) and would not catch a per-layer reload regression. Add a `@pytest.mark.core_model` `@pytest.mark.cpu` test that monkeypatches `kernels.get_kernel`, clears `_hub_modules`, builds two `FlashAttentionHubImpl` instances, and asserts `get_kernel` was called once for `kernels-community/flash-attn2`. (evidence: flash_attn_hub.py:42-46 `def _get_hub_module(repo_id: str): + with _hub_lock: + if repo_id not in _hub_modules: + _hub_modules[repo_id] = _load_hub_module(repo_id) + return _hub_modules[repo_id]`; flash_attn_hub.py:119 `hub_module = _get_hub_module("kernels-community/flash-attn2")`; unchanged by this diff, present in PR-time tree: test_kernels_hub.py:81 `fa_hub_impl = FlashAttentionHubImpl(` — only test-site construction of `FlashAttentionHubImpl` in `tests/`) + +```suggestion +@pytest.mark.core_model +@pytest.mark.cpu +def test_hub_kernel_cached_across_impl_constructions(monkeypatch): + calls = [] + + def fake_get_kernel(repo_id, version=None): + calls.append((repo_id, version)) + mod = types.ModuleType(repo_id) + mod.flash_attn_func = lambda *a, **k: a[0] + return mod + + kernels_mod = types.ModuleType("kernels") + kernels_mod.get_kernel = fake_get_kernel + monkeypatch.setitem(sys.modules, "kernels", kernels_mod) + + from vllm_omni.diffusion.attention.backends import flash_attn_hub + flash_attn_hub._hub_modules.clear() + + FlashAttentionHubImpl(num_heads=8, head_size=64, softmax_scale=0.125) + FlashAttentionHubImpl(num_heads=8, head_size=64, softmax_scale=0.125) + assert len(calls) == 1 + assert calls[0][0] == "kernels-community/flash-attn2" +``` + +`?:34` [minor] — PR Purpose still claims this adds `trust_remote_code=True` for hub kernel loading, but head (9947f414) dropped that kwarg because kernels 0.13.x `get_kernel()` rejects it and fallback attempts failed on older installs. Please update the description to match the final behavior. (evidence: flash_attn_hub.py:34 `return get_kernel(repo_id, version=version)` — no trust_remote_code kwarg at PR head; flash_attn_hub.py:35 `return get_kernel(repo_id)` — same; PR #4977 Purpose (live body via gh pr view): "adding `trust_remote_code=True` for more reliable Hub access" — stale after 9947f414) + +`flash_attn_hub.py:34` [minor] — PR description mismatch: body claims adding trust_remote_code=True, but head _load_hub_module calls get_kernel(repo_id, version=version) / get_kernel(repo_id) with no trust_remote_code (flash_attn_hub.py:33-35). (evidence: flash_attn_hub.py:34 `return get_kernel(repo_id, version=version)`; flash_attn_hub.py:35 `return get_kernel(repo_id)`; unchanged by this diff, present in PR-time tree: PR #4977 body states "adding `trust_remote_code=True` for more reliable Hub access") + +`?:33` [minor] — PR description still claims adding trust_remote_code=True, but 9947f414 dropped that kwarg from _load_hub_module() because kernels 0.13.x get_kernel() does not accept it; head only calls get_kernel(repo_id, version=version) / get_kernel(repo_id). (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:33 `return get_kernel(repo_id, version=version)` — no trust_remote_code at PR head; vllm_omni/diffusion/attention/backends/flash_attn_hub.py:34 `return get_kernel(repo_id)`; unchanged by final diff, present on GitHub PR #4977 body (not in repo tree): `adding trust_remote_code=True for more reliable Hub access`; commit 9947f414 message: `kernels 0.13.x get_kernel() does not accept trust_remote_code`) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~45` [minor] — _load_hub_module returns on the first successful get_kernel without checking flash_attn_func/flash_attn_varlen_func; _get_hub_module caches that module (line 45) before __init__ validation (lines 122-123). If version 1 succeeds with a function-less module, version 2/None are never tried and the bad entry persists for the process lifetime. (evidence: flash_attn_hub.py:33-35 `if version is not None: return get_kernel(repo_id, version=version)` / `return get_kernel(repo_id)` — success returns without attr check; flash_attn_hub.py:44-45 `if repo_id not in _hub_modules: _hub_modules[repo_id] = _load_hub_module(repo_id)` — caches before validation; flash_attn_hub.py:122-123 `if self.flash_attn_func is None and self.flash_attn_varlen_func is None: raise RuntimeError(...)` — validation only in __init__, not before cache) + +```suggestion +if version is not None: + mod = get_kernel(repo_id, version=version) + else: + mod = get_kernel(repo_id) + if getattr(mod, "flash_attn_func", None) or getattr(mod, "flash_attn_varlen_func", None): + return mod + last_error = RuntimeError( + f"{repo_id} version {version} missing flash_attn_func/flash_attn_varlen_func" + ) +``` + +`tests/diffusion/attention/test_kernels_hub.py:~58` [minor] — This diff adds process-wide hub kernel caching in `flash_attn_hub.py` (`_hub_modules` + `_get_hub_module`), but `test_kernels_hub_execution`—the only test that constructs `FlashAttentionHubImpl`—is not on any Buildkite pytest path. CUDA jobs cover `tests/diffusion/{offloader,cache,batching,distributed,models}` and top-level `tests/diffusion/*.py` (not `tests/diffusion/attention/`). CPU CI runs `tests/diffusion` with `-m 'core_model and cpu'`, which picks up `test_kernels_hub_platform_fallback` but not this execution test. Consider a cpu-marked cache test on a CI-selected path or extend the Buildkite suite. (evidence: flash_attn_hub.py:22 `_hub_modules: dict[str, object] = {}`; flash_attn_hub.py:42-46 `def _get_hub_module(repo_id: str):` / `with _hub_lock:` / `_hub_modules[repo_id] = _load_hub_module(repo_id)`; test_kernels_hub.py:56-58 `@pytest.mark.core_model` / `@hardware_test(res={"cuda": "L4"}, num_cards=1)` / `def test_kernels_hub_execution():` (no `@pytest.mark.cpu`); test_kernels_hub.py:81 `fa_hub_impl = FlashAttentionHubImpl(`; unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:145 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` (glob excludes `attention/` subdirectory); grep `test_kernels_hub` across `.buildkite/**`: no matches) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:34` [nit] — This diff calls `get_kernel(repo_id, version=version)` and `get_kernel(repo_id)` with no `trust_remote_code` kwarg (flash_attn_hub.py:33-35), and commit 9947f414 dropped that argument because kernels 0.13.x rejects it; update the PR description, which still claims `trust_remote_code=True` was added for more reliable Hub access, so reviewers are not misled about the final contract. (evidence: Commit 9947f414 message: 'kernels 0.13.x get_kernel() does not accept trust_remote_code'; flash_attn_hub.py:34-35 calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` with no extra kwargs; pyproject.toml:70 pins `kernels<0.15.0`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr4977.trace.json.gz new file mode 100644 index 000000000..6dda7df2a Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_train/pr5009.cost.json b/eval/dataset/arms/copilot_v17cb_train/pr5009.cost.json new file mode 100644 index 000000000..e1b5894ea --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr5009.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 767.5, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_train/runs/pr5009/run-20260817-132147-b213f2", + "recorded_at": "2026-08-17T13:34:34", + "attempts": [ + { + "invocation_id": "inv-2e9142b4", + "outcome": "run:run-20260817-132147-b213f2", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-132147-b213f2", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-132147-b213f2", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3114, + "minutes": 12.78, + "ci_minutes": 0, + "input_tokens": 738683, + "output_tokens": 101796, + "tool_calls": 434, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5174, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.04, + "gate": 1.71, + "review": 761.05, + "report": 0.06 + }, + "time_to_first_result_s": 766.84 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 23821 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_train/pr5009.md b/eval/dataset/arms/copilot_v17cb_train/pr5009.md new file mode 100644 index 000000000..979e79f3f --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_train/pr5009.md @@ -0,0 +1,83 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 5 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Part 1 vs Part 2 interaction for Qwen — after Part 2, Qwen RMSNorm bypasses IR op priority entirely (`nn.RMSNorm`); Part 1 still needed fleet-wide for other diffusion models and matches pre-#4709 global default +- [resolved] Cosmos3 native-only need — `get_cosmos3_ir_op_priority_func` forces `rms_norm=["native"]`; residual: any future model needing native must register its own hook +- [resolved] Qwen Part 1 relevance at HEAD — Part 2 uses nn.RMSNorm which does not consume IR op priority; Part 1 Qwen profiler evidence is Part-1-only intermediate state +- [resolved] PR Cosmos3 native-only claim — registry + get_cosmos3_ir_op_priority_func at :620-621; residual: no other per-model overrides exist +- [resolved] Cosmos3 native-only concern — `get_cosmos3_ir_op_priority_func` (`pipeline_cosmos3.py:620-621`) overrides platform default to `["native"]`; residual: other vllm `RMSNorm` diffusion models inherit restored `vllm_c` priority. +- [resolved] Part 1 vs Part 2 for Qwen — after nn.RMSNorm at all five sites Qwen norms bypass IR op priority; Part 1 still restores fleet default for ~14 other diffusion models still importing vllm RMSNorm; residual: Qwen perf/accuracy evidence is Part 2-driven +- [claim-verified] PR latency −26.9%: (1.2613−1.7263)/1.7263×100 = −26.94% +- [claim-verified] PR Part-1 profiler −16%: (1080−1288)/1288×100 = −16.15% +- [claim-verified] Cosmos3 per-model native override exists and is tested in tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py:748 +- [claim-verified] PR latency −26.9%: calc (1.7263−1.2613)/1.7263×100 = 26.94% +- [claim-verified] Part 1 profiler −16%: calc (1288−1080)/1288×100 = 16.15% +- [claim-verified] PR scope 'all CUDA diffusion using vLLM RMSNorm' — worker is diffusion-only; AR grep has zero get_default_ir_op_priority hits outside tests +- [claim-verified] PR Part 2 accuracy path — tests/e2e/accuracy/test_qwen_image.py:test_qwen_image_matches_diffusers exists; not re-run here +- [claim-verified] `nn.RMSNorm` dependency — already used in `hidream_image_transformer.py` and `ltx2_transformer.py`; vLLM 0.24 install path pins torch via vllm wheel (docs/getting_started/quickstart.md). + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — This hunk restores `torch.nn.RMSNorm` for Qwen-Image norms but adds no merge-gate regression test (the platform fix alone gets `tests/platforms/test_cuda_ir_op_priority.py`). A future accidental revert to `vllm.model_executor.layers.layernorm.RMSNorm` would likely pass L2/L3 online_serving jobs and is only guarded by nightly `tests/e2e/accuracy/test_qwen_image.py::test_qwen_image_matches_diffusers` (`full_model`, SSIM 0.94). Consider a lightweight `isinstance(..., nn.RMSNorm)` unit test under `tests/diffusion/models/qwen_image/`. (evidence: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)`; tests/platforms/test_cuda_ir_op_priority.py:26 `assert priority.rms_norm == ["vllm_c", "native"]` (platform-only regression, unchanged consumer for Qwen-Image norms); tests/e2e/accuracy/test_qwen_image.py:20 `pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]`; tests/e2e/accuracy/test_qwen_image.py:32 `SSIM_THRESHOLD = 0.94`; tests/e2e/accuracy/test_qwen_image.py:197 `def test_qwen_image_matches_diffusers(accuracy_artifact_root: Path) -> None:`; unchanged by this diff, present in PR-time tree: .buildkite/test-nightly.yml:919 `pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model` (no matching accuracy entry in .buildkite/test-ready.yml or .buildkite/test-merge.yml)) + +```suggestion +# tests/diffusion/models/qwen_image/test_qwen_image_rmsnorm.py +import torch.nn as nn +from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import QwenImageTransformer2DModel + +def test_qwen_image_uses_torch_rmsnorm_not_vllm_custom_op(): + m = QwenImageTransformer2DModel.__new__(QwenImageTransformer2DModel) + # minimal init path or fixture that builds one attention block + assert isinstance(m.txt_norm, nn.RMSNorm) # extend to norm_q/norm_k after construction +``` + +`vllm_omni/platforms/cuda/platform.py:252` [minor] — This hunk restores default=["vllm_c", "native"] in CudaOmniPlatform.get_default_ir_op_priority (removing #4709's inductor-active native-only branch), affecting every CUDA diffusion worker via diffusion_worker._resolve_ir_op_priority → kernel_config.ir_op_priority (diffusion_worker.py:290). Only Cosmos3 overrides via _DIFFUSION_IR_OP_PRIORITY_FUNCS (registry.py:549). Qwen-Image at this head uses nn.RMSNorm (qwen_image_transformer.py:557), so its norms no longer route through IR rms_norm priority after Part 2. Clarify in the PR description which profiler/E2E tables were captured with Part 1 only versus the final merged Part 1+Part 2 state. (evidence: platform.py:253 `default = ["vllm_c", "native"]`; unchanged by this diff, present in PR-time tree: diffusion_worker.py:175 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)` and diffusion_worker.py:290 `vllm_config.kernel_config.ir_op_priority = _resolve_ir_op_priority(self.od_config, vllm_config)`; unchanged by this diff, present in PR-time tree: registry.py:549 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",`; qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`) + +```suggestion +"""Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops. + + Restores pre-#4709 Omni defaults for all CUDA diffusion workers. + Models needing native-only RMSNorm must register via + ``_DIFFUSION_IR_OP_PRIORITY_FUNCS`` (e.g. Cosmos3).""" +``` + +`vllm_omni/platforms/cuda/platform.py:253` [minor] — This hunk sets CUDA diffusion default IR op priority to `["vllm_c", "native"]` unconditionally (no inductor branch), reversing the prior inductor-active `native`-only default. `diffusion_worker._resolve_ir_op_priority` is the sole caller of `get_default_ir_op_priority`; only Cosmos3 is registered in `_DIFFUSION_IR_OP_PRIORITY_FUNCS` to force `native`, while ~14 other diffusion architectures still use vLLM `RMSNorm` and inherit this path. PR validation is Qwen-focused — before merge, confirm at least one non-Qwen CUDA diffusion model (e.g. Wan/Flux/SD3) still compiles and matches baseline under default compile. (evidence: vllm_omni/platforms/cuda/platform.py:252 `default = ["vllm_c", "native"]` — unconditional, no inductor/mode branch in get_default_ir_op_priority; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/worker/diffusion_worker.py:175 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/registry.py:545-549 `_DIFFUSION_IR_OP_PRIORITY_FUNCS = { ... "Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func", }` (sole entry); unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:620-621 `priority_kwargs["rms_norm"] = ["native"]` / `priority_kwargs["fused_add_rms_norm"] = ["native"]`) + +`?:557` [minor] — Part 1 vs Part 2 for Qwen: Part 2 switches Qwen RMSNorm to torch.nn.RMSNorm (qwen_image_transformer.py:557, :1009), which bypasses kernel_config.ir_op_priority rms_norm dispatch. Part 1 still matters fleet-wide—e.g. flux_transformer.py:19 still uses vllm.model_executor.layers.layernorm.RMSNorm—and restores unconditional ["vllm_c", "native"] (platform.py:253; test_cuda_ir_op_priority.py:26). (evidence: qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; qwen_image_transformer.py:1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)`; unchanged by this diff, present in PR-time tree: flux_transformer.py:19 `from vllm.model_executor.layers.layernorm import RMSNorm`; platform.py:253 `default = ["vllm_c", "native"]`; forward_context.py:205 `vllm_config.kernel_config.ir_op_priority.set_priority()`; test_cuda_ir_op_priority.py:26 `assert priority.rms_norm == ["vllm_c", "native"]`) + +`?:620` [minor] — Cosmos3 native-only need — `get_cosmos3_ir_op_priority_func` (registered in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`) forces `rms_norm`/`fused_add_rms_norm` to `["native"]`; residual: any future model needing native must register its own hook in the diffusion IR-op-priority registry. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:620-621 `priority_kwargs["rms_norm"] = ["native"]` and `priority_kwargs["fused_add_rms_norm"] = ["native"]`; vllm_omni/diffusion/registry.py:549 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func"`; vllm_omni/diffusion/worker/diffusion_worker.py:174-178 platform default then optional per-model `ir_op_priority_func` merge) + +`?:620` [minor] — Cosmos3 remains the only per-model IR op priority override: registered at `registry.py:549` and enforced by `get_cosmos3_ir_op_priority_func`, which forces `rms_norm`/`fused_add_rms_norm` to native-only at `pipeline_cosmos3.py:620-621`; no other static `_DIFFUSION_IR_OP_PRIORITY_FUNCS` entries exist. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/registry.py:549 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",` — sole `_DIFFUSION_IR_OP_PRIORITY_FUNCS` entry; vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:620 `priority_kwargs["rms_norm"] = ["native"]`; vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:621 `priority_kwargs["fused_add_rms_norm"] = ["native"]`; vllm_omni/diffusion/worker/diffusion_worker.py:174-178 platform default is merged then overridden by `ir_op_priority_func` when registered) + +`pipeline_cosmos3.py:620` [minor] — Cosmos3 native-only concern is resolved: `get_cosmos3_ir_op_priority_func` (pipeline_cosmos3.py:620-621) overrides platform default to `["native"]` for rms_norm/fused_add_rms_norm. Residual: other diffusion models still using vllm `RMSNorm` (no pipeline-specific override in registry) inherit the restored `["vllm_c", "native"]` platform priority from this PR. (evidence: unchanged by this diff, present in PR-time tree: pipeline_cosmos3.py:620 `priority_kwargs["rms_norm"] = ["native"]`; pipeline_cosmos3.py:621 `priority_kwargs["fused_add_rms_norm"] = ["native"]`; changed in this PR: platform.py:253 `default = ["vllm_c", "native"]`; unchanged by this diff: registry.py:549 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func"` (sole pipeline IR-op-priority override)) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [nit] — This hunk replaces vllm.model_executor.layers.layernorm.RMSNorm with torch.nn.RMSNorm at norm_q/norm_k (557-558), norm_added_q/norm_added_k (595-596), and txt_norm (1009). ltx2_transformer.py:642-643 already documents diffusers parity for the same choice — add a brief comment at these five sites to discourage reverting to vLLM RMSNorm. (evidence: qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; qwen_image_transformer.py:558 `self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; qwen_image_transformer.py:595 `self.norm_added_q = nn.RMSNorm(head_dim, eps=eps)`; qwen_image_transformer.py:596 `self.norm_added_k = nn.RMSNorm(head_dim, eps=eps)`; qwen_image_transformer.py:1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)` — unchanged by this diff, present in PR-time tree: ltx2_transformer.py:642 `# At TP=1, use torch.nn.RMSNorm which is numerically identical to`; ltx2_transformer.py:643 `# the diffusers reference (verified via hook-based comparison).`) + +```suggestion +# torch.nn.RMSNorm for diffusers parity under torch.compile (see ltx2_transformer.py; #4074/#3588). + self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity() +``` + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_train/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v17cb_train/pr5009.trace.json.gz new file mode 100644 index 000000000..9ab6a4a70 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_train/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_val/manifest.json b/eval/dataset/arms/copilot_v17cb_val/manifest.json new file mode 100644 index 000000000..15f4a2b10 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v17cb_val", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 5, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4810.cost.json b/eval/dataset/arms/copilot_v17cb_val/pr4810.cost.json new file mode 100644 index 000000000..d3c36d467 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4810.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 666.8, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_val/runs/pr4810/run-20260817-123559-b35345", + "recorded_at": "2026-08-17T12:47:05", + "attempts": [ + { + "invocation_id": "inv-ed74f0cc", + "outcome": "run:run-20260817-123559-b35345", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-123559-b35345", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-123559-b35345", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2723, + "minutes": 11.1, + "ci_minutes": 0, + "input_tokens": 623543, + "output_tokens": 94534, + "tool_calls": 420, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4627, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.26, + "gate": 1.81, + "review": 659.56, + "report": 0.06 + }, + "time_to_first_result_s": 665.68 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 23453 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4810.md b/eval/dataset/arms/copilot_v17cb_val/pr4810.md new file mode 100644 index 000000000..d06fd3477 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4810.md @@ -0,0 +1,81 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] mammoth/mimo old per-weight `get_cache_scale` + `loaded_weight[0]` branch removed; scales now flow through mapper + `maybe_remap_kv_scale_name` like upstream standard loaders. Residual: no L4 quantized checkpoint load shown. +- [resolved] Mammoth/MiMo direct-loader mapper insertion mirrors upstream vLLM #43167 contract; residual: mapper=None with quant_config set silently skips rename (same as no-op mapper) +- [resolved] mimo_audio_llm.py:1162-1163 model. prefix doubling — pre-existing, not introduced by this diff +- [resolved] MammothModa2 double-mapper concern: second apply is no-op on already-renamed .attn.*_scale names; checked idempotency via replace pattern +- [resolved] PR merge commit f9665cc2 rebased onto main — author's pre-merge failure log may not describe head; re-run `pytest -q tests/model_executor/models/test_kv_cache_scale_mapper.py` at f9665cc2 before merge. +- [resolved] hunyuan removed-branch loaded_weight[0] indexing — residual checked: scales now reach maybe_remap_kv_scale_name/default_weight_loader after outer mapper, same as upstream hunyuan_v1.py:882 +- [claim-verified] PR lists four model_executor loaders; grep shows no `.get_cache_scale(` remains in those four files at head. +- [claim-verified] _STALE_API_FILES source-scan guard — all four listed model_executor files pass .get_cache_scale( absence check +- [claim-verified] Repo-wide grep at head: zero .get_cache_scale( in the four changed model_executor files +- [claim-verified] Repo grep at head: only remaining production `get_cache_scale` caller is `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238`; four PR-touched model-executor files are clean. +- [claim-verified] `get_cache_scale_mapper` requires vLLM ≥0.23.0 (per PR body); `docker/Dockerfile.rocm` pins `vllm/vllm-openai-rocm:v0.24.0` — compatible with direct call (no getattr). +- [claim-verified] PR scope is four model_executor loaders — grep shows only diffusion hunyuan_image3_transformer.py:2238 still has .get_cache_scale( +- [claim-verified] test_kv_cache_scale_mapper pytestmark core_model+cpu — collected in .buildkite/test-ready.yml tests/model_executor lane +- [claim-refuted] PR scope 'all affected custom loaders' — repo grep at head f9665cc: only production get_cache_scale caller left is vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:~2238` [major] — This PR removes manual get_cache_scale handling from four model_executor loaders (e.g. hunyuan_image3.py now relies on the outer AutoWeightsLoader mapper at lines 225–226) but leaves hunyuan_image3_transformer.py:2238 calling quant_config.get_cache_scale(name)—the sole remaining production caller in vllm_omni. HunyuanImage3Pipeline.load_weights (pipeline_hunyuan_image3.py:458–462) uses AutoWeightsLoader on self, which reaches HunyuanImage3Model.load_weights; with quant_config set, the stale branch AttributeErrors on removed vLLM API before maybe_remap_kv_scale_name at 2348 runs. Apply the same fix here or track as an explicit follow-up. (evidence: unchanged by this diff, present in PR-time tree: hunyuan_image3_transformer.py:2238 `if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`; repo-wide grep shows only this production `.get_cache_scale(` caller; pipeline_hunyuan_image3.py:458 `loader = AutoWeightsLoader(` / :462 `return loader.load_weights(weights)`; hunyuan_image3.py:225-226 `# KV-cache scales are renamed to .attn.{k,v,q}_scale by the` / `# quant_config mapper in the outer AutoWeightsLoader.` (AR loader fixed in this PR)) + +```suggestion +# KV-cache scales are renamed to .attn.{k,v,q}_scale by the + # quant_config mapper in the outer AutoWeightsLoader. + is_found = False +``` + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:112` [minor] — test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale (line 112) only feeds pre-mapped _MAPPED_SCALE_NAME into HunyuanModel.load_weights / Qwen2Model.load_weights. Production uses AutoWeightsLoader at the parent entry (hunyuan_image3.py:2178, qwen_old.py:444) with unmapped checkpoint names (_SOURCE_SCALE_NAME). Consider adding one case that drives _SOURCE_SCALE_NAME through AutoWeightsLoader or the parent load_weights and asserts the inner scale param loads; test_direct_custom_loaders_apply_cache_scale_mapper already covers the direct-loader path at line 144. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:22 `SOURCE_SCALE_NAME = "layers.0.self_attn.k_proj.output_scale"`; tests/model_executor/models/test_kv_cache_scale_mapper.py:23 `_MAPPED_SCALE_NAME = "layers.0.self_attn.attn.k_scale"`; tests/model_executor/models/test_kv_cache_scale_mapper.py:112 `loaded = load_weights(fake_model, [(_MAPPED_SCALE_NAME, torch.ones(()))])`; tests/model_executor/models/test_kv_cache_scale_mapper.py:144 `loaded = load_weights(fake_model, [(_SOURCE_SCALE_NAME, torch.ones(()))])`; vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py:2178-2183 `loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes,); loaded_params = loader.load_weights(weights)`; vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py:443-448 `def load_weights(...): loader = AutoWeightsLoader(self, ...); return loader.load_weights(weights)`) + +```suggestion +from vllm.model_executor.models.utils import AutoWeightsLoader + +# Inside delegated test, after building fake_model: +loader = AutoWeightsLoader(fake_model) +loaded = loader.load_weights([( _SOURCE_SCALE_NAME, torch.ones(()))]) +assert loaded == {_MAPPED_SCALE_NAME} +torch.testing.assert_close(fake_model.param.data, torch.ones(())) +``` + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:15` [minor] — _STALE_API_FILES (lines 15–20) scans four model_executor loaders for `.get_cache_scale(` but omits `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py`, which still uses the removed API at line 2238 (unchanged by this PR). After migrating that loader, add the diffusion path to `_STALE_API_FILES` so the source-scan regression covers all affected custom loaders. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:15 `_STALE_API_FILES = [` — lists only hunyuan_image3.py, mammoth_moda2.py, mimo_audio_llm.py, qwen2_old.py; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 `if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`) + +```suggestion +_STALE_API_FILES = [ + "vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py", + "vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py", + "vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py", + "vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py", + "vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py", +] +``` + +`?:?` [minor] — mammoth_moda2/mimo_audio: removed per-weight get_cache_scale + loaded_weight[0] handling; KV scales now go through quant_config.get_cache_scale_mapper().apply() then maybe_remap_kv_scale_name in the standard loader loop (same pattern as other upstream-aligned loaders). Residual: PR adds only CPU/mock mapper tests—no L4 quantized checkpoint load shown. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:487 `cache_scale_mapper = self.quant_config.get_cache_scale_mapper()`; mammoth_moda2.py:489 `weights = cache_scale_mapper.apply(weights)`; mammoth_moda2.py:507 `name = maybe_remap_kv_scale_name(name, params_dict)`; mimo_audio_llm.py:1157-1159 `cache_scale_mapper = self.quant_config.get_cache_scale_mapper()` / `weights = cache_scale_mapper.apply(weights)`; mimo_audio_llm.py:1188 `name = maybe_remap_kv_scale_name(name, params_dict)`; tests/model_executor/models/test_kv_cache_scale_mapper.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`) + +`?:486` [minor] — MammothModa2 (`mammoth_moda2.py:486-489`) and MiMo-Audio (`mimo_audio_llm.py:1156-1159`) direct loaders mirror upstream AutoWeightsLoader (#43167): apply `get_cache_scale_mapper().apply(weights)` only when the mapper is non-None; `quant_config` set with `mapper=None` skips rename via the mapper path, same as upstream. (evidence: mammoth_moda2.py:486-489 `if self.quant_config is not None: cache_scale_mapper = self.quant_config.get_cache_scale_mapper(); if cache_scale_mapper is not None: weights = cache_scale_mapper.apply(weights)`; mimo_audio_llm.py:1156-1159 identical guard; unchanged upstream contract (not in this diff): vllm/model_executor/models/utils.py AutoWeightsLoader `if cache_scale_mapper is not None: weights = mapper.apply(weights)`) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:3` [nit] — This PR adds tests/model_executor/models/test_kv_cache_scale_mapper.py with pytestmark core_model+cpu (line 12) for merge-gate CI, but the PR description says the file may be removed after maintainer validation. Resolve the contradiction: keep it as permanent #4809 regression coverage (recommended) and update the PR body accordingly. (evidence: PR description Test Result section: 'Will remove it from the PR once this is validated by a maintainer as I don't think this coverage is necessary!') + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v17cb_val/pr4810.trace.json.gz new file mode 100644 index 000000000..f0e536527 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_val/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4816.cost.json b/eval/dataset/arms/copilot_v17cb_val/pr4816.cost.json new file mode 100644 index 000000000..2fd310299 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4816.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 452.4, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_val/runs/pr4816/run-20260817-125210-a2dba3", + "recorded_at": "2026-08-17T12:59:42", + "attempts": [ + { + "invocation_id": "inv-6874e4e4", + "outcome": "run:run-20260817-125210-a2dba3", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-125210-a2dba3", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-125210-a2dba3", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2097, + "minutes": 7.53, + "ci_minutes": 0, + "input_tokens": 474860, + "output_tokens": 74116, + "tool_calls": 322, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3466, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.62, + "gate": 1.56, + "review": 446.49, + "report": 0.06 + }, + "time_to_first_result_s": 451.71 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 16574 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4816.md b/eval/dataset/arms/copilot_v17cb_val/pr4816.md new file mode 100644 index 000000000..1177d063d --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4816.md @@ -0,0 +1,87 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Stale-consumer concern after rename: no survivors in `tests/**`, `docs/**`, `examples/**`, or `**/*.md` +- [resolved] Dual error paths both aligned: handler-missing uses `getattr(state, "serving_tokenization")`; speech voice/list/delete errors use `base(raw_request)` — residual checked: pure-diffusion still sets `serving_tokenization = None` (`api_server.py:754`) so `base()` can return `None` on speech error paths; pre-existing, unchanged by this diff +- [resolved] AttributeError `'State' object has no attribute 'serving_tokenization'` on omni speech servers — fixed by setting attribute at init; residual checked: pure diffusion still sets `serving_tokenization=None` at :754 while `openai_serving_speech` is active at :784, so `_create_speech_error_json_response` ValueError paths can still fail when `base()` returns None (pre-existing, out of rename scope) +- [resolved] pure-diffusion branch still sets serving_tokenization=None (api_server.py:754); voice 404 via base() would still fail if base returns None — checked residual: speech handler also None, endpoints unsupported, same failure class pre-PR +- [resolved] AttributeError on `base(raw_request)` when state key mismatched — fixed by init/fallback rename; residual checked: `list_voices`/`upload_voice` no-handler still route through `_create_speech_error_json_response` → `base()` at api_server.py:376, which now finds `serving_tokenization` when init ran. +- [resolved] Pure-diffusion `serving_tokenization = None` at api_server.py:754 — pre-existing; speech handler also None so unsupported endpoints remain 404/500 class unchanged by this rename-only diff. +- [claim-verified] PR body: upstream `base()` reads `request.app.state.serving_tokenization` — `api_server.py:376` `err = base(raw_request).create_error_response(...)` in `_create_speech_error_json_response` +- [claim-verified] PR body: mismatch turned caught ValueError into 500 — `upload_voice` `except ValueError` at `api_server.py:1482-1483` routes through `_create_speech_error_json_response` → `base()` +- [claim-verified] PR scope: rename only — `diff_stat` vs main: 2 files, 9 insertions/9 deletions, no other hunks +- [claim-verified] PR claim of misalignment with upstream `instrumentator.basic.tokenization` — all in-repo producers/consumers renamed; no stale docs/yaml refs +- [claim-verified] PR description mechanism (AttributeError in base() → 500 instead of 400) matches api_server.py:376 and omni_init_app_state setter sites +- [claim-verified] PR claim that upstream now reads `request.app.state.serving_tokenization`: repo grep shows 0 `openai_serving_tokenization` and 7 `serving_tokenization` refs, all in the two changed files. +- [validated] Repo-wide grep `openai_serving_tokenization`: 0 matches at head +- [validated] All `serving_tokenization` writers: `api_server.py:754` (pure diffusion `None`), `api_server.py:1017` (`ServingTokenization(...)`) + +`tests/entrypoints/openai_api/test_serving_speech.py:~2382` [minor] — This PR renames Omni app state to `serving_tokenization` (e.g. `omni_init_app_state` at api_server.py:1017), matching upstream `base(raw_request)` used by `_create_speech_error_json_response` (api_server.py:376). Speech error tests—including `test_api_server_upload_voice_value_error_returns_400` (test_serving_speech.py:2382)—all call `_patch_api_server_base`, which patches `api_server_module.base` (line 2352) and would pass even if init still wrote `openai_serving_tokenization`. Add one regression test that does not patch `base`, sets only `raw_request.app.state.serving_tokenization` with a `create_error_response` stub, triggers `upload_voice` with a handler `ValueError`, and asserts HTTP 400. (evidence: test_serving_speech.py:2352 `mocker.patch.object(api_server_module, "base", return_value=fake_base)`; test_serving_speech.py:2382-2383 `def test_api_server_upload_voice_value_error_returns_400(mocker: MockerFixture):` / `_patch_api_server_base(mocker)`; api_server.py:376 `err = base(raw_request).create_error_response(`; api_server.py:1017 `state.serving_tokenization = ServingTokenization(`; unchanged by this diff, present in upstream vLLM main vllm/entrypoints/serve/instrumentator/basic.py `return request.app.state.serving_tokenization`) + +```suggestion +def test_api_server_upload_voice_value_error_uses_serving_tokenization_state(mocker: MockerFixture): + handler = mocker.MagicMock() + handler.upload_voice = mocker.AsyncMock(side_effect=ValueError("Could not decode audio file")) + raw_request = _make_api_server_request(handler) + tokenization = mocker.MagicMock() + tokenization.create_error_response.side_effect = lambda **kwargs: ErrorResponse( + error=ErrorInfo(message=kwargs["message"], type=kwargs.get("err_type", "BadRequestError"), param=None, code=400) + ) + raw_request.app.state.serving_tokenization = tokenization + + response = asyncio.run( + api_server_module.upload_voice( + raw_request, + audio_sample=mocker.MagicMock(), + speaker_embedding=None, + consent="cons_test", + name="probe", + ) + ) + + _assert_openai_error_response(response, status_code=400, message="Could not decode audio file") +``` + +`api_server.py:754` [minor] — Aligned on serving_tokenization: handler-missing fallbacks use getattr(state, "serving_tokenization") (e.g. api_server.py:1248); voice/list/delete errors use base(raw_request) through _create_speech_error_json_response (api_server.py:376). Residual pre-existing: pure-diffusion sets serving_tokenization = None (api_server.py:754), so upstream base() returns None and AttributeError is possible on voice error paths without a None guard. (evidence: api_server.py:754 `state.serving_tokenization = None` — pure-diffusion init; api_server.py:376 `err = base(raw_request).create_error_response(` — voice/list/delete error helper; api_server.py:1248 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)` — handler-missing fallback; unchanged by this diff, present in PR-time tree (upstream vllm): `return request.app.state.serving_tokenization` in tokenization() called by base()) + +`?:754` [minor] — Rename fix is correct: init now always sets `serving_tokenization`, matching upstream `base()`/`tokenization()`. Residual (pre-existing, out of scope): pure diffusion sets `serving_tokenization=None` (api_server.py:754) while `openai_serving_speech` is active (api_server.py:784), so `_create_speech_error_json_response` still fails via `base(raw_request).create_error_response(...)` when `base()` returns None — affects voice upload/delete validation and ValueError handlers, not only ValueError. (evidence: unchanged by this diff, present in PR-time tree: vllm/entrypoints/serve/instrumentator/basic.py:23 `return request.app.state.serving_tokenization`; api_server.py:754 `state.serving_tokenization = None`; api_server.py:784 `state.openai_serving_speech = OmniOpenAIServingSpeech.for_diffusion(`; api_server.py:376 `err = base(raw_request).create_error_response(`; api_server.py:1482-1483 `except ValueError as e:` / `return _create_speech_error_json_response(raw_request, str(e))`) + +`vllm_omni/entrypoints/openai/api_server.py:~1157` [minor] — Rename is applied in `create_chat_completion`'s no-handler fallback (`serving_tokenization`), and speech/audio no-handler tests were updated, but there is still no API-server test covering the chat path with `Omnichat` unset and `serving_tokenization` stubbed—add a sibling test to lock the renamed getattr contract. (evidence: vllm_omni/entrypoints/openai/api_server.py:1157 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)`; tests/entrypoints/openai_api/test_serving_speech.py:2530 `def test_api_server_create_speech_without_handler_returns_404(mocker: MockerFixture):` and :2533 `raw_request.app.state.serving_tokenization = fake_base`; repo-wide grep under tests/ finds no `test_api_server_create_chat_completion_without_handler` or other chat no-handler test setting `serving_tokenization`.) + +`vllm_omni/entrypoints/openai/api_server.py:~376` [nit] — Gate report shows buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, and buildkite/vllm-omni-npu-ci failing at head, but this PR only renames an app.state key in api_server.py and matching test fixtures—no platform worker or kernel files changed. Can you confirm whether those three lanes are also red on main, or if there is an unexpected coupling? (evidence: gate_report lists failing checks buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, buildkite/vllm-omni-npu-ci; diff_stat shows only vllm_omni/entrypoints/openai/api_server.py and tests/entrypoints/openai_api/test_serving_speech.py changed.) + +`tests/entrypoints/openai_api/test_serving_speech.py:~2531` [nit] — In `test_api_server_create_speech_without_handler_returns_404` (and the batch/audio-generate siblings at :2544/:2612), `_patch_api_server_base` is redundant: when `Omnispeech`/`OmniAudioGenerate` returns None, the handler-missing branch calls `getattr(raw_request.app.state, "serving_tokenization")` and `base_server.create_error_response` (api_server.py:1248–1258, :1286–1297, :1329–1339); `base()` is only used by `_create_speech_error_json_response` (api_server.py:376), which these endpoints do not hit. Replace `_patch_api_server_base` with a direct `serving_tokenization` mock. (evidence: unchanged by this diff, present in the PR-time tree: api_server.py:1248 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)`; api_server.py:1254 `err = base_server.create_error_response(`; api_server.py:376 `err = base(raw_request).create_error_response(`; test_serving_speech.py:2352 `mocker.patch.object(api_server_module, "base", return_value=fake_base)`) + +```suggestion +def test_api_server_create_speech_without_handler_returns_404(mocker: MockerFixture): + fake_base = mocker.MagicMock() + fake_base.create_error_response.side_effect = lambda **kw: ErrorResponse(error=ErrorInfo(message=kw["message"], type=kw.get("err_type", "NotFoundError"), code=404)) + raw_request = _make_api_server_request(None, path="/v1/audio/speech") + raw_request.app.state.serving_tokenization = fake_base +``` + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v17cb_val/pr4816.trace.json.gz new file mode 100644 index 000000000..5b144488f Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_val/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4825.cost.json b/eval/dataset/arms/copilot_v17cb_val/pr4825.cost.json new file mode 100644 index 000000000..131699367 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4825.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 350.9, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_val/runs/pr4825/run-20260817-124619-9cb2d4", + "recorded_at": "2026-08-17T12:52:10", + "attempts": [ + { + "invocation_id": "inv-b9e66f89", + "outcome": "run:run-20260817-124619-9cb2d4", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-124619-9cb2d4", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-124619-9cb2d4", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2555, + "minutes": 5.84, + "ci_minutes": 0, + "input_tokens": 633948, + "output_tokens": 76628, + "tool_calls": 287, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3182, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.84, + "gate": 1.75, + "review": 344.55, + "report": 0.06 + }, + "time_to_first_result_s": 350.24 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 19090 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4825.md b/eval/dataset/arms/copilot_v17cb_val/pr4825.md new file mode 100644 index 000000000..048a0d20c --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4825.md @@ -0,0 +1,82 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4825, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac3a2ce5a9b9) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] Concern: `to_out.0` WeightsMapper added in `632efb42` then reverted in `ac3a2ce5` (`weights_mapper=None` at manager.py:309); SDXL attn stores output proj as `to_out.0` (`sdxl_unet.py:151-155,243`) while PEFT keys often use `to_out`; `_get_lora_weights` (manager.py:499-509) has no `.to_out.0`↔`.to_out` alias — residual for adapters heavy on `to_out` targets; author's manual cosmic-tf test suggests primary gap was missing `unet` scan. +- [resolved] Commit ac3a2ce5 reverted `.to_out.0.` WeightsMapper from 632efb42; SDXL UNet registers output proj as `to_out.0` (`sdxl_unet.py:151-154`), so keeping mapper would desync LoRAModel keys (`...to_out.`) from `_lora_modules` paths (`...to_out.0`) during activation +- [resolved] ac3a2ce5 reverted `weights_mapper` `.to_out.0.` normalization and its test — residual: SDXL `sdxl_unet.py` uses `to_out` ModuleList (`to_out[0]` RowParallelLinear); `_match_target_modules` matches `.*\.to_out$` not `*.to_out.0` +- [resolved] at head, `weights_mapper=None` in `_load_adapter` (manager.py:306) — PEFT keys ending in `to_out.0` may not load for those layers; author manual test still shows LoRA effect via other targets +- [resolved] ac3a2ce5 reverted `WeightsMapper` `to_out.0` normalization and its unit test; residual: SDXL `to_out` is `nn.ModuleList([RowParallelLinear])` (sdxl_unet.py) — adapters keyed as `to_out.0` untested at head +- [claim-verified] PR body: SDXL denoiser is `unet` — `pipeline_sdxl.py:110` sets `self.unet = SDXLUNet2DConditionModel(...)`; pre-change `default_components` omitted `unet` (`file_at_base` manager.py:371). +- [claim-verified] Root cause: `_compute_supported_lora_modules` scans the whole pipeline (manager.py:124) so adapters could load, but `_replace_layers_with_lora` skipped `unet`, yielding zero wrapped layers and base-identical output. +- [claim-verified] Commit timeline `revert to out chnage` — at head, diff is only `unet` in `default_components` + unet test; no `to_out` hunk remains. +- [claim-verified] PR body SDXL before/after images claim LoRA effect; at head diff only adds unet scan + unit test, no remaining to_out normalization code +- [claim-verified] Commit timeline `revert to out chnage` matches net diff: no WeightsMapper or `test_lora_manager_load_adapter_uses_to_out_normalization_mapper` at head +- [claim-verified] PR SDXL manual test (stabilityai/stable-diffusion-xl-base-1.0 + cosmic-tf adapter) shows visual delta; no in-repo automated SDXL LoRA regression test +- [claim-verified] `tests/diffusion/lora/test_diffusion_lora.py` is Z-Image-only (`advanced_model`, transformer path) — does not cover SDXL/unet +- [claim-verified] `.buildkite/test-merge.yml` runs `tests/diffusion/lora/ -m 'advanced_model and cuda'` (Z-Image E2E) — no SDXL LoRA lane +- [claim-refuted] No `sdxl`/`SDXL` references under `tests/` — PR SDXL LoRA image claim has no automated in-repo regression + +`tests/diffusion/lora/test_lora_manager.py:580` [minor] — This diff adds `"unet"` to `default_components` to fix SDXL LoRA applying to zero layers, but the only new automation is `test_lora_manager_discovers_unet_component` (line 580), which calls `_replace_layers_with_lora` directly with monkeypatched `from_layer_diffusion` and `_FakeLinearBase` rather than exercising real SDXL `RowParallelLinear`/`QKVParallelLinear` or the `add_adapter`→`_load_adapter`→`generate()` path. Existing CI LoRA image coverage targets Z-Image Turbo (`test_images_generations_lora.py`, `test_diffusion_lora.py`); `tests/e2e/online_serving/test_sdxl.py` covers base SDXL generation only — there is no SDXL LoRA regression under `tests/diffusion/lora/`. Add an SDXL `advanced_model` regression (load a tiny on-disk adapter and assert `generate()` output differs from base, following `test_diffusion_lora.py`) so a future `"unet"` revert or SDXL layer-type breakage is caught in CI. (evidence: tests/diffusion/lora/test_lora_manager.py:580 `def test_lora_manager_discovers_unet_component(monkeypatch):`; tests/diffusion/lora/test_lora_manager.py:593 `monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion)`; tests/diffusion/lora/test_lora_manager.py:614 `manager._replace_layers_with_lora(peft_helper)`; tests/diffusion/lora/test_diffusion_lora.py:20 `models = ["Tongyi-MAI/Z-Image-Turbo"]`; tests/diffusion/lora/test_diffusion_lora.py:24 `@pytest.mark.advanced_model`; tests/diffusion/lora/test_images_generations_lora.py:30 `MODEL = "Tongyi-MAI/Z-Image-Turbo"`; unchanged by this diff, present in PR-time tree: tests/e2e/online_serving/test_sdxl.py:40 `model="stabilityai/stable-diffusion-xl-base-1.0"` (base T2I, no LoRA); vllm_omni/diffusion/lora/manager.py:376 `"unet"` in `default_components`) + +`vllm_omni/diffusion/lora/manager.py:~309` [minor] — This diff adds `"unet"` to `default_components` (manager.py:375-380) so SDXL-style pipelines scan the denoiser, but ac3a2ce5 reverted the scoped `WeightsMapper({".to_out.0.": ".to_out."})`, so `_load_adapter` still passes `weights_mapper=None` (manager.py:309). SDXL attention stores the output projection in a `to_out` `ModuleList` (`sdxl_unet.py:151-155`; forward uses `to_out[0]` at line 243), so live module paths end in `to_out.0` while many PEFT checkpoints/`target_modules` use bare `to_out`. `_match_target_modules` suffix-matches only the bare target (`utils.py:24-26`), and `_get_lora_weights` does not normalize `to_out.0`↔`to_out` (manager.py:499-509). Please confirm the PR-tested adapter does not rely on this mapping, or restore the scoped mapper so `to_out`-targeting SDXL adapters are not silently skipped after `unet` discovery. (evidence: unchanged by unet diff, present at PR head: manager.py:309 `weights_mapper=None,`; manager.py:375-380 `"unet",` in `default_components`; sdxl_unet.py:151 `self.to_out = nn.ModuleList(`; sdxl_unet.py:243 `attn_out, _ = self.attn1.to_out[0](attn_out)`; utils.py:25 `re.match(rf".*\.{target_module}$", module_name) or target_module == module_name`; manager.py:508 `module_suffix = full_module_name.split(".")[-1]`) + +```suggestion +weights_mapper = WeightsMapper( + orig_to_new_substr={ + ".to_out.0.": ".to_out.", + } + ) + + lora_model = LoRAModel.from_local_checkpoint( + lora_path, + expected_lora_modules=self._expected_lora_modules, + peft_helper=peft_helper, + lora_model_id=lora_request.lora_int_id, + device="cpu", + dtype=self.dtype, + model_vocab_size=None, + tensorizer_config_dict=lora_request.tensorizer_config_dict, + weights_mapper=weights_mapper, + ) +``` + +`manager.py:309` [minor] — Residual (minor): `weights_mapper=None` at manager.py:309 after ac3a2ce5 reverted the 632efb42 WeightsMapper; SDXL attn modules register as `...to_out.0` (sdxl_unet.py:151-155,243) while many PEFT keys use `...to_out`; `_get_lora_weights` (manager.py:499-509) has no `.to_out.0`↔`.to_out` alias—may still miss `to_out`-heavy adapters even with `unet` scanning. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/lora/manager.py:309 `weights_mapper=None,`; vllm_omni/diffusion/lora/manager.py:499-509 `_get_lora_weights` ends with `module_suffix = full_module_name.split(".")[-1]` / `return lora_model.get_lora(module_suffix)` (no `to_out.0`↔`to_out` alias); vllm_omni/diffusion/models/sdxl/sdxl_unet.py:151-155 `self.to_out = nn.ModuleList([RowParallelLinear(inner_dim, dim, bias=True),])`; sdxl_unet.py:243 `attn_out, _ = self.attn1.to_out[0](attn_out)`) + +`?:25` [minor] — Residual after ac3a2ce5 revert: SDXL UNet attention uses `to_out` ModuleList with `RowParallelLinear` at `to_out[0]` (`sdxl_unet.py`), so scanned module paths end in `to_out.0`; `_match_target_modules` suffix-matches only `.*\.to_out$`, and `_load_adapter` no longer normalizes `.to_out.0.` checkpoint keys (`weights_mapper=None`). PEFT adapters targeting `to_out` may skip attention output projections. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/sdxl/sdxl_unet.py:151 `self.to_out = nn.ModuleList(` / :153 `RowParallelLinear(inner_dim, dim, bias=True),`; vllm_omni/diffusion/models/sdxl/sdxl_unet.py:243 `attn_out, _ = self.attn1.to_out[0](attn_out)`; vllm_omni/diffusion/lora/utils.py:25 `re.match(rf".*\.{target_module}$", module_name) or target_module == module_name`; vllm_omni/diffusion/lora/manager.py:309 `weights_mapper=None,`) + +`manager.py:309` [minor] — At head, `_load_adapter` passes `weights_mapper=None` to `LoRAModel.from_local_checkpoint` (manager.py:309). The LoRA path has no `to_out.0` key remapping, while other diffusion loaders remap that suffix (e.g. qwen_image_transformer.py:56-57), so PEFT keys ending in `to_out.0` may be dropped and those layers stay at base weights; other targeted modules can still show LoRA effect. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/lora/manager.py:309 `weights_mapper=None,`; vllm_omni/diffusion/lora/utils.py:38-40 `ensures these sublayer keys are not dropped when loading a LoRA checkpoint.`; vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:56-57 `if ".to_out.0." in name:` / `name = name.replace(".to_out.0.", ".to_out.")`) + +`?:309` [minor] — At head ac3a2ce5, `DiffusionLoRAManager._load_adapter` passes `weights_mapper=None` after reverting the `.to_out.0.`→`.to_out.` WeightsMapper and its unit test. SDXL UNet attention defines `to_out` as `nn.ModuleList([RowParallelLinear])` (`sdxl_unet.py:151-178`), so PEFT adapters keyed as `to_out.0` are untested and may not map to vLLM module names without that mapper. (evidence: unchanged by this PR's unet-scan diff, present at PR head: vllm_omni/diffusion/lora/manager.py:309 `weights_mapper=None,`; vllm_omni/diffusion/models/sdxl/sdxl_unet.py:151 `self.to_out = nn.ModuleList(` and :153 `RowParallelLinear(inner_dim, dim, bias=True),`; commit ac3a2ce5 removed `weights_mapper = WeightsMapper(orig_to_new_substr={".to_out.0.": ".to_out."})` and `test_lora_manager_load_adapter_uses_to_out_normalization_mapper`; tests/diffusion/lora has zero `to_out` matches at head) + +`vllm_omni/diffusion/lora/manager.py:370` [minor] — Adding "unet" to default_components (lines 375–381) means SDXL pipelines with a top-level unet now get LoRA layer replacement; update line 370—"other pipelines unchanged" is stale. Pipelines without matching component attribute names are skipped via hasattr. (evidence: manager.py:370 `# ``_lora_components`` (opt-in; other pipelines unchanged).`; manager.py:380 `"unet",`; unchanged by this diff, present in PR-time tree: pipeline_sdxl.py:110 `self.unet = SDXLUNet2DConditionModel(od_config=od_config)`) + +```suggestion +# Default denoising components plus any a pipeline opts into via + # ``_lora_components``. Pipelines without these attribute names are skipped. +``` + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v17cb_val/pr4825.trace.json.gz new file mode 100644 index 000000000..d06861032 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_val/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4837.cost.json b/eval/dataset/arms/copilot_v17cb_val/pr4837.cost.json new file mode 100644 index 000000000..b6548c96b --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4837.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 682.5, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_val/runs/pr4837/run-20260817-124705-0398ec", + "recorded_at": "2026-08-17T12:58:27", + "attempts": [ + { + "invocation_id": "inv-8526e887", + "outcome": "run:run-20260817-124705-0398ec", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-124705-0398ec", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-124705-0398ec", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3071, + "minutes": 11.37, + "ci_minutes": 0, + "input_tokens": 689018, + "output_tokens": 110060, + "tool_calls": 517, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4844, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.41, + "gate": 1.63, + "review": 675.78, + "report": 0.07 + }, + "time_to_first_result_s": 681.87 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 17663 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4837.md b/eval/dataset/arms/copilot_v17cb_val/pr4837.md new file mode 100644 index 000000000..6efc3eea8 --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4837.md @@ -0,0 +1,78 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4837, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] Base had `if already_submitted and len(diffusion_prompt)==1` — update path unwrapped, initial path passed list to submit_initial; residual len>1 still fails at stage_pool by design +- [resolved] Old outputs[0]+hasattr unpack assumed indexable/tuple return; incompatible with DiffusionOutput dataclass return type +- [resolved] already_submitted gating removed — residual checked: submit_initial vs submit_update still selected at 1295-1309; only payload shape changes +- [resolved] Old outputs[0] indexing — DiffusionOutput has no __getitem__; PR correctly uses .output +- [resolved] Ming outputs[0]/hasattr fallback removed — ZImagePipeline.forward returns DiffusionOutput at pipeline_z_image.py:720; dataclass has no __getitem__ (data.py:1180); residual checked: diffusion_model_runner still consumes `.output`. +- [resolved] Concern that update-only paths still unwrap: same if len(diffusion_prompt)==1 block serves both submit_initial and submit_update — residual len>1 still passes list and fails at stage_pool +- [resolved] hasattr `.images` / `outputs[0]` fallbacks removed — ZImagePipeline contract is `.output` tensor at pipeline_z_image.py:720; residual checked: diffusion_model_runner consumes `.output` via `_normalize_pipeline_outputs` (diffusion_model_runner.py:57-59). +- [resolved] `isinstance(raw, torch.Tensor)` guard retained at :466-467 — non-tensor parent output still fails loud with `RuntimeError`. +- [claim-verified] PR root cause matches #4839: list-prompt ValueError on first diffusion submit when bridge returns list[dict] +- [claim-verified] PR manual test uses ming_flash_omni_image deploy config and chat completions with modalities image +- [claim-verified] thinker2imagegen returns [{"prompt":"","extra":...}] at ming_flash_omni.py:435; Ming imagegen initial forward was the broken path +- [claim-verified] PR resolves linked issue #4839 list-prompt ValueError on initial submit +- [claim-verified] PR claim: thinker2imagegen returns list[dict] and stage_pool rejects lists — ming_flash_omni.py:435 `return [{"prompt": "", "extra": extra}]`; stage_pool.py:951-955 raises on `isinstance(request, list)`. +- [claim-verified] PR claim: unwrap was gated on already_submitted at main — orchestrator.py base had `already_submitted and len(diffusion_prompt) == 1`; head removes the guard. + +`vllm_omni/engine/orchestrator.py:1290` [minor] — This hunk changes orchestrator.py:1290 to unwrap any single-element bridge list before submit_initial (previously gated on already_submitted). Ming thinker2imagegen returns `[{"prompt": "", "extra": extra}]` (ming_flash_omni.py:435) and StagePool.submit_initial rejects lists (stage_pool.py:951-955), but tests/engine/test_orchestrator_kv_sender_info.py only covers empty-list terminal errors—not this initial-submit unwrap. Add a CPU L2 test stubbing custom_process_input_func to return a one-element list and asserting submit_initial receives a dict. (evidence: orchestrator.py:1290 `if len(diffusion_prompt) == 1:`; orchestrator.py:1298 `await next_pool.submit_initial(`; ming_flash_omni.py:435 `return [{"prompt": "", "extra": extra}]`; stage_pool.py:951-953 `if isinstance(request, list):` / `raise ValueError("Diffusion list-prompt batch requests are no longer supported.`; tests/engine/test_orchestrator_kv_sender_info.py:196 `diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: []`) + +```suggestion +def test_forward_to_diffusion_unwraps_single_element_bridge_list_on_initial_submit(): + orchestrator = object.__new__(Orchestrator) + diffusion_stage = _DummyDiffusionStage(engine_input_source=[0]) + bridge_prompt = {"prompt": "", "extra": {"thinker_hidden_states": "stub"}} + diffusion_stage.custom_process_input_func = lambda *_a, **_k: [bridge_prompt] + sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151}) + diffusion_pool = StagePool(1, diffusion_stage) + orchestrator.num_stages = 2 + orchestrator.stage_pools = [sender_pool, diffusion_pool] + orchestrator._cfg_tracker = CfgCompanionTracker() + params = OmniDiffusionSamplingParams() + req_state = OrchestratorRequestState( + request_id="req-bridge", + prompt={"prompt": "draw a cat"}, + sampling_params_list=[SamplingParams(max_tokens=4), params], + final_stage_id=1, + ) + output = SimpleNamespace(request_id="req-bridge", finished=True) + asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-bridge", 0, output, req_state)) + assert len(diffusion_stage.calls) == 1 + assert diffusion_stage.calls[0]["prompt"] == bridge_prompt + assert not isinstance(diffusion_stage.calls[0]["prompt"], list) +``` + +`?:1290` [minor] — Base gated unwrap on `already_submitted` (removed in this PR): only submit_update unwrapped `len==1` lists; submit_initial could receive a one-element list. After this change, both paths unwrap at orchestrator.py:1290; `len>1` lists still hit stage_pool ValueError by design. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/engine/orchestrator.py:1196 `already_submitted = self._next_stage_already_submitted(src_stage_id, req_state)`; vllm_omni/engine/orchestrator.py:1290-1301 `if len(diffusion_prompt) == 1:` / `diffusion_prompt = diffusion_prompt[0]` then `if already_submitted:` → `submit_update` else `submit_initial(..., diffusion_prompt, ...)`; PR diff removes base guard `if already_submitted and len(diffusion_prompt) == 1:`; vllm_omni/engine/stage_pool.py:951-955 `if isinstance(request, list):` `raise ValueError("Diffusion list-prompt batch requests are no longer supported. ...")`) + +`?:1290` [minor] — orchestrator.py:1290-1291 — `already_submitted` removed from single-element unwrap; submit path still gated by `already_submitted` at 1295-1309. Effect: first submit now unwraps `[prompt]` to `prompt` (required because stage_pool.py:950-955 rejects list diffusion requests). (evidence: orchestrator.py:1290 `if len(diffusion_prompt) == 1:` — unwrap no longer requires `already_submitted`; orchestrator.py:1295 `if already_submitted:` — still selects `submit_update` vs `submit_initial`; stage_pool.py:951 `if isinstance(request, list):` — diffusion `submit_initial` rejects list payloads (unchanged by this diff, present in PR-time tree).) + +`pipeline_z_image.py:720` [minor] — Ming outputs[0]/hasattr fallback removal is correct — ZImagePipeline.forward returns DiffusionOutput at pipeline_z_image.py:720; DiffusionOutput is a dataclass with no __getitem__ (data.py:1180); downstream diffusion_engine reads output.output at diffusion_engine.py:298 (diffusion_model_runner passes the whole DiffusionOutput as result). (evidence: pipeline_z_image.py:720 `return DiffusionOutput(output=image, stage_durations=stage_durations)` — ZImagePipeline.forward returns DiffusionOutput, not a list/tuple; data.py:1180 `@dataclass` / data.py:1186 `output: torch.Tensor | tuple[Any, ...] | dict[str, Any] | None = None` — no __getitem__; pipeline_ming_imagegen.py:465 `raw = outputs.output` / pipeline_ming_imagegen.py:475 `return outputs` — Ming uses .output and returns full DiffusionOutput; unchanged by this diff, present in PR-time tree: diffusion_engine.py:298 `output_data = output.output` — downstream post-process reads .output (diffusion_model_runner itself does not reference .output)) + +`vllm_omni/engine/orchestrator.py:1290` [nit] — gate_report lists buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci as failing at PR head, while this diff only changes vllm_omni/engine/orchestrator.py (line 1290 unwrap guard) and vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py with no AMD/NPU/platform yaml edits. Confirm whether those gate failures reproduce on main or are unrelated before merge. (evidence: gate_report lists failing checks buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci; diff_stat shows only vllm_omni/engine/orchestrator.py and vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py changed.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v17cb_val/pr4837.trace.json.gz new file mode 100644 index 000000000..e505b7622 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_val/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4893.cost.json b/eval/dataset/arms/copilot_v17cb_val/pr4893.cost.json new file mode 100644 index 000000000..1b05d360f --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4893.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 620.9, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17cb_val/runs/pr4893/run-20260817-123558-8d6a28", + "recorded_at": "2026-08-17T12:46:19", + "attempts": [ + { + "invocation_id": "inv-30040e32", + "outcome": "run:run-20260817-123558-8d6a28", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-123558-8d6a28", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-123558-8d6a28", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4394, + "minutes": 10.34, + "ci_minutes": 0, + "input_tokens": 1128629, + "output_tokens": 122463, + "tool_calls": 479, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5153, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.85, + "gate": 1.76, + "review": 614.68, + "report": 0.06 + }, + "time_to_first_result_s": 620.34 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 25937 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4893.md b/eval/dataset/arms/copilot_v17cb_val/pr4893.md new file mode 100644 index 000000000..5ac84a9ab --- /dev/null +++ b/eval/dataset/arms/copilot_v17cb_val/pr4893.md @@ -0,0 +1,93 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4893, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Prior alias vllm_parallel_state._PCP = _SP removed — test asserts _PCP is not _SP (test_expert_parallel_layout.py:112); residual: extra process groups created for all MoE+EP layouts including cfg=1. +- [resolved] cfg_parallel_size > 1 guard removed for vllm _DP remap — now always remapped when use_moe_parallel_mapping; residual checked: cfg=1 dp=1 yields world_size 1 so _set_forward_context_dp_metadata early-returns. +- [resolved] Removed cfg_parallel_size>1 gate — intentional for tp2_sp2 (cfg=1): vllm TP/PCP/EP companions still required; cfg=1 cfg-dp groups collapse to dp-only rank sets; residual: no unit test pins that 4-GPU tp2_sp2 layout. +- [resolved] NPU hunyuan_fused_moe.py lacks dp_metadata — uses Ascend ALLGATHER/ALLTOALL comm methods, not vLLM 0.24 CUDA all2all dp_metadata; not a regression from this GPU-focused fix. +- [resolved] cfg_parallel_size>1 guard removed (diff hunk parallel_state.py:882) — concern was skipping vLLM DP coordinator when cfg=1; residual checked: tp2_sp2 needs separate PCP/TP/EP coordinators even when cfg-dp ws=1; dp_metadata no-ops at ws<=1 +- [resolved] PCP aliasing to diffusion _SP — parallel_state.py:859-864 now builds separate vLLM PCP; residual: real vLLM coordinator APIs unverified in CI +- [resolved] kept comment line 92 spy/wrap use_device_communicator=True — orthogonal to hunk 19/184; not re-litigated. +- [resolved] kept comment cfg=1 parametrization — orthogonal to these hunks; not re-litigated. +- [claim-verified] PR scope matches #4885 tp2_cfgp2/tp2_sp2 perf recipes (tests/dfx/perf/tests/test_hunyuan_image_tp2_*.json); L2 pytest files marked core_model+cpu. +- [claim-verified] PR 11 passed — cannot re-run here; tests exist with @pytest.mark.cpu and @pytest.mark.core_model collected by test-ready.yml L13 core_model+cpu lane. +- [claim-verified] Remote tp2_cfgp2/tp2_sp2 smoke — author artifacts only; perf JSONs confirm CLI overrides (tp2_sp2.json ulysses-degree=2; tp2_cfgp2.json cfg-parallel-size=2). +- [claim-verified] PR claims 11 unit tests passed — tests are @pytest.mark.core_model @pytest.mark.cpu in touched files +- [claim-verified] PR remote tp2_cfgp2/tp2_sp2 smoke — external artifact URLs only; not re-run here +- [claim-verified] PR 11 passed — matches two touched test modules in merge-gate CPU lane + +`tests/diffusion/distributed/test_expert_parallel_layout.py:~101` [minor] — This diff drops the `cfg_parallel_size > 1` guard in `initialize_model_parallel`, so vLLM `_PCP`/`_TP`/`_DP`/`_EP` are always built via `init_vllm_model_parallel_group` when `use_moe_parallel_mapping` is set (fixing cfg=1 MoE+EP layouts such as tp2_sp2). `test_moe_ep_maps_diffusion_sp_cfg_dp_to_vllm_groups` only hardcodes `cfg_parallel_size=2` on 32 ranks. Add a `cfg_parallel_size=1` case with `tensor_parallel_size=2`, `sequence_parallel_size=2`, `world_size=4`, `enable_expert_parallel=True` asserting vLLM `_PCP` is not aliased to diffusion `_SP`, coordinators expose `device_communicator`/`reduce_scatter`, and created group names include `vllm_pcp`/`vllm_tp`/`vllm_ep`. (evidence: vllm_omni/diffusion/distributed/parallel_state.py:868 `if use_moe_parallel_mapping:` — vLLM _TP uses `init_vllm_model_parallel_group` with no `cfg_parallel_size > 1` guard (string absent repo-wide in this file). vllm_omni/diffusion/distributed/parallel_state.py:882 `if use_moe_parallel_mapping:` — vLLM _DP likewise always remapped via `init_vllm_model_parallel_group`. tests/diffusion/distributed/test_expert_parallel_layout.py:106 `cfg_parallel_size=2,` — sole MoE+EP layout test uses cfg=2 only; no `cfg_parallel_size=1` with `enable_expert_parallel=True` anywhere under tests/.) + +`tests/diffusion/distributed/test_expert_parallel_layout.py:92` [minor] — Production init_vllm_model_parallel_group (parallel_state.py:515-526) delegates to upstream init_model_parallel_group with use_device_communicator=True, but test_expert_parallel_layout.py monkeypatches the entire wrapper with fake_init_vllm_model_parallel_group (lines 62-77), which injects device_communicator/reduce_scatter on _FakeGroup without calling production code. Spy/wrap the real wrapper and assert upstream is invoked with use_device_communicator=True so CI cannot pass if that kwarg is dropped. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/distributed/parallel_state.py:525 `use_device_communicator=True,`; tests/diffusion/distributed/test_expert_parallel_layout.py:62-77 `def fake_init_vllm_model_parallel_group(...): ... device_communicator=object(), reduce_scatter=lambda tensor, **kwargs: tensor`; tests/diffusion/distributed/test_expert_parallel_layout.py:92 `monkeypatch.setattr(parallel_state, "init_vllm_model_parallel_group", fake_init_vllm_model_parallel_group)`) + +```suggestion +real_init = parallel_state.init_vllm_model_parallel_group + def spy_init_vllm_model_parallel_group(group_ranks, local_rank, backend, group_name): + with patch.object(parallel_state.vllm_parallel_state, "init_model_parallel_group", wraps=parallel_state.vllm_parallel_state.init_model_parallel_group) as mock_init: + result = real_init(group_ranks, local_rank, backend, group_name) + mock_init.assert_called_once_with( + group_ranks=group_ranks, + local_rank=local_rank, + backend=backend, + group_name=group_name, + use_device_communicator=True, + ) + return result + monkeypatch.setattr(parallel_state, "init_vllm_model_parallel_group", spy_init_vllm_model_parallel_group) +``` + +`test_expert_parallel_layout.py:112` [minor] — MoE+EP mapping no longer aliases `_PCP` to diffusion `_SP` (test_expert_parallel_layout.py:112); instead always builds four vLLM coordinators (pcp/tp/dp/ep) when `use_moe_parallel_mapping`, including cfg=1—residual extra process groups vs prior aliasing. (evidence: test_expert_parallel_layout.py:112 `assert parallel_state.vllm_parallel_state._PCP is not parallel_state._SP`; parallel_state.py:859 `vllm_parallel_state._PCP = init_vllm_model_parallel_group(`; parallel_state.py:882-890 `if use_moe_parallel_mapping:` then `vllm_parallel_state._DP = init_vllm_model_parallel_group(` with `group_name="dp"` (no cfg_parallel_size guard); parallel_state.py:902-909 `if use_moe_parallel_mapping:` then `vllm_parallel_state._EP = init_vllm_model_parallel_group(` — unchanged by this diff at base: prior `vllm_parallel_state._PCP = _SP` alias and `if use_moe_parallel_mapping and cfg_parallel_size > 1` DP gate per PR diff) + +`?:882` [minor] — vLLM `_DP` remap no longer gated on `cfg_parallel_size > 1`: when `use_moe_parallel_mapping`, `vllm_parallel_state._DP` is always rebuilt from cfg-dp ranks (parallel_state.py:882–890). With cfg=1 and dp=1, that group has world_size 1, so `_set_forward_context_dp_metadata` early-returns at hunyuan_fused_moe.py:38 and skips `all_gather_object`. (evidence: parallel_state.py:882 `if use_moe_parallel_mapping:` — no `cfg_parallel_size > 1` guard; parallel_state.py:885 `vllm_parallel_state._DP = init_vllm_model_parallel_group(` with `group_ranks=rank_generator.get_ranks("cfg-dp")`; hunyuan_fused_moe.py:38 `if dp_group is None or getattr(dp_group, "world_size", 1) <= 1:`) + +`?:882` [minor] — parallel_state.py:882 — removed `cfg_parallel_size > 1` gate on vLLM `_DP`; with `use_moe_parallel_mapping`, PCP/TP/DP/EP companions are always built. For cfg=1, `get_ranks("cfg-dp")` collapses to dp-only rank sets. Residual: no unit test pins the 4-GPU tp2_sp2 (tp=2, sp=2, cfg=1) MoE+EP layout. (evidence: unchanged by this diff, present in PR-time tree base: parallel_state.py (pre-PR) `if use_moe_parallel_mapping and cfg_parallel_size > 1:` — gate removed in PR; parallel_state.py:882 `if use_moe_parallel_mapping:` — vLLM `_DP` now always created for MoE mapping; parallel_state.py:853-861 `vllm_parallel_state._PCP = init_vllm_model_parallel_group(...)`; parallel_state.py:868-874 `vllm_parallel_state._TP = init_vllm_model_parallel_group(...)`; parallel_state.py:886 `group_ranks=rank_generator.get_ranks("cfg-dp")`; parallel_state.py:902-909 `vllm_parallel_state._EP = init_vllm_model_parallel_group(...)`; test_expert_parallel_layout.py:100-110 `tensor_parallel_size=2, sequence_parallel_size=2, cfg_parallel_size=2, data_parallel_size=2` with `world_size = 32` — no cfg=1 / 4-GPU tp2_sp2 case) + +`parallel_state.py:882` [minor] — [resolved] parallel_state.py:882 — removed `cfg_parallel_size > 1` guard on vLLM `_DP` init; PCP/TP/EP vLLM coordinators (853/868/902) remain required under `use_moe_parallel_mapping` even when cfg-dp ws=1; `_set_forward_context_dp_metadata` no-ops at dp_group.world_size<=1 (hunyuan_fused_moe.py:38). (evidence: PR-time tree parallel_state.py:882 `if use_moe_parallel_mapping:` — vLLM `_DP` init no longer checks cfg_parallel_size; PR diff removed `and cfg_parallel_size > 1` from this block. unchanged sibling gates: parallel_state.py:853 `if use_moe_parallel_mapping:` + `:859` `vllm_parallel_state._PCP = init_vllm_model_parallel_group(`; parallel_state.py:868 `:869` `vllm_parallel_state._TP = init_vllm_model_parallel_group(`; parallel_state.py:902 `:904` `vllm_parallel_state._EP = init_vllm_model_parallel_group(`. unchanged consumer: hunyuan_fused_moe.py:38-39 `if dp_group is None or getattr(dp_group, "world_size", 1) <= 1:` / `return` — dp_metadata gather skipped when ws<=1.) + +`vllm_omni/diffusion/distributed/parallel_state.py:515` [minor] — This diff changes shared `initialize_model_parallel` MoE coordinator creation (`use_moe_parallel_mapping` blocks at parallel_state.py:853-908): PCP/TP/DP/EP now use `init_vllm_model_parallel_group` (device communicator + reduce_scatter). That path is entered from `diffusion_worker.py:294-305` for any diffusion worker with `enable_expert_parallel` on a MoE model, and NPU Hunyuan calls `get_expert_parallel_group_ranks()` during `prepare_hunyuan_fused_moe_runtime` (platforms/npu/models/hunyuan_fused_moe.py:79-85). Merge gate reports AMD/NPU CI red — confirm whether those jobs fail on main or only on this branch before merge. (evidence: parallel_state.py:515 `def init_vllm_model_parallel_group(` — new helper used for MoE vLLM coordinators; parallel_state.py:904 `vllm_parallel_state._EP = init_vllm_model_parallel_group(` — _EP creation switched to vLLM GroupCoordinator path; unchanged by this diff, present in PR-time tree: vllm_omni/platforms/npu/models/hunyuan_fused_moe.py:79 `if vllm_config.parallel_config.enable_expert_parallel:` and :85 `group_ranks=get_expert_parallel_group_ranks(),` — NPU (non-GPU) consumer of EP rank layout from shared parallel_state; diffusion_worker.py:294 `initialize_model_parallel(` with :304 `enable_expert_parallel=parallel_config.enable_expert_parallel,` — shared diffusion worker entry, not Hunyuan-GPU-only) + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:~88` [minor] — The diff wires `_set_forward_context_dp_metadata(hidden_states.shape[0])` into the HunyuanFusedMoEDefault forward pre-hook (hunyuan_fused_moe.py:88-89) alongside `_set_forward_context_num_tokens`, but the new tests only call `_set_forward_context_dp_metadata` directly (test_hunyuan_fused_moe.py:73, :87). Add a test that a mocked MoE runner forward triggers both helpers so a regression that drops the pre-hook call still fails CI. (evidence: unchanged by this diff, present in PR-time tree: hunyuan_fused_moe.py:88 `_set_forward_context_num_tokens(hidden_states.shape[0])`; hunyuan_fused_moe.py:89 `_set_forward_context_dp_metadata(hidden_states.shape[0])`; test_hunyuan_fused_moe.py:73 `hunyuan_moe._set_forward_context_dp_metadata(1024)` — direct helper call, not pre-hook path) + +```suggestion +def test_pre_hook_sets_num_tokens_and_dp_metadata(self, mocker): + import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe + mock_num = mocker.patch.object(hunyuan_moe, "_set_forward_context_num_tokens") + mock_dp = mocker.patch.object(hunyuan_moe, "_set_forward_context_dp_metadata") + mocker.patch("vllm.model_executor.layers.fused_moe.FusedMoE", return_value=mocker.MagicMock()) + runner = hunyuan_moe.HunyuanFusedMoEDefault(prefix="") + hidden = mocker.MagicMock(shape=(17, 64)) + runner(hidden_states=hidden) + mock_num.assert_called_once_with(17) + mock_dp.assert_called_once_with(17) +``` + +**Additional observations (beyond the comment budget):** +- `tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json:3` [minor] HunyuanImage tp2_cfgp2 and tp2_sp2 perf recipes exist (tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json, test_hunyuan_image_tp2_sp2.json), but merge-gate CI does not run dfx perf benchmarks and nightly wires other diffusion perf JSONs (e.g. test_qwen_image_vllm_omni.json) without any test_hunyuan_image_tp2_* […] +- `tests/diffusion/distributed/test_expert_parallel_layout.py:22` [minor] Extend test_moe_ep_maps_diffusion_sp_cfg_dp_to_vllm_groups with paired negative assertions on diffusion _SP: assert parallel_state._SP.device_communicator is None and assert not hasattr(parallel_state._SP, "reduce_scatter"). The test already checks vLLM _PCP/_DP/_EP have those APIs (lines 121-126) and that _PCP is not […] +- `tests/diffusion/distributed/test_expert_parallel_layout.py:187` [minor] The test inventories four vLLM companion groups including vllm_tp (line 188), and MoE initialize_model_parallel builds _TP via init_vllm_model_parallel_group (parallel_state.py:869-874). Lines 121-126 assert device_communicator and reduce_scatter on _PCP/_DP/_EP only; extend the same checks to vllm_parallel_state._TP […] + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17cb_val/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v17cb_val/pr4893.trace.json.gz new file mode 100644 index 000000000..5b6a01119 Binary files /dev/null and b/eval/dataset/arms/copilot_v17cb_val/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/manifest.json b/eval/dataset/arms/copilot_v17ds_train/manifest.json new file mode 100644 index 000000000..7d4a6f577 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v17ds_train", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4804.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4804.cost.json new file mode 100644 index 000000000..bdddd855a --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4804.cost.json @@ -0,0 +1,115 @@ +{ + "wall_s": 2912.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4804/run-20260817-115016-1b55ed", + "recorded_at": "2026-08-17T12:38:47", + "attempts": [ + { + "invocation_id": "inv-d7b32c2f", + "outcome": "run:run-20260817-115016-1b55ed", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-100117-3c96ff", + "llm_calls": 230, + "input_tokens": 2428029, + "output_tokens": 368995, + "usd": 1.906295, + "source": "spans" + }, + { + "run_dir": "run-20260817-115016-1b55ed", + "llm_calls": 242, + "input_tokens": 2642131, + "output_tokens": 446314, + "usd": 2.040144, + "source": "spans" + } + ], + "attempt_usd_total": 3.946439, + "llm_calls": 242, + "input_tokens": 2642131, + "output_tokens": 446314, + "usd": 2.040144, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-115016-1b55ed", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 2.0401, + "minutes": 48.52, + "ci_minutes": 0, + "input_tokens": 2642131, + "output_tokens": 446314, + "tool_calls": 387, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.6207, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 242, + "by_role": { + "agent": { + "usd": 1.974391, + "input_tokens": 2571474, + "output_tokens": 403948, + "calls": 239 + }, + "reducer": { + "usd": 0.065753, + "input_tokens": 70657, + "output_tokens": 42366, + "calls": 3 + } + }, + "cache_read_tokens": 30956416, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.7, + "gate": 1.84, + "review": 2903.3, + "report": 0.06 + }, + "time_to_first_result_s": 2910.89 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 151023 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4804.md b/eval/dataset/arms/copilot_v17ds_train/pr4804.md new file mode 100644 index 000000000..4c6fb9016 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4804.md @@ -0,0 +1,116 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | 7 finding(s) below | + +**Validated:** +- [resolved] empty finish sentinel reaches _finish_empty_streaming_requests before n_vq divisibility guard; residual: starved request finishing before slot drops buffered codes with only a warning. +- [claim-verified] PR body 'make async chunk indeed work': talker2codec_raw_async_chunk emits threshold-based chunks; talker2codec_delay_async_chunk still accumulates until finish (chunk_frames=1<<30), consistent with its 'complex mode delay' scope. +- [claim-verified] stereo support: v2 config defaults 48000/2 channels; codec infers channels 2 for v2 paths; serving_speech writes tts_local_seed. +- [claim-refuted] Commit timeline says 'Fix leak issues when aborting', but gpu_generation_model_runner.py:157 gates `_update_request_states` on `async_chunk and num_scheduled_tokens`; an abort step with 0 scheduled tokens never calls the new on_requests_finished hook. +- [validated] Seed rename is complete: grep `qwen3_tts_request_seed` returns no matches; serving_speech.py:3715 writes `tts_local_seed` and gpu_model_runner.py:1826 reads it; tests/worker/test_omni_gpu_model_runner.py updated to `tts_local_seed`. +- [validated] `talker2codec_async_chunk` removed and no stale references (grep: none); pipeline.py:31/62/91 route delay→delay_async_chunk, realtime/local→raw_async_chunk. +- [validated] serving_speech.py:3708-3711 guards `if request.seed is not None and sampling_params_list:` before indexing [0]; unconditional deepcopy added at 3711 avoids mutating the engine default list. +- [validated] _save_seed_tts_eval_audio catches OSError and increments seed_tts_save_audio_failed (seed_tts_eval.py:588-596); path building strips unsafe chars via _safe_filename_part. +- [validated] talker2codec_async_chunk rename: repo-wide grep found no remaining reference; pipeline.py wires delay→talker2codec_delay_async_chunk and realtime/local→talker2codec_raw_async_chunk (pipeline.py:34,65,94). +- [validated] seed rename qwen3_tts_request_seed→tts_local_seed: grep found zero qwen3_tts_request_seed references; consumer gpu_model_runner.py:1826 and tests/worker/test_omni_gpu_model_runner.py:278,322,326 all updated. +- [validated] WER PCM normalization in patch.py:137 matches the existing chat-path convention (AudioResampler(target_sr=24000).resample(pcm_f32, orig_sr=...) is already used in patch.py), so no new dependency/API gap. +- [validated] grep qwen3_tts_request_seed: 0 matches at head; tts_local_seed now only in serving_speech.py:3715, gpu_model_runner.py:1826, and the renamed tests. +- [validated] NPU execute_model passes seq_token_counts (npu_generation_model_runner.py:299) but NPU _dummy_run does not, while GPU _dummy_run gained it. +- [validated] tests/worker/test_omni_gpu_model_runner.py:319 — read in situ; test_talker_mtp_forward_keeps_explicit_seeded_requests_scalar drives the scalar seed-split branch of _talker_mtp_forward and asserts batch sizes [1,1], distinct generators, and restored input buffers. + +`vllm_omni/worker/gpu_generation_model_runner.py:81` [blocker] — Confirmed: the hook is inside `_update_request_states`, which `execute_model` only calls when `async_chunk` is set and `num_scheduled_tokens > 0` (gpu_generation_model_runner.py:157-158). A finish/abort step with zero scheduled tokens skips it, so `_stream_req_slots`/`_stream_pending_codes` leak and repeated aborts exhaust the configured slots. The AR runner already calls the hook unconditionally right after `_update_states` (gpu_ar_model_runner.py:942-943). Additionally, stream state is keyed by `_runtime_request_key` (preferring `meta.req_id`, i.e. external_req_id) while `on_requests_finished` looks up `str(finished_req_id)` (engine id), so cleanup misses when the two id spaces differ. Move the hook next to `_update_states` and make the id space consistent. (evidence: Added by this diff — vllm_omni/worker/gpu_generation_model_runner.py:80-81: +` if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` +` self.model.on_requests_finished(scheduler_output.finished_req_ids)` +Pre-existing in PR-time tree, NOT in the diff — vllm_omni/worker/gpu_generation_model_runner.py:157-158: +` if self.model_config.async_chunk and num_scheduled_tokens:` +` self._update_request_states(scheduler_output)` +Unchanged by this diff, present in PR-time tree — vllm_omni/worker/gpu_ar_model_runner.py:942-943 (unconditional, right after `_update_states`): +` if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` +` self.model.on_requests_finished(scheduler_output.finished_req_ids)` +Id-space mismatch — vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:422 (stream slots keyed by `_runtime_request_key`): +` meta.get("req_id"),` +while `on_requests_finished` iterates `finished_req_ids` and uses `request_id = str(req_id)`.) + +```suggestion +deferred_state_corrections_fn = self._update_states(scheduler_output) + if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): + self.model.on_requests_finished(scheduler_output.finished_req_ids) +``` + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:~729` [blocker] — `_build_codec` (modeling_moss_tts_codec.py:729) now unconditionally constructs `MossAudioTokenizerV2Config`/`MossAudioTokenizerV2Model` first and only falls back to the legacy codec on exception. Since both config classes share `model_type = "moss-audio-tokenizer"` and the v2 config accepts the v1 config's keys as named parameters (popping model_type), a v1 checkpoint's config never raises, so MossTTSDelay/Realtime — whose defaults still point at the v1 repo `OpenMOSS-Team/MOSS-Audio-Tokenizer` — build the v2 architecture and the new strict loader raises on any resulting missing/skipped/shape-mismatched weights. Select v2 vs legacy explicitly (the existing 'v2' path marker or `audio_tokenizer_name_or_path`) instead of try-instantiate-and-fallback. (evidence: ['modeling_moss_tts_codec.py:729 `codec_cfg = MossAudioTokenizerV2Config.from_pretrained(codec_path)`', 'modeling_moss_tts_codec.py:730 `codec = MossAudioTokenizerV2Model(codec_cfg)` (in `_build_codec`; v2 is tried first, legacy only in the `except Exception` fallback below)', 'configuration_moss_audio_tokenizer_v2.py:68 `model_type = "moss-audio-tokenizer"` (new file in this diff)', 'audio_tokenizer.py:37 `model_type = "moss-audio-tokenizer"` (unchanged by this diff, present in the PR-time tree — legacy codec)', 'configuration_moss_tts.py:50 `codec_model_name_or_path: str = "OpenMOSS-Team/MOSS-Audio-Tokenizer"` (unchanged by this diff; MossTTSDelay default is the v1 repo)', 'configuration_moss_tts.py:172 `codec_model_name_or_path: str = "OpenMOSS-Team/MOSS-Audio-Tokenizer"` (unchanged by this diff; MossTTSRealtime default is the v1 repo)', 'modeling_moss_tts_codec.py `_build_codec` catch-all: `except Exception:` then `logger.exception(...)` and `codec_cfg = MossAudioTokenizerConfig.from_pretrained(codec_path)` / `codec = MossAudioTokenizerModel(codec_cfg)` — fallback only executes if v2 instantiation raises.']) + +```suggestion +def _build_codec(self, codec_path: str) -> tuple[Any, nn.Module]: + use_v2 = "v2" in codec_path.lower() or bool( + getattr(self.vllm_config.model_config.hf_config, "audio_tokenizer_name_or_path", None) + ) + if use_v2: + codec_cfg = MossAudioTokenizerV2Config.from_pretrained(codec_path) + codec = MossAudioTokenizerV2Model(codec_cfg) + logger.info("Using vendored MOSS Audio Tokenizer v2 classes from %s", codec_path) + return codec_cfg, codec + codec_cfg = MossAudioTokenizerConfig.from_pretrained(codec_path) + codec = MossAudioTokenizerModel(codec_cfg) + return codec_cfg, codec +``` + +`vllm_omni/worker/gpu_generation_model_runner.py:~799` [major] — This diff adds the shared code2wav contract seq_token_counts to the GPU generation runner's dummy/profile path, but the NPU twin was not co-changed: npu_generation_model_runner._dummy_run only sets runtime_additional_information in its Omni section and never passes seq_token_counts. MossTTSCodecDecoder now hard-fails when that kwarg is absent (`raise RuntimeError("MossTTS codec requires seq_token_counts...")`), so any MossTTS codec dummy/profile/coordination run on the NPU lane breaks. Add the same synthetic segment to npu_generation_model_runner._dummy_run (its real execute path already sets it at line 299), or the red vllm-omni-npu-ci lane will keep exercising a contract this PR made mandatory only on GPU. (evidence: gpu_generation_model_runner.py:799 `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]`; npu_generation_model_runner.py:299 `model_kwargs["seq_token_counts"] = tokens` (real path only); modeling_moss_tts_codec.py:262-264 `raise RuntimeError("MossTTS codec requires seq_token_counts; otherwise concatenated codec tokens cannot be split per request.")`.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:266` [major] — Confirmed at PR-time tree. `MossTTSCodecDecoder.forward` now requires `sum(seq_token_counts) == ids_flat.numel()`, but the two producers disagree: the real path sets `seq_token_counts` to unpadded per-request counts (`tokens`, sum == `total_num_scheduled_tokens`) while `_preprocess` is passed `num_tokens_padded` and produces padded input_ids; the dummy path self-consistently passes `[num_tokens_padded]`. On any batch where `num_tokens_padded > num_tokens_unpadded` (full CUDA graph / DP padding) the real path raises RuntimeError. Reject only `sum(token_counts) > ids_flat.shape[0]`, then slice `ids_flat = ids_flat[:sum(token_counts)]` before decoding. (evidence: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:266-267 (added by this diff): `if sum(token_counts) != int(ids_flat.shape[0]):` / `raise RuntimeError(` — strict equality against the flat input tensor length. vllm_omni/worker/gpu_generation_model_runner.py:316 (added by this diff): `model_kwargs["seq_token_counts"] = tokens` — `tokens` is the unpadded per-request list `[scheduler_output.num_scheduled_tokens[i] for i in req_ids]` whose sum is `scheduler_output.total_num_scheduled_tokens`. vllm_omni/worker/gpu_generation_model_runner.py:799 (added by this diff): `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]` — dummy path uses padded length. Decisive outside-diff link, unchanged context in this diff, gpu_generation_model_runner.py:309-314: `) = self._preprocess(` / `scheduler_output,` / `num_tokens_padded,` / `intermediate_tensors,` — the real path feeds `_preprocess` the padded token count, so `ids_flat.shape[0] == num_tokens_padded` while `sum(tokens) == num_tokens_unpadded`.) + +```suggestion +total_tokens = sum(token_counts) + if total_tokens > int(ids_flat.shape[0]): + raise RuntimeError( + "MossTTS codec seq_token_counts mismatch: " + f"counts={token_counts}, sum={total_tokens}, input_tokens={int(ids_flat.shape[0])}." + ) + ids_flat = ids_flat[:total_tokens] +``` + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:~48` [major] — The GPU generation runner now calls model.on_requests_finished in _update_request_states before removing finished requests (gpu_generation_model_runner.py:81), but the NPU twin _update_request_states (npu_generation_model_runner.py:48) only removes requests with no model hook. MossTTSCodecDecoder now defines on_requests_finished to release _stream_req_slots/_stream_pending_codes (modeling_moss_tts_codec.py:564), so on NPU an aborted/disconnected MOSS-TTS Local streaming request would leak codec stream slots and starve later requests. Mirror the GPU hook before the removal loop. (evidence: ['vllm_omni/worker/gpu_generation_model_runner.py:81-82 (in this diff): `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"):` / ` self.model.on_requests_finished(scheduler_output.finished_req_ids)`', 'vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:48-50 (unchanged by this diff, present in PR-time tree): ` def _update_request_states(self, scheduler_output: SchedulerOutput):` / ` # remove requests` / ` for req_id in scheduler_output.finished_req_ids:` — no on_requests_finished call exists in this file (repo-wide grep confirms).', 'vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:564 (in this diff): ` def on_requests_finished(self, finished_req_ids: set[str] | list[str]) -> None:` — the stage-1 codec hook that releases _stream_req_slots/_stream_pending_codes.']) + +```suggestion +if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): + self.model.on_requests_finished(scheduler_output.finished_req_ids) + for req_id in scheduler_output.finished_req_ids: + self.input_batch.remove_request(req_id) +``` + +`tests/worker/test_omni_gpu_model_runner.py:278` [major] — Confirmed: the only committed test change renames qwen3_tts_request_seed -> tts_local_seed in tests/worker/test_omni_gpu_model_runner.py. grep over tests/ finds no coverage of talker2codec_raw_async_chunk, _MossCodecStreamSession, _decode_streaming_batch, or MossTTSCodecDecoder.on_requests_finished (the only on_requests_finished tests are for voxcpm2), and the existing MOSS make_omni_output test targets the delay variant, not the rewritten Local/Realtime path. Add CPU regression tests for the raw processor (chunk accumulation, initial->steady threshold, finished flush) and codec slot acquire/release/cleanup. (evidence: tests/worker/test_omni_gpu_model_runner.py:278 ` extra_args={"tts_local_seed": 42},` — the cited anchor. PR diff (the only test-file hunk): `- extra_args={"qwen3_tts_request_seed": 42},` / `+ extra_args={"tts_local_seed": 42},` (same at lines 322 and 326 for the second test). tests/model_executor/models/moss_tts/test_per_request_codes.py:32-33 (unchanged by this diff, present in the PR-time tree): `from vllm_omni.model_executor.models.moss_tts.modeling_moss_tts_talker import ( + MossTTSDelayTalkerForGeneration, +)` — proves the existing MOSS make_omni_output regression test exercises the delay variant only. Grep of tests/ for `talker2codec_raw_async_chunk`, `_MossCodecStreamSession`, and `_decode_streaming_batch` returns no matches; grep for `on_requests_finished` matches only tests/model_executor/models/voxcpm2/test_talker_state_eviction.py (different model).) + +`?:298` [minor] — In modeling_moss_tts_codec.py the control-only finish branch at line 298 calls _finish_empty_streaming_requests before the n_vq divisibility guard at line 302; when a starved request (pending buffered codes, no stream slot) finishes, lines 395-401 log only a warning and _finish_stream_request drops the buffered codes. (evidence: vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:298 `if streaming_enabled and finished and code_flat_numel is not None and int(code_flat_numel) == 0:` -> line 299 `for _, wav in self._finish_empty_streaming_requests([info]).items():` -> line 301 `continue`; the n_vq guard follows at line 302 `if seg.numel() % self._n_vq != 0:` (unchanged context, present in PR-time tree). In _finish_empty_streaming_requests: line 395 `if pending and slot is None:` -> lines 396-398 `logger.warning("MOSS codec stream request %s finished before a codec stream slot became available; " "dropping buffered codes instead of offline-decoding a non-delta waveform.", req_key,)` -> line 401 `self._finish_stream_request(req_key, session, slot)` which pops _stream_pending_codes.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:369` [minor] — `_finish_empty_streaming_requests` always returns `{}`, so the two `for ... .items()` loops in `forward` never execute (call sites at lines 249 and 299; helper defined at line 369). Cleanup already happens inside the helper via `_finish_stream_request`, so this is dead/misleading code only; remove the dead loops or have the helper return values. (evidence: modeling_moss_tts_codec.py:369 `def _finish_empty_streaming_requests(self, info_list: list[dict[str, Any]]) -> dict[int, torch.Tensor]:`; modeling_moss_tts_codec.py:378 `outputs: dict[int, torch.Tensor] = {}`; modeling_moss_tts_codec.py:408 `return outputs` — `outputs` is never assigned any entries, so both consumer loops `for i, wav in self._finish_empty_streaming_requests(info_list).items():` (line 249) and `for _, wav in self._finish_empty_streaming_requests([info]).items():` (line 299) iterate zero times.) + +**Additional observations (beyond the comment budget):** +- `tests/worker/test_omni_gpu_model_runner.py:280` [minor] The updated seed tests in tests/worker/test_omni_gpu_model_runner.py exercise the runner seed path through CaptureTalkerMTP (line 280 and line 328) and never enter the newly added MossTTSLocalTalkerForGeneration.talker_mtp body. Add a direct unit test for talker_mtp so its sampling defaults, stop logic, and […] +- `vllm_omni/benchmarks/patch/patch.py:163` [minor] The normalization failure branch returns the raw captured PCM as if it were the normalized value, and seed_tts_eval then writes it as a 24 kHz mono WAV and feeds it to WER. If _pcm_s16le_to_seed_tts_wer_bytes raises (resampler failure, unexpected dtype), the saved WAV is mislabeled (possibly a different rate/channel […] +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1524` [minor] talker_mtp accepts temperature/top_k/top_p (and the runner reads subtalker_sampling_params at gpu_model_runner.py:1817) but the body hardcodes audio_temperature=1.7, audio_top_k=25, audio_top_p=0.8 and never references the parameters, so user-specified MOSS-TTS Local subtalker sampling values silently no-op while the […] +- `vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [minor] The new vendored v2 files (audio_tokenizer_v2.py and configuration_moss_audio_tokenizer_v2.py) begin with only the Apache copyright line and lack the vendored-source provenance header present on the legacy audio_tokenizer.py sibling. Selection between v2 and legacy is implicit: _build_codec first attempts […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4804.trace.json.gz new file mode 100644 index 000000000..f140f5067 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4817.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4817.cost.json new file mode 100644 index 000000000..d94be9f4d --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4817.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1126.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4817/run-20260817-102534-49ae06", + "recorded_at": "2026-08-17T10:44:18", + "attempts": [ + { + "invocation_id": "inv-0ec997be", + "outcome": "run:run-20260817-102534-49ae06", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-102534-49ae06", + "llm_calls": 99, + "input_tokens": 339269, + "output_tokens": 161018, + "usd": 0.387319, + "source": "spans" + } + ], + "attempt_usd_total": 0.387319, + "llm_calls": 99, + "input_tokens": 339269, + "output_tokens": 161018, + "usd": 0.387319, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-102534-49ae06", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3873, + "minutes": 18.73, + "ci_minutes": 0, + "input_tokens": 339269, + "output_tokens": 161018, + "tool_calls": 205, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6657, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 99, + "by_role": { + "planner": { + "usd": 0.00244, + "input_tokens": 3, + "output_tokens": 2155, + "calls": 1 + }, + "agent": { + "usd": 0.349514, + "input_tokens": 320450, + "output_tokens": 131401, + "calls": 95 + }, + "reducer": { + "usd": 0.035365, + "input_tokens": 18816, + "output_tokens": 27462, + "calls": 3 + } + }, + "cache_read_tokens": 4392448, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.41, + "gate": 1.76, + "review": 1117.46, + "report": 0.06 + }, + "time_to_first_result_s": 1123.7 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4817.trace.json.gz", + "trace_bytes": 31737 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4817.md b/eval/dataset/arms/copilot_v17ds_train/pr4817.md new file mode 100644 index 000000000..aa9dead85 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4817.md @@ -0,0 +1,88 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4817, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head fc423e5a9e86) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Red AMD/NPU lanes: AMD runs tests/diffusion+tests/model_executor and NPU runs one TTS test; neither collects tests/quantization, and the predicate returns False on ROCm cc 9.x/NPU, so the diff does not obviously explain those failures — asked author to confirm pre-existing. +- [claim-verified] PR body claim 'the env escape hatch is preserved (env is checked first)': quack_fp8.py:38-40 checks `override = os.environ.get(...)` before `_is_quack_capable()`. +- [claim-verified] PR body claim 'sm_120/121 default to native FlashInfer': with `== 10`, (12,x)->False so install_quack_fp8_patch returns early at quack_fp8.py:102-103, leaving FlashInfer untouched. +- [claim-verified] docs are the only quack reference: grep 'quack' shows fp8.md, pyproject.toml extra, quack_fp8.py, patch.py, and the new test — no other doc/README needs updating. +- [claim-verified] The new test would fail on the old gate: `(12,0)->False` would receive True under `>=10`, so it is a genuine regression guard. +- [claim-verified] 'No behavior change on sm_100.x': both old `>=10` and new `==10` return True for cc 10.x. +- [claim-verified] PR claim 'test fails on old >= 10 and passes on fix': old `12 >= 10`→True vs expected False, new `12 == 10`→False; arithmetic confirms. +- [claim-verified] PR claim 'no behavior change on sm_100.x': `== 10` keeps cc (10,0)/(10,1)/(10,3) enabled; confirmed. +- [claim-refuted] PR says the test 'runs on CI without a Blackwell GPU', but the file has no pytest marker and no .buildkite command selects tests/quantization, so `-m 'core_model and cpu'` lanes deselect it. +- [claim-refuted] PR root cause lists sm_110a as a tcgen05.mma target, but `== 10` excludes cc 11.x; vllm_omni/platforms/cuda/platform.py:83 treats major 11 as Blackwell — gate/narrative inconsistent. +- [validated] diff_stat: 3 files changed (docs fp8.md, new tests/quantization/test_quack_fp8_gate.py, vllm_omni/quantization/quack_fp8.py) — matches PR scope; no stray files. +- [validated] file_at_base vllm_omni/quantization/quack_fp8.py: old gate was `return torch.cuda.get_device_capability()[0] >= 10` — confirms the regression the PR describes. +- [validated] grep '_is_blackwell' repo-wide: zero remaining references after the rename to `_is_quack_capable` — no stale caller. +- [validated] grep 'quack_enabled': only callers are quack_fp8.py:102 (install_quack_fp8_patch) and the new test; env-override precedence (quack_fp8.py:37-41) unchanged. + +`tests/quantization/test_quack_fp8_gate.py:23` [major] — tests/quantization/test_quack_fp8_gate.py declares no `pytestmark`, so the `-m 'core_model and cpu'` selector in .buildkite/test-ready.yml:29 deselects this new regression test and it never runs in CI. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (as tests/model_executor/quantization/test_modelopt_nvfp4_nan_clamp.py:30 does) so the hardware-free guard is actually executed. (evidence: tests/quantization/test_quack_fp8_gate.py:23 `_ENV = "VLLM_OMNI_USE_QUACK_FP8"` — the new test file's body contains no `pytestmark` anywhere (verified via read_file, and `grep pytestmark tests/quantization` returned no matches). Unchanged by this diff, present in the PR-time tree: .buildkite/test-ready.yml:29 `"timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`. Unchanged by this diff, present in the PR-time tree: tests/model_executor/quantization/test_modelopt_nvfp4_nan_clamp.py:30 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`.) + +```suggestion +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_ENV = "VLLM_OMNI_USE_QUACK_FP8" +``` + +`vllm_omni/quantization/quack_fp8.py:32` [minor] — quack_fp8.py:32 gates on `get_device_capability()[0] == 10`, which auto-enables every cc 10.x variant and excludes cc 11.x. The new test only exercises (10,0), (10,3), (12,0), (12,1), (9,0), (8,9), so (10,1) and (11,0) are unguarded. The PR body says PTX tcgen05.mma targets sm_100a/101a/103a/110a while platform.py:83 classifies major 11 as Blackwell, so the sm_110 scope is ambiguous. Make the supported set explicit (e.g. {(10,0),(10,1),(10,3)} plus sm_110 if in scope) so undocumented 10.x/11.x parts fail closed to native FlashInfer. (evidence: vllm_omni/quantization/quack_fp8.py:32 ` return torch.cuda.get_device_capability()[0] == 10` — auto-enables every major-10 variant and excludes major 11. tests/quantization/test_quack_fp8_gate.py:33-39 ` [ + ((10, 0), True), # sm_100 datacenter Blackwell — tcgen05 present + ((10, 3), True), # sm_103 datacenter Blackwell + ((12, 0), False), # sm_120 workstation Blackwell — the regression + ((12, 1), False), # sm_121 consumer Blackwell + ((9, 0), False), # Hopper — CUTLASS already fuses bias, quack unused + ((8, 9), False), # Ada + ],` — no (10,1) or (11,0) case. vllm_omni/platforms/cuda/platform.py:83 ` is_blackwell = major in (10, 11, 12)` — unchanged by this diff, present in the PR-time tree; shows the platform treats 11.x as Blackwell while this gate excludes it.) + +```suggestion +capability = torch.cuda.get_device_capability() + return (capability[0], capability[1]) in {(10, 0), (10, 1), (10, 3)} +``` + +`docs/user_guide/quantization/fp8.md:47` [minor] — fp8.md:47-48 enumerates sm_100/sm_101/sm_103, but quack_fp8.py:32 `_is_quack_capable` accepts any compute capability 10.x (`major == 10`), so a 10.x part outside that list would silently auto-enable quack. Align by either gating on an explicit {sm_100, sm_101, sm_103} allowlist or documenting the condition as 'compute capability 10.x'. (evidence: vllm_omni/quantization/quack_fp8.py:32 ` return torch.cuda.get_device_capability()[0] == 10` (within the PR diff) — accepts every cc 10.x regardless of minor version. docs/user_guide/quantization/fp8.md:47-48 `It is enabled automatically once installed (no flag needed) and is **datacenter Blackwell only** (`sm_100` / `sm_101` / `sm_103`, compute capability `10.x`, e.g. B200):` (within the PR diff) — documents only three sm_100/101/103 parts while the gate is the broader `10.x` condition.) + +`pyproject.toml:102` [minor] — Update the quack extra comment to reflect the narrowed gate: pyproject.toml:102 still says 'Optional Blackwell-only fused-bias FP8 GEMM', but after `>=10` → `==10` the default only auto-enables on compute capability 10.x (datacenter Blackwell), not the full Blackwell family (12.x). Reword to 'datacenter Blackwell-only (compute capability 10.x)' so packaging metadata doesn't advertise a broader default. (evidence: Unchanged by this diff, present in the PR-time tree: pyproject.toml:102 `# Optional Blackwell-only fused-bias FP8 GEMM; package is `quack-kernels` (imported` ; vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10` — the `== 10` gate means cc 12.x Blackwell no longer auto-enables quack, so the 'Blackwell-only' comment is broader than the actual default.) + +`tests/quantization/test_quack_fp8_gate.py:3` [minor] — The two failing checks buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci (both FAILURE in gh_ci_read; main buildkite/vllm-omni is SUCCESS) do not intersect this diff surface at file level: .buildkite/test-amd.yaml runs only tests/e2e, tests/diffusion, tests/engine and tests/entrypoints targets (none under tests/quantization), and .buildkite/test-npu-ready.yaml runs a single e2e file (tests/e2e/online_serving/test_qwen3_tts_customvoice.py). The gate flip '>= 10' → '== 10' is also a no-op on AMD MI325 (cc 9.4 → False either way). Please confirm these two are pre-existing red on main rather than introduced here. (evidence: Anchor verified at tests/quantization/test_quack_fp8_gate.py:3 `"""Unit tests for the quack FP8 auto-enable gate.` — matches the cited snippet. CI check states from gh_ci_read(4817): buildkite/vllm-omni-amd-ci FAILURE, buildkite/vllm-omni-npu-ci FAILURE, buildkite/vllm-omni SUCCESS. PR-time CI lane definitions (unchanged by this diff, present in the PR-time tree): .buildkite/test-amd.yaml invokes only pytest targets under tests/e2e, tests/diffusion, tests/engine and tests/entrypoints (e.g. `pytest -s -v tests/entrypoints/test_omni_sleep_mode.py ...`); .buildkite/test-npu-ready.yaml invokes `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`. Gate no-op proven by vllm_omni/quantization/quack_fp8.py:33 `return torch.cuda.get_device_capability()[0] == 10` — for cc 9.x this is False exactly as the old `>= 10` would have been.) + +`tests/quantization/test_quack_fp8_gate.py:36` [minor] — tests/quantization/test_quack_fp8_gate.py:36 — Confirmed the new (10, 3) case only exercises the gate: _fake_cuda() monkeypatches only torch.cuda.is_available/get_device_capability (lines 28-29), and the test asserts only _is_quack_capable()/quack_enabled() (lines 50-51); it never calls _load_quack() or quack_scaled_fp8_mm(), so it proves routing, not that quack/CUTLASS actually loads on sm_101/sm_103. The PR body itself notes CuteDSL BlockScaledMmaOp hardcodes admissible_archs = [Arch.sm_100a], so please add a kernel-load smoke test or provide an sm_101/sm_103 run log showing 'Quack FP8 fused-bias GEMM enabled' and no 'Quack FP8 GEMM failed' flood to close the gap. (evidence: tests/quantization/test_quack_fp8_gate.py:28-29 ` monkeypatch.setattr(torch.cuda, "is_available", lambda: True)` / ` monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: capability)` — only torch.cuda probes are mocked; tests/quantization/test_quack_fp8_gate.py:36 ` ((10, 3), True), # sm_103 datacenter Blackwell`; tests/quantization/test_quack_fp8_gate.py:50-51 ` assert quack_fp8._is_quack_capable() is expected` / ` assert quack_fp8.quack_enabled() is expected` — the sm_103 case never calls _load_quack()/quack_scaled_fp8_mm(). PR body (verified via gh_pr_view, not repo code): 'the CuteDSL `BlockScaledMmaOp` hardcodes `admissible_archs = [Arch.sm_100a]`'.) + +`docs/user_guide/quantization/fp8.md:54` [nit] — Documenting `VLLM_OMNI_USE_QUACK_FP8=1` as 'force quack on' is misleading on the very GPUs this PR protects: quack_fp8.py:39-40 lets the env override bypass `_is_quack_capable()`, and on sm_120/sm_121 `install_quack_fp8_patch` (quack_fp8.py:101-105) still patches the kernel, so every GEMM raises and falls back to FlashInfer one call at a time (quack_fp8.py:118-124). Add the caveat that on 12.x `=1` only forces the slow per-GEMM fallback and should be used only once CuteDSL ships sm_120a support — as tests/quantization/test_quack_fp8_gate.py:63 already notes. (evidence: docs/user_guide/quantization/fp8.md:54 (added by this PR) `VLLM_OMNI_USE_QUACK_FP8=1` to force quack on, or `VLLM_OMNI_USE_QUACK_FP8=0` to force`; vllm_omni/quantization/quack_fp8.py:39-40 (unchanged context in this diff) `if override is not None:` / `return override.lower() in _TRUTHY` — env override bypasses `_is_quack_capable()`; vllm_omni/quantization/quack_fp8.py:101-105 (unchanged by this diff) `def install_quack_fp8_patch() -> None:` / `if not quack_enabled(): return` / `if _load_quack() is None: return`; vllm_omni/quantization/quack_fp8.py:118-124 (unchanged by this diff) `try:` / `out = quack_scaled_fp8_mm(A, B, As, Bs, out_dtype, bias)` / `except Exception as exc:` / `logger.warning_once("Quack FP8 GEMM failed (%s); using FlashInfer.", exc)` / `return original(self, A=A, B=B, out_dtype=out_dtype, As=As, Bs=Bs, bias=bias, output_shape=output_shape)`; tests/quantization/test_quack_fp8_gate.py:63 (added by this PR) `# sm_120 auto-disables, but an explicit truthy override forces quack on`.) + +```suggestion +`VLLM_OMNI_USE_QUACK_FP8=1` to force quack on (on `sm_120`/`sm_121` this only forces the +per-GEMM FlashInfer fallback — use only once CuteDSL ships `sm_120a` support), or +`VLLM_OMNI_USE_QUACK_FP8=0` to force the FlashInfer path. +``` + +`docs/user_guide/quantization/fp8.md:~31` [nit] — The body this diff rewrites now says quack auto-enables on datacenter Blackwell only, but the section heading (`### Faster FP8 GEMM on Blackwell (quack)`) and the following `On Blackwell (SM 100+)` framing still present quack as a general Blackwell feature — exactly the misunderstanding this PR fixes. Tighten the heading/framing to datacenter Blackwell so a 12.x reader is not steered back into the broken path. (evidence: Unchanged by this diff, present in the PR-time tree: docs/user_guide/quantization/fp8.md:31 `### Faster FP8 GEMM on Blackwell (quack)` and docs/user_guide/quantization/fp8.md:33 `On Blackwell (SM 100+), vLLM runs FP8 linears through the FlashInfer kernel, which`. The diff's rewritten body narrows the scope: docs/user_guide/quantization/fp8.md:47-48 `It is enabled automatically once installed (no flag needed) and is **datacenter +Blackwell only** (`sm_100` / `sm_101` / `sm_103`, compute capability `10.x`, e.g.` — proving the heading/framing now conflict with the body.) + +```suggestion +### Faster FP8 GEMM on datacenter Blackwell (quack) +``` + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4817.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4817.trace.json.gz new file mode 100644 index 000000000..6dd5aa9cc Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4817.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4859.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4859.cost.json new file mode 100644 index 000000000..be93b869a --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4859.cost.json @@ -0,0 +1,115 @@ +{ + "wall_s": 2717.6, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4859/run-20260817-121822-1c30f6", + "recorded_at": "2026-08-17T13:03:37", + "attempts": [ + { + "invocation_id": "inv-dd68d7d4", + "outcome": "run:run-20260817-121822-1c30f6", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-105025-8aca82", + "llm_calls": 1, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "spans" + }, + { + "run_dir": "run-20260817-121822-1c30f6", + "llm_calls": 192, + "input_tokens": 1799576, + "output_tokens": 376525, + "usd": 1.33443, + "source": "spans" + } + ], + "attempt_usd_total": 1.33443, + "llm_calls": 192, + "input_tokens": 1799576, + "output_tokens": 376525, + "usd": 1.33443, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-121822-1c30f6", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.3344, + "minutes": 45.25, + "ci_minutes": 0, + "input_tokens": 1799576, + "output_tokens": 376525, + "tool_calls": 386, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3838, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 192, + "by_role": { + "agent": { + "usd": 1.276334, + "input_tokens": 1771616, + "output_tokens": 330639, + "calls": 189 + }, + "reducer": { + "usd": 0.058096, + "input_tokens": 27960, + "output_tokens": 45886, + "calls": 3 + } + }, + "cache_read_tokens": 16087680, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.07, + "gate": 1.71, + "review": 2709.02, + "report": 0.07 + }, + "time_to_first_result_s": 2714.84 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4859.trace.json.gz", + "trace_bytes": 52031 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4859.md b/eval/dataset/arms/copilot_v17ds_train/pr4859.md new file mode 100644 index 000000000..5d5ec3742 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4859.md @@ -0,0 +1,106 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4859, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 6ff4ce59ed87) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] removed `_audio_pcm_format` has no remaining callers at this head; grep found only `defs.stream_pcm_format_from_env` used in patch.py:1163 and seed_tts_eval.py:189. +- [resolved] `_pcm_s16le_to_f32_16k` signature change is backward compatible for the sole in-repo caller (seed_tts_eval.py:616 passes pcm only). +- [resolved] patch.py import cleanup verified: exactly one `logger = init_logger(__name__)` at line 52, no duplicate. +- [claim-verified] diff_stat: 6 non-test files changed, 0 test files touched. +- [claim-verified] commit timeline shows merge 6ff4ce59 of upstream/main while the PR body says tested on 0455bffc — reported numbers predate the merge; needs re-validation at head. +- [claim-verified] include_language removal leaves no stale in-repo caller (repo-wide grep). +- [claim-refuted] PR-body WER 1.83% is computed with VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 on a path that double-resamples normalized 24k PCM (patch.py capture -> seed_tts_eval decode), so the number is not trustworthy at this head. +- [claim-refuted] PR's 1.83% WER: Test Plan exports VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100; at this head the WER decoder then resamples 24 kHz-normalized PCM with orig_sr=44100 (target_len = len*16000/44100 ≈ 0.36*len), so the headline number is computed on wrong-rate audio. +- [claim-refuted] PR WER 1.83% evidence path: capture normalizes PCM to 24k (patch.py), but eval resamples at env 44100 (seed_tts_eval.py:616 + :189) — 16000/44100 vs 16000/24000 gives 0.544x duration, so the reported number is computed on mis-resampled audio. +- [claim-refuted] PR test plan sets VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100; capture normalizes to 24 kHz (patch.py:428, 1215) then eval re-reads 44100 (seed_tts_eval.py:188) → double resample to ~0.544× correct duration. +- [validated] patch.py:113 docstring `Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER.` confirms the eval consumer must decode at 24000, not env rate. +- [validated] test_ming_shared_modules.py pins the new parser output only for language='Auto'/flash default; the removed non-Auto language→方言 path is untested. +- [validated] no test file touched (diff evidence) and `tests/model_executor/models/ming_tts/test_ming_shared_modules.py` still calls `_parse_ming_instruction_fields`/`_parse_ming_instruction` with compatible args; no stale call site for the removed kwarg. +- [validated] stream_pcm_format_from_env consumers — grep shows only patch.py:1163 and seed_tts_eval.py:189; old _audio_pcm_format is fully gone (grep 0 hits). + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:141` [major] — Verified: `self.encoder = Qwen2Model(config)` (audio_vae.py:131, unchanged) passes the SAME Qwen2Config into the encoder before the hunk; the hunk then executes `config.num_hidden_layers = 4` and `self.aggregator = Qwen2Model(config)` (audio_vae.py:141-142). Because Qwen2Model retains the config by reference, `self.encoder.config.num_hidden_layers` now reports 4 while the encoder still has the checkpoint's real layer count — a silent config-state corruption for any later reader (weight loading, validation, or future code). No in-repo consumer of that field was found, but the invariant violation is real; build the aggregator from a distinct copy as the old `_qwen2_config({**encoder_args, "num_hidden_layers": 4})` did. (evidence: audio_vae.py:131 (unchanged by this diff, present in PR-time tree): `self.encoder = Qwen2Model(config)` — passes the Qwen2Config to the encoder first. audio_vae.py:141 (in this diff): `config.num_hidden_layers = 4` — mutates that same object. audio_vae.py:142 (in this diff): `self.aggregator = Qwen2Model(config)` — reuses the mutated object. Since Qwen2Model stores the config by reference, `self.encoder.config.num_hidden_layers` becomes 4 even though the encoder was built with the checkpoint's real layer count.) + +```suggestion +aggregator_config = _qwen2_config({**encoder_args, "num_hidden_layers": 4}) + self.aggregator = Qwen2Model(aggregator_config) +``` + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:188` [major] — `_pcm_s16le_to_f32_16k` defaults to env-derived `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`/`VLLM_OMNI_BENCH_AUDIO_CHANNELS`, but its only input `out.tts_output_pcm_bytes` has already been normalized to 24 kHz mono by patch.py capture (`AudioResampler(target_sr=24000)`; field doc: 'Raw PCM s16le mono at 24 kHz for Seed-TTS WER'). When the env is overridden (e.g. 44100), the eval resamples 24 kHz samples as 44.1 kHz — `target_len = int(len(raw) * 16000 / 44100)` — and measures WER on time-warped audio. Fix: hardcode the eval path to 24 kHz mono (or pass 24000/1 explicitly at the `wav_16k = _pcm_s16le_to_f32_16k(pcm)` call), delete the now-unneeded `stream_pcm_format_from_env` import (line 64), and remove the module-docstring lines 42-43 that promise env-driven eval decoding. (evidence: seed_tts_eval.py:188 ` if pcm_sample_rate is None or channels is None:` → :189 ` env_sample_rate, env_channels = stream_pcm_format_from_env()` → :200 ` target_len = int(len(raw) * 16000 / pcm_sample_rate)` (new in this diff). Consumer call site: seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)` with seed_tts_eval.py:586 `pcm = getattr(out, "tts_output_pcm_bytes", None)` (grep-confirmed). Producer — unchanged by this diff, present in the PR-time tree: patch.py:112 `def _pcm_s16le_to_seed_tts_wer_bytes(` with patch.py:130 `resampler = AudioResampler(target_sr=24000)` and patch.py:133 `return (pcm_f32 * 32767).astype(np.int16).tobytes()`; invariant documented at patch.py:428 `#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or`.) + +```suggestion +if pcm_sample_rate is None: + pcm_sample_rate = 24000 + if channels is None: + channels = 1 +``` + +`examples/online_serving/text_to_speech/ming_tts/README.md:~101` [major] — The diff removes `include_language` from `_parse_ming_instruction_fields` and `_parse_ming_instruction` no longer maps `request.language` to `方言`, so the OpenAI `language` field (and the example client's `--dialect`) now silently has no effect for Ming dense. README.md:101 still documents `language` as Ming `方言` control and openai_speech_client.py:214 still sends `payload["language"] = args.dialect`. Update the README row and the client `--dialect` help if the removal is intentional (HF alignment), or restore the `include_language=True` branch. (evidence: ['vllm_omni/entrypoints/openai/serving_speech.py:2619-2624 (changed by this diff): ` def _parse_ming_instruction_fields(` / ` self,` / ` request,` / ` *,` / ` include_voice=False,` / ` plain_text_passthrough=False,` — the `include_language` parameter is gone; a repo-wide grep for `方言` in this file returns no matches, so no remaining code maps `language` to `方言`.', 'vllm_omni/entrypoints/openai/serving_speech.py:2648-2652 (changed by this diff): ` return self._parse_ming_instruction_fields(` / ` request,` / ` include_voice=True,` / ` plain_text_passthrough=True,` — `_parse_ming_instruction` no longer passes `include_language=True`.', 'examples/online_serving/text_to_speech/ming_tts/README.md:101 (unchanged by this diff, present in the PR-time tree): `| `language` | Ming `方言` control |` — the docs still promise the removed mapping.', 'examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py:214 (unchanged by this diff, present in the PR-time tree): ` payload["language"] = args.dialect` — the example client still sends the now-ignored field for `--dialect`.']) + +```suggestion +| `language` | no longer mapped to Ming `方言` for dense; use `--instruction-json '{"方言":"广粤话"}'` | +``` + +`vllm_omni/model_executor/models/ming_tts/prompt_assembly.py:~107` [major] — patch_emission.py:130 now requires `min_required = max(min_stop_step + 2, min_steps)`, but `estimate_decode_step_window_for_duration` (prompt_assembly.py:104-108, unchanged by this diff) still produces `max_steps = max(min_steps, target_steps + 3)`. With the default `stop_head_min_steps=3`, a `Duration:` request with target_steps=1 (<=0.32 s at 14112 samples/step @44.1 kHz) yields max_steps=4 < min_required=5, so `_validate_ming_decode_window` raises and the request fails. Clamp `max_steps` to `min_stop_step + 2` in the window producer (or reject short durations explicitly) and add a regression test. (evidence: changed by this diff: vllm_omni/model_executor/models/ming_tts/patch_emission.py:130 ` min_required = max(min_stop_step + 2, min_steps)`; vllm_omni/model_executor/models/ming_tts/patch_emission.py:131 ` if max_steps < min_required:`. Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/ming_tts/prompt_assembly.py:106 ` min_steps = max(1, target_steps - 3)`; vllm_omni/model_executor/models/ming_tts/prompt_assembly.py:107 ` max_steps = max(min_steps, target_steps + 3)`; vllm_omni/model_executor/models/ming_tts/constants.py:51 `STOP_HEAD_MIN_STEPS = 3`.) + +```suggestion +def estimate_decode_step_window_for_duration( + duration_seconds: float, *, min_stop_step: int = STOP_HEAD_MIN_STEPS +) -> tuple[int, int]: + target_steps = estimate_decode_steps_for_duration(duration_seconds) + min_steps = max(1, target_steps - 3) + max_steps = max(min_steps, target_steps + 3, min_stop_step + 2) + return min_steps, max_steps +``` + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:~351` [minor] — This hunk switches the prompt-latent path from deterministic mean to reparameterized sampling with `torch.randn_like(mean)`, and `encode_latent(self, waveform, waveform_length)` exposes no generator/seed parameter. Both callers (prompt_encoder.py:137 and voice_presets.py:257) pass no generator, so identical prompt audio now encodes to a different latent on each call; reproducibility depends solely on global torch RNG. If HF parity requires sampling, add an explicit `generator` parameter (defaulting to a seeded per-call generator) so eval can be made deterministic. (evidence: audio_vae.py:350 `std = F.softplus(scale) + 1e-4`; audio_vae.py:351 `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)`; audio_vae.py:329 `def encode_latent(self, waveform, waveform_length):` (no generator parameter). Unchanged by this diff, present in the PR-time tree: prompt_encoder.py:137 `latents, _ = encoder.encode_latent(waveform, waveform_length)`; voice_presets.py:257 `prompt_wav_lat, _ = self._audio_vae.encode_latent(`.) + +```suggestion +noise = torch.randn_like(mean) if generator is None else torch.randn( + mean.shape, generator=generator, device=mean.device, dtype=mean.dtype + ) + latent = (mean + std * noise).transpose(1, 2) +``` + +`vllm_omni/model_executor/models/ming_tts/patch_emission.py:130` [minor] — The stop-head gate shifts from `min_stop_step + 1` to `+ 2` in both `_validate_ming_decode_window` (line 130) and `_resolve_ming_stop_decision` (line 151), changing the default minimum decode length from 4 to 5 steps (`STOP_HEAD_MIN_STEPS = 3`). No test covers either function, so this correctness-alignment change can regress silently. Add a focused unit test asserting: `max_decode_steps = min_stop_step + 1` now raises/does-not-stop, and `min_stop_step + 2` is the first step where `should_stop_head` can become true. (evidence: patch_emission.py:130 `min_required = max(min_stop_step + 2, min_steps)`; patch_emission.py:151 `min_required_decode_steps = max(min_stop_step + 2, min_decode_steps)`; patch_emission.py:158 (unchanged by this diff, present in PR-time tree) `should_stop_head = ((step + 1) >= min_required_decode_steps) and stop_prob > stop_threshold`; constants.py:51 (unchanged by this diff, present in PR-time tree) `STOP_HEAD_MIN_STEPS = 3`; ming_tts_llm.py:227 (unchanged by this diff, present in PR-time tree) `min_stop_step=int(self.ming_config.stop_head_min_steps)`.) + +`vllm_omni/model_executor/models/ming_tts/patch_emission.py:130` [minor] — Diff changes both Ming decode-window sites from `min_stop_step + 1` to `min_stop_step + 2` (patch_emission.py:130 and :151) with no named constant, test, or rationale. With STOP_HEAD_MIN_STEPS=3 (constants.py:51), min_required_decode_steps becomes 5, so any request whose KEY_MAX_DECODE_STEPS (serving_speech.py:2679, from request.max_new_tokens) is <5 now raises, and the minimum generated length silently shifts. Please extract the grace step into a named constant, use it in both sites, and add a one-line comment tying it to the HF path. (evidence: vllm_omni/model_executor/models/ming_tts/patch_emission.py:130 `min_required = max(min_stop_step + 2, min_steps)` (changed by this diff) +vllm_omni/model_executor/models/ming_tts/patch_emission.py:151 `min_required_decode_steps = max(min_stop_step + 2, min_decode_steps)` (changed by this diff) +vllm_omni/model_executor/models/ming_tts/constants.py:51 `STOP_HEAD_MIN_STEPS = 3` (unchanged by this diff, present in PR-time tree) +vllm_omni/model_executor/models/ming_tts/ming_tts_llm.py:136 `min_stop_step=int(self.ming_config.stop_head_min_steps),` (unchanged by this diff; same at :227) +vllm_omni/entrypoints/openai/serving_speech.py:2679 `runtime_controls[KEY_MAX_DECODE_STEPS] = request.max_new_tokens` (unchanged by this diff; links max_decode_steps to user-supplied max_new_tokens)) + +```suggestion +min_required = max(min_stop_step + MING_STOP_HEAD_GRACE_STEPS, min_steps) +``` + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:188` [minor] — PR body reports WER tested on 0455bffc, but head 6ff4ce59 is a merge of upstream/main that resolved conflicts in seed_tts_eval.py and patch.py — re-run the 1088-utterance Seed-TTS WER at head before relying on the 1.83% number. (evidence: PR metadata, not in the diff: PR body — '**vLLM-Omni Commit:** this PR, tested on current branch based on `0455bffc3768e1fe616fea37885fb0a84cd5985b`'; commit 6ff4ce59ed87 — 'Merge remote-tracking branch 'upstream/main' into fix_ming_tts_moe_precision' (with conflict resolutions in `vllm_omni/benchmarks/data_modules/seed_tts_eval.py` and `vllm_omni/benchmarks/patch/patch.py`).) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/common/ming/audio_vae.py:351` [minor] The softplus VAE posterior-sampling change at audio_vae.py:349-351 is real and in the diff, but no HF Ming reference is vendored in this checkout, so the parity claim is unverified. Please link the HF Ming implementation that uses `std = F.softplus(scale) + 1e-4` before merging. +- `vllm_omni/metrics/definitions.py:317` [minor] Add a unit test for `stream_pcm_format_from_env` in metrics/definitions.py (line 317); grep shows no test references, and this parser is the replacement for `_audio_pcm_format` with no coverage. +- `.buildkite/test-amd-ready.yaml:19` [minor] `buildkite/vllm-omni-amd-ci` is red at head 6ff4ce59ed87 while `buildkite/vllm-omni` is green; the AMD lane's 'core_model and cpu' collection includes `tests/model_executor/models/ming_tts/test_ming_shared_modules.py` (marker at line 27, not 22), which calls the changed […] +- `vllm_omni/entrypoints/openai/serving_speech.py:2623` [minor] This hunk removes `include_language` and the `request.language -> instruction_dict["方言"]` mapping, so a non-'Auto' `language` (e.g. `粤语`) is now silently dropped for Ming dense (`_parse_ming_instruction` now passes only `include_voice=True, plain_text_passthrough=True`) and remains dropped for Ming flash […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4859.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4859.trace.json.gz new file mode 100644 index 000000000..fd6978fb9 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4859.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4870.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4870.cost.json new file mode 100644 index 000000000..2fbb64fc0 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4870.cost.json @@ -0,0 +1,115 @@ +{ + "wall_s": 1685.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4870/run-20260817-115017-8afc10", + "recorded_at": "2026-08-17T12:18:19", + "attempts": [ + { + "invocation_id": "inv-2b30d4ef", + "outcome": "run:run-20260817-115017-8afc10", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-102517-86395f", + "llm_calls": 163, + "input_tokens": 1934211, + "output_tokens": 165698, + "usd": 1.30053, + "source": "spans" + }, + { + "run_dir": "run-20260817-115017-8afc10", + "llm_calls": 187, + "input_tokens": 2130404, + "output_tokens": 284534, + "usd": 1.340805, + "source": "spans" + } + ], + "attempt_usd_total": 2.641335, + "llm_calls": 187, + "input_tokens": 2130404, + "output_tokens": 284534, + "usd": 1.340805, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-115017-8afc10", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.3408, + "minutes": 28.04, + "ci_minutes": 0, + "input_tokens": 2130404, + "output_tokens": 284534, + "tool_calls": 272, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.164, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 187, + "by_role": { + "agent": { + "usd": 1.307613, + "input_tokens": 2115670, + "output_tokens": 258017, + "calls": 185 + }, + "reducer": { + "usd": 0.033192, + "input_tokens": 14734, + "output_tokens": 26517, + "calls": 2 + } + }, + "cache_read_tokens": 16763264, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.5, + "gate": 1.49, + "review": 1676.44, + "report": 0.06 + }, + "time_to_first_result_s": 1682.47 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 58132 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4870.md b/eval/dataset/arms/copilot_v17ds_train/pr4870.md new file mode 100644 index 000000000..68912c1d3 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4870.md @@ -0,0 +1,75 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] seq_len parameter removal has no remaining repo callers at this head — grep for _build_omni_model_runner_output_from_snapshot returns only sample_tokens and the updated tests; residual: NPU twin does not call this function and was not audited. +- [claim-verified] PR body 'four files changed' — gh_pr_view files array lists exactly 4 modified files (2 tests + 2 source). +- [claim-verified] PR body test additions — worker regression test and serving-speech unit tests are present at the cited locations. +- [claim-verified] diff shows the `seq_len` parameter removed from the signature and from the single internal call site and all test call sites; no in-repo consumer left stale. +- [claim-verified] diff_stat/diff surface matches the four claimed files; no probe file or benchmark artifact is included. +- [claim-verified] PR body states the E2E probe used `stream=False`, so the new stream=True + async_chunk=False path has no end-to-end validation. +- [validated] serving_speech.py:3504-3506 read in tree — new gate computes async_chunk from model_config, sets qwen3_full_payload only for qwen3_tts + async_chunk False, and ANDs it out of is_streaming_request. +- [validated] audio.py:302-304 — is_streaming() is raw-audio-stream or SSE stream, so stream_format='audio' with response_format='pcm' reaches the changed gate. +- [validated] entrypoints/utils.py:759 — coerce_param_message_types takes the computed is_streaming bool; serving tests assert the bool they expect. +- [validated] utils/mm_outputs.py to_payload_element — exact dim-0 match on seq_len is what gates per-request slicing; a mismatched tensor falls to the whole-tensor clone branch (cross-request contamination risk). +- [validated] qwen3_tts_talker.py:535-605 — make_omni_output returns text_hidden_states=hidden untruncated and codes.audio=torch.cat(...), so dim0==total scheduled tokens; new mm_seq_len matches this producer. +- [validated] tests/worker/test_gpu_ar_model_runner.py:438-478 — new regression test reproduces hidden shape 1 vs total 3 and asserts codes[0:1]/[1:2]/[2:3] per request; it exercises the changed seq_len. +- [validated] tests/entrypoints/openai_api/test_serving_speech.py:3453/3473 — both new serving tests drive _prepare_speech_generation and assert coerce bool False (qwen3_tts) vs True (voxtral non-qwen3). +- [validated] Read `vllm_omni/utils/mm_outputs.py:131` — `to_payload_element` slices only when `element.shape[0] == seq_len`; `total_num_scheduled_tokens` is the correct axis for concatenated per-token tensors. + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:~905` [major] — The NPU twin still derives `seq_len = hidden_states.shape[0]` (vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905) and passes it to `to_payload_element` at :1144, while the GPU runner now computes `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` (vllm_omni/worker/gpu_ar_model_runner.py:1734) and forwards it at :1790. Qwen3-TTS runs on NPU via `apply_qwen3_tts_code2wav_patch()` (platforms/npu/platform.py:41) and `apply_310p_patches()` (:42). Mirror the scheduled-token substitution in the NPU runner or explicitly declare NPU out of scope and track it. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905 ` seq_len = hidden_states.shape[0]`; vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:1144 ` seq_len=seq_len,`; changed by this PR: vllm_omni/worker/gpu_ar_model_runner.py:1734 ` mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` and vllm_omni/worker/gpu_ar_model_runner.py:1790 ` seq_len=mm_seq_len,`; NPU platform wiring: vllm_omni/platforms/npu/platform.py:41 ` apply_qwen3_tts_code2wav_patch()` and vllm_omni/platforms/npu/platform.py:42 ` apply_310p_patches()`) + +```suggestion +seq_len = int(scheduler_output.total_num_scheduled_tokens) +``` + +`tests/worker/test_gpu_ar_model_runner.py:~478` [major] — The test force-disables the full-payload accumulator that the PR description names as the actual failing path, so the accumulation result is never asserted. (evidence: Summary says the accumulation result is never asserted.) + +`tests/entrypoints/openai_api/test_serving_speech.py:~3453` [minor] — Confirmed at tests/entrypoints/openai_api/test_serving_speech.py:3453-3471 (qwen3) and :3473+ (voxtral): both new tests patch `coerce_param_message_types` and assert only the injected boolean (`False`/`True`), proving routing but not the resulting `SamplingParams.output_kind`. The qwen3 request uses `stream_format='audio'` (not `stream=True`), and no test exercises the real `coerce_param_message_types` FINAL_ONLY output or the raw-audio streaming response path for `async_chunk=False`. Strengthen the qwen3 case to assert `output_kind == SamplingParams.OutputKind.FINAL_ONLY` (DELTA otherwise). (evidence: serving_speech.py:3507 `async_chunk = getattr(self.model_config, "async_chunk", True)`; serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; serving_speech.py:3509 `is_streaming_request = request.is_streaming() and not qwen3_full_payload`; serving_speech.py:3510 `sampling_params_list = coerce_param_message_types(sampling_params_list, is_streaming_request)`. tests/entrypoints/openai_api/test_serving_speech.py:3463-3466 `mock_coerce = mocker.patch("vllm_omni.entrypoints.openai.serving_speech.coerce_param_message_types", return_value=qwen3_tts_server.engine_client.default_sampling_params_list)`; tests/entrypoints/openai_api/test_serving_speech.py:3471 `mock_coerce.assert_called_once_with(qwen3_tts_server.engine_client.default_sampling_params_list, False)`.) + +```suggestion +Strengthen the qwen3 case to construct the real SamplingParams path and assert `output_kind == SamplingParams.OutputKind.FINAL_ONLY` for qwen3_tts and DELTA otherwise. +``` + +`vllm_omni/entrypoints/openai/serving_speech.py:3507` [minor] — The fallback default at vllm_omni/entrypoints/openai/serving_speech.py:3507 is True, inverted from the OmniModelConfig default (`async_chunk: bool = False` at vllm_omni/config/model.py:121) and the worker read (`getattr(self.model_config, "async_chunk", False)` at vllm_omni/worker/gpu_ar_model_runner.py:293). Since `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` (line 3509), a missing attribute would re-enable DELTA coercion for qwen3_tts streaming and defeat the FINAL_ONLY fix. Use `getattr(self.model_config, "async_chunk", False)` for consistency. (evidence: vllm_omni/entrypoints/openai/serving_speech.py:3507 (added in this diff) ` async_chunk = getattr(self.model_config, "async_chunk", True)`; vllm_omni/entrypoints/openai/serving_speech.py:3509 (added in this diff) ` qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; unchanged by this diff, present in the PR-time tree: vllm_omni/config/model.py:121 ` async_chunk: bool = False`; unchanged by this diff, present in the PR-time tree: vllm_omni/worker/gpu_ar_model_runner.py:293 ` self._async_chunk = getattr(self.model_config, "async_chunk", False)`.) + +```suggestion +async_chunk = getattr(self.model_config, "async_chunk", False) +``` + +`?:?` [minor] — Confirmed: at this head `_build_omni_model_runner_output_from_snapshot` has no `seq_len` parameter and its only non-test caller (`sample_tokens`) no longer passes it; repo-wide grep shows no NPU-platform caller, so the NPU twin remains unaudited as noted. (evidence: Repo-wide grep for `_build_omni_model_runner_output_from_snapshot` at head returns only: `vllm_omni/worker/gpu_ar_model_runner.py:1662` `def _build_omni_model_runner_output_from_snapshot(` (definition), `vllm_omni/worker/gpu_ar_model_runner.py:2027` `return self._build_omni_model_runner_output_from_snapshot(` (the only non-test caller, inside `sample_tokens`), and `tests/worker/test_gpu_ar_model_runner.py` lines 250, 295, 348, 408, 454, 652. No occurrence exists under `vllm_omni/platforms/npu/**`. At the call site, `:2041` `kv_extracted_req_ids=kv_extracted_req_ids,` is followed directly by `:2042` `num_scheduled_tokens_np=num_scheduled_tokens_np,` — the `seq_len=seq_len,` argument this diff removed is absent.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3507` [minor] — qwen3_full_payload (serving_speech.py:3508) gates request.is_streaming() for both raw-audio and SSE transports, but the PR's new regression test only covers stream_format='audio' + response_format='pcm'. Either scope the FINAL_ONLY gate to the intended transport, or add an SSE (stream=true / stream_format='sse') assertion for the async_chunk=False chunking and first-byte contract. (evidence: Changed by this diff, present in PR-time tree: vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)` and the following line `is_streaming_request = request.is_streaming() and not qwen3_full_payload` — no stream_format/SSE vs raw-audio distinction. Unchanged by this diff, present in PR-time tree: vllm_omni/entrypoints/openai/protocol/audio.py:302-303 `def is_streaming(self) -> bool:` / `return self.is_raw_audio_stream() or self.is_sse_stream()` — both transports therefore reach the gate.) + +`?:?` [minor] — buildkite/vllm-omni-amd-ci is FAILURE at head 231b3ddb97c1 while buildkite/vllm-omni and buildkite/vllm-omni-npu-ci pass; this PR modifies vllm_omni/worker/gpu_ar_model_runner.py and tests/worker/test_gpu_ar_model_runner.py. Attribute the red lane to main vs this PR (or re-run it) so the changed GPU AR path is provably green. (PR is already merged, so treat as post-merge follow-up.) (evidence: CI metadata, not a code line (gh_ci_read(4870), PR-time head): `{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}`; `{"bucket":"pass","name":"buildkite/vllm-omni-npu-ci","state":"SUCCESS"}`; `{"bucket":"pass","name":"buildkite/vllm-omni","state":"SUCCESS"}`. PR file list (gh_pr_view(4870)) includes `vllm_omni/worker/gpu_ar_model_runner.py` (2 additions / 4 deletions) and `tests/worker/test_gpu_ar_model_runner.py` (45 additions / 5 deletions); diff headers confirm both: `diff --git a/vllm_omni/worker/gpu_ar_model_runner.py` and `diff --git a/tests/worker/test_gpu_ar_model_runner.py`.) + +`?:?` [minor] — The PR body's H20 before/after E2E cosine (0.228 -> 1.000/1.000/0.987) and '1 passed in 194.78s' are author evidence only; the PR body itself says the E2E probe file was used only for validation and is not included in this PR diff, and the PR-time tree contains no test or artifact that reproduces those numbers. Do not treat them as verified. (evidence: PR body via gh_pr_view(4870): 'Cosine similarity was `0.228`.' and '`1 passed, 17 warnings in 194.78s`. Three concurrent responses had ASR cosine similarities `1.000`, `1.000`, and `0.987`.' PR body via gh_pr_view(4870): 'The E2E probe used real remote H20 serving with Qwen3-TTS CustomVoice, `--no-async-chunk`, `stream=False`, and `request_num=3`. The probe file was used only for validation and is not included in this PR diff.' Repo-wide grep at PR-time head: '194.78' -> no matches; 'test_voice_001_nonstream_concurrent_no_async_chunk' -> no matches; '0.228' -> only unrelated vllm_omni/diffusion/models/lance/wan_vae.py:625 `-0.2289,`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/worker/gpu_ar_model_runner.py:1790` [minor] The diff replaces the `seq_len` split argument with `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` and passes it through to `to_payload_element` (vllm_omni/worker/gpu_ar_model_runner.py:1734, passed at 1790). `to_payload_element` only slices when `element.shape[0] == seq_len`; on mismatch it silently […] +- `tests/worker/test_gpu_ar_model_runner.py:441` [minor] Minor: test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only schedules all three requests with one token each and only asserts uniform slices codes[0:1]/codes[1:2]/codes[2:3]. Consider a non-uniform case (e.g. {r1:1, r2:2, r3:1}) to exercise variable start/end slicing through seq_len=mm_seq_len. + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4870.trace.json.gz new file mode 100644 index 000000000..39eb1effe Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4923.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4923.cost.json new file mode 100644 index 000000000..5dab990fc --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1439.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4923/run-20260817-100118-ac97ca", + "recorded_at": "2026-08-17T10:25:14", + "attempts": [ + { + "invocation_id": "inv-9d04b9ec", + "outcome": "run:run-20260817-100118-ac97ca", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-100118-ac97ca", + "llm_calls": 136, + "input_tokens": 1155977, + "output_tokens": 274293, + "usd": 0.918693, + "source": "spans" + } + ], + "attempt_usd_total": 0.918693, + "llm_calls": 136, + "input_tokens": 1155977, + "output_tokens": 274293, + "usd": 0.918693, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-100118-ac97ca", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9187, + "minutes": 23.94, + "ci_minutes": 0, + "input_tokens": 1155977, + "output_tokens": 274293, + "tool_calls": 279, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.964, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 136, + "by_role": { + "agent": { + "usd": 0.867909, + "input_tokens": 1128240, + "output_tokens": 235000, + "calls": 133 + }, + "reducer": { + "usd": 0.050784, + "input_tokens": 27737, + "output_tokens": 39293, + "calls": 3 + } + }, + "cache_read_tokens": 11291008, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.48, + "gate": 1.65, + "review": 1431.4, + "report": 0.06 + }, + "time_to_first_result_s": 1436.57 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 39780 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4923.md b/eval/dataset/arms/copilot_v17ds_train/pr4923.md new file mode 100644 index 000000000..20ba22172 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4923.md @@ -0,0 +1,109 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] async-output vs prefix-cache exclusivity: `_should_use_async_omni_output` returns False when `omni_prefix_cache is not None` (gpu_ar_model_runner.py:1541-1543); disabling prefix caching in the yaml is what actually enables async output — residual: a user who re-enables prefix caching silently falls back to sync output with no warning. +- [claim-verified] PR says stage 1 consumes codec codes not hidden states: talker.py:317 pre-existing omni_pooler_payload_include_hidden=False with that rationale; _build_omni_pooler_payload omits hidden payload when flag is False. +- [claim-verified] PR body/comment admit seed loss: talker.py:334-335 'under full cudagraphs, per-request tts_local_seed is / not reproducible' and :345-346 'mtp seeding / is silently dropped when the decode batch > 1 under full cudagraphs'. +- [claim-verified] qwen3_tts.yaml `platforms.npu.stages[0/1].compilation_config.cudagraph_mode: PIECEWISE` present as described. +- [claim-verified] `has_full_cudagraphs()` already used at gpu_model_runner.py:209, so the pinned vLLM supports it; no new external-library kwarg needs a dependency-version check (item 10 clean). +- [claim-verified] PR perf numbers (talker_mtp 39ms->9ms, RTF 0.54->0.19) are reported-only and not independently reproducible here; note the benchmark bundles graph/async changes with capacity bumps (max_num_seqs 10->64, max_num_batched_tokens 512->32768), so attribution is not isolated. +- [claim-refuted] PR table says NPU PIECEWISE 'Fix NPU startup crash from nested graph capture', yet gh_ci_read shows buildkite/vllm-omni-npu-ci still FAILURE at head (AMD CI also FAILURE) — remaining failure needs attribution. +- [claim-refuted] The new code comment says seeded MTP is 'silently dropped'; the runner keeps a scalar fallback branch, but full-graph replay discards the per-call generator, so `tts_local_seed` is still non-reproducible — with no user-facing warning. +- [claim-refuted] PR body says the scalar fallback fires when talker_mtp_accepts_per_row_generators=True; runner actually gates it on `not` that flag (gpu_model_runner.py:1854), so the new False-under-full-cudagraphs value triggers N scalar calls for any seeded batch. +- [claim-refuted] PR body says talker 'did not set ... omni_pooler_payload_include_hidden'; the base file already sets it (head line 317, introduced by 6d50348e/#4879) — the added line 353 is a duplicate assignment. +- [validated] talker.py:327 sets talker_mtp_graph_safe=True; gpu_model_runner.py:209 wraps talker_mtp only when cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe) — matches PR claim. +- [validated] qwen3_tts_code_predictor_vllm.py:38 use_cuda_graphs=current_omni_platform.is_npu(); qwen3_code_predictor.py:552-556 confirms GPU with use_cuda_graphs=False takes torch.compile-only branch (no _capture_cuda_graphs). +- [validated] gpu_model_runner._store_value clones GPU-resident keys (codes,audio) before storing, so the graph static output buffer written into model_intermediate_buffer is not aliased. +- [validated] gpu_model_runner.py:209 wraps talker_mtp only when `cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe)`. + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~350` [major] — Confirmed. Under full cudagraphs qwen3_tts_talker.py:350 sets talker_mtp_accepts_per_row_generators=False, so gpu_model_runner.py:1854 falls back to per-row scalar calls and the graph-wrapped talker_mtp drops the per-row generator, silently making request.seed non-reproducible for residual codebooks (serving_speech.py:3694). Emit logger.warning_once in this mode and add a seeded-batch determinism test, or keep the eager per-row path for seeded batches instead of documenting the silent drop. (evidence: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped` +vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:334-335 `# Consequence: under full cudagraphs, per-request ``tts_local_seed`` is` / `# not reproducible (matching Qwen3-Omni, which has no per-row seeding);` +vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:345-346 `# This also means mtp seeding` / `# is silently dropped when the decode batch > 1 under full cudagraphs;` +unchanged by this diff, present in the PR-time tree: vllm_omni/worker/gpu_model_runner.py:1854 `and not getattr(self.model, "talker_mtp_accepts_per_row_generators", False)` +unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/serving_speech.py:3694 `stage0_params.extra_args["tts_local_seed"] = request.seed`) + +```suggestion +self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped + if talker_mtp_graph_wrapped: + logger.warning_once( + "Qwen3-TTS talker_mtp is graph-wrapped (full cudagraphs): per-request " + "tts_local_seed (request.seed) is ignored for residual codebooks." + ) +``` + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39` [major] — The new `use_cuda_graphs=current_omni_platform.is_npu()` default disables inner code-predictor graphs on every non-NPU platform (ROCm/XPU/MUSA), while the PR body only names GPU and NPU. Since `talker_mtp_graph_safe` is now True and `get_graph_wrapper_cls()` defaults to vLLM `CUDAGraphWrapper`, ROCm (and any other non-NPU platform running full cudagraphs) also gets the outer full wrap — so its code-predictor execution path silently changes from inner graphs to outer-wrap-only. Either gate the inner/outer graph behavior to the platforms actually validated, or add ROCm/XPU/MUSA coverage to this PR. (evidence: Changed by this diff: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu(),`. Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/interface.py:43 `return self._omni_enum == OmniPlatformEnum.NPU` (enum values ROCM/XPU/MUSA at interface.py:23,25,26). Changed by this diff: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:327 `self.talker_mtp_graph_safe = True`. Unchanged by this diff, present in the PR-time tree: vllm_omni/worker/gpu_model_runner.py:209-210 `if cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe):` / `graph_wrapper_cls = current_omni_platform.get_graph_wrapper_cls()`; vllm_omni/platforms/interface.py:216 `return CUDAGraphWrapper`.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:~39` [major] — On GPU, `use_cuda_graphs=current_omni_platform.is_npu()` is False, which also disables the prefix-graph path: `CodePredictorWrapper.__init__` computes `_prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu` (qwen3_code_predictor.py:477). `qwen3_tts_high_concurrency.yaml` still requests `code_predictor_prefix_graphs: true` + `code_predictor_prefix_graph_buckets: [64]`, so those keys now no-op on GPU and the wrapper logs 'requested but disabled'. Gate `use_cuda_graphs` to non-full-cudagraph GPU modes (keep inner graphs when the runner is not wrapping `talker_mtp`), or update and re-validate the high-concurrency profile in this PR. (evidence: Changed by this diff: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu(),`. Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/common/qwen3_code_predictor.py:477 `self._prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu`; vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:27 `code_predictor_prefix_graphs: true`; vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:28 `code_predictor_prefix_graph_buckets: [64]`.) + +```suggestion +use_cuda_graphs=( + current_omni_platform.is_npu() + or not vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs() + ), +``` + +`vllm_omni/deploy/qwen3_tts.yaml:41` [minor] — vllm_omni/deploy/qwen3_tts.yaml lines 41-45 still claim prefix caching is 'on by default' and a 'net throughput win' that should only be disabled when memory is tight, but line 46 sets `enable_prefix_caching: false`. The actual reason is that this PR enables async omni output (`self.use_async_omni_output = True` in qwen3_tts_talker.py), and `_should_use_async_omni_output` returns False whenever the omni prefix cache is enabled, so the reference profile must disable prefix caching to use async output. Rewrite the comment to match the value and reason. (evidence: vllm_omni/deploy/qwen3_tts.yaml:41 ` # Prefix caching on by default. The HS opt-out (talker only consumes the`; vllm_omni/deploy/qwen3_tts.yaml:46 ` enable_prefix_caching: false`; unchanged by this diff, present in the PR-time tree: vllm_omni/worker/gpu_ar_model_runner.py:1540-1541 ` if self.omni_prefix_cache is not None:` / ` return False` inside `_should_use_async_omni_output`; PR diff enables the async path via qwen3_tts_talker.py `self.use_async_omni_output = True`.) + +```suggestion +# Prefix caching is disabled so async omni output can run: the runner's + # `_should_use_async_omni_output` returns False whenever the omni prefix + # cache is enabled, and this stage opts into async output. +``` + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — Confirmed test gap. The new producer path in vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:346-350 sets `talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped` (False under full cudagraphs), and vllm_omni/worker/gpu_model_runner.py:1854 consumes it via `getattr(..., False)`. Existing tests at tests/worker/test_omni_gpu_model_runner.py:373-374 fake the flag on a SimpleNamespace and never construct this __init__ path, so add a model-level unit test for FULL vs PIECEWISE/eager gating as suggested. (evidence: Added by this diff — vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:348-350 `talker_mtp_graph_wrapped = ( + self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs() +) +self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`. Unchanged by this diff, present in the PR-time tree — vllm_omni/worker/gpu_model_runner.py:1854 `and not getattr(self.model, "talker_mtp_accepts_per_row_generators", False)`. Unchanged by this diff, present in the PR-time tree — tests/worker/test_omni_gpu_model_runner.py:373-374 `runner.model = SimpleNamespace( + talker_mtp_output_key=("codes", "audio"), + talker_mtp_accepts_per_row_generators=True, +)`.) + +`vllm_omni/deploy/qwen3_tts.yaml:48` [minor] — qwen3_tts.yaml:6-8 still says Stage 0 sets max_num_batched_tokens=512 and explains it as async-chunk first-chunk latency tuning, but the stage now sets 32768 at line 48. The PR body rationale 'Allow larger decode batches with graph capture' misstates the field: max_num_batched_tokens is the prefill budget (docs/configuration/stage_configs.md:44 'Prefill budget.'), while decode batch is bounded by max_num_seqs. The reported RTF/TTFP benchmark runs the whole PR (capacity + graph + async changes), so it does not isolate this retune. Update the header comment and either split the capacity retune or state that the benchmark used exactly this updated yaml. (evidence: vllm_omni/deploy/qwen3_tts.yaml:6-8 (unchanged by this diff, present in PR-time tree): '# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk' / '# latency tuning (not correctness) — small per-step batches keep' / '# first-chunk latency low.'; vllm_omni/deploy/qwen3_tts.yaml:48: ' max_num_batched_tokens: 32768'; docs/configuration/stage_configs.md:44 (unchanged by this diff): '| max_num_batched_tokens | int | optional | 32768 | Prefill budget. |'; PR body rationale table: 'max_num_batched_tokens (stage 0) | 512 | 32768 | Allow larger decode batches with graph capture'.) + +```suggestion +# Stage 0 (talker) raises max_num_batched_tokens to 32768 to reduce prefill + # queueing under concurrent load; first-chunk latency is now bounded by the + # per-request prompt length and max_num_seqs, not by this token budget. + max_num_batched_tokens: 32768 +``` + +`vllm_omni/deploy/qwen3_tts.yaml:~99` [minor] — Confirmed: the NPU override pins both stages to cudagraph_mode: PIECEWISE (qwen3_tts.yaml:99,102), which keeps talker_mtp unwrapped because has_full_cudagraphs() is false for PIECEWISE. This re-encodes in YAML the same wrap decision the model derives at qwen3_tts_talker.py:346-350, and the code's own TODO states that read belongs in the runner. Prefer moving the gating into the runner (models expose only talker_mtp_graph_safe) and dropping the per-platform pin, or document explicitly why NPU's main model must run PIECEWISE rather than the platform default. (evidence: qwen3_tts.yaml:99 ` cudagraph_mode: PIECEWISE` and qwen3_tts.yaml:102 ` cudagraph_mode: PIECEWISE` (both added by this diff under `platforms: npu: stages:`). qwen3_tts_talker.py:343-345 ` # TODO(#4923): the model should not read ``cudagraph_mode`` — it is an + # upstream (runner) concern, and the runner already re-derives the same + # wrap decision in ``_init_talker_mtp``.`; qwen3_tts_talker.py:346-350 ` cudagraph_mode = getattr(vllm_config.compilation_config, "cudagraph_mode", None) + talker_mtp_graph_wrapped = ( + self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs() + ) + self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`.) + +`gpu_ar_model_runner.py:1539` [minor] — async-output vs prefix-cache exclusivity: `_should_use_async_omni_output` returns False when `omni_prefix_cache is not None` (gpu_ar_model_runner.py:1539-1541); disabling prefix caching in the yaml (qwen3_tts.yaml stage 0 `enable_prefix_caching: false`) is what actually enables the newly added `use_async_omni_output = True` async path — residual: a user who re-enables prefix caching silently falls back to sync output with no warning. (evidence: vllm_omni/worker/gpu_ar_model_runner.py:1537-1541 `def _should_use_async_omni_output(self) -> bool: / if not self.use_async_scheduling: return False / if self.omni_prefix_cache is not None: return False / if self.speculative_config is not None: return False` — unchanged by this diff, present in the PR-time tree; vllm_omni/deploy/qwen3_tts.yaml (diff) changes stage 0 to `enable_prefix_caching: false`; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:351 `self.use_async_omni_output = True` (added by this diff).) + +**Additional observations (beyond the comment budget):** +- `docs/user_guide/examples/online_serving/text_to_speech.md:334` [minor] text_to_speech.md:334 still documents stage 1 (Code2Wav) at max_num_seqs: 10, but vllm_omni/deploy/qwen3_tts.yaml now ships stage_id 1 with max_num_seqs: 64. Update this paragraph (and the stage-overrides example) so the documented capacity matches the shipped profile. +- `vllm_omni/worker/gpu_model_runner.py:1854` [minor] Correct the PR-body 'Per-row generator fallback' bullet: the runner's scalar fallback is gated on `not talker_mtp_accepts_per_row_generators` (gpu_model_runner.py:1854), so it fires when the flag is False, not True. Under this PR, full cudagraphs set the flag to False via `self.talker_mtp_accepts_per_row_generators = […] +- `tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py:481` [minor] This PR changes the Qwen3-TTS code-predictor constructor to `use_cuda_graphs=current_omni_platform.is_npu()` (qwen3_tts_code_predictor_vllm.py:39), but `test_tts_config` (test_code_predictor_dtype.py:477-488) still builds `CodePredictorWrapperConfig(use_cuda_graphs=True, ...)` and asserts the value it just injected — […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4923.trace.json.gz new file mode 100644 index 000000000..13d462ed7 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4926.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4926.cost.json new file mode 100644 index 000000000..27b786634 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4926.cost.json @@ -0,0 +1,123 @@ +{ + "wall_s": 2322.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4926/run-20260817-120618-557198", + "recorded_at": "2026-08-17T12:44:58", + "attempts": [ + { + "invocation_id": "inv-75b1f334", + "outcome": "run:run-20260817-120618-557198", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-104958-7295a6", + "llm_calls": 1, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "spans" + }, + { + "run_dir": "run-20260817-105011-b1bdc1", + "llm_calls": 1, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "spans" + }, + { + "run_dir": "run-20260817-120618-557198", + "llm_calls": 201, + "input_tokens": 874597, + "output_tokens": 410168, + "usd": 1.007272, + "source": "spans" + } + ], + "attempt_usd_total": 1.007272, + "llm_calls": 201, + "input_tokens": 874597, + "output_tokens": 410168, + "usd": 1.007272, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-120618-557198", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.0073, + "minutes": 38.67, + "ci_minutes": 0, + "input_tokens": 874597, + "output_tokens": 410168, + "tool_calls": 364, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.1978, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 201, + "by_role": { + "agent": { + "usd": 0.949428, + "input_tokens": 835961, + "output_tokens": 367135, + "calls": 198 + }, + "reducer": { + "usd": 0.057844, + "input_tokens": 38636, + "output_tokens": 43033, + "calls": 3 + } + }, + "cache_read_tokens": 11849856, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.02, + "gate": 1.55, + "review": 2314.31, + "report": 0.07 + }, + "time_to_first_result_s": 2319.92 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 72754 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4926.md b/eval/dataset/arms/copilot_v17ds_train/pr4926.md new file mode 100644 index 000000000..58ae9471e --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4926.md @@ -0,0 +1,126 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [resolved] commit 332da8f 'Fall back to varlen kernel in Hub piecewise attention' — _flash_wrapper:117-123 now falls back to varlen when flash_attn_func is None; residual: masked paths :147/:326 still call varlen_func without a None guard (finding #4). +- [resolved] The fallback test simulation is sound: Python 3.12 import_module delegates to _gcd_import (importlib/__init__.py:77-83), and importing a module whose sys.modules entry is None raises ModuleNotFoundError — so `monkeypatch.setitem(sys.modules,'kernels',None)` exercises the ImportError branch. +- [resolved] Piecewise fallback to varlen (commit 332da8f2): _flash_wrapper routes attn_func=None → _run_varlen_dense (flash_attn_hub.py:117-124); piecewise_attn passes per-segment uniform-batch q/k/v, which matches _run_varlen_dense's cu_seqlens construction. +- [claim-verified] Non-CUDA fallback matches PR body: musa/npu/rocm/xpu platform.py each add the identical HUB→FLASH_ATTN branch (read all four in-tree) and cuda falls back when `importlib.import_module("kernels")` raises ImportError (cuda/platform.py:124-138). +- [claim-verified] FLASH_ATTN_3_HUB→FLASH_ATTN_HUB fallback on pre-Hopper exists (cuda/platform.py:140-148) and is exercised by test_kernels_hub_platform_fallback:50-53. +- [claim-verified] cuda/platform.py:124-138: `kernels` import failure -> local FLASH_ATTN fallback works and is unit-tested. +- [claim-verified] cuda/platform.py:140-147: FA3_HUB pre-Hopper -> FLASH_ATTN_HUB as documented. +- [claim-verified] docs 'sm_90+' vs gate `compute_capability.major >= 9` is internally consistent (admits sm_100/120 Blackwell as 'sm_90+'). +- [claim-verified] diff_stat: 9 files, 601 insertions, no automated e2e z-image test shipped despite PR body's E2E mention (only manual image attachments). +- [claim-verified] PR's reported max diff 0.00390625 < 1e-2 threshold (checked with calc) for the FA2/FA3 dense assertions as reported. +- [claim-verified] PR body's non-CUDA fallback claim holds: musa/npu/rocm/xpu platform.py all redirect both hub names to FLASH_ATTN with a warning. +- [claim-refuted] PR body claims 'E2E test using z-image t2i generation with kernels FA3' — diff_stat shows the only test added is tests/diffusion/attention/test_kernels_hub.py; no z-image E2E test, the FA3 result is two screenshots. +- [claim-refuted] 'falls back gracefully' covers only import failure: get_kernel fetch failure at flash_attn_hub.py:96-103 / 275-282 raises at model load (3rd call is outside try/except). +- [claim-refuted] 'All unit tests pass' — test_kernels_hub_execution (L4) is never selected by any .buildkite lane; grep `tests/diffusion/attention` across .buildkite returns no matches. + +`vllm_omni/platforms/cuda/platform.py:140` [major] — vllm_omni/platforms/cuda/platform.py:140-147 rewrites an explicit FLASH_ATTN_3_HUB request to FLASH_ATTN_HUB on pre-Hopper GPUs, and lines 126-136 rewrite any HUB request to local FLASH_ATTN when `kernels` is not importable. Both fallbacks execute a different kernel with different numerics, contradicting the PR's stated purpose of perfect train/rollout numerical consistency, with only a WARNING log (and the doc's fallback note) as signal. Make explicit HUB selection fail hard or require an explicit allow-fallback flag, or expose the effective backend in user-visible config/response. (evidence: vllm_omni/platforms/cuda/platform.py:126 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):`; vllm_omni/platforms/cuda/platform.py:128 `importlib.import_module("kernels")`; vllm_omni/platforms/cuda/platform.py:132 `backend_upper = "FLASH_ATTN"`; vllm_omni/platforms/cuda/platform.py:140 `if backend_upper == "FLASH_ATTN_3_HUB":`; vllm_omni/platforms/cuda/platform.py:145 `"Falling back to FLASH_ATTN_HUB."`; vllm_omni/platforms/cuda/platform.py:147 `backend_upper = "FLASH_ATTN_HUB"` — all present in the PR diff; the requested kernel is replaced with a numerically different one without hard failure.) + +```suggestion +if backend_upper == "FLASH_ATTN_3_HUB": + fa3_hub_supported = compute_capability is not None and compute_capability.major >= 9 + if not fa3_hub_supported: + raise ValueError( + "FLASH_ATTN_3_HUB requires compute capability >= 9.0; " + "refusing to silently degrade train/rollout numerical alignment. " + "Select FLASH_ATTN_HUB explicitly if fallback is intended." + ) +``` + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96` [major] — FlashAttentionHubImpl.__init__ (and the FlashAttention3HubImpl twin at line 270+) calls kernels.get_kernel once per Attention layer — layer.py:95 `self.attention = self.attn_impl_cls(` — so model load performs an unmemoized Hub resolution per layer. The platform selector only probes importability (cuda/platform.py:126 `importlib.import_module("kernels")`), so a kernels install that imports but cannot fetch (offline/air-gapped, rate-limit, cache miss) passes selection and then crashes at model load: the third get_kernel attempt (flash_attn_hub.py:103) is unwrapped, and the 'no functions found' RuntimeError is likewise unrecoverable. Memoize the loaded module keyed by repo id and wrap the final attempt so a fetch failure falls back or raises a clear, actionable error. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)`; vllm_omni/diffusion/attention/backends/flash_attn_hub.py:103 `hub_module = get_kernel("kernels-community/flash-attn2")` — final fallback call is not wrapped in its own try/except (the enclosing block is the `except Exception as e2:` handler, not a try); vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92 `from kernels import get_kernel` — resolution happens in every FlashAttentionHubImpl.__init__. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/attention/layer.py:95 `self.attention = self.attn_impl_cls(` — each Attention layer instantiates the impl, so the fetch runs per layer. vllm_omni/platforms/cuda/platform.py:126 `importlib.import_module("kernels")` — selector probes importability only, so an importable-but-unfetchable kernels package passes selection.) + +```suggestion +from functools import lru_cache + + @lru_cache(maxsize=None) + def _load_hub_kernel(repo_id: str): + from kernels import get_kernel + + try: + return get_kernel(repo_id, version=1) + except Exception: + try: + return get_kernel(repo_id, version=2) + except Exception: + return get_kernel(repo_id) + + hub_module = _load_hub_kernel("kernels-community/flash-attn2") +``` + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96` [major] — The Hub constructors resolve kernels through an unpinned three-rung fallback — version=1, then version=2, then the no-version default — and log only at INFO level on failure. A transient failure of version=1 will silently run a different kernel revision, directly contradicting the documented 'perfect numerical consistency' guarantee. Pin an immutable revision (commit SHA/ref) and fail loudly (or at minimum emit a prominent warning) instead of silently switching kernels. The same applies to the flash-attn3 path at lines 275-282. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)`; flash_attn_hub.py:100 `hub_module = get_kernel("kernels-community/flash-attn2", version=2)`; flash_attn_hub.py:103 `hub_module = get_kernel("kernels-community/flash-attn2")` — no revision/SHA is passed, and exceptions between rungs only call `logger.info(...)` before switching kernels. docs/user_guide/diffusion/attention_backends.md:214 `To achieve perfect numerical consistency between **training** (typically using HuggingFace Diffusers with Hub kernels) and **serving/rollout** (using vLLM-Omni), you can use the Hub-based attention backends. This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout.` — the documented guarantee is contradicted by the unpinned silent fallback in the changed code.) + +```suggestion +try: + hub_module = get_kernel("kernels-community/flash-attn2", version=1) + except Exception as e: + raise RuntimeError( + "Failed to load pinned kernels-community/flash-attn2 (version=1) from the Hub; " + "refusing to silently switch to a different kernel revision." + ) from e +``` + +`vllm_omni/platforms/cuda/platform.py:124` [major] — The Hub branch at vllm_omni/platforms/cuda/platform.py:124-148 never consults `compute_supported` or `flash_attn_supported`; it only probes `importlib.import_module("kernels")` and, for FLASH_ATTN_3_HUB, checks `compute_capability.major >= 9`. With `kernels` installed on an sm_75 GPU, FLASH_ATTN_HUB (and FLASH_ATTN_3_HUB, via the FA3->FA2 fallback) reaches the enum lookup and returns the Hub backend path, whose FA2 kernel cannot launch below sm_80 — bypassing the TORCH_SDPA fallback that local FLASH_ATTN gets at line 149. Gate the Hub backends on `compute_supported` after the FA3->FA2 fallback and fall back to TORCH_SDPA. (evidence: vllm_omni/platforms/cuda/platform.py:80 `compute_supported = capability >= 80`; vllm_omni/platforms/cuda/platform.py:124 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):`; vllm_omni/platforms/cuda/platform.py:125 `importlib.import_module("kernels")` — this branch never tests `compute_supported`; vllm_omni/platforms/cuda/platform.py:149 `if backend_upper == "FLASH_ATTN" and not flash_attn_supported:` is the only sm<80 fallback and it cannot match a retained Hub backend, so the Hub path falls through to the enum lookup/return at the end of the selected_backend block.) + +```suggestion +if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB") and not compute_supported: + logger.warning( + "HuggingFace kernels-backed FlashAttention requires GPU with compute capability >= 8.0. " + "Falling back to TORCH_SDPA backend." + ) + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() +``` + +`tests/diffusion/attention/test_kernels_hub.py:42` [major] — New @pytest.mark.cpu test calls CudaOmniPlatform.get_diffusion_attn_backend_cls, which unconditionally evaluates torch.backends.cudnn.version() (vllm_omni/platforms/cuda/platform.py:100) before the selected_backend branch. On ROCm torch, torch.backends.cudnn is absent, so test_kernels_hub_platform_fallback errors during the AMD merge lane (.buildkite/test-amd-merge.yml:21: pytest -sv tests/diffusion -m 'core_model and cpu' on mi325_1). The NPU ready lane only runs test_qwen3_tts_customvoice.py (test-npu-ready.yaml:21), so it cannot be affected by this file. Make the test portable by monkeypatching torch.backends.cudnn or skipping unless CUDA. (evidence: tests/diffusion/attention/test_kernels_hub.py:42 ` backend_path = CudaOmniPlatform.get_diffusion_attn_backend_cls("FLASH_ATTN_HUB", head_size=64)` (added by this PR, in the diff); unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/cuda/platform.py:100 ` cudnn_version = torch.backends.cudnn.version() or 0` executes unconditionally before the selected_backend branch; unchanged by this diff, present in the PR-time tree: .buildkite/test-amd-merge.yml:21 ` - "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` on ROCm agent mi325_1; unchanged by this diff, present in the PR-time tree: .buildkite/test-npu-ready.yaml:21 ` - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`.) + +```suggestion +monkeypatch.setattr( + torch.backends, + "cudnn", + types.SimpleNamespace(version=lambda: 90500), + raising=False, + ) +``` + +`vllm_omni/platforms/cuda/platform.py:141` [minor] — The gate at cuda/platform.py:141 (`major >= 9`) lets Blackwell (sm_10x/sm_12x) through, contradicting the 'Hopper (sm_90+) only' docs and the only execution test (L4, sm_89), which skips the FA3 path entirely — so FLASH_ATTN_3_HUB is unverified on every admitted architecture above sm_90. Either narrow to `major == 9` until Blackwell is validated, or document Blackwell support and add a Blackwell result. (evidence: vllm_omni/platforms/cuda/platform.py:141 `fa3_hub_supported = compute_capability is not None and compute_capability.major >= 9` — admitted Blackwell major 10/12 passes this gate (unchanged context in same file: `is_blackwell = major in (10, 11, 12)`); docs/user_guide/diffusion/attention_backends.md:24 `| \`FLASH_ATTN_3_HUB\` | FlashAttention 3 from HuggingFace \`kernels\` library. CUDA Hopper (sm_90+) only; falls back to \`FLASH_ATTN_HUB\` on older GPUs. |`; tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)`; tests/diffusion/attention/test_kernels_hub.py:92 `if major < 9:` followed by line 93 `pytest.skip("FLASH_ATTN_3_HUB execution requires Hopper-class GPU (compute capability >= 9.0)")`) + +```suggestion +fa3_hub_supported = compute_capability is not None and compute_capability.major == 9 +``` + +`?:?` [minor] — vllm_omni/diffusion/attention/backends/flash_attn_hub.py:147 and :326 — _forward_varlen_masked calls self.flash_attn_varlen_func with no None guard. __init__ (lines 106-108) only raises when both flash_attn_func and flash_attn_varlen_func are None, so when the Hub module exposes flash_attn_func but not flash_attn_varlen_func, any masked attention (attention_mask with False entries) raises TypeError. Add the same fallback/guard used in _flash_wrapper (lines 117-123). Severity: minor. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:106 ` self.flash_attn_varlen_func = getattr(hub_module, "flash_attn_varlen_func", None)`; :108 ` if self.flash_attn_func is None and self.flash_attn_varlen_func is None:` (guards only the both-None case, so varlen may be None while flash_attn_func exists); :147 ` out_unpad = self.flash_attn_varlen_func(` and :326 ` out_unpad = self.flash_attn_varlen_func(` — the masked paths call the possibly-None varlen function directly; contrast :118 ` if attn_func is not None:` in _flash_wrapper, which handles the None case before falling back.) + +`vllm_omni/platforms/musa/platform.py:81` [minor] — Add unit coverage for the four non-CUDA HUB→FLASH_ATTN fallbacks (musa:81, npu:110, rocm:91, xpu:45), mirroring test_kernels_hub_platform_fallback. None of these branches is exercised in this PR — the only new test mocks CudaOmniPlatform — and each fallback is a warn-only route to local FLASH_ATTN, so a regression would silently diverge from the feature's kernel-consistency goal. (evidence: vllm_omni/platforms/musa/platform.py:81 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):` (present in this PR's diff) followed by warning `"not supported on MUSA. Falling back to local " "FLASH_ATTN."`; vllm_omni/platforms/npu/platform.py:110 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):`; vllm_omni/platforms/rocm/platform.py:91 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):`; vllm_omni/platforms/xpu/platform.py:45 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):` — all four branch blocks set `backend_upper = "FLASH_ATTN"` after a warning. The PR's only new test, tests/diffusion/attention/test_kernels_hub.py:42 `backend_path = CudaOmniPlatform.get_diffusion_attn_backend_cls("FLASH_ATTN_HUB", head_size=64)`, mocks only CudaOmniPlatform; grep for MUSAOmniPlatform/NPUOmniPlatform/RocmOmniPlatform/XPUOmniPlatform in tests/diffusion/attention returns no matches.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/attention/layer.py:95` [minor] layer.py constructs the Hub attention impl unconditionally (line 95: self.attention = self.attn_impl_cls(...)) and only decides to use sdpa_fallback for float32 at forward time (lines 285-290). FlashAttentionHubImpl.__init__ eagerly imports kernels and calls get_kernel('kernels-community/flash-attn2', ...) […] +- `tests/diffusion/attention/test_kernels_hub.py:57` [minor] test_kernels_hub_execution is live-Hub-dependent and only exercises the dense path: FlashAttentionHubImpl.__init__ calls kernels.get_kernel(...) on instantiation (flash_attn_hub.py:96), and on L4 (sm_89) the FA3 half always skips (test_kernels_hub.py:92-93), while the metadata uses attn_mask=None (dense path only). […] +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:147` [minor] `_forward_varlen_masked` calls `self.flash_attn_varlen_func` unguarded, but `__init__` only raises when BOTH `flash_attn_func` and `flash_attn_varlen_func` are None, while `supports_attention_mask` returns True unconditionally. If a hub module ever exposes only the dense `flash_attn_func`, a masked request (e.g. SP […] +- `tests/diffusion/attention/test_kernels_hub.py:92` [minor] The FA3 forward is never exercised by merge-gate CI: the only command that recurses tests/diffusion/attention is the 'core_model and cpu' lane (test-ready.yml:13), which skips test_kernels_hub_execution because it has no cpu marker; the cuda lanes restrict to […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4926.trace.json.gz new file mode 100644 index 000000000..0cf8735a5 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4950.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4950.cost.json new file mode 100644 index 000000000..becd3958c --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4950.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 334.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4950/run-20260817-124500-6cff39", + "recorded_at": "2026-08-17T12:50:32", + "attempts": [ + { + "invocation_id": "inv-1d315eda", + "outcome": "run:run-20260817-124500-6cff39", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-124500-6cff39", + "llm_calls": 24, + "input_tokens": 198320, + "output_tokens": 37319, + "usd": 0.110955, + "source": "spans" + } + ], + "attempt_usd_total": 0.110955, + "llm_calls": 24, + "input_tokens": 198320, + "output_tokens": 37319, + "usd": 0.110955, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-124500-6cff39", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.111, + "minutes": 5.53, + "ci_minutes": 0, + "input_tokens": 198320, + "output_tokens": 37319, + "tool_calls": 45, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2455, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 24, + "by_role": { + "agent": { + "usd": 0.110955, + "input_tokens": 198320, + "output_tokens": 37319, + "calls": 24 + } + }, + "cache_read_tokens": 605824, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.5, + "gate": 1.95, + "review": 326.05, + "report": 0.06 + }, + "time_to_first_result_s": 331.54 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4950.trace.json.gz", + "trace_bytes": 11725 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4950.md b/eval/dataset/arms/copilot_v17ds_train/pr4950.md new file mode 100644 index 000000000..429b3ffeb --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4950.md @@ -0,0 +1,49 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4950, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 48d4fd8a5e98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Concern that the Gradio README contradicts the new curl guidance is resolved: the demo uses the SDK extra_body path, which the new 'OpenAI Python SDK may use extra_body' sentence covers; residual: the README does not link to the recipe for raw-HTTP users. +- [claim-verified] Server reads top-level field: vllm_omni/entrypoints/openai/serving_chat.py:373 'chat_template_kwargs=request.chat_template_kwargs,' and :381 'merged_template_kwargs = self._effective_chat_template_kwargs(request)'. +- [claim-verified] extra_body is not merged into chat-template kwargs in the AR path; repo-wide grep found no extra_body->chat_template_kwargs merge, and the image path reads extra_body only for image params (serving_chat.py:441). +- [claim-verified] OpenAI SDK extra_body flattening matches in-repo comment serving_chat.py:438-441 and gradio_demo.py:223 which sends extra_body={'chat_template_kwargs': {'use_tts_template': True}, ...}. +- [claim-verified] Two-choices response shape matches chat_completion_full_generator: text and audio build separate choices then extend one list (serving_chat.py:2159 and :2569). +- [claim-verified] 151703 is the 4.5 tts_bos id: vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:94 'tts_bos_id, tts_eos_id = 151703, 151704'. +- [sweep] Diff hunk 1 (lines 102-107) read in situ in recipes/OpenBMB/MiniCPM-o-4_5.md: curl example now uses top-level chat_template_kwargs. +- [sweep] Diff hunk 2 (lines 113-123) read in situ: SDK extra_body note and two-choices response note present. +- [sweep] Diff hunk 3 (lines 216-225) read in situ: TTS-trigger note corrected to 'returns silent audio (not text-only)'. +- [sweep] Repo-wide grep use_tts_template: only this recipe, gradio_demo.py:223, and examples/online_serving/minicpmo/README.md:58; both examples use the SDK extra_body path and are consistent with the new SDK note. +- [sweep] examples/online_serving/minicpmo/README.md:58 'extra_body.chat_template_kwargs.use_tts_template=True' stays accurate for the demo's OpenAI-SDK client; no stale consumer found. +- [sweep] No test files touched (per sweep_targets); docs-only PR, runtime verification is manual per the PR body. + +`recipes/OpenBMB/MiniCPM-o-4_5.md:~222` [minor] — Confirmed statically: omitting use_tts_template leaves tts_token_ids/tts_hidden_states unset (stage_input_processors/minicpmo_4_5_omni.py:104-118), the talker forward takes the dummy zero path (minicpmo_4_5_omni_tts.py:406-409), and the wrapper packages multimodal_outputs=None (minicpmo_4_5_omni.py:320-331) — no waveform is generated. The doc's 'returns silent audio (not text-only)' may overstate the behavior; please verify the serving layer actually synthesizes an audio choice from final_output_type='audio' when multimodal_outputs is None, or rewrite to 'produces no speech output'. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:104-105 `tts_token_ids_slice = tts_hidden_slice = None` / `if tts_bos_idx is not None and full_hidden.shape[0] > tts_bos_idx:` — slices stay None when no tts_bos token exists; vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_tts.py:406-409 `if tts_token_ids is None or tts_hidden_states is None:` ... `return self._dummy_hidden_states(input_ids, positions, inputs_embeds)` — no waveform on the missing-flag path; vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni.py:327 `mm_out["model_outputs"] = [waveform]` and later `return OmniOutput(text_hidden_states=dummy_hidden, multimodal_outputs=mm_out if mm_out else None)` — the dummy path produces multimodal_outputs=None, not a silent WAV.) + +`recipes/OpenBMB/MiniCPM-o-4_5.md:107` [nit] — At recipes/OpenBMB/MiniCPM-o-4_5.md:105, the parenthetical is true but indirect: the server reads `chat_template_kwargs` from the request root (`serving_chat.py:373` uses `request.chat_template_kwargs`), so nested `extra_body.chat_template_kwargs` is ignored for raw curl. Reword to: `field (the server reads chat_template_kwargs from the request root, not from extra_body):` (evidence: Unchanged by this diff, present in the PR-time tree — vllm_omni/entrypoints/openai/serving_chat.py:373 `chat_template_kwargs=request.chat_template_kwargs,` and vllm_omni/entrypoints/openai/serving_chat.py:381 `merged_template_kwargs = self._effective_chat_template_kwargs(request)`.) + +```suggestion +field (the server reads `chat_template_kwargs` from the request root, not from `extra_body`): +``` + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4950.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4950.trace.json.gz new file mode 100644 index 000000000..d591adca8 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4950.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4970.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4970.cost.json new file mode 100644 index 000000000..a151331bb --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4970.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 411.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4970/run-20260817-123850-b80b95", + "recorded_at": "2026-08-17T12:45:39", + "attempts": [ + { + "invocation_id": "inv-ded84b1c", + "outcome": "run:run-20260817-123850-b80b95", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-123850-b80b95", + "llm_calls": 19, + "input_tokens": 241846, + "output_tokens": 35026, + "usd": 0.125558, + "source": "spans" + } + ], + "attempt_usd_total": 0.125558, + "llm_calls": 19, + "input_tokens": 241846, + "output_tokens": 35026, + "usd": 0.125558, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-123850-b80b95", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1256, + "minutes": 6.81, + "ci_minutes": 0, + "input_tokens": 241846, + "output_tokens": 35026, + "tool_calls": 41, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2885, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 19, + "by_role": { + "agent": { + "usd": 0.125558, + "input_tokens": 241846, + "output_tokens": 35026, + "calls": 19 + } + }, + "cache_read_tokens": 804864, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.36, + "gate": 1.76, + "review": 403.25, + "report": 0.06 + }, + "time_to_first_result_s": 408.4 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4970.trace.json.gz", + "trace_bytes": 11716 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4970.md b/eval/dataset/arms/copilot_v17ds_train/pr4970.md new file mode 100644 index 000000000..71f76e8bb --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4970.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4970, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 4992e10b7367) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Concern 'deleting a reference-profile value needs a tombstone + docs sweep': at this head no in-file tombstone was added; residual — docs merely point at `--deploy-config qwen3_tts.yaml` without claiming determinism, but a future maintainer copying the profile may re-add `seed: 42` and silently regress throughput; filed as minor. +- [claim-verified] Per-request determinism survives removal: serving_speech.py sets `stage0_params.seed = request.seed` and `extra_args["tts_local_seed"] = request.seed` when `request.seed is not None`. +- [claim-verified] "no test references the deploy-level seed" at this head: the only default-seed test (test_serving_speech.py:3347) checks the propagation contract, not the YAML value. +- [validated] serving_speech.py:3698-3706 is qwen3_tts-gated: `default_seed = getattr(stage0_params, "seed", None)` then `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))` — with the YAML seed gone, `default_seed` is None and no request carries `tts_local_seed`. +- [validated] gpu_model_runner.py:1826 `seed = extra_args.get("tts_local_seed")` feeds `_row_generator`; all-None row_generators skip the `decode_batch_size > 1 and any(...)` scalar fallback, returning to the single batched-multinomial path. +- [sweep] diff_stat: 1 file, 2 deletions (`vllm_omni/deploy/qwen3_tts.yaml`); no test files touched — matches the provided diff. +- [sweep] Read `vllm_omni/deploy/qwen3_tts.yaml` in full and verified both hunks in situ: stage-0 `default_sampling_params` now reads `min_tokens: 2` -> `repetition_penalty: 1.05`; stage-1 reads `max_tokens: 65536` -> `repetition_penalty: 1.0`; no `seed:` remains. +- [sweep] Repo-wide grep for `tts_local_seed`: consumers are gpu_model_runner.py (reader), tests/worker/test_omni_gpu_model_runner.py (per-request seeds 11/22/42), and tests/entrypoints/openai_api/test_serving_speech.py:3347 (propagation unit test) — the unit test exercises serving code with a mocked seed and is not stale after this YAML-only change. +- [sweep] Sibling contrast: `vllm_omni/deploy/qwen3_tts_high_concurrency.yaml` already has no `seed:` in either stage (matching #4869); `qwen3_tts.yaml` was the only remaining qwen3_tts deploy profile with a default seed. +- [sweep] Fleet exposure: grep `seed: ?[0-9]+` in `vllm_omni/deploy` finds ~60 yamls still shipping seed 42/52, but the `tts_local_seed` propagation is `if self._tts_model_type == "qwen3_tts"`-gated, so only qwen3_tts profiles are exposed to THIS mechanism — both are now seed-free. +- [sweep] CI lanes loading qwen3_tts.yaml: .buildkite/test-npu-ready.yaml:21, test-amd-merge.yml:177/192, test-merge.yml:174/201, test-ready.yml:328/354 — the red AMD/NPU gates intersect this config's coverage (config-only is not inert). +- [sweep] qwen3_tts.yaml consumers enumerated (e2e online/offline qwen3_tts tests, websocket/speaker-embedding/batch tests, dfx invalid-param test, benchmarks/tts/README, docs/serving/speech_api.md, recipes/Qwen/Qwen3-TTS.md); none asserts a deploy-level seed. + +`vllm_omni/deploy/qwen3_tts.yaml:60` [minor] — Deleting `seed: 42` from stage 0's `default_sampling_params` makes default serve nondeterministic; that absence is intentional because `stage0_params.seed` is propagated to `extra_args["tts_local_seed"]` (serving_speech.py:3694/3698/3706) and consumed by the runner (gpu_model_runner.py:1826), which selects the per-row seeded path implicated in #4963. Consider adding a tombstone comment near `min_tokens: 2` explaining why no default seed is set and that deterministic output is available per-request via the `seed` field. (evidence: vllm_omni/deploy/qwen3_tts.yaml (changed by this diff): stage 0 `default_sampling_params` now ends with ` min_tokens: 2` then ` repetition_penalty: 1.05` with no `seed` line (the diff removed `- seed: 42`). Unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/serving_speech.py:3698 `default_seed = getattr(stage0_params, "seed", None)`; vllm_omni/entrypoints/openai/serving_speech.py:3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`; vllm_omni/entrypoints/openai/serving_speech.py:3694 `stage0_params.extra_args["tts_local_seed"] = request.seed`; vllm_omni/worker/gpu_model_runner.py:1826 `seed = extra_args.get("tts_local_seed")`.) + +```suggestion +min_tokens: 2 + # No default seed: a deploy-level seed propagates to tts_local_seed + # (serving_speech.py) and forces the per-row seeded talker-MTP path + # (~40% audio_throughput regression; #4963). Use the per-request + # `seed` field for deterministic output. + repetition_penalty: 1.05 +``` + +`.buildkite/test-npu-ready.yaml:~21` [minor] — The NPU lane (`.buildkite/test-npu-ready.yaml:21`) and AMD lane (`.buildkite/test-amd-merge.yml:177`) both run `tests/e2e/online_serving/test_qwen3_tts_customvoice.py`, which loads the changed `qwen3_tts.yaml` via `get_deploy_config_path("qwen3_tts.yaml")` at line 22 (not line 44). Both CI checks are currently failing, so before merge confirm whether those failures are pre-existing on main or introduced by this config-only change. (evidence: Unchanged by this diff, present in the PR-time tree: `.buildkite/test-npu-ready.yaml:21` ` - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; `tests/e2e/online_serving/test_qwen3_tts_customvoice.py:21-22` `_STAGE_CONFIG = modify_stage_config(` / ` get_deploy_config_path("qwen3_tts.yaml"),`; `.buildkite/test-amd-merge.yml:177` ` pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"`; `.buildkite/test-amd-merge.yml:192` ` pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'`.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4970.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4970.trace.json.gz new file mode 100644 index 000000000..0efd24c72 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4970.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4977.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr4977.cost.json new file mode 100644 index 000000000..e365cc952 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4977.cost.json @@ -0,0 +1,123 @@ +{ + "wall_s": 960.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr4977/run-20260817-115017-f8c68c", + "recorded_at": "2026-08-17T12:06:15", + "attempts": [ + { + "invocation_id": "inv-85217531", + "outcome": "run:run-20260817-115017-f8c68c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-104420-8edf85", + "llm_calls": 53, + "input_tokens": 157925, + "output_tokens": 66259, + "usd": 0.166646, + "source": "spans" + }, + { + "run_dir": "run-20260817-104946-ed83da", + "llm_calls": 1, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "spans" + }, + { + "run_dir": "run-20260817-115017-f8c68c", + "llm_calls": 100, + "input_tokens": 360468, + "output_tokens": 191653, + "usd": 0.396826, + "source": "spans" + } + ], + "attempt_usd_total": 0.563472, + "llm_calls": 100, + "input_tokens": 360468, + "output_tokens": 191653, + "usd": 0.396826, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-115017-f8c68c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3968, + "minutes": 15.96, + "ci_minutes": 0, + "input_tokens": 360468, + "output_tokens": 191653, + "tool_calls": 200, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6195, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 100, + "by_role": { + "agent": { + "usd": 0.371069, + "input_tokens": 347922, + "output_tokens": 171358, + "calls": 98 + }, + "reducer": { + "usd": 0.025757, + "input_tokens": 12546, + "output_tokens": 20295, + "calls": 2 + } + }, + "cache_read_tokens": 3284480, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.29, + "gate": 1.52, + "review": 952.5, + "report": 0.07 + }, + "time_to_first_result_s": 957.36 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 45557 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4977.md b/eval/dataset/arms/copilot_v17ds_train/pr4977.md new file mode 100644 index 000000000..8e1dba672 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr4977.md @@ -0,0 +1,102 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Lock/deadlock concern: `_hub_lock` is released by the context manager on exception; `get_kernel` is external with no in-repo re-entry into `_get_hub_module`; no deadlock path found. +- [resolved] trust_remote_code compat: commit 9947f414 removed the kwarg after an earlier revision passed `trust_remote_code=True` which kernels 0.13.x `get_kernel()` rejects; residual = PR description still advertises the removed kwarg. +- [resolved] head flash_attn_hub.py:33-35 calls `get_kernel(repo_id, version=version)`/`get_kernel(repo_id)` with no `trust_remote_code`; commit 9947f414 dropped the kwarg, so the PR description's `trust_remote_code=True` claim is stale at this head (kept comment #1 covers). +- [claim-verified] single-load cache: flash_attn_hub.py:44 `if repo_id not in _hub_modules:` plus layer.py:95 `self.attention = self.attn_impl_cls(...)` (one impl per Attention layer) means get_kernel is now called once per process instead of once per layer. +- [claim-verified] v1→v2→default fallback preserved: flash_attn_hub.py:31 `for version in (1, 2, None):` with version=1/2 calling `get_kernel(repo_id, version=version)` and None calling `get_kernel(repo_id)` matches the base nested try/except exactly. +- [claim-verified] forward behavior and backend selection unchanged: diff_stat shows only flash_attn_hub.py changed; forward_* methods and vllm_omni/platforms/cuda/platform.py:56 `get_diffusion_attn_backend_cls` are untouched. +- [claim-verified] motivation is real: commit 528804e8 message states 'hundreds of HuggingFace Hub API requests per rollout worker and 429 rate limits at startup', which the cache directly addresses. +- [claim-verified] fallback order v1→v2→default preserved: `for version in (1, 2, None):` at flash_attn_hub.py:31 matches the old nested try/except chain. +- [claim-verified] load-once-per-process mechanism present: `_hub_modules` dict + `_hub_lock` + conditional assignment at flash_attn_hub.py:44-46 ensure one load per repo_id. +- [claim-verified] 'Attention forward behavior and backend selection unchanged': the diff only replaces __init__ loading plus adds helpers; forward_cuda/_forward_varlen_* paths are untouched. +- [claim-verified] per-process dedup: `_get_hub_module` returns cached module for both repo_ids (flash_attn_hub.py:119 and :284). +- [claim-refuted] PR body says 'adding trust_remote_code=True for more reliable Hub access' but at this head there is no `trust_remote_code` anywhere in the file; commit 9947f414 dropped it because 'kernels 0.13.x get_kernel() does not accept trust_remote_code'. The PR description was not updated. +- [claim-refuted] PR body says 'adding `trust_remote_code=True`' but `grep trust_remote_code vllm_omni/diffusion/attention/backends/flash_attn_hub.py` → no matches; commit timeline records '9947f414 Drop trust_remote_code from hub kernel loading'. +- [claim-refuted] PR description says 'adding trust_remote_code=True', but final head calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` at flash_attn_hub.py:34-35 with no trust_remote_code; commit 9947f414 'Drop trust_remote_code from hub kernel loading' states kernels 0.13.x does not accept it. + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~33` [minor] — Head flash_attn_hub.py:34-35 calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` with no `trust_remote_code`, but the PR description still says this change adds `trust_remote_code=True` for more reliable Hub access. Commit 9947f414 dropped that kwarg because kernels 0.13.x `get_kernel()` rejects it, so every fallback attempt failed on older installs. Update the PR description to remove the claim or state that it is intentionally omitted for kernels 0.13.x compatibility. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:33-35 (PR-time tree) ` if version is not None: + return get_kernel(repo_id, version=version) + return get_kernel(repo_id)` — no `trust_remote_code` is passed. PR #4977 description (unchanged by this diff, read via gh_pr_view): `... speeds up worker initialization while preserving the existing version fallback behavior (`v1` → `v2` → default) and adding `trust_remote_code=True` for more reliable Hub access.` Commit 9947f41430aa message: `kernels 0.13.x get_kernel() does not accept trust_remote_code, so passing it made every fallback attempt fail on older installs. Match main's call signature; caching remains unchanged.`) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:44` [minor] — The diff centralizes Hub loading into `_hub_modules`/`_get_hub_module` (vllm_omni/diffusion/attention/backends/flash_attn_hub.py:19-46) so each repo_id is loaded once per process, replacing the per-construction `get_kernel` calls in `FlashAttentionHubImpl.__init__` and `FlashAttention3HubImpl.__init__`. No test exercises that path: `test_kernels_hub_platform_fallback` (tests/diffusion/attention/test_kernels_hub.py:20) mocks `sys.modules['kernels']` and only asserts the platform fallback returns FLASH_ATTN without importing `_get_hub_module`; `test_kernels_hub_execution` (tests/diffusion/attention/test_kernels_hub.py:58) is `@hardware_test(res={"cuda": "L4"})` and asserts shape/NaN/reference-diff only, not load count. `.buildkite/test-merge.yml` runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`, which deselects the L4/CUDA-gated test. Add a CPU unit test that monkeypatches `kernels.get_kernel` with a call recorder, calls `_get_hub_module('repo-a')` twice, and asserts identical object and exactly one `get_kernel` call; add a fallback case asserting `[('repo-a', 1), ('repo-a', 2)]` when version=1 raises. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:41-45 (in the diff): `with _hub_lock:` / `if repo_id not in _hub_modules:` / `_hub_modules[repo_id] = _load_hub_module(repo_id)` / `return _hub_modules[repo_id]`. vllm_omni/diffusion/attention/backends/flash_attn_hub.py:31-34 (in the diff): `for version in (1, 2, None):` / `if version is not None:` / `return get_kernel(repo_id, version=version)`. Unchanged by this diff, present in the PR-time tree: tests/diffusion/attention/test_kernels_hub.py:20-21 `def test_kernels_hub_platform_fallback(monkeypatch: pytest.MonkeyPatch):` and its body only asserts `backend_path == DiffusionAttentionBackendEnum.FLASH_ATTN.get_path()`; tests/diffusion/attention/test_kernels_hub.py:58 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` on `test_kernels_hub_execution`; .buildkite/test-merge.yml:13 `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`.) + +```suggestion +def test_get_hub_module_caches_single_load(monkeypatch): + import sys + import types + import vllm_omni.diffusion.attention.backends.flash_attn_hub as fah + + calls = [] + fake = types.ModuleType("kernels") + def get_kernel(repo_id, version=None): + calls.append((repo_id, version)) + return object() + fake.get_kernel = get_kernel + monkeypatch.setitem(sys.modules, "kernels", fake) + fah._hub_modules.clear() + + a = fah._get_hub_module("repo-a") + b = fah._get_hub_module("repo-a") + assert a is b + assert calls == [("repo-a", 1)] + + +def test_get_hub_module_falls_back(monkeypatch): + import sys + import types + import vllm_omni.diffusion.attention.backends.flash_attn_hub as fah + + calls = [] + fake = types.ModuleType("kernels") + def get_kernel(repo_id, version=None): + calls.append((repo_id, version)) + if version == 1: + raise RuntimeError("v1 unavailable") + return object() + fake.get_kernel = get_kernel + monkeypatch.setitem(sys.modules, "kernels", fake) + fah._hub_modules.clear() + + assert fah._get_hub_module("repo-a") is not None + assert calls == [("repo-a", 1), ("repo-a", 2)] +``` + +`?:34` [minor] — PR description still advertises `trust_remote_code=True`, but head commit 9947f414 removed it: `_load_hub_module` now calls `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` with no kwarg. Update the PR description to match the merged code. (evidence: flash_attn_hub.py:34 `return get_kernel(repo_id, version=version)` and flash_attn_hub.py:35 `return get_kernel(repo_id)` — no `trust_remote_code` kwarg; grep for `trust_remote_code` in the file returns no matches. PR description (via gh_pr_view): '...preserving the existing version fallback behavior (`v1` → `v2` → default) and adding `trust_remote_code=True` for more reliable Hub access.' The description still advertises the removed kwarg; commit 9947f414 message confirms the removal: 'Drop trust_remote_code from hub kernel loading' / 'kernels 0.13.x get_kernel() does not accept trust_remote_code'.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:45` [minor] — Make the pinned-version behavior explicit: `_hub_modules[repo_id]` is assigned only after the first successful `get_kernel` load, so a transient v1 failure that falls through to v2 permanently pins v2 for the whole process; the base code re-ran the v1→v2→default chain on every layer construction. If one-time v1 unavailability should not disable v1 for the process lifetime, store the chosen version and re-probe v1 on later misses; otherwise add a comment stating sticky fallback is intentional. (evidence: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:44-46 `if repo_id not in _hub_modules:` / ` _hub_modules[repo_id] = _load_hub_module(repo_id)` / `return _hub_modules[repo_id]` — cache populated only after a successful load; vllm_omni/diffusion/attention/backends/flash_attn_hub.py:31-33 `for version in (1, 2, None):` / ` try:` / ` if version is not None:` / ` return get_kernel(repo_id, version=version)` / ` return get_kernel(repo_id)` — first successful version returns and becomes the cached module. Base behavior (unchanged by this diff, present pre-PR): vllm_omni/diffusion/attention/backends/flash_attn_hub.py (merge-base) ~`hub_module = get_kernel("kernels-community/flash-attn2", version=1)` then nested excepts retrying version=2 and default on every `__init__`.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:44` [minor] — Please state the process-wide scope in the PR description: `_hub_modules`/`_hub_lock` are module-level in `flash_attn_hub.py` and are shared by every CUDA diffusion model selecting FLASH_ATTN_HUB or FLASH_ATTN_3_HUB via `--diffusion-attention-backend` / `DIFFUSION_ATTENTION_BACKEND` (docs/user_guide/diffusion/attention_backends.md:239-242), so the new cache and the sticky version-1→2→default fallback are not limited to the PR-tested model. Scope the cache per backend/model only if any caller is expected to keep per-construction version probing. (evidence: PR-time tree, unchanged outside diff except the new helper: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:22 `_hub_modules: dict[str, object] = {}`; :23 `_hub_lock = threading.Lock()`; :42-46 `def _get_hub_module(repo_id: str):` / ` with _hub_lock:` / ` if repo_id not in _hub_modules:` / ` _hub_modules[repo_id] = _load_hub_module(repo_id)` / ` return _hub_modules[repo_id]`; :119 `hub_module = _get_hub_module("kernels-community/flash-attn2")`; :284 `hub_module = _get_hub_module("kernels-community/flash-attn3")`. Sticky fallback at :27-33 `for version in (1, 2, None):` ... `return get_kernel(repo_id, version=version)` / `return get_kernel(repo_id)`. Global backend selection docs (unchanged by diff): docs/user_guide/diffusion/attention_backends.md:239 `export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN_HUB`; :242 `vllm-omni serve --diffusion-attention-backend FLASH_ATTN_HUB`.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:284` [minor] — Confirmed. flash_attn_hub.py:284 (`hub_module = _get_hub_module("kernels-community/flash-attn3")`) is reachable only via `FlashAttention3HubImpl.__init__`, whose sole constructor in the tree is `test_kernels_hub_execution` (tests/diffusion/attention/test_kernels_hub.py:58). No `.buildkite` lane selects `tests/diffusion/attention` or this test: the only lane selecting `tests/diffusion` runs `-m 'core_model and cpu'` (.buildkite/test-merge.yml:13), which deselects this test because it carries `core_model` plus gpu/cuda/L4 marks but no `cpu` mark. Even on its own L4 resource the test skips before FA3 (`major < 9` at test_kernels_hub.py:93, L4 is sm_89), so a wrong FA3 repo_id or bad `get_kernel` call in the FA3 half would ship green. Please run `pytest tests/diffusion/attention/test_kernels_hub.py::test_kernels_hub_execution` on an sm_90/H100 node, or extend the CPU monkeypatch test to call `_get_hub_module('kernels-community/flash-attn3')` and assert `get_kernel` receives exactly that repo_id. (evidence: Decisive lines, all present in the PR-time tree; test/CI files are unchanged by this diff: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:284 `hub_module = _get_hub_module("kernels-community/flash-attn3")`; tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)`; tests/diffusion/attention/test_kernels_hub.py:93 `pytest.skip("FLASH_ATTN_3_HUB execution requires Hopper-class GPU (compute capability >= 9.0)")`; .buildkite/test-merge.yml:13 `timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml`; grep for `FlashAttention3HubImpl` hits only flash_attn_hub.py and tests/diffusion/attention/test_kernels_hub.py:95/97; grep for `diffusion/attention` and `test_kernels_hub` in .buildkite returns no matches.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [nit] — The diff only touches `vllm_omni/diffusion/attention/backends/flash_attn_hub.py` (adds `_hub_modules`/`_hub_lock`, `_load_hub_module`, and `_get_hub_module`). On NPU, `NPUOmniPlatform.get_diffusion_attn_backend_cls` remaps `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` to local `FLASH_ATTN` before the Hub module is imported, and `.buildkite/test-npu-ready.yaml:21` runs only a Qwen3-TTS e2e. The failing `buildkite/vllm-omni-npu-ci` gate therefore appears unrelated to this diff; confirm the failure is pre-existing on main before merging. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/npu/platform.py:110-117 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):` / `logger.warning("HuggingFace kernels-backed FlashAttention is " "not supported on NPU. Falling back to local " "FLASH_ATTN.")` / `backend_upper = "FLASH_ATTN"`; .buildkite/test-npu-ready.yaml:21 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; diff scope: `diff --stat` shows only `vllm_omni/diffusion/attention/backends/flash_attn_hub.py` changed (30 insertions, 30 deletions).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:45` [nit] — `_get_hub_module` acquires the process-wide `_hub_lock` (line 43) even when `repo_id` is already cached, and calls `_load_hub_module` (line 45) while still holding it, so an in-flight Hub download for one repo blocks loads for other repos. A raise in `_load_hub_module` exits before the line-45 assignment, so a persistently unreachable repo re-runs the v1→v2→default chain on every attempt. Per-repo locks and/or a cached failure sentinel, or a short comment that success-only caching and cross-repo serialization are intentional, would address this. (evidence: flash_attn_hub.py:23 `_hub_lock = threading.Lock()` — single process-wide lock (added by this diff). flash_attn_hub.py:43 ` with _hub_lock:` — acquired on every `_get_hub_module` call, including cache hits. flash_attn_hub.py:45 ` _hub_modules[repo_id] = _load_hub_module(repo_id)` — download/load performed while holding the lock, and assignment only occurs after a successful return. flash_attn_hub.py:39 ` raise RuntimeError(f"Failed to load HuggingFace Hub kernel {repo_id!r}") from last_error` — raised before the assignment, so nothing is cached on failure. flash_attn_hub.py:119 ` hub_module = _get_hub_module("kernels-community/flash-attn2")` and flash_attn_hub.py:282 ` hub_module = _get_hub_module("kernels-community/flash-attn3")` — the two in-diff callers.) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr4977.trace.json.gz new file mode 100644 index 000000000..484b75e93 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_train/pr5009.cost.json b/eval/dataset/arms/copilot_v17ds_train/pr5009.cost.json new file mode 100644 index 000000000..be300a073 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1456.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_train/runs/pr5009/run-20260817-100117-24f934", + "recorded_at": "2026-08-17T10:25:31", + "attempts": [ + { + "invocation_id": "inv-05f32e67", + "outcome": "run:run-20260817-100117-24f934", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-100117-24f934", + "llm_calls": 136, + "input_tokens": 949989, + "output_tokens": 269131, + "usd": 0.853745, + "source": "spans" + } + ], + "attempt_usd_total": 0.853745, + "llm_calls": 136, + "input_tokens": 949989, + "output_tokens": 269131, + "usd": 0.853745, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-100117-24f934", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8537, + "minutes": 24.23, + "ci_minutes": 0, + "input_tokens": 949989, + "output_tokens": 269131, + "tool_calls": 288, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9457, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 136, + "by_role": { + "agent": { + "usd": 0.820636, + "input_tokens": 932732, + "output_tokens": 243305, + "calls": 134 + }, + "reducer": { + "usd": 0.033109, + "input_tokens": 17257, + "output_tokens": 25826, + "calls": 2 + } + }, + "cache_read_tokens": 11155712, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.94, + "gate": 2.08, + "review": 1447.68, + "report": 0.07 + }, + "time_to_first_result_s": 1453.74 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 39498 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_train/pr5009.md b/eval/dataset/arms/copilot_v17ds_train/pr5009.md new file mode 100644 index 000000000..986c4f033 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_train/pr5009.md @@ -0,0 +1,88 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] `vllm_config` is now unused inside get_default_ir_op_priority, but the parameter is required by the OmniPlatform interface and diffusion_worker.py:175; not a defect. +- [resolved] VLLM_USE_OINK_OPS guarded read is inside the method body (platform.py:259), not at module import, so monkeypatch.setattr(envs, ...) is effective — the oink test cannot silently pass. +- [resolved] `import vllm.envs as envs` exists at cuda/platform.py:6, so `envs.VLLM_USE_OINK_OPS` still resolves after the diff dropped the local CompilationMode import. +- [claim-verified] Part 1 archaeology: search_history shows `using_inductor` added by a560ed18 (#4709) and removed by b73e0355 (this PR); file_at_base shows the pre-PR `default = ["native"] if using_inductor else ["vllm_c", "native"]`. +- [claim-verified] Part 2 archaeology: search_history shows 35ee3c75 (#4074) added `nn.RMSNorm`, 99234881 (#3588) reverted it, b5589481 restores it; show_commit 35ee3c75 touches exactly norm_q/norm_k, norm_added_q/norm_added_k and txt_norm. +- [claim-verified] Only Cosmos3 has an IR-op-priority override: vllm_omni/diffusion/registry.py:545-549 `_DIFFUSION_IR_OP_PRIORITY_FUNCS` contains a single Cosmos3OmniDiffusersPipeline entry. +- [claim-verified] Cosmos3 override pins only `rms_norm`/`fused_add_rms_norm` to native and inherits `default` (pipeline_cosmos3.py:619-622). +- [claim-verified] The five changed norm sites take no quant_config (qwen_image_transformer.py:557-558/595-596/1009), consistent with 'W4A16 targets Linear only'; the two autoround W4A16 tests exist. +- [claim-verified] Accuracy/perf test names exist: tests/e2e/accuracy/test_qwen_image.py:197 `test_qwen_image_matches_diffusers`; tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py:52/73. +- [claim-verified] PR arithmetic: (1.7263-1.2613)/1.7263=26.9%; (2.4960-2.4502)/2.4960=1.8%; (2.4502-2.2847)/2.2847=+7.2%; (1288-1080)/1288=16.1% — all internally consistent. +- [claim-verified] search_history: `using_inductor` in vllm_omni/platforms/cuda/platform.py was introduced by a560ed18 (#4709 v0.24 rebase) and removed by b73e0355 (this PR) — matches the PR's 'pre-#4709 always vllm_c' claim. +- [claim-verified] grep: the only in-repo caller of `get_default_ir_op_priority` is `_resolve_ir_op_priority` at vllm_omni/diffusion/worker/diffusion_worker.py:175 (`ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`); tests only mock it. +- [claim-verified] per-model override mechanism exists: `_DIFFUSION_IR_OP_PRIORITY_FUNCS` at vllm_omni/diffusion/registry.py:545-549 maps Cosmos3 to `get_cosmos3_ir_op_priority_func` (forces rms_norm native at pipeline_cosmos3.py:612-624). +- [claim-verified] PR latency arithmetic reproduces: 1.7263→1.2613 = −26.9%, 2.4960→2.4502 = −1.8%, 2.4502 vs 2.2847 = +7.2% (calc). + +`vllm_omni/platforms/cuda/platform.py:~253` [major] — This diff deletes the `using_inductor` branch in `CudaOmniPlatform.get_default_ir_op_priority`; `default` is now unconditionally `["vllm_c", "native"]` (platform.py:253) and `IrOpPriorityConfig.with_default(default, ...)` (platform.py:258) applies it to every wrapped IR op, not just rms_norm/fused_add_rms_norm. `diffusion_worker.py:175` consumes this via `current_omni_platform.get_default_ir_op_priority(vllm_config)`, and `_DIFFUSION_IR_OP_PRIORITY_FUNCS` has only the Cosmos3 override (registry.py:549), so every other CUDA diffusion model gets the flip with no per-model validation, while `rocm/platform.py:184` and `xpu/platform.py:116` still keep the inductor-aware `native` branch. Please scope the change through the per-model `ir_op_priority_func` hook for the model that actually needs `vllm_c`, or add per-model evidence for all inheritors, and document the CUDA divergence from ROCm/XPU. (evidence: ['vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]` — changed by this diff (deletes the `using_inductor` branch).', 'vllm_omni/platforms/cuda/platform.py:258 `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)` — applies `default` as the base priority for all wrapped IR ops.', 'Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/worker/diffusion_worker.py:175 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`.', 'Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/registry.py:549 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",` — the only entry in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`.', 'Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/rocm/platform.py:184 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` (followed by `default = ["native"] if using_inductor else ["vllm_c", "native"]`).', 'Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/xpu/platform.py:116 `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` (followed by `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`).']) + +```suggestion +"""Copied from upstream CudaPlatformBase with inductor-aware logic. + + When inductor is active (compiling) use native as the default; + otherwise prefer vllm_c kernels where available. Models that need + vllm_c under inductor register a per-model ir_op_priority_func + (see _DIFFUSION_IR_OP_PRIORITY_FUNCS). + """ + from vllm.config.compilation import CompilationMode + + cc = vllm_config.compilation_config + using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE + default = ["native"] if using_inductor else ["vllm_c", "native"] + + # Use oink if enabled for rms_norm + # TODO(Laurawly/luka): remove this env var, + # users can just use IR op priority directly + rms_norm = default + if envs.VLLM_USE_OINK_OPS: + rms_norm = ["oink"] + default + + return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm) +``` + +`tests/platforms/test_cuda_ir_op_priority.py:7` [minor] — Confirmed: tests/platforms/test_cuda_ir_op_priority.py:7 imports CudaOmniPlatform at module scope and line 9 marks the file core_model/cpu. The Intel XPU lane runs pathless `pytest -v -s -m "core_model and xpu and B60"` (.buildkite/scripts/hardware_ci/run-xpu-test.sh:54), so pytest imports this module during collection before marker deselection; if that CUDA-platform import is not XPU-safe, collection aborts the lane. AMD merge lanes only target tests/diffusion and tests/model_executor and the NPU ready lane only runs test_qwen3_tts_customvoice.py, so they never reach tests/platforms; the general CPU lane (.buildkite/test-merge.yml:29) does collect it. Sibling CUDA-platform tests already lazy-import (tests/diffusion/attention/test_sage_attn3.py:93, tests/diffusion/attention/test_kernels_hub.py:24). Recommend moving the import inside both test functions. (evidence: tests/platforms/test_cuda_ir_op_priority.py:7 `from vllm_omni.platforms.cuda.platform import CudaOmniPlatform`; tests/platforms/test_cuda_ir_op_priority.py:9 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; .buildkite/scripts/hardware_ci/run-xpu-test.sh:54 `pytest -v -s -m "core_model and xpu and B60"` (pathless, so the new file is imported at collection before deselection); .buildkite/test-merge.yml:29 `"timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (collects tests/platforms); .buildkite/test-amd-merge.yml:13/21/32 only target tests/diffusion and tests/model_executor (unchanged by this diff); .buildkite/test-npu-ready.yaml:21 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'` (unchanged by this diff); sibling lazy imports: tests/diffusion/attention/test_sage_attn3.py:93 ` from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (inside test function) and tests/diffusion/attention/test_kernels_hub.py:24 ` from vllm_omni.platforms.cuda.platform import CudaOmniPlatform` (inside test function).) + +`vllm_omni/platforms/cuda/platform.py:253` [minor] — CudaOmniPlatform.get_default_ir_op_priority now returns `["vllm_c", "native"]` unconditionally (platform.py:253), so the inductor-active default changes from `["native"]` to `["vllm_c", "native"]`. Cosmos3's get_cosmos3_ir_op_priority_func (pipeline_cosmos3.py:619-621) copies every field of the incoming config and overrides only `rms_norm`/`fused_add_rms_norm` to `["native"]`; any other IR-op priority fields therefore inherit the new platform default, and the preservation test confirms those other fields are kept. This is a real behavior change for Cosmos3 non-RMSNorm IR ops under inductor, but harm is not yet demonstrated — verify vllm_c is safe for Cosmos3 under inductor or pin the base/default field to `["native"]` in the override before landing. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:619 ` priority_kwargs = {field.name: list(getattr(ir_op_priority, field.name)) for field in fields(ir_op_priority)}`; vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:620 ` priority_kwargs["rms_norm"] = ["native"]`; vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:621 ` priority_kwargs["fused_add_rms_norm"] = ["native"]`. Changed by this diff: vllm_omni/platforms/cuda/platform.py:253 ` default = ["vllm_c", "native"]`. Unchanged by this diff, present in the PR-time tree: tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py:772 ` assert merged.custom_op == ["platform_kernel", "native"]`.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — This diff changes Qwen-Image denoise-loop numerics by replacing vLLM `RMSNorm` with `torch.nn.RMSNorm` at `qwen_image_transformer.py:557` (also `norm_added_q/k` and `txt_norm` later in the hunk). The merge-gate path for this model is a 2-step smoke test with no pixel/similarity assertion (`tests/e2e/online_serving/test_qwen_image.py:85`), while the parity tests `test_qwen_image_matches_diffusers` and `test_qwen_image_2512_matches_diffusers_pixelwise` run only in the nightly full_model lane (`.buildkite/test-nightly.yml:919`). Attach `pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m full_model --run-level full_model` (including the 2512 pixelwise case) to this PR, or state explicitly that the nightly lane is the accepted backstop. (evidence: vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557: `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; tests/e2e/online_serving/test_qwen_image.py:85: `"num_inference_steps": 2,` with no similarity assertion; .buildkite/test-nightly.yml:919: `pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model`.) + +`diffusion_worker.py:251` [minor] — `vllm_config` is now unused inside get_default_ir_op_priority, but the parameter is required by the OmniPlatform interface and is still passed at diffusion_worker.py:175; not a defect. (evidence: vllm_omni/platforms/cuda/platform.py:251 `def get_default_ir_op_priority(cls, vllm_config: VllmConfig) -> IrOpPriorityConfig:` — parameter retained but not referenced in the body; vllm_omni/platforms/cuda/platform.py:253-260 `default = ["vllm_c", "native"]` ... `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)`; unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/worker/diffusion_worker.py:175 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`; unchanged by this diff, present in the PR-time tree: tests/diffusion/test_diffusion_plugin_hooks.py:140 `mock_platform.get_default_ir_op_priority.assert_called_once_with(vllm_config)`.) + +`cuda/platform.py:7` [minor] — `import vllm.envs as envs` exists at vllm_omni/platforms/cuda/platform.py:7, so `envs.VLLM_USE_OINK_OPS` still resolves after the diff dropped the local CompilationMode import. (evidence: Unchanged by this diff, present in the PR-time tree: vllm_omni/platforms/cuda/platform.py:7 `import vllm.envs as envs`; consumer at vllm_omni/platforms/cuda/platform.py:259 `if envs.VLLM_USE_OINK_OPS:`.) + +`vllm_omni/platforms/cuda/platform.py:253` [minor] — PR body FLUX.1-dev row Δ vs A is miscomputed: (0.5402−0.6166)/0.6166 = −12.39%, not −14.1%; 0.0764/0.5402 = +14.1% is the speedup relative to the new value, opposite the table's (B−A)/A convention used for the −26.9% and −1.8% rows. (evidence: PR #5009 description (not in the diff): `| A — post-#4709 / current main | ["native"] only | 0.6166s | — |` and `| B — Part 1 (this PR) | ["vllm_c", "native"] | 0.5402s | **−14.1%** |`. Calculation: (0.5402−0.6166)/0.6166 = −0.1239 = −12.39%; (0.6166−0.5402)/0.5402 = +0.1414 = +14.1%. The table's own delta convention elsewhere: `**This PR vs main: latency_mean −26.9%** (1.7263s → 1.2613s)` — (1.2613−1.7263)/1.7263 = −26.9% — confirming Δ is computed as (new−old)/old.) + +`tests/platforms/test_cuda_ir_op_priority.py:26` [minor] — tests/platforms/test_cuda_ir_op_priority.py:26-27 — the new test only asserts `priority.rms_norm` and `priority.fused_add_rms_norm`, but the changed code also sets the generic `default` field to `["vllm_c", "native"]` (vllm_omni/platforms/cuda/platform.py:253,262). Add `assert priority.default == ["vllm_c", "native"]` to cover the wider default-field behavior. (evidence: tests/platforms/test_cuda_ir_op_priority.py:26 `assert priority.rms_norm == ["vllm_c", "native"]` and :27 `assert priority.fused_add_rms_norm == ["vllm_c", "native"]` — the test asserts only these two rms fields; vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]` and :262 `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)` — the changed code sets and passes the generic default field, which the test never asserts.) + +```suggestion +assert priority.default == ["vllm_c", "native"] + assert priority.rms_norm == ["vllm_c", "native"] +``` + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_train/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v17ds_train/pr5009.trace.json.gz new file mode 100644 index 000000000..8d4cffc2c Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_train/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_val/manifest.json b/eval/dataset/arms/copilot_v17ds_val/manifest.json new file mode 100644 index 000000000..b0365d27c --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v17ds_val", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 5, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4810.cost.json b/eval/dataset/arms/copilot_v17ds_val/pr4810.cost.json new file mode 100644 index 000000000..c81c1f8bc --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4810.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2539.5, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_val/runs/pr4810/run-20260817-100115-1c6d23", + "recorded_at": "2026-08-17T10:43:32", + "attempts": [ + { + "invocation_id": "inv-6f915259", + "outcome": "run:run-20260817-100115-1c6d23", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-100115-1c6d23", + "llm_calls": 195, + "input_tokens": 1484305, + "output_tokens": 342317, + "usd": 1.334028, + "source": "spans" + } + ], + "attempt_usd_total": 1.334028, + "llm_calls": 195, + "input_tokens": 1484305, + "output_tokens": 342317, + "usd": 1.334028, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-100115-1c6d23", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.334, + "minutes": 42.27, + "ci_minutes": 0, + "input_tokens": 1484305, + "output_tokens": 342317, + "tool_calls": 355, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.3508, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 195, + "by_role": { + "agent": { + "usd": 1.304432, + "input_tokens": 1466372, + "output_tokens": 319854, + "calls": 193 + }, + "reducer": { + "usd": 0.029596, + "input_tokens": 17933, + "output_tokens": 22463, + "calls": 2 + } + }, + "cache_read_tokens": 20619136, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.95, + "gate": 2.17, + "review": 2530.19, + "report": 0.07 + }, + "time_to_first_result_s": 2536.36 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 40671 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4810.md b/eval/dataset/arms/copilot_v17ds_val/pr4810.md new file mode 100644 index 000000000..8ec92bdd2 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4810.md @@ -0,0 +1,67 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] initially suspected the Mammoth local mapper was redundant because `MammothModa2ARForConditionalGeneration` extends `Qwen2_5_VLForConditionalGeneration` with no load_weights override (mammoth_moda2.py:540); but registry.py:92's standalone `MammothModa2Qwen2ForCausalLM` entry proves a direct-call path exists, so the local mapper is needed. Residual: in the AR-wrapper path the outer AutoWeightsLoader may re-apply it (idempotent). +- [resolved] Mammoth nested path (stage dispatcher load_weights -> self.ar.load_weights -> Qwen2_5_VL AutoWeightsLoader -> MammothModa2Qwen2ForCausalLM) double-applies the mapper, but the mapper only rewrites source suffix patterns that are gone after the first apply, so it is idempotent and harmless. +- [resolved] Survivor is not a new regression: issue #4809 comment (lishunyang12) states #4808 removes the DiT transformer branch and #4810 covers the other 4 sites. +- [claim-verified] `test_model_loaders_do_not_call_removed_get_cache_scale` passes for the 4 files in `_STALE_API_FILES` at head (grep confirms no `.get_cache_scale(` in them). +- [claim-verified] PR body 'on current main' run lists `...hunyuan_image3_transformer.py` as a FAILING static-test case; final diff's `_STALE_API_FILES` no longer contains that path, so the failure was dropped rather than fixed. +- [claim-refuted] repo-wide grep for `get_cache_scale` at head returns exactly one caller: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238. +- [claim-refuted] PR body's own main-branch failure list includes `...hunyuan_image3_transformer.py`, but the final `_STALE_API_FILES` (test file lines 15-20) omits it and leaves the caller in place — the PR does not fully resolve #4809. +- [claim-refuted] PR body's pre-fix test run lists `test_model_loaders_do_not_call_removed_get_cache_scale[vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py]` as FAILED, but the final test `_STALE_API_FILES` (tests/model_executor/models/test_kv_cache_scale_mapper.py:13-18) drops that path and the diff never fixes it. +- [claim-refuted] PR body test output lists `vllm_omni/diffusion/.../hunyuan_image3_transformer.py` as a FAILED parametrization, but committed `_STALE_API_FILES` has only 4 entries; pasted test table no longer matches the final diff. +- [claim-refuted] PR body 'Fix custom model loaders that still call the removed vLLM get_cache_scale API' is incomplete: the diffusion HunyuanImage3Model.load_weights still calls `self.quant_config.get_cache_scale(name)` (grep 2238). +- [claim-refuted] PR title 'custom loaders still call the removed get_cache_scale API': head grep still finds `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` calling `get_cache_scale(name)`; the PR leaves this known failing path unfixed and out of `_STALE_API_FILES`. +- [validated] hunyuan_image3.py:2178 uses `AutoWeightsLoader` for the top-level HunyuanImage3 model, so removing the manual branch and relying on the outer mapper + pre-existing `maybe_remap_kv_scale_name` fallback is sound for the AR loader. +- [validated] registry.py:92 registers `MammothModa2Qwen2ForCausalLM` as a standalone architecture, so its `load_weights` can be invoked directly; the new pre-loop mapper at mammoth_moda2.py:486-488 is required for that direct path. +- [validated] registry.py maps `MiMoAudioLLMModel` to `MiMoAudioLLMForConditionalGeneration` whose custom `load_weights` is the direct model loader; the new pre-loop mapper at mimo_audio_llm.py:1156-1158 is required. + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:~2238` [major] — Confirmed: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 (inside load_weights, unchanged by this PR) still calls the removed quant_config.get_cache_scale(name) directly, and the new _STALE_API_FILES regression list omits it. Apply the same migration (get_cache_scale_mapper() before the weight loop, or drop the branch if reached via an outer AutoWeightsLoader) or add this path to the stale-API list. (evidence: Decisive code line is UNCHANGED by this diff, present in the PR-time tree: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 ` if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`. The branch sits inside a loader: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2125 ` def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):`. The new test's stale-API list contains only the four model_executor paths and omits the diffusion path: tests/model_executor/models/test_kv_cache_scale_mapper.py:15-20 `_STALE_API_FILES = [` -> `"vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py",`, `"vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py",`, `"vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py",`, `"vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py",`.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:12` [minor] — The new test file `tests/model_executor/models/test_kv_cache_scale_mapper.py:12` adds `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, and `.buildkite/test-amd-merge.yml:32` runs `pytest -sv tests/model_executor -m 'core_model and cpu'`, so this file — whose tests `__import__` the changed model modules at lines 106 and 137 — is collected in the AMD lane. `buildkite/vllm-omni-amd-ci` is FAILURE at this head; please confirm the AMD failure is pre-existing on main or run the lane's exact pytest command on this head to show this test/import is not the cause. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:12 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (file added by this diff); tests/model_executor/models/test_kv_cache_scale_mapper.py:106 `module = __import__(module_name, fromlist=[class_name])` and :137 `module = __import__(module_name, fromlist=[class_name])` (test bodies import the changed model modules); .buildkite/test-amd-merge.yml:32 `- "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"` (unchanged by this diff, present in the PR-time tree).) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:~110` [minor] — test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale calls HunyuanModel.load_weights/Qwen2Model.load_weights directly with the already-mapped `_MAPPED_SCALE_NAME` (tests/model_executor/models/test_kv_cache_scale_mapper.py:112), so it only proves the delegated loaders no longer expect `get_cache_scale`; it never drives `_SOURCE_SCALE_NAME` through the outer AutoWeightsLoader used by HunyuanImage3ForConditionalGeneration.load_weights (hunyuan_image3.py:2178) and Qwen2ForCausalLM.load_weights (qwen2_old.py:444). Add an AutoWeightsLoader-routed test with `_SOURCE_SCALE_NAME` and assert the renamed `_MAPPED_SCALE_NAME` parameter receives the weight, so a missing mapper application would fail. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:110 ` load_weights = getattr(module, class_name).load_weights` and :112 ` loaded = load_weights(fake_model, [(_MAPPED_SCALE_NAME, torch.ones(()))])` — direct delegated-loader call with the already-mapped name; `_SOURCE_SCALE_NAME` never enters this test. vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py:2178 ` loader = AutoWeightsLoader(` (unchanged by this diff, present in PR-time tree) and vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py:444 ` loader = AutoWeightsLoader(` (unchanged by this diff) — the outer mapper path the test does not exercise.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:81` [minor] — The static guard at tests/model_executor/models/test_kv_cache_scale_mapper.py:81 asserts only `.get_cache_scale(` is absent, but the pattern removed from qwen2_old.py was `getattr(self.quant_config, "get_cache_scale", None)` followed by a separate `get_cache_scale(name)`, which contains no `.get_cache_scale(` and was therefore not flagged in the PR body's pre-fix run. Broaden to `assert "get_cache_scale(" not in source` and add `assert 'getattr(self.quant_config, "get_cache_scale",' not in source` so the static guard covers both removed-API call styles. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:81 ` assert ".get_cache_scale(" not in source`; PR diff removed from vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py: `- get_cache_scale = getattr(self.quant_config, "get_cache_scale", None)` (no ".get_cache_scale(" substring) and later `- if get_cache_scale is not None and (scale_name := get_cache_scale(name)):` (contains `get_cache_scale(` but no dot prefix), so line 81 cannot detect that removed-API style.) + +```suggestion +assert ".get_cache_scale(" not in source + assert 'getattr(self.quant_config, "get_cache_scale",' not in source +``` + +`mammoth_moda2.py:540` [minor] — mammoth_moda2.py:540 `class MammothModa2ARForConditionalGeneration(Qwen2_5_VLForConditionalGeneration)` has no load_weights override (only defs at lines 476 and 810), so the AR wrapper inherits the parent loader; however registry.py:92 registers `MammothModa2Qwen2ForCausalLM` standalone, which calls the class's own load_weights directly and needs the local cache-scale mapper added at mammoth_moda2.py:486-489. Residual about the outer AutoWeightsLoader re-applying the mapper remains speculative and non-blocking. (evidence: mammoth_moda2.py:540 `class MammothModa2ARForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):`; registry.py:92 ` "MammothModa2Qwen2ForCausalLM": (` (standalone registry entry, unchanged by this diff, present in PR-time tree); mammoth_moda2.py:486-489 `cache_scale_mapper = self.quant_config.get_cache_scale_mapper()` / `weights = cache_scale_mapper.apply(weights)` (local mapper added by this diff).) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:137` [minor] — Add a `model.`-prefixed MiMo case to the direct-loader test: production MiMo weights arrive as `model.`-prefixed names, while the test feeds unprefixed `layers.0...`, so the mapper-before-`name.startswith("model.")` ordering at vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1153-1162 is currently untested. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:22 `_SOURCE_SCALE_NAME = "layers.0.self_attn.k_proj.output_scale"`; tests/model_executor/models/test_kv_cache_scale_mapper.py:144 `loaded = load_weights(fake_model, [(_SOURCE_SCALE_NAME, torch.ones(()))])`; vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1153-1158 `if self.quant_config is not None: cache_scale_mapper = self.quant_config.get_cache_scale_mapper(); if cache_scale_mapper is not None: weights = cache_scale_mapper.apply(weights)` then `for name, loaded_weight in weights: if name.startswith("model."): name = "model." + name` — mapper runs before the prefix fix, and the test's MiMo input omits `model.`.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:112` [minor] — Extend the delegated-loader cases to include a non-scalar/per-head KV-scale weight: line 112 passes only torch.ones(()) (0-dim scalar), so the delegated loaders' handling of per-head KV-scale shapes is left uncovered (note these loaders receive the pre-mapped name and never call the cache-scale mapper themselves). (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:112 `loaded = load_weights(fake_model, [(_MAPPED_SCALE_NAME, torch.ones(()))])` — 0-dim scalar only; tests/model_executor/models/test_kv_cache_scale_mapper.py:50 `self.param = torch.nn.Parameter(torch.zeros(()), requires_grad=False)` — fake parameter is also scalar; tests/model_executor/models/test_kv_cache_scale_mapper.py:144 `loaded = load_weights(fake_model, [(_SOURCE_SCALE_NAME, torch.ones(()))])` — even the mapper-driving direct-custom-loader test uses a scalar.) + +`?:32` [minor] — Please confirm the `buildkite/vllm-omni-npu-ci` failure is pre-existing on main. Grep-verified: the NPU lanes (test-npu-ready.yaml, pipeline-npu.yaml, pipeline-npu-a3.yaml, test-npu-nightly.yaml) contain no `tests/model_executor` runner, so this diff does not intersect that gate. `.buildkite/test-amd-merge.yml:32` does run `tests/model_executor -m 'core_model and cpu'`, so the new test collects on AMD; both AMD and NPU gates are red at this head, and attribution cannot be resolved from the snapshot. (evidence: Unchanged by this diff, present in the PR-time tree — `.buildkite/test-amd-merge.yml:32` `- "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`. NPU lanes: `.buildkite/test-npu-ready.yaml:21` only `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; `.buildkite/test-npu-nightly.yaml:24` only `- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"`; `.buildkite/pipeline-npu.yaml` and `.buildkite/pipeline-npu-a3.yaml` contain no `pytest` invocations. gh_ci_read(4810): `{"name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"}` and `{"name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"}`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1159` [minor] The mapper now runs before the loop's `if name.startswith("model."): name = "model." + name`, replacing the old branch that indexed non-scalar scales via `loaded_weight[0]`. The direct-loader test feeds only an unprefixed scalar (`_SOURCE_SCALE_NAME`, `torch.ones(())` at […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v17ds_val/pr4810.trace.json.gz new file mode 100644 index 000000000..373ecf078 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_val/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4816.cost.json b/eval/dataset/arms/copilot_v17ds_val/pr4816.cost.json new file mode 100644 index 000000000..49e0e0fbf --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4816.cost.json @@ -0,0 +1,129 @@ +{ + "wall_s": 1344.8, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_val/runs/pr4816/run-20260817-115014-9430a8", + "recorded_at": "2026-08-17T12:12:36", + "attempts": [ + { + "invocation_id": "inv-86ad6c9f", + "outcome": "run:run-20260817-115014-9430a8", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-103717-fbc8bf", + "llm_calls": 83, + "input_tokens": 651325, + "output_tokens": 103587, + "usd": 0.493027, + "source": "spans" + }, + { + "run_dir": "run-20260817-104946-e1d77c", + "llm_calls": 2, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "spans" + }, + { + "run_dir": "run-20260817-115014-9430a8", + "llm_calls": 134, + "input_tokens": 1416585, + "output_tokens": 206840, + "usd": 0.930494, + "source": "spans" + } + ], + "attempt_usd_total": 1.423521, + "llm_calls": 134, + "input_tokens": 1416585, + "output_tokens": 206840, + "usd": 0.930494, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-115014-9430a8", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.9305, + "minutes": 22.36, + "ci_minutes": 0, + "input_tokens": 1416585, + "output_tokens": 206840, + "tool_calls": 248, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9414, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 134, + "by_role": { + "planner": { + "usd": 0.000661, + "input_tokens": 87, + "output_tokens": 532, + "calls": 1 + }, + "agent": { + "usd": 0.907628, + "input_tokens": 1404056, + "output_tokens": 189216, + "calls": 131 + }, + "reducer": { + "usd": 0.022205, + "input_tokens": 12442, + "output_tokens": 17092, + "calls": 2 + } + }, + "cache_read_tokens": 11870080, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.28, + "gate": 2.12, + "review": 1336.22, + "report": 0.06 + }, + "time_to_first_result_s": 1341.67 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 52689 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4816.md b/eval/dataset/arms/copilot_v17ds_val/pr4816.md new file mode 100644 index 000000000..8ebc3c1bc --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4816.md @@ -0,0 +1,121 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Multi-stage (Qwen3-Omni) upload_voice ValueError -> `_create_speech_error_json_response` (api_server.py:376) -> upstream `base()` now finds the new key set at api_server.py:1017; the described 500 is fixed for the multi-stage path. Residual: pure-diffusion branch (754) is not fixed, see review comment. +- [resolved] For LLM/multi-stage the AttributeError is fixed: state.serving_tokenization is a real ServingTokenization there, so upstream base() returns it. Residual checked: pure diffusion still sets it to None and the helper does not guard that. +- [resolved] Multi-stage fix works: api_server.py:1017 `state.serving_tokenization = ServingTokenization(...)` now satisfies upstream `base()` -> `request.app.state.serving_tokenization`; residual is pure-diffusion None (kept major @376). +- [claim-verified] repo-wide grep for `openai_serving_tokenization` returns 0 matches at head; all 9 occurrences now use `serving_tokenization`. +- [claim-verified] file_at_base shows pre-PR api_server.py already imported `base` from `vllm.entrypoints.serve.instrumentator.basic` while state used `openai_serving_tokenization` — the reported mismatch existed. +- [claim-verified] The PR body's 'switch Omni from state.openai_serving_tokenization -> state.serving_tokenization' claim is fully realized at this head: no old-name reference survives. +- [claim-verified] PR description's premise that upstream base/tokenization reads request.app.state.serving_tokenization matches the imported helper at api_server.py:67 and the quoted traceback. +- [claim-verified] PR description trace `AttributeError: 'State' object has no attribute 'serving_tokenization'` matches upstream `base()` reading `request.app.state.serving_tokenization`; diff adds that key at 1017. +- [claim-verified] gh_ci_read: buildkite/vllm-omni (CUDA), pre-commit, DCO, docs all SUCCESS; buildkite/vllm-omni-amd-ci, -intel-ci, -npu-ci FAILURE. +- [claim-refuted] PR-body claim that the rename 'keeps things from turning into a 500' holds for LLM/multi-stage mode but not for pure-diffusion speech error routes (residual above). +- [validated] api_server.py:754 pure-diffusion branch now writes `state.serving_tokenization = None`; api_server.py:1017 LLM branch writes a real `ServingTokenization`. +- [validated] api_server.py:1157/1248/1286/1329 — all four fallback `getattr(..., "serving_tokenization", None)` sites guard None before calling create_error_response. +- [validated] test_serving_speech.py:2533/2546/2615 — the three updated tests set `state.serving_tokenization = fake_base` and assert 404, exercising the renamed fallback. +- [validated] api_server.py:1017 constructs ServingTokenization on the new key; the import of `base` and `ServingTokenization` is unchanged by this diff. + +`vllm_omni/entrypoints/openai/api_server.py:~376` [major] — api_server.py:376 `_create_speech_error_json_response` calls `base(raw_request).create_error_response(...)` without a None guard, while api_server.py:754 now sets `state.serving_tokenization = None` in pure-diffusion mode. Since upstream `base()` returns `request.app.state.serving_tokenization`, any speech error handled through this helper in pure-diffusion mode (e.g. the upload_voice ValueError at line 1483) raises `AttributeError: 'NoneType' object has no attribute 'create_error_response'` and returns 500 instead of the intended error response. Add a None guard here or construct a non-None error-response provider in pure diffusion. (evidence: unchanged by this diff, present in the PR-time tree: vllm_omni/entrypoints/openai/api_server.py:376 ` err = base(raw_request).create_error_response(` — helper dereferences base() result with no None check. changed by this diff: vllm_omni/entrypoints/openai/api_server.py:754 ` state.serving_tokenization = None` — pure-diffusion sentinel now set to None. caller: vllm_omni/entrypoints/openai/api_server.py:1483 ` return _create_speech_error_json_response(raw_request, str(e))` — speech exception path routes through the unguarded helper. `base` is imported from upstream vllm's `vllm.entrypoints.serve.instrumentator.basic` (dependency, not in this repo) and returns `request.app.state.serving_tokenization`, so in pure-diffusion mode `base(raw_request)` is None and line 376 raises AttributeError.) + +```suggestion +base_server = base(raw_request) + if base_server is None: + from vllm.entrypoints.openai.engine.protocol import ErrorInfo + err = ErrorResponse(error=ErrorInfo(message=message, type=err_type, param=None, code=int(status_code))) + return _error_response_to_json_response(err, status_code=status_code) + err = base_server.create_error_response( + message=message, + err_type=err_type, + status_code=status_code, + ) +``` + +`?:?` [major] — Merge blocker: on this head, gh_ci_read shows buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, and buildkite/vllm-omni-npu-ci all FAILURE while buildkite/vllm-omni is SUCCESS; diff_stat shows only vllm_omni/entrypoints/openai/api_server.py and tests/entrypoints/openai_api/test_serving_speech.py changed (no hardware/config files). Resolve the lane failures or document with evidence that they pre-exist on main. (evidence: gh_ci_read(4816): [{"bucket":"fail","name":"buildkite/vllm-omni-amd-ci","state":"FAILURE"},{"bucket":"fail","name":"buildkite/vllm-omni-intel-ci","state":"FAILURE"},{"bucket":"fail","name":"buildkite/vllm-omni-npu-ci","state":"FAILURE"},{"bucket":"pass","name":"buildkite/vllm-omni","state":"SUCCESS"}]; diff_stat: merge-base a560ed184d91, 2 files changed, 9 insertions(+), 9 deletions(-): tests/entrypoints/openai_api/test_serving_speech.py and vllm_omni/entrypoints/openai/api_server.py; gh_pr_view: state MERGED. Changed-code anchor (PR-time tree): vllm_omni/entrypoints/openai/api_server.py:754 `state.serving_tokenization = None`; vllm_omni/entrypoints/openai/api_server.py:1017 `state.serving_tokenization = ServingTokenization(`; tests/entrypoints/openai_api/test_serving_speech.py:2533 `raw_request.app.state.serving_tokenization = fake_base`.) + +`tests/entrypoints/openai_api/test_serving_speech.py:~2533` [minor] — Confirmed as written. Minor test-coverage gap: the three 404 tests update `app.state.serving_tokenization` but still mock `api_server_module.base` via `_patch_api_server_base`, so the real `_create_speech_error_json_response` -> `base(raw_request)` path is never exercised. Consider adding a regression that does not patch `base` and either asserts `omni_init_app_state` sets `app.state.serving_tokenization` or drives upload/delete error responses through an initialized state. (evidence: tests/entrypoints/openai_api/test_serving_speech.py:2339 `def _patch_api_server_base(mocker: MockerFixture):` ... (body) `mocker.patch.object(api_server_module, "base", return_value=fake_base)` — this mocks the real base, so the three 404 tests never hit it; vllm_omni/entrypoints/openai/api_server.py:376 `err = base(raw_request).create_error_response(` — the real bug route the comment says is unexercised; both present in PR-time tree, api_server.py:376 unchanged by this diff.) + +```suggestion +def test_api_server_upload_voice_decode_error_reads_serving_tokenization(mocker: MockerFixture): + handler = mocker.MagicMock() + handler.upload_voice = mocker.AsyncMock(side_effect=ValueError("Could not decode audio file")) + raw_request = _make_api_server_request(handler) + + def _fake_create_error_response(message, err_type="BadRequestError", status_code=400, param=None): + return ErrorResponse(error=ErrorInfo(message=message, type=err_type, param=param, code=getattr(status_code, "value", status_code))) + + fake_base = mocker.MagicMock() + fake_base.create_error_response.side_effect = _fake_create_error_response + raw_request.app.state.serving_tokenization = fake_base + + response = asyncio.run( + api_server_module.upload_voice( + raw_request, + audio_sample=mocker.MagicMock(), + speaker_embedding=None, + consent="cons_test", + name="probe", + ) + ) + _assert_openai_error_response(response, status_code=400, message="Could not decode audio file") +``` + +`api_server.py:754` [minor] — The rename fixes the multi-stage 500: `_create_speech_error_json_response` (api_server.py:376) delegates to upstream `base()`, and the multi-stage branch now sets `state.serving_tokenization = ServingTokenization(...)` (api_server.py:1017). Residual: the pure-diffusion branch still sets `state.serving_tokenization = None` (api_server.py:754), so any speech/audio error routed through `base()` in pure-diffusion mode still has no handler. (evidence: api_server.py:376 `err = base(raw_request).create_error_response(`; api_server.py:1017 `state.serving_tokenization = ServingTokenization(`; api_server.py:754 `state.serving_tokenization = None`; api_server.py:1248 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)` — all present in the PR-time tree; lines 754 and 1017 are the changed lines from the diff.) + +`?:1017` [minor] — api_server.py:1017 now assigns a real ServingTokenization to state.serving_tokenization in the LLM/multi-stage branch, resolving the AttributeError. Residual (minor, checked): api_server.py:754 still sets state.serving_tokenization = None in pure-diffusion mode, and _create_speech_error_json_response calls base(raw_request).create_error_response(...) without a None guard. (evidence: vllm_omni/entrypoints/openai/api_server.py:1017 ` state.serving_tokenization = ServingTokenization(` — LLM/multi-stage creates a real instance; vllm_omni/entrypoints/openai/api_server.py:754 ` state.serving_tokenization = None` — pure diffusion still sets None; vllm_omni/entrypoints/openai/api_server.py:1157 ` base_server = getattr(raw_request.app.state, "serving_tokenization", None)` — endpoint handlers guard the None case.) + +`vllm_omni/entrypoints/openai/api_server.py:~1163` [minor] — Confirmed. The chat fallback branch at api_server.py:1157-1163 now reads the renamed `serving_tokenization` key, returns a bare `ErrorResponse` (not wrapped via `_error_response_to_json_response` as create_speech does), and has no test coverage: `grep 'does not support Chat Completions API' tests/` has no matches, while the only create_chat_completion test callers exercise the handler-present path. Add a regression test that sets `raw_request.app.state.openai_serving_chat = None` and `raw_request.app.state.serving_tokenization = fake_base`, and decide whether line 1163 should wrap its error response like the speech paths. (evidence: vllm_omni/entrypoints/openai/api_server.py:1157 `base_server = getattr(raw_request.app.state, "serving_tokenization", None)`; vllm_omni/entrypoints/openai/api_server.py:1163 `return base_server.create_error_response(message="The model does not support Chat Completions API")`; tests/ grep for the literal `does not support Chat Completions API` returns no matches — the create_chat_completion fallback is untested, and the bare ErrorResponse return contrasts with the speech paths' `_error_response_to_json_response` wrapping.) + +```suggestion +def test_api_server_create_chat_completion_without_handler_returns_error(mocker: MockerFixture): + fake_base = _patch_api_server_base(mocker) + raw_request = _make_api_server_request(None, path="/v1/chat/completions") + raw_request.app.state.openai_serving_chat = None + raw_request.app.state.serving_tokenization = fake_base + request = ChatCompletionRequest(messages=[], model="m") + response = asyncio.run(api_server_module.create_chat_completion(request, raw_request)) + assert isinstance(response, ErrorResponse) + assert response.error.message == "The model does not support Chat Completions API" +``` + +`vllm_omni/entrypoints/openai/api_server.py:1017` [nit] — The diff renames `state.openai_serving_tokenization = ServingTokenization(...)` to `state.serving_tokenization = ServingTokenization(...)` (api_server.py:1017) but declares no minimum vLLM version for this upstream layout: `requirements/common.txt` has no vllm requirement and the docs pin `vllm==0.24.0`. On a vLLM whose instrumentator still reads `request.app.state.openai_serving_tokenization`, this rename makes `base()` raise AttributeError again. Either document/enforce the minimum vLLM version that reads `serving_tokenization`, or set `state.openai_serving_tokenization = state.serving_tokenization` as a transition alias. (evidence: vllm_omni/entrypoints/openai/api_server.py:1017 ` state.serving_tokenization = ServingTokenization(`; api_server.py:369-371 `# vLLM moved \`base\` from openai.basic.api_router to serve.instrumentator.basic.\n# Keep a fallback for older/newer upstream layouts during rebase windows.\nfrom vllm.entrypoints.serve.instrumentator.basic import base`; docs/getting_started/installation/gpu/cuda.inc.md:23 `uv pip install vllm==0.24.0 --torch-backend=auto`; `requirements/common.txt` (read in full) declares no vllm requirement.) + +```suggestion +state.serving_tokenization = ServingTokenization( + state.openai_serving_models, + state.openai_serving_render, + request_logger=request_logger, + chat_template=resolved_chat_template, + chat_template_content_format=args.chat_template_content_format, + default_chat_template_kwargs=args.default_chat_template_kwargs, + trust_request_chat_template=args.trust_request_chat_template, + ) + # Compat alias for upstream vLLM layouts that still read openai_serving_tokenization. + state.openai_serving_tokenization = state.serving_tokenization +``` + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v17ds_val/pr4816.trace.json.gz new file mode 100644 index 000000000..89b2c0a01 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_val/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4825.cost.json b/eval/dataset/arms/copilot_v17ds_val/pr4825.cost.json new file mode 100644 index 000000000..f3f6e0afa --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4825.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 992.5, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_val/runs/pr4825/run-20260817-100115-0623d1", + "recorded_at": "2026-08-17T10:17:45", + "attempts": [ + { + "invocation_id": "inv-bd78efc7", + "outcome": "run:run-20260817-100115-0623d1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-100115-0623d1", + "llm_calls": 107, + "input_tokens": 534709, + "output_tokens": 181650, + "usd": 0.531861, + "source": "spans" + } + ], + "attempt_usd_total": 0.531861, + "llm_calls": 107, + "input_tokens": 534709, + "output_tokens": 181650, + "usd": 0.531861, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-100115-0623d1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5319, + "minutes": 16.49, + "ci_minutes": 0, + "input_tokens": 534709, + "output_tokens": 181650, + "tool_calls": 220, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6867, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 107, + "by_role": { + "agent": { + "usd": 0.505422, + "input_tokens": 521387, + "output_tokens": 160922, + "calls": 105 + }, + "reducer": { + "usd": 0.026439, + "input_tokens": 13322, + "output_tokens": 20728, + "calls": 2 + } + }, + "cache_read_tokens": 6950912, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.79, + "gate": 1.85, + "review": 983.73, + "report": 0.06 + }, + "time_to_first_result_s": 989.41 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 32580 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4825.md b/eval/dataset/arms/copilot_v17ds_val/pr4825.md new file mode 100644 index 000000000..31614c950 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4825.md @@ -0,0 +1,89 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4825, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac3a2ce5a9b9) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] packed-mapping timing: diffusion_worker.py calls load_model() before init_lora_manager(); sdxl_unet.load_weights sets `self.stacked_params_mapping = stacked_params_mapping`, so _compute_packed_modules_mapping sees to_qkv->to_q/to_k/to_v. +- [resolved] commit timeline 'Normalize to_out.0. to .to_out.' added then reverted: at head the net diff contains only the unet default + test (diff_stat 2 files), so no dangling to_out normalization code. +- [claim-verified] PR body 'include unet ... so replacement can occur on SDXL-style pipelines': pipeline_sdxl.py:110 `self.unet = SDXLUNet2DConditionModel(od_config=od_config)` and base manager had only ("transformer","transformer_2","dit","bagel") (file_at_base manager.py:371-378). +- [claim-verified] PR body 'before: base and lora produce same output' mechanism: at base no default component matched SDXL, `_lora_modules` stayed empty and `_activate_adapter` iterates zero layers — base-identical output. +- [claim-verified] archaeology: 632efb42 added WeightsMapper `{'.to_out.0.': '.to_out.'}` + test; ac3a2ce5 'revert to out chnage' removed both; at head manager.py:309 `weights_mapper=None` — the `to_out.0` LoRA naming is now unhandled and untested. +- [claim-verified] test module declares `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (test_lora_manager.py:16), so the new test runs in the `core_model and cpu` diffusion lanes (.buildkite/test-ready.yml:13) — collection is not silently deselected. +- [claim-verified] manager.py:380 adds "unet" to default_components — matches the PR's stated purpose. +- [claim-verified] PR body manual SDXL before/after image difference is consistent with attention LoRA now applying, though the attached images are not verifiable from this checkout. +- [claim-verified] PR body claim 'include unet in default component scanning' matches diff (manager.py:368-381 adds "unet"). +- [validated] blast radius: grep `self\.unet\s*=` finds only sdxl/pipeline_sdxl.py:110 at pipeline top level; ovis_image/flux2_klein `self.unet` occurrences are docstring-only and soulx_singer's is a nested rmvpe submodule, so the default widening currently reaches SDXL only. +- [validated] real SDXL layers are QKVParallelLinear/ColumnParallelLinear/RowParallelLinear/MergedColumnParallelLinear (sdxl_unet.py), all handled by from_layer_diffusion in lora/utils.py. +- [validated] the new test can fail (asserts `"down_block.proj" in replace_calls` plus isinstance), so it is not a tautology. +- [validated] test selection: file pytestmark = [core_model, cpu]; .buildkite/test-merge.yml:13 runs `tests/diffusion tests/model_executor -m 'core_model and cpu'`, so the test executes in the main CPU gate (buildkite/vllm-omni is SUCCESS). +- [validated] Blast radius: `self.unet =` at top-level exists only in vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py:110 among registered pipelines; soulx_singer/modules/preprocess/rmvpe.py:236 is a nested submodule, not a pipeline attr, so the new scan effectively fires for StableDiffusionXLPipeline only. + +`tests/diffusion/lora/test_lora_manager.py:586` [minor] — New test `test_lora_manager_discovers_unet_component` proves only that `_replace_layers_with_lora` walks a top-level `unet` and records/replaces a mocked `_FakeLinearBase`; both `from_layer_diffusion` (line 593) and `replace_submodule` (line 597) are monkeypatched, so no real SDXL LinearBase is converted. The synthetic `peft_helper` (line 613) also has no `target_modules`, so `_matches_target` is not exercised. Real SDXL still uses `to_out = nn.ModuleList([...])` (child `to_out.0`, sdxl_unet.py:151/243) while head passes `weights_mapper=None` (manager.py:309) because 632efb42 was reverted by ac3a2ce5. Consider a CPU regression using a real SDXL-shaped unet (e.g. `SDXLBasicTransformerBlock`) without mocking `from_layer_diffusion`, asserting replacement/activation for `to_out.0` and `attn1.to_q`. (evidence: tests/diffusion/lora/test_lora_manager.py:593 monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion) — mocks the real converter. tests/diffusion/lora/test_lora_manager.py:597 lambda root, name, sub: fake_replace_submodule(root, name, sub, replace_calls), — mocks replacement. tests/diffusion/lora/test_lora_manager.py:613 peft_helper = type("_PH", (), {"r": 1})() — no target_modules, so `_matches_target` is not exercised. vllm_omni/diffusion/lora/manager.py:309 weights_mapper=None, — unchanged by this diff, present in the PR-time tree (632efb42 added a `.to_out.0` -> `.to_out` WeightsMapper; ac3a2ce5, the head, reverted it). vllm_omni/diffusion/models/sdxl/sdxl_unet.py:151 self.to_out = nn.ModuleList( and vllm_omni/diffusion/models/sdxl/sdxl_unet.py:243 attn_out, _ = self.attn1.to_out[0](attn_out) — outside this diff; real SDXL exposes the `to_out.0` child.) + +```suggestion +peft_helper = type("_PH", (), {"r": 1, "target_modules": ["proj"]})() +``` + +`vllm_omni/diffusion/lora/manager.py:~380` [minor] — manager.py:380 adds 'unet' to the shared default_components tuple while manager.py:370 still claims the list is 'opt-in; other pipelines unchanged'. Blast radius today is only StableDiffusionXLPipeline (top-level self.unet at pipeline_sdxl.py:110), which already declares _dit_modules = ["unet"] (pipeline_sdxl.py:46) resolved by ModuleDiscovery, and _lora_components already exists as a per-pipeline opt-in (pipeline_sensenova_u1.py:543). Prefer scoping to SDXL via _lora_components or deriving scan components from _dit_modules/ModuleDiscovery instead of broadening the framework default. (The only other self.unet in the repo is E2E0.__init__ at rmvpe.py:236, a preprocessing module, not a diffusion pipeline.) (evidence: Added by this diff: vllm_omni/diffusion/lora/manager.py:380 ` "unet",` inside `default_components`; contradicted by the unchanged line vllm_omni/diffusion/lora/manager.py:370 ` # ``_lora_components`` (opt-in; other pipelines unchanged).`. Unchanged by this diff, present in the PR-time tree: vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py:110 ` self.unet = SDXLUNet2DConditionModel(od_config=od_config)`; vllm_omni/diffusion/models/sdxl/pipeline_sdxl.py:46 ` _dit_modules: list[str] = ["unet"]`; vllm_omni/diffusion/offloader/module_collector.py:110 ` dit_attrs = pipeline._dit_modules`; vllm_omni/diffusion/models/sensenova_u1/pipeline_sensenova_u1.py:543 ` _lora_components: ClassVar[list[str]] = ["language_model"]`.) + +```suggestion +default_components = ( + "transformer", + "transformer_2", + "dit", + "bagel", + ) +``` + +`.buildkite/test-amd-ready.yaml:~19` [minor] — Confirmed: `.buildkite/test-amd-ready.yaml:19` runs `pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'`, and `tests/diffusion/lora/test_lora_manager.py:19` marks the file `core_model`+`cpu`, so the new `test_lora_manager_discovers_unet_component` is collected by that AMD lane. `.buildkite/test-amd.yaml:23` only runs `tests/diffusion/lora/test_images_generations_lora.py`, and `.buildkite/test-npu-ready.yaml:21` only runs `tests/e2e/online_serving/test_qwen3_tts_customvoice.py`; gh_ci_read shows both `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` FAILURE while `buildkite/vllm-omni` passes. Please link the failing job logs or confirm the failures are pre-existing on main. (evidence: PR-time tree, unchanged by this diff: `.buildkite/test-amd-ready.yaml:19` `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`. PR-time tree, unchanged by this diff: `tests/diffusion/lora/test_lora_manager.py:19` `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. PR-time tree, unchanged by this diff: `.buildkite/test-amd.yaml:23` `- pytest -s -v tests/diffusion/lora/test_images_generations_lora.py`. PR-time tree, unchanged by this diff: `.buildkite/test-npu-ready.yaml:21` `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`.) + +`vllm_omni/diffusion/lora/manager.py:~498` [minor] — This diff adds `"unet"` to `default_components`, so SDXL UNet layers now flow through `_replace_layers_with_lora` and `_activate_adapter`. SDXL FF is stored internally as `ff.geglu.proj` / `ff.out_proj`, while diffusers/PEFT checkpoints use `ff.net.0.proj` / `ff.net.2` (remapped only in `SDXLUNet2DConditionModel.load_weights`). `_matches_target` (manager.py:408) and `_get_lora_weights` (manager.py:498-508) perform no equivalent rename, so an SDXL adapter whose `target_modules` includes `ff.net.0.proj` / `ff.net.2` misses selection and can silently drop FF weights. Add the same diffusers->internal rename in both paths, or confirm FF is intentionally unsupported and document it. (evidence: Unchanged by this diff, present in the PR-time tree: +vllm_omni/diffusion/lora/manager.py:408 `return _match_target_modules(module_name, target_modules_list)` — `module_name` is the internal name (e.g. `unet.down_blocks.1.attentions.0.transformer_blocks.0.ff.geglu.proj`); no rename before matching. +vllm_omni/diffusion/lora/utils.py:18-21 `return any(re.match(rf".*\.{target_module}$", module_name) or target_module == module_name for target_module in target_modules)` — literal suffix match, so `ff.net.0.proj` cannot match `...ff.geglu.proj`. +vllm_omni/diffusion/lora/manager.py:498-507 `lora_weights = lora_model.get_lora(full_module_name)` then `component_relative_name = full_module_name.split(".", 1)[-1] ...` then `module_suffix = full_module_name.split(".")[-1]` — lookup only tries full/relative/suffix; no `ff.net.0.proj`/`ff.net.2` -> `ff.geglu.proj`/`ff.out_proj` remap. +vllm_omni/diffusion/models/sdxl/sdxl_unet.py:835-838 `if ".ff.net.0.proj." in name:` / `name = name.replace(".ff.net.0.proj.", ".ff.geglu.proj.")` and `elif ".ff.net.2." in name:` / `name = name.replace(".ff.net.2.", ".ff.out_proj.")` — proves internal FF names differ from diffusers/PEFT checkpoint names.) + +```suggestion +lora_weights = lora_model.get_lora(full_module_name) + if lora_weights is not None: + return lora_weights + + component_relative_name = full_module_name.split(".", 1)[-1] if "." in full_module_name else full_module_name + lora_weights = lora_model.get_lora(component_relative_name) + if lora_weights is None: + # SDXL LoRA checkpoints keep diffusers FF names; the model renames + # them to ff.geglu.proj / ff.out_proj in load_weights (sdxl_unet.py). + diffusers_name = component_relative_name.replace("ff.geglu.proj", "ff.net.0.proj").replace( + "ff.out_proj", "ff.net.2" + ) + lora_weights = lora_model.get_lora(diffusers_name) + if lora_weights is not None: + return lora_weights + + module_suffix = full_module_name.split(".")[-1] + return lora_model.get_lora(module_suffix) +``` + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v17ds_val/pr4825.trace.json.gz new file mode 100644 index 000000000..d441accfd Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_val/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4837.cost.json b/eval/dataset/arms/copilot_v17ds_val/pr4837.cost.json new file mode 100644 index 000000000..946f3acf7 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4837.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1169.4, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_val/runs/pr4837/run-20260817-101748-2cd7bd", + "recorded_at": "2026-08-17T10:37:14", + "attempts": [ + { + "invocation_id": "inv-0a6a1c6d", + "outcome": "run:run-20260817-101748-2cd7bd", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-101748-2cd7bd", + "llm_calls": 142, + "input_tokens": 1175252, + "output_tokens": 199074, + "usd": 0.884284, + "source": "spans" + } + ], + "attempt_usd_total": 0.884284, + "llm_calls": 142, + "input_tokens": 1175252, + "output_tokens": 199074, + "usd": 0.884284, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-101748-2cd7bd", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.8843, + "minutes": 19.43, + "ci_minutes": 0, + "input_tokens": 1175252, + "output_tokens": 199074, + "tool_calls": 264, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.873, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 142, + "by_role": { + "agent": { + "usd": 0.846904, + "input_tokens": 1163506, + "output_tokens": 168016, + "calls": 140 + }, + "reducer": { + "usd": 0.03738, + "input_tokens": 11746, + "output_tokens": 31058, + "calls": 2 + } + }, + "cache_read_tokens": 12888320, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.92, + "gate": 1.82, + "review": 1160.12, + "report": 0.06 + }, + "time_to_first_result_s": 1165.94 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 32440 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4837.md b/eval/dataset/arms/copilot_v17ds_val/pr4837.md new file mode 100644 index 000000000..beb561914 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4837.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4837, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] 'regression imported in #4079': search_history shows c45ac74d (#4079) introduced the StagePool list rejection; the pre-existing orchestrator guard predates it. +- [claim-verified] PR claim 'Both of the two submit paths reject a list' confirmed at vllm_omni/engine/stage_pool.py:953 and :1024 ('Diffusion list-prompt batch requests are no longer supported.'). +- [claim-verified] PR description says DiffusionOutput issue is 'trivial' and forward returns DiffusionOutput; verified super().forward returns DiffusionOutput, so the old `outputs[0]` unpacking was the bug being fixed. +- [claim-refuted] PR body test-result image: at this head MingImagePipeline passes kwargs to a single-arg ZImagePipeline.forward (see review comment) — the image claim is not reproducible from this tree. +- [claim-refuted] PR body test image is from commit 5c51782c (pre-merge); head is merge 78cc4dcb, so the image does not validate the merged state. +- [claim-refuted] PR's 'trivial DiffusionOutput issue' fix does not survive the merge: at head pipeline_z_image.py:413 is `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput`, but Ming still calls `super().forward(z_req, prompt=None, height=..., ..., return_dict=True)` -> TypeError. +- [claim-refuted] PR body's Test Plan shows only a manual single-request curl and an image; it does not show any automated test run, so the claim of verification does not extend to automated regression coverage. +- [validated] StagePool.submit_initial (stage_pool.py:947-951) and submit_update (stage_pool.py:1027-1031) both raise 'Diffusion list-prompt batch requests are no longer supported' for list prompts — matches the PR claim. +- [validated] thinker2imagegen (ming_flash_omni.py:346+) returns a single-element list[dict] for one parent output — the unwrap targets the right producer. +- [validated] hunyuan_image3.ar2diffusion (hunyuan_image3.py:94+) returns list[dict]; a single AR output yields len==1, so the fix covers Hunyuan too (issue comment confirmed). +- [validated] glm_image.ar2diffusion (glm_image.py:215+) returns a plain dict, not list, so the changed branch does not affect GLM-Image. +- [validated] ZImagePipeline.forward returns a single DiffusionOutput(output=image, stage_durations=...) (pipeline_z_image.py:720); old `outputs[0]` would TypeError because DiffusionOutput is not subscriptable — a14c43ab's fix direction is correct. +- [validated] Returning `outputs` directly preserves stage_durations/custom fields that the old `DiffusionOutput(output=raw)` re-wrap silently dropped. +- [validated] stage_pool.py:950-954 rejects any list for diffusion stages with ValueError ("Diffusion list-prompt batch requests are no longer supported"), so len==1 unwrap is the correct fix. + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:~449` [blocker] — Confirmed at PR-time head. ZImagePipeline.forward (pipeline_z_image.py:413) accepts only `req` — no **kwargs — while MingImagePipeline.forward still calls `super().forward(z_req, prompt=None, height=..., width=..., num_inference_steps=..., guidance_scale=..., prompt_embeds=..., negative_prompt_embeds=..., generator=..., output_type="pt", return_dict=True)` at pipeline_ming_imagegen.py:449-459, so Python raises TypeError: forward() got an unexpected keyword argument 'prompt' before `raw = outputs.output` is reached. Independent of the kwargs, _ZPipelineSamplingParams lacks the parent-read fields `latents`, `cfg_normalize`, `extra_args`, and `output_type`, and the parent's encode_prompt path uses self.tokenizer/self.text_encoder, both set to None in MingImagePipeline.__init__. Port MingImagePipeline to the new req-based parent contract (inject precomputed embeddings/output_type through the request shim and complete the shim fields, or keep a parent path that accepts precomputed prompt_embeds), then re-run at head. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413 ` def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput:` (only parameter `req`; no **kwargs). changed by this diff: vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449 ` outputs: DiffusionOutput = super().forward(`; :451 ` prompt=None,`; :459 ` return_dict=True,` — passing `prompt=None` plus other kwargs to a method accepting only `req` raises `TypeError: ZImagePipeline.forward() got an unexpected keyword argument 'prompt'` before the parent body runs.) + +`vllm_omni/engine/orchestrator.py:~1290` [minor] — Removing the `already_submitted and` guard means `if len(diffusion_prompt) == 1:` (orchestrator.py:1290) now unwraps a single-element list returned by `custom_process_input_func` on the initial-submit path, avoiding StagePool.submit_initial's list-prompt ValueError (stage_pool.py:953). No test covers this branch: test_orchestrator_kv_sender_info.py:196 only covers the empty-list terminal error. Add a regression test that sets `diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: [{"prompt": "hello"}]`, calls `_forward_to_next_stage`, asserts the submitted prompt is the unwrapped dict, and adds a second case asserting a len>1 list still raises the StagePool ValueError. (evidence: ['vllm_omni/engine/orchestrator.py:1290 ` if len(diffusion_prompt) == 1:` — changed by this PR; the diff removed the `already_submitted and ` prefix, so this unwrap now runs on the initial-submit path.', 'vllm_omni/engine/stage_pool.py:953 ` "Diffusion list-prompt batch requests are no longer supported. "` — unchanged by this diff, present in the PR-time tree; StagePool.submit_initial raises ValueError when a list prompt reaches it.', 'tests/engine/test_orchestrator_kv_sender_info.py:196 ` diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: []` — the only custom_process_input_func test branch, covering only the empty-list terminal error; no test returns a single-element list.']) + +```suggestion +Add a test in tests/engine/test_orchestrator_kv_sender_info.py that sets `diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: [{"prompt": "hello"}]`, calls `_forward_to_next_stage`, and asserts `diffusion_stage.calls[0]["prompt"] == {"prompt": "hello"}` (not a list). +``` + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:~475` [minor] — The diff replaces `return DiffusionOutput(output=raw)` with `return outputs` (pipeline_ming_imagegen.py:475), relying on the parent ZImagePipeline.forward returning a DiffusionOutput whose .output is the tensor; the code guards this by raising RuntimeError if `outputs.output` is not a torch.Tensor. Grep finds no test that imports MingImagePipeline — tests/ only has string assertions in tests/test_config_factory.py (lines 1344, 1382) — and tests/diffusion/models has no ming_flash_omni directory. Add a focused test under tests/diffusion/models (or a z_image-family test) that calls MingImagePipeline.forward and asserts outputs.output is a torch.Tensor; run it through the AMD diffusion lane command `pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py`. (evidence: vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:475 `return outputs` (unchanged-in-context PR-time tree region; diff replaced `return DiffusionOutput(output=raw)`); vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:473 `raw = outputs.output`; vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:474 `if not isinstance(raw, torch.Tensor):`; vllm_omni/diffusion/models/z_image/pipeline_z_image.py:720 `return DiffusionOutput(output=image, stage_durations=stage_durations)` (unchanged by this diff, present in the PR-time tree); tests/test_config_factory.py:1344 `assert s.model_arch == "MingImagePipeline"` and tests/test_config_factory.py:1382 `assert stages[1].yaml_engine_args["model_arch"] == "MingImagePipeline"` (string assertions only; no test imports/calls MingImagePipeline).) + +`vllm_omni/engine/orchestrator.py:1290` [nit] — CI shows buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci failing while buildkite/vllm-omni passes. AMD lane runs tests/engine/test_async_omni_engine_abort.py (.buildkite/test-amd-ready.yaml:148) and diffusion tests; NPU lane runs tests/e2e/online_serving/test_qwen3_tts_customvoice.py (.buildkite/test-npu-ready.yaml:21). Neither test references the changed symbols directly, and the changed files are the ming_flash_omni diffusion pipeline and the orchestrator. Please confirm on main whether these failures predate this PR before attributing them to this diff. (evidence: vllm_omni/engine/orchestrator.py:1290 `if len(diffusion_prompt) == 1:` (changed by this PR, the `already_submitted and` guard was removed); .buildkite/test-amd-ready.yaml:148 `pytest -s -v tests/engine/test_async_omni_engine_abort.py`; .buildkite/test-npu-ready.yaml:21 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; gh_ci_read(4837) returns buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci as FAILURE while buildkite/vllm-omni is SUCCESS.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v17ds_val/pr4837.trace.json.gz new file mode 100644 index 000000000..e76f05571 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_val/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4893.cost.json b/eval/dataset/arms/copilot_v17ds_val/pr4893.cost.json new file mode 100644 index 000000000..2ca687fac --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4893.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2222.9, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v17ds_val/runs/pr4893/run-20260817-100115-84160c", + "recorded_at": "2026-08-17T10:38:15", + "attempts": [ + { + "invocation_id": "inv-8f822c0d", + "outcome": "run:run-20260817-100115-84160c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-100115-84160c", + "llm_calls": 221, + "input_tokens": 1574467, + "output_tokens": 369633, + "usd": 1.235705, + "source": "spans" + } + ], + "attempt_usd_total": 1.235705, + "llm_calls": 221, + "input_tokens": 1574467, + "output_tokens": 369633, + "usd": 1.235705, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260817-100115-84160c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 1.2357, + "minutes": 37.0, + "ci_minutes": 0, + "input_tokens": 1574467, + "output_tokens": 369633, + "tool_calls": 404, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 2.2563, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 221, + "by_role": { + "agent": { + "usd": 1.189949, + "input_tokens": 1552887, + "output_tokens": 333399, + "calls": 218 + }, + "reducer": { + "usd": 0.045757, + "input_tokens": 21580, + "output_tokens": 36234, + "calls": 3 + } + }, + "cache_read_tokens": 14963072, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.1, + "gate": 1.69, + "review": 2213.97, + "report": 0.06 + }, + "time_to_first_result_s": 2219.8 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 42681 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4893.md b/eval/dataset/arms/copilot_v17ds_val/pr4893.md new file mode 100644 index 000000000..c6f9c0c00 --- /dev/null +++ b/eval/dataset/arms/copilot_v17ds_val/pr4893.md @@ -0,0 +1,110 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4893, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 9 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] prior `_PCP is _SP` assertion flipped to `is not _SP` and `world_size==2` still checked (test_expert_parallel_layout.py:109-112) — matches the new separate coordinator. +- [resolved] Concern that the separate vllm _DP coordinator would be double-destroyed: parallel_state.py:919-921 guards with `vllm_parallel_state._DP is not _DP`, so it is destroyed exactly once. +- [resolved] Non-EP AttributeError concern: diffusion GroupCoordinator has cpu_group/world_size (group_coordinator.py:99-101), so _set_forward_context_dp_metadata cannot AttributeError when _vllm_ps._DP is the diffusion DP group. +- [resolved] Stale dp_metadata concern: pipeline re-enters vllm set_forward_context(None, self.vllm_config) per forward (pipeline_hunyuan_image3.py:1728), so the 'is not None' guard cannot leak across denoise steps. +- [resolved] NPU sibling: platforms/npu/models/hunyuan_fused_moe.py uses AscendFusedMoE/moe_comm_type, not vLLM DP all2all, so it does not need the dp_metadata change. +- [claim-verified] diff_stat confirms the PR's named test files are both touched. +- [claim-verified] PR's '11 passed' unit tests are present in the diff and marked core_model/cpu; remote smoke images cannot be verified from this checkout. +- [claim-verified] PR '11 passed' = 8 tests in test_hunyuan_fused_moe.py + 3 in test_expert_parallel_layout.py. +- [validated] destroy_model_parallel symmetry (parallel_state.py:928-950): vLLM `_DP`/`_PCP`/`_TP`/`_EP` are destroyed and None'd; `_DP is not _DP` guard prevents double destroy. +- [validated] destroy_model_parallel (parallel_state.py:925-942) destroys vllm _PCP/_TP/_EP and the distinct vllm _DP; lifecycle is symmetric for the newly created coordinators. +- [validated] tests/diffusion/distributed/test_expert_parallel_layout.py asserts vllm PCP/DP/EP device_communicator and reduce_scatter presence and pins vllm_pcp/tp/dp/ep group order. +- [validated] tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py covers dp_metadata population and existing-metadata no-op. +- [validated] consumers of changed groups: ltx2_transformer.py:304 uses _TP.all_reduce (vLLM coordinator provides it); platforms/npu/models/hunyuan_fused_moe.py:85 uses get_expert_parallel_group_ranks (rank metadata unchanged). +- [validated] resource lifecycle: destroy_model_parallel (parallel_state.py:919-945) destroys vllm _DP/_PCP/_TP/_EP including the new separate coordinators, with is-not guards preventing double-free. + +`vllm_omni/diffusion/distributed/parallel_state.py:~515` [blocker] — init_vllm_model_parallel_group always passes use_device_communicator=True but does not pass use_all2all. vLLM DeviceCommunicatorBase only builds the all2all manager when self.use_all2all is true, so the EP coordinator created by this PR will lack the all2all path needed by the very fusion dispatch it modifies. (evidence: parallel_state.py:515-527 call site; DeviceCommunicatorBase sets use_all2all = is_ep_communicator and use_all2all and builds the manager under if self.use_all2all.) + +```suggestion +vllm_parallel_state.init_model_parallel_group( + ... + use_device_communicator=True, + use_all2all=use_all2all, +) +``` + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:~91` [minor] — The pre-hook at hunyuan_fused_moe.py:91 calls _set_forward_context_dp_metadata for every HunyuanImage3 fused MoE forward, not just EP/MoE-mapping runs. With use_moe_parallel_mapping=False, parallel_state.py:816 leaves vllm_parallel_state._DP aliased to the diffusion DP GroupCoordinator (cpu_group/world_size present, device_communicator absent), so a non-EP run with data_parallel_size>1 performs a blocking all_gather_object and sets ForwardContext.dp_metadata — a new collective outside the PR's EP scope and not covered by the new tests, which inject a fake vLLM DP group directly. Gate the call on getattr(_vllm_ps._DP, 'device_communicator', None) is not None. (evidence: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:91 `_set_forward_context_dp_metadata(hidden_states.shape[0])` — unconditional call in the forward pre-hook (added by this diff). vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:37-46 `dp_group = getattr(_vllm_ps, "_DP", None)` / `if dp_group is None or getattr(dp_group, "world_size", 1) <= 1:` / `torch.distributed.all_gather_object(... group=dp_group.cpu_group)` — blocking collective over whatever _vllm_ps._DP currently is (added by this diff). vllm_omni/diffusion/distributed/parallel_state.py:816 `vllm_parallel_state._DP = _DP` — unchanged by this diff, present in the PR-time tree; aliases vLLM DP to the diffusion GroupCoordinator before any MoE-mapping branch. vllm_omni/diffusion/distributed/parallel_state.py diff hunk `@@ -838,23 +853,40 @@` — the vLLM companion DP group replaces that alias only inside `if use_moe_parallel_mapping:` (`vllm_parallel_state._DP = init_vllm_model_parallel_group(... group_name="dp")`). vllm_omni/diffusion/distributed/group_coordinator.py GroupCoordinator.__init__ sets `self.cpu_group`/`self.world_size` and defines no `device_communicator` attribute (diffusion DP coordinator).) + +```suggestion +if hidden_states is not None: + _set_forward_context_num_tokens(hidden_states.shape[0]) + if getattr(_vllm_ps._DP, "device_communicator", None) is not None: + _set_forward_context_dp_metadata(hidden_states.shape[0]) +``` + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:~91` [minor] — The integration point for the DP-metadata half of the fix (hunyuan_fused_moe.py:91) is untested: grep for `_num_tokens_pre_hook` in tests/ returns no matches, and the two new tests call `_set_forward_context_dp_metadata` directly. Note that `_num_tokens_pre_hook` is a local closure inside `HunyuanFusedMoEDefault.__new__` (line 85), so the suggested test `hook = hunyuan_moe.HunyuanFusedMoEDefault._num_tokens_pre_hook` would fail — instead build the runner via `HunyuanFusedMoEDefault.__new__` and invoke the registered hook with `kwargs={"hidden_states": torch.zeros(4, ...)}`, asserting `_set_forward_context_dp_metadata` is called with `4`. Also cover the world_size<=1 no-op and the RuntimeError branch. (evidence: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:85 `def _num_tokens_pre_hook(module: Any, args: Any, kwargs: Any) -> None:` ; :91 `_set_forward_context_dp_metadata(hidden_states.shape[0])` ; :93 `moe_runner.register_forward_pre_hook(_num_tokens_pre_hook, with_kwargs=True)` — the hook is a local closure inside `__new__`, not a class attribute. tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:74 and :89 both call the helper directly: `hunyuan_moe._set_forward_context_dp_metadata(1024)`. `grep -r '_num_tokens_pre_hook' tests/` returns no matches in the PR-time tree.) + +```suggestion +def test_pre_hook_sets_dp_metadata(self, mocker): + import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe + module = mocker.MagicMock() + hidden = torch.zeros(4, 8) + mocker.patch.object(hunyuan_moe, "_set_forward_context_num_tokens") + mocker.patch.object(hunyuan_moe, "_set_forward_context_dp_metadata") as dp_md + hook = hunyuan_moe.HunyuanFusedMoEDefault._num_tokens_pre_hook + hook(module, (), {"hidden_states": hidden}) + dp_md.assert_called_once_with(4) +``` + +`test_expert_parallel_layout.py:112` [minor] — `test_expert_parallel_layout.py:112` flips the prior `_PCP is _SP` assertion to `is not _SP`, and `:113` still checks `_PCP.world_size == 2` — consistent with the new separate vLLM coordinator created at `parallel_state.py:859`. (evidence: tests/diffusion/distributed/test_expert_parallel_layout.py:112 ` assert parallel_state.vllm_parallel_state._PCP is not parallel_state._SP`; tests/diffusion/distributed/test_expert_parallel_layout.py:113 ` assert parallel_state.vllm_parallel_state._PCP.world_size == 2`; vllm_omni/diffusion/distributed/parallel_state.py:859 ` vllm_parallel_state._PCP = init_vllm_model_parallel_group(`) + +`pyproject.toml:?` [minor] — vLLM is not declared as a dependency anywhere in the package install metadata: pyproject.toml sets dependencies dynamic and requirements/*.txt contain no vllm line (only an unrelated comment), so no minimum vLLM version is enforced. Since this PR calls vLLM APIs (`init_model_parallel_group(use_device_communicator=True)` and `_vllm_fc.DPMetadata`), add `vllm>=0.24.0` (matching the docs pin) or a runtime version guard. (evidence: pyproject.toml:11 `dynamic = ["version", "dependencies"]` — dependencies are not enumerated in pyproject. setup.py:249 `install_requires=install_requires,` — install_requires is populated from platform-specific requirements files via load_requirements(). requirements/common.txt:11 `# transformers_keys_to_ignore_compat (vllm_omni/model_executor/models/utils.py).` — the only occurrence of 'vllm' across requirements/ is this comment; no vllm requirement line exists in any platform file.) + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:?` [minor] — The PR's new automated tests pin only cfg_parallel_size=2: tests/diffusion/distributed/test_expert_parallel_layout.py:106, :240, and :305 all pass cfg_parallel_size=2, and the DP-metadata unit tests use a 2-rank group. The smoke/perf matrix only names test_hunyuan_image_tp2_cfgp2 and test_hunyuan_image_tp2_sp2. Since cfg_parallel_size=3 is accepted by config tests (tests/config/test_omni_config.py:452), add a cfg>2 layout/DP-metadata case or explicitly document cfg>2 as unsupported. (evidence: tests/diffusion/distributed/test_expert_parallel_layout.py:106 ` cfg_parallel_size=2,`; tests/diffusion/distributed/test_expert_parallel_layout.py:240 ` cfg_parallel_size=2,`; tests/diffusion/distributed/test_expert_parallel_layout.py:305 ` cfg_parallel_size=2,` — all three new/updated layout cases pin cfg=2. Outside this diff but present in the PR-time tree: tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json:3 ` "test_name": "test_hunyuan_image_tp2_cfgp2",` and tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json:3 ` "test_name": "test_hunyuan_image_tp2_sp2",`. New DP-metadata unit tests in tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py use `fake_group.world_size = 2`, so no cfg>2 path is exercised.) + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:~77` [minor] — The value assertion checks output fabricated by the mock (`output[:] = [obj, obj + 1]`), so the test proves route plumbing but not the real all2all token-count contract. Note the mock side effect already asserts `group is fake_group.cpu_group`; the valuable additions are asserting the local object value passed to all_gather_object and the pre-allocated output list length. (evidence: tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:65-68 (in this diff): `def _all_gather_object(output, obj, group): + assert group is fake_group.cpu_group + output[:] = [obj, obj + 1]`; tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:77 (in this diff): `assert mock_ctx.dp_metadata.num_tokens_across_dp_cpu.tolist() == [1024, 1025]`.) + +```suggestion +call = hunyuan_moe.torch.distributed.all_gather_object.call_args + assert call.kwargs["group"] is fake_group.cpu_group + assert call.args[1] == 1024 + assert len(call.args[0]) == fake_group.world_size +``` + +`tests/diffusion/distributed/test_expert_parallel_layout.py:~92` [minor] — Confirmed at tests/diffusion/distributed/test_expert_parallel_layout.py:92 — the layout test monkeypatches init_vllm_model_parallel_group with a fake, so the real wrapper (vllm_omni/diffusion/distributed/parallel_state.py:515-527) is never executed and its use_device_communicator=True forwarding to vLLM's GroupCoordinator is not covered. Add an unpatched/integration path that calls the real init_vllm_model_parallel_group for the EP arm and asserts the returned coordinator actually constructs its device communicator/all2all manager. Note: use_all2all does not appear anywhere in the PR-time tree; verify the installed vLLM signature before asserting that exact kwarg. (evidence: tests/diffusion/distributed/test_expert_parallel_layout.py:92 `monkeypatch.setattr(parallel_state, "init_vllm_model_parallel_group", fake_init_vllm_model_parallel_group)` — the helper under test is replaced by a fake. vllm_omni/diffusion/distributed/parallel_state.py:521-526 `return vllm_parallel_state.init_model_parallel_group( + group_ranks=group_ranks, + local_rank=local_rank, + backend=backend, + group_name=group_name, + use_device_communicator=True, +)` — the real wrapper is never executed by the layout test, so its forwarding/all2all manager construction is uncovered (the use_all2all kwarg appears nowhere in the PR-time tree; the external vLLM signature is not inspectable from this repo).) + +```suggestion +Add a test that calls the real init_vllm_model_parallel_group (or a thin unpatched wrapper) with EP enabled and asserts the created EP coordinator/device communicator has use_all2all True and the all2all manager is not None. +``` + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/distributed/test_expert_parallel_layout.py:102` [minor] The layout test at test_expert_parallel_layout.py:98-106 already covers a combined cfg+sp rank layout (SP=2, CFG=2), but it only asserts group mapping. The new _set_forward_context_dp_metadata producer/consumer path is not exercised for that layout: the metadata unit tests use a flat 2-rank fake DP group. Add a […] +- `vllm_omni/diffusion/distributed/parallel_state.py:882` [minor] This diff removes the `and cfg_parallel_size > 1` guard, so `vllm_parallel_state._DP` is now rebuilt as a separate vLLM coordinator even when `cfg_parallel_size == 1`, yet the only MoE-mapping unit test drives `cfg_parallel_size=2` (test_expert_parallel_layout.py:106). Add a CPU test for `cfg_parallel_size=1` to pin […] +- `vllm_omni/diffusion/distributed/parallel_state.py:526` [minor] Confirmed: test_expert_parallel_layout.py:92 monkeypatches init_vllm_model_parallel_group away, and the fake at lines 73-74 supplies device_communicator=object() and reduce_scatter=lambda..., which the test then asserts at lines 121-126. The real wrapper at parallel_state.py:526 (use_device_communicator=True) is never […] + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. +- **vllm-omni-async-omni-output-review-contract**: Reviewing vllm-omni docs/code that mention async Omni output materialization: verify the doc's guard, opt-in, platform, and fallback claims against gpu_ar_model_runner.py and the model/deploy files. +- **vllm-omni-docs-image-reference-convention**: When reviewing docs PRs in vllm-omni, verify figure references render on both the MkDocs site and GitHub preview; docs/design/feature/ conventionally uses raw.githubusercontent.com absolute URLs. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/copilot_v17ds_val/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v17ds_val/pr4893.trace.json.gz new file mode 100644 index 000000000..d26fea9e0 Binary files /dev/null and b/eval/dataset/arms/copilot_v17ds_val/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r1/manifest.json b/eval/dataset/arms/copilot_v7_goal_r1/manifest.json new file mode 100644 index 000000000..4e3445a15 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v7_goal_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4804.cost.json b/eval/dataset/arms/copilot_v7_goal_r1/pr4804.cost.json new file mode 100644 index 000000000..add6788bc --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 360.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r1/runs/pr4804/run-20260812-172349-a2e0bc", + "recorded_at": "2026-08-12T17:29:47", + "attempts": [ + { + "invocation_id": "inv-00db2ed7", + "outcome": "run:run-20260812-172349-a2e0bc", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-172349-a2e0bc", + "llm_calls": 41, + "input_tokens": 724319, + "output_tokens": 49112, + "usd": 0.340783, + "source": "spans" + } + ], + "attempt_usd_total": 0.340783, + "llm_calls": 41, + "input_tokens": 724319, + "output_tokens": 49112, + "usd": 0.340783, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-172349-a2e0bc", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3408, + "minutes": 5.96, + "ci_minutes": 0, + "input_tokens": 724319, + "output_tokens": 49112, + "tool_calls": 68, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3564, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 41, + "by_role": { + "agent": { + "usd": 0.313998, + "input_tokens": 667871, + "output_tokens": 38618, + "calls": 40 + }, + "reducer": { + "usd": 0.026784, + "input_tokens": 56448, + "output_tokens": 10494, + "calls": 1 + } + }, + "cache_read_tokens": 3377536, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.11, + "gate": 1.65, + "review": 351.98, + "report": 0.07 + }, + "time_to_first_result_s": 357.78 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 67197 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4804.md b/eval/dataset/arms/copilot_v7_goal_r1/pr4804.md new file mode 100644 index 000000000..e825dc6cc --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4804.md @@ -0,0 +1,54 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 5 finding(s) below | + +**Validated:** +- [sweep] serving_speech.py:3705-3715 — seed extra_arg guard removed; was `if self._tts_model_type == "qwen3_tts"`, now unconditional; blast radius affects all _SAMPLING_MAX_TOKENS_TTS_MODEL_TYPES and MOSS_TTS_MODEL_STAGES +- [sweep] serving_speech.py:3708-3709 — old `if sampling_params_list is self.engine_client.default_sampling_params_list` copy guard removed; copy.deepcopy now runs on every request +- [sweep] modeling_moss_tts_talker.py:1392-1470 — talker_mtp methods moved from make_omni_output to runner-owned talker_mtp; sampling params changed 0.8→1.7 temp, 30→25 top_k, 0.6→0.8 top_p per model-card comment +- [sweep] modeling_moss_tts_talker.py:1639-1696 — make_omni_output simplified to package codes populated by talker_mtp; per_req_codes now emits current frame only (not accumulated); meta.finished removed +- [sweep] stage_input_processors/moss_tts.py:94-231 — talker2codec_async_chunk renamed to talker2codec_delay_async_chunk; skip_dedelay path removed; talker2codec_raw_async_chunk added +- [sweep] stage_input_processors/moss_tts.py:220-231 — raw async chunk processor's non-empty finish sentinel uses codes=CodesStruct(audio=torch.tensor([0])); code_flat_numel=0 tells codec this is control-only +- [sweep] modeling_moss_tts_codec.py:37-125 — _MossCodecStreamSession: acquire/release/reset/close lifecycle; checked that on_requests_finished is wired in gpu_generation_model_runner.py:78-79 +- [sweep] modeling_moss_tts_codec.py:243-393 — forward() now uses seq_token_counts contract; raises on missing/mismatch; validated runner side passes tokens via model_kwargs + +`vllm_omni/entrypoints/openai/serving_speech.py:3711` [major] — Diff removes the `if self._tts_model_type == "qwen3_tts"` guard in `_prepare_speech_generation` (line 3710) and unconditionally sets `tts_local_seed` in `extra_args` for every TTS model. The worker-side reader (`gpu_model_runner.py:1826`) reads `tts_local_seed` only for models with a `talker_mtp` that supports per-request seeding; the blast radius now covers all TTS models. Either restore the guard (or document that `tts_local_seed` is a universal extra_arg) and confirm that every other model safely ignores it. (evidence: grep for `tts_local_seed` in the repo returns only this diff's changed files and the test file; no other model reads this key.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [major] — New vendored file `audio_tokenizer_v2.py` (line 1) lacks the required `# Vendored from @ ` provenance header. The existing `audio_tokenizer.py` in the same directory carries such a header. Add the upstream source commit/tag reference after the copyright line. (evidence: Checked `audio_tokenizer.py` in the same directory — it has an explicit vendored-from line; this new file does not.) + +`vllm_omni/benchmarks/patch/patch.py:1249` [major] — In `_pcm_s16le_to_seed_tts_wer_bytes` (line 147), the `except Exception` fallback assigns raw `bytes(pcm_capture)` — the original multi-channel, original-sample-rate PCM — as `tts_output_pcm_bytes`. Downstream `compute_seed_tts_wer_metrics` expects 24 kHz mono PCM; feeding it unnormalized bytes produces silently wrong WER scores. The fallback should either raise (so the utterance is marked failed) or set `output.success = False` so `compute_seed_tts_wer_metrics` skips it rather than computing bogus WER on mismatched audio. (evidence: The normalization function converts to mono + resamples to 24 kHz; the fallback skips both transforms. `compute_seed_tts_wer_metrics` then calls `_pcm_s16le_to_f32_16k` on those bytes, which interprets raw int16 at whatever sample rate it was captured at — yielding wrong duration and spectral content for WER.) + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:?` [major] — The GPU runner's `_update_request_states` (line 78-79) gained an `on_requests_finished` hook to release MOSS codec streaming slots on abort. The NPU runner's `_update_request_states` (line 46) does not call `self.model.on_requests_finished(scheduler_output.finished_req_ids)`. On NPU, MOSS-TTS codec streaming slots will leak when requests are aborted, eventually starving the slot pool. Add the same hook before the `remove_request` loop. (evidence: grep of npu_generation_model_runner.py shows no on_requests_finished call; gpu_generation_model_runner.py diff adds it at line 78-79) + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:?` [major] — The GPU runner's `_dummy_run` (line 799) adds `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]` to satisfy `MossTTSCodecDecoder`'s hard requirement for `seq_token_counts`. The NPU runner's `_dummy_run` (line 620) does not set this kwarg. On NPU, any MOSS-TTS dummy/profiling run will hit the `seq_token_counts` RuntimeError (modeling_moss_tts_codec.py:260-264). Add the same line after the `runtime_additional_information` block. (evidence: gpu_generation_model_runner.py diff adds seq_token_counts at line 799; npu_generation_model_runner.py _dummy_run has no such line; modeling_moss_tts_codec.py:262-264 raises RuntimeError when seq_token_counts is None) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1524` [major] — The new `talker_mtp` method (line 1461) hardcodes sampling parameters (temperature=1.7, top_k=25, top_p=0.8) that differ from the old `make_omni_output` defaults (0.8/30/0.6). This is a silent behavior change for all MOSS-TTS Local users. Either document the rationale and provide before/after WER/SIM parity evidence, or make these configurable connector parameters. (evidence: Read talker_mtp at modeling_moss_tts_talker.py lines ~1466-1476; compared with old make_omni_output sampling params (temperature=0.8, top_k=30, top_p=0.6) visible in the removed code in the diff.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1494` [major] — The new `talker_mtp` method (line 1461) is the core streaming change for `MossTTSLocalTalkerForGeneration`, but zero unit tests exercise it. grep confirms no test in `tests/` imports or calls `MossTTSLocalTalkerForGeneration.talker_mtp`. Add a CPU test that calls `talker_mtp` on a `MossTTSLocalTalkerForGeneration` instance with mock weights and verifies: (a) audio embedding is added to `input_embeds_out`, (b) `req_infos` audio_codes are populated with `emit=True`, (c) the stop condition triggers `is_stopping`. (evidence: grep for 'talker_mtp' in tests/ shows only tests/worker/test_omni_gpu_model_runner.py (fake modules) and tests/model_executor/models/fish_speech/test_fish_speech_fast_ar_runtime_buffers.py (FishSpeech, not MOSS-TTS); grep for 'MossTTSLocalTalkerForGeneration' in tests/ returns zero matches.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:225` [major] — The new `talker2codec_raw_async_chunk` processor (line 220) is the streaming entry point for the raw/local codec path, but zero unit tests exercise it. grep for `talker2codec_raw_async_chunk` in `tests/` returns zero matches. Add a CPU unit test that exercises: (a) chunk accumulation below threshold returns None, (b) reaching threshold emits a chunk with correct `codec_streaming=True` metadata, (c) `is_finished=True` with empty pending emits a finish sentinel with `code_flat_numel=0`. (evidence: grep for 'talker2codec_raw_async_chunk' in tests/ returns zero matches; grep for 'moss_tts_local' in tests/ returns zero matches. The pipeline.py diff shows MossTTSLocalTalkerForGeneration now uses talker2codec_raw_async_chunk, but no test exercises this path.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:564` [major] The `on_requests_finished` hook in `MossTTSCodecDecoder` (line 564) has no unit test. No test in `tests/model_executor/models/moss_tts/` exercises the abort/slot-release path. Add a unit test that acquires a stream slot, +- `vllm_omni/entrypoints/openai/serving_speech.py:3708` [minor] Diff removes the `if sampling_params_list is self.engine_client.default_sampling_params_list:` guard (line 3708), causing `copy.deepcopy(sampling_params_list)` on every speech request even when the list was already a per +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:731` [minor] The `except Exception` catch-all in `_build_codec` (line 731) silences any v2 config/model construction error and falls back to legacy codec. If a v2 checkpoint fails due to a missing config key or shape mismatch (not a +- `vllm_omni/deploy/moss_tts_local.yaml:42` [minor] `max_num_seqs` changed from 1 to 8 in `moss_tts_local.yaml` (line 42) without benchmark rationale or memory-budget justification. The stage-1 codec decoder with streaming state per slot has GPU memory cost proportional t +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:260` [minor] The old forward path used `kwargs.get("num_scheduled_tokens")` with a graceful fallback; the new code (line 260) raises a hard RuntimeError when `seq_token_counts` is missing. While the GPU and NPU runners set it, any te +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:130` [minor] The class docstring for `MossTTSCodecDecoder` (line 130) states 'mono waveform', but the v2 codec (audio_tokenizer_v2.py) and the `_n_channels` inference (line ~698: `inferred_channels = 2 if 'v2' in codec_path.lower() e +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:76` [minor] `_MossCodecStreamSession.close()` (line 76) is defined but never called (grep shows zero callers). The ExitStack cleanup is left to Python GC, which is non-deterministic for GPU resources. `MossTTSCodecDecoder` should ex +- `vllm_omni/worker/gpu_generation_model_runner.py:78` [minor] `on_requests_finished` is only called inside `_update_request_states` (line 78), which is gated on `self.model_config.async_chunk and num_scheduled_tokens` in `execute_model`. In the non-async-chunk path or when `num_sch + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r1/pr4804.trace.json.gz new file mode 100644 index 000000000..43b119a6e Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r1/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4870.cost.json b/eval/dataset/arms/copilot_v7_goal_r1/pr4870.cost.json new file mode 100644 index 000000000..4070983ff --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 342.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r1/runs/pr4870/run-20260812-173803-916641", + "recorded_at": "2026-08-12T17:43:40", + "attempts": [ + { + "invocation_id": "inv-99e7891b", + "outcome": "run:run-20260812-173803-916641", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-173803-916641", + "llm_calls": 41, + "input_tokens": 1076526, + "output_tokens": 46113, + "usd": 0.413237, + "source": "spans" + } + ], + "attempt_usd_total": 0.413237, + "llm_calls": 41, + "input_tokens": 1076526, + "output_tokens": 46113, + "usd": 0.413237, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-173803-916641", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4132, + "minutes": 5.61, + "ci_minutes": 0, + "input_tokens": 1076526, + "output_tokens": 46113, + "tool_calls": 87, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3727, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 41, + "by_role": { + "agent": { + "usd": 0.403552, + "input_tokens": 1068588, + "output_tokens": 39276, + "calls": 40 + }, + "reducer": { + "usd": 0.009685, + "input_tokens": 7938, + "output_tokens": 6837, + "calls": 1 + } + }, + "cache_read_tokens": 2661120, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.23, + "gate": 1.66, + "review": 330.56, + "report": 0.07 + }, + "time_to_first_result_s": 336.58 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 23674 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4870.md b/eval/dataset/arms/copilot_v7_goal_r1/pr4870.md new file mode 100644 index 000000000..4542276e7 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4870.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] gpu_ar_model_runner.py:1734 — mm_seq_len derived from scheduler_output.total_num_scheduled_tokens matches the multimodal output tensor's first dimension (scheduled tokens, not hidden-state shape); verified all existing callers in tests/worker/test_gpu_ar_model_runner.py updated to remove seq_len kwarg. +- [validated] gpu_ar_model_runner.py:1876 removed — the old seq_len = hidden_states.shape[0] was dead after the signature change; its only consumer was _build_omni_model_runner_output_from_snapshot, now self-computing mm_seq_len internally. +- [validated] serving_speech.py:3505-3508 — the qwen3_full_payload gate correctly combines model_type check + async_chunk falsiness; the bool() wrapper is belt-and-suspenders and harmless. All four branch combinations (qwen3/non-qwen3 × async_chunk T/F) produce the right coercion boolean. +- [sweep] serving_speech.py:3505 — getattr(self.model_config, 'async_chunk', True) defaults to True, so models without explicit async_chunk=False keep the old DELTA behavior. This matches the PR's stated scope. +- [sweep] test_gpu_ar_model_runner.py:435-480 — new test test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only correctly mocks tail-only path (omni_pooler_payload_include_hidden=False, _should_accumulate returns False, _async_chunk=False), sets num_scheduled_tokens={'r1':1,'r2':1,'r3':1}, and asserts per-request codes.audio split via torch.equal. +- [sweep] test_serving_speech.py:3450-3498 — both new serving tests correctly assert coerce_param_message_types called with False (qwen3) / True (voxtral), validating the scoping. +- [validated] All 5 existing test call sites for _build_omni_model_runner_output_from_snapshot in test_gpu_ar_model_runner.py updated to drop seq_len—no stale callers remain (grep confirms only test file calls this static method). +- [validated] gpu_ar_model_runner.py:1679 removed — the seq_len parameter is fully removed from the function signature; a repo-wide grep confirms no other callers or overrides reference this parameter. + +`vllm_omni/entrypoints/openai/serving_speech.py:3508` [minor] — The FINAL_ONLY streaming mode is gated on qwen3_tts only (line 3507). Since async_chunk defaults to False, this gate is live for all async_chunk=False TTS models (moss_tts, voxcpm2, higgs_audio, etc.). The PR validates voxtral keeps DELTA coercion, but other models are untested. Add a documentation note explaining why DELTA is appropriate for them or open a tracking issue to verify those models. (evidence: Grep of OmniModelConfig shows async_chunk defaults to False; review checklist states 'Any gate or coercion keyed on it applies to EVERY async_chunk=false model'. The PR test only covers qwen3_tts and voxtral.) + +`tests/worker/test_gpu_ar_model_runner.py:454` [minor] — The new test (line 455) verifies per-request multimodal splitting with uniform num_scheduled_tokens={'r1':1,'r2':1,'r3':1}. Consider adding a variant with heterogeneous counts (e.g., {'r1':2,'r2':1,'r3':3}) to exercise the general slicing formula and catch off-by-one errors. (evidence: The test uses uniform num_scheduled_tokens={'r1':1,'r2':1,'r3':1}; the bug in the linked issue involved concurrent requests which may have had heterogeneous token counts. A heterogeneous test would increase confidence.) + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [nit] — The async_chunk default differs between serving_speech.py (True, line 3506) and GPUARModelRunner.__init__ (False, line 293). The True default ensures FINAL_ONLY activates only for explicit async_chunk=False, but inconsistent defaults can be confusing. Consider adding a comment explaining the intentional difference. (evidence: GPUARModelRunner.__init__ line 293: `self._async_chunk = getattr(self.model_config, 'async_chunk', False)`. The serving_speech gate `not bool(async_chunk)` with default True means it only activates for explicitly-disabled async_chunk, which is the intended behavior.) + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [nit] — The int() cast on total_num_scheduled_tokens at line 1734 is redundant because total_num_scheduled_tokens is already an integer; consider removing it for clarity. (evidence: Read the scheduler_output type definition; total_num_scheduled_tokens is declared as int in SchedulerOutput.) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [nit] — The new mm_seq_len (line 1734) uses total_num_scheduled_tokens for batch-level payload slicing. Since multimodal outputs have dim[0] == total_num_scheduled_tokens, this is correct. Consider adding a brief comment explaining why total_num_scheduled_tokens is the right dimension. (evidence: `build_mm_cpu` at mm_outputs.py:67 flattens `multimodal_outputs` without changing dim[0]; `to_payload_element` at mm_outputs.py:141 slices tensors when `element.shape[0] == seq_len`. The codes.audio tensor produced by Qwen3-TTS talker has dim[0] == total scheduled tokens.) + +`tests/worker/test_gpu_ar_model_runner.py:478` [nit] — The new test verifies tail-only hidden states splitting (line 478). The 0-scheduled-tokens case is guarded in execute_model but not in _build_omni_model_runner_output_from_snapshot. Consider verifying the unreachable case or adding an assertion. (evidence: Read _build_omni_model_runner_output_from_snapshot at line 1734: `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`. The early-return at line ~1908 guards `if not num_scheduled_tokens`, so the 0-token case should never reach this branch, but the guard is in execute_model, not in the called method.) + +`N/A:?` [nit] — The AMD CI check (buildkite/vllm-omni-amd-ci) is failing, but the PR does not modify AMD-specific code. Please confirm whether this failure is pre-existing by checking the CI log; if so, note it as known. (evidence: Gate report shows FAILING CHECKS: ['buildkite/vllm-omni-amd-ci']. All other CI checks (build 3.11/3.12, pre-commit, DCO, omni-release, vllm-omni, intel-ci, npu-ci, docs) pass.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r1/pr4870.trace.json.gz new file mode 100644 index 000000000..6aa6f24ad Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r1/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4923.cost.json b/eval/dataset/arms/copilot_v7_goal_r1/pr4923.cost.json new file mode 100644 index 000000000..4d42d872c --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 489.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r1/runs/pr4923/run-20260812-172949-79af18", + "recorded_at": "2026-08-12T17:37:57", + "attempts": [ + { + "invocation_id": "inv-c296537b", + "outcome": "run:run-20260812-172949-79af18", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-172949-79af18", + "llm_calls": 44, + "input_tokens": 776235, + "output_tokens": 70338, + "usd": 0.34568, + "source": "spans" + } + ], + "attempt_usd_total": 0.34568, + "llm_calls": 44, + "input_tokens": 776235, + "output_tokens": 70338, + "usd": 0.34568, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-172949-79af18", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3457, + "minutes": 8.12, + "ci_minutes": 0, + "input_tokens": 776235, + "output_tokens": 70338, + "tool_calls": 91, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4204, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.331369, + "input_tokens": 768113, + "output_tokens": 59341, + "calls": 43 + }, + "reducer": { + "usd": 0.01431, + "input_tokens": 8122, + "output_tokens": 10997, + "calls": 1 + } + }, + "cache_read_tokens": 2174976, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.66, + "gate": 2.03, + "review": 481.44, + "report": 0.12 + }, + "time_to_first_result_s": 487.18 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 24342 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4923.md b/eval/dataset/arms/copilot_v7_goal_r1/pr4923.md new file mode 100644 index 000000000..f49a70106 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4923.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] qwen3_tts_code_predictor_vllm.py:36 — use_cuda_graphs=current_omni_platform.is_npu() correctly gates inner graphs to NPU only; GPU path relies on outer full-graph wrapper +- [validated] qwen3_tts_talker.py:327-350 — talker_mtp_graph_safe=True and the cudagraph_mode.has_full_cudagraphs() gate on talker_mtp_accepts_per_row_generators mirrors the runner's _init_talker_mtp decision in gpu_model_runner.py:209 +- [validated] qwen3_tts_talker.py:344 — getattr(vllm_config.compilation_config, 'cudagraph_mode', None) has a safe None fallback before calling .has_full_cudagraphs(); no AttributeError risk +- [validated] qwen3_tts_talker.py:351-352 — use_async_omni_output and eager_omni_postprocess_before_async_output flags match the Qwen3-Omni talker pattern (qwen3_omni.py:162-163) +- [validated] deploy/qwen3_tts.yaml platforms.npu section — PIECEWISE cudagraph_mode for both stages on NPU is consistent with the PR's NPU handling +- [sweep] grep for has_full_cudagraphs confirms the method exists on upstream vllm.config.CUDAGraphMode and is used consistently in gpu_model_runner.py:209 and talker.py:348 +- [sweep] grep for talker_mtp_graph_safe shows fish_speech_slow_ar.py also sets it (line 198); this PR's pattern is consistent with the multi-model adoption plan mentioned in the TODO +- [sweep] grep for use_async_omni_output confirms the flag is consumed in gpu_ar_model_runner.py:_should_use_async_omni_output and test_gpu_ar_model_runner.py + +`None:None` [blocker] — CI gates buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are failing on head commit 3b2d87c9. The NPU failure is especially concerning because the PR adds NPU-specific cudagraph_mode: PIECEWISE and changes code-predictor CUDA graph usage to keep internal graphs only on NPU; the PR description expects this to work but the check is still red. The AMD failure is not mentioned in the PR description. Both gates must pass (or be confirmed as pre-existing flakes) before merge. (evidence: gh_ci_read(pr=4923) shows FAILURE for buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [major] — The diff bumps max_num_seqs from 10 to 64 and max_num_batched_tokens from 512 to 32768 in qwen3_tts.yaml for both stages. The PR description provides no benchmark evidence or memory analysis to justify these specific capacity increases. Please supply performance data supporting the new defaults, or split the capacity retune into a separate PR with explicit rationale. (evidence: diff hunks in qwen3_tts.yaml; PR description only discusses graph/async changes, not these capacity knobs.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [major] — This PR introduces talker_mtp_graph_safe=True, gates talker_mtp_accepts_per_row_generators on cudagraph_mode.has_full_cudagraphs(), and enables async output (use_async_omni_output, eager_omni_postprocess_before_async_output, omni_pooler_payload_include_hidden), but no test files are modified. Existing e2e tests (test_qwen3_tts_base.py) use enforce_eager=true and never exercise full-cudagraphs. The worker test test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models hardcodes talker_mtp_accepts_per_row_generators=True and does not cover the disabled path. Add at least one test that runs talker_mtp under cudagraph_mode=FULL to validate the new graph-safe + async-output paths, and confirm the existing test suite passes with the new deploy defaults. (evidence: grep of tests/ for qwen3_tts: e2e tests use enforce_eager; worker test sets talker_mtp_accepts_per_row_generators=True directly; no test exercises has_full_cudagraphs() gating.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The comment block at lines 41-47 in qwen3_tts.yaml says 'Prefix caching on by default' and describes it as a throughput win, but the diff changes enable_prefix_caching from true to false. According to the PR description, this was disabled to 'avoid prefix-cache bookkeeping interfering with full-graph MTP path at high concurrency'. Update the comment to reflect the actual disabled state and rationale. (evidence: diff changes enable_prefix_caching: true → false; the preceding comment block (lines 43-48) still describes benefits of having it enabled) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — In qwen3_tts_talker.py __init__, the assignment `self.omni_pooler_payload_include_hidden = False` added at line 353 duplicates the same assignment already present earlier (around line 317). Remove the duplicate to keep a single source of truth. (evidence: grep shows two assignments in __init__: line 317 (pre-existing) and line 353 (added by this PR); both set to False) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The top-of-file comment in vllm_omni/deploy/qwen3_tts.yaml (lines 5-10) still says 'Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk' with rationale about low first-chunk latency, but the diff changes max_num_batched_tokens to 32768. Update the comment to reflect the new value and the new rationale (allowing larger decode batches under graph capture). (evidence: diff changes line 46: max_num_batched_tokens: 512 → 32768; the comment block at lines 5-10 still describes the old value) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:344` [minor] — The TODO at lines 340-345 in qwen3_tts_talker.py describes a determinism regression (per-request tts_local_seed dropped under full cudagraphs) and references '#4923' as the tracking issue. Since this PR is #4923, the TODO should reference a dedicated follow-up issue so the regression is not lost after merge. (evidence: The PR's own behavior-change table documents 'Not reproducible per request (same as Qwen3-Omni)' under full cudagraphs; the TODO comment at line 340-345 describes the mechanism but uses self-referential '#4923'.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r1/pr4923.trace.json.gz new file mode 100644 index 000000000..16b11fbc7 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r1/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4926.cost.json b/eval/dataset/arms/copilot_v7_goal_r1/pr4926.cost.json new file mode 100644 index 000000000..8c36d5787 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 624.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r1/runs/pr4926/run-20260812-173024-c575e1", + "recorded_at": "2026-08-12T17:40:47", + "attempts": [ + { + "invocation_id": "inv-643fcf74", + "outcome": "run:run-20260812-173024-c575e1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-173024-c575e1", + "llm_calls": 43, + "input_tokens": 323280, + "output_tokens": 92155, + "usd": 0.229855, + "source": "spans" + } + ], + "attempt_usd_total": 0.229855, + "llm_calls": 43, + "input_tokens": 323280, + "output_tokens": 92155, + "usd": 0.229855, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-173024-c575e1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2299, + "minutes": 10.38, + "ci_minutes": 0, + "input_tokens": 323280, + "output_tokens": 92155, + "tool_calls": 96, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4269, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.216914, + "input_tokens": 307282, + "output_tokens": 84336, + "calls": 42 + }, + "reducer": { + "usd": 0.012941, + "input_tokens": 15998, + "output_tokens": 7819, + "calls": 1 + } + }, + "cache_read_tokens": 1525888, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.85, + "gate": 1.79, + "review": 617.15, + "report": 0.06 + }, + "time_to_first_result_s": 622.83 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 31916 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4926.md b/eval/dataset/arms/copilot_v7_goal_r1/pr4926.md new file mode 100644 index 000000000..c20b1a338 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4926.md @@ -0,0 +1,51 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [sweep] flash_attn_hub.py:20-48 _run_varlen_dense — compared against flash_attn.py:127-154 _forward_varlen_dense; structurally identical except uses passed-in varlen_func instead of importing from fa module; correct +- [sweep] flash_attn_hub.py:72-108 FlashAttentionHubImpl.__init__ — version cascade (v1→v2→default) with broad except Exception; retry-on-failure intentional for network robustness; RuntimeError if no functions found is correct terminal outcome +- [sweep] flash_attn_hub.py:110-112 _unwrap_flash_output comment says 'FA3 may return (out, lse)' but this is in the FA2 class; comment is misleading (though the tuple-unwrap logic is correct defensively for both) +- [sweep] flash_attn_hub.py:114-124 _flash_wrapper — checked; attn_func is None when only varlen is available (commit 332da8f2 fix); fallback to _run_varlen_dense handles this; correct +- [sweep] flash_attn_hub.py:189-202 forward_cuda piecewise path — uses partial(self._flash_wrapper, attn_func=self.flash_attn_func); matches flash_attn.py:211-218 pattern; correct +- [sweep] flash_attn_hub.py:205-219 masked and direct paths in forward_cuda — checked; attention_mask gating (torch.any(~attention_mask)), flash_attn_func preferred over varlen; correct +- [sweep] flash_attn_hub.py:250-404 FlashAttention3HubImpl — structurally identical to FA2 impl except kernel loading (flash-attn3); all branches verified +- [sweep] registry.py:64-65 — two new enum entries FLASH_ATTN_HUB and FLASH_ATTN_3_HUB; correctly map to the new backend classes + +`:?` [blocker] — Buildkite CI lanes `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are FAILING on this PR. These failures must be investigated and resolved before merge. (evidence: gh_ci_read for PR #4926 shows buildkite/vllm-omni-amd-ci (FAILURE) and buildkite/vllm-omni-npu-ci (FAILURE).) + +`tests/diffusion/attention/test_kernels_hub.py:57` [blocker] — The test file `tests/diffusion/attention/test_kernels_hub.py` is never executed by any Buildkite CI job. The 'Diffusion · Other Test' lane only globs `tests/diffusion/*.py` (top‑level), and the 'Simple' lane runs `core_model and cpu` which excludes CUDA/L4 markers. This means the Hub backends have zero CI coverage. Add a dedicated lane or adjust the test selection. (evidence: Searched .buildkite/test-ready.yml and .buildkite/test-merge.yml — no job runs `tests/diffusion/attention/` with `core_model and cuda`. The 'Diffusion · Other Test' uses `tests/diffusion/*.py` glob (top-level files only). The `@hardware_test` decorator adds `cuda` and `L4` markers incompatible with the `cpu` Simple lane.) + +`tests/diffusion/attention/test_kernels_hub.py:70` [major] — The test creates `k = q.clone()` and `v = q.clone()` (lines 77‑78), making Q, K, V identical. This degenerate input cannot detect attention bugs because the attention output is trivially Q (modulo softmax). Use independently‑sampled tensors for K and V, e.g. `torch.randn(...)`, so the forward pass exercises cross‑token attention. (evidence: Read tests/diffusion/attention/test_kernels_hub.py lines 78-80; attention output with K=V=Q is trivially Q (modulo softmax over identical vectors), so the max-diff < 1e-2 assertion only proves the kernel didn't crash, not that it computed correctly.) + +`pyproject.toml:?` [major] — The `kernels` dependency in `pyproject.toml` line 70 has only an upper bound `<0.15.0` with no lower bound. The code calls `get_kernel(..., version=1)` (flash_attn_hub.py:96) which may not exist in older kernels releases. Add a lower bound such as `kernels>=0.14.0,<0.15.0`. (evidence: Read pyproject.toml line 70; grep for 'kernels' in pyproject.toml shows only this one occurrence under dev extras with no lower bound. The code at flash_attn_hub.py:96 calls get_kernel(..., version=1).) + +`vllm_omni/platforms/rocm/platform.py:?` [major] — Buildkite AMD CI (`buildkite/vllm-omni-amd-ci`) is FAILING. The ROCm platform fallback block (rocm/platform.py lines 91‑97) may interact incorrectly with aiter‑based FLASH_ATTN routing or the new enum members. Root‑cause and fix the failure. (evidence: gh_ci_read for PR #4926 shows buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci in FAILURE state; rocm/platform.py and npu/platform.py are both touched by this diff) + +`vllm_omni/platforms/cuda/platform.py:?` [major] — The gate `major >= 9` (cuda/platform.py line 140) admits SM 10.x+ (Blackwell and beyond), but documentation line 26 restricts FLASH_ATTN_3_HUB to Hopper (sm_90+) only. Either tighten the check to `major == 9` or update the docs to clarify future support. (evidence: Read cuda/platform.py line 140; docs at docs/user_guide/diffusion/attention_backends.md line 26 say 'CUDA Hopper (sm_90+) only'.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:113` [minor] — The comment on `FlashAttentionHubImpl._unwrap_flash_output` (line 111) reads '# FA3 may return (out, lse), FA2 returns out', but this is the FA2 wrapper class. Rewrite to describe what *this* method handles, e.g. '# Hub kernels may return (output, lse) tuple or raw tensor'. (evidence: Read flash_attn_hub.py line 111; the class is FlashAttentionHubImpl (FA2) but comment says 'FA3 may return'.) + +`docs/user_guide/diffusion/attention_backends.md:228` [minor] — Documentation at line 219 pins `pip install kernels==0.14.1`, but `pyproject.toml` line 70 allows any version under 0.15.0 without a lower bound. This mismatch may lead users to install an unsupported older version. Align the two: either pin the doc to `kernels>=0.14.1,<0.15.0` or document the acceptable range. (evidence: Read docs/user_guide/diffusion/attention_backends.md line 219 (pip install kernels==0.14.1) vs pyproject.toml line 70 (kernels<0.15.0).) + +**Additional observations (beyond the comment budget):** +- `docs/user_guide/diffusion/attention_backends.md:217` [minor] The documentation at line 217 claims 'perfect numerical consistency' between training and rollout, but the test only checks <1e‑2 max diff against SDPA — not bit‑exact parity with diffusers training output. Qualify as 'n +- `tests/diffusion/attention/test_kernels_hub.py:92` [minor] The test file is marked `@hardware_test(res={'cuda': 'L4'})` (line 58) — L4 is sm_89 (major=8) — and the FA3 execution guard at line 92 skips when `major < 9`. This means the FA3 Hub path has zero CI coverage. Either req +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:92` [minor] The `from kernels import get_kernel` lazy import inside `__init__` (line 92) has no timeout or backoff. If the HF Hub is unreachable, `get_kernel` could hang or raise an unhandled exception. Wrap with a timeout guard or +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:69` [nit] `FlashAttentionHubImpl` (lines 69‑225) and `FlashAttention3HubImpl` (lines 248‑404) are ~95% copy‑pasted. This duplication creates a maintenance risk where every forward‑path fix must be applied twice. Consider extractin +- `vllm_omni/platforms/cuda/platform.py:129` [nit] The fallback chain Hub→FLASH_ATTN (lines 124‑148) and FLASH_ATTN→SDPA (lines 151‑157) is correct but implicit. A brief comment linking the two layers would help maintainers. + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r1/pr4926.trace.json.gz new file mode 100644 index 000000000..f67ec4690 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r1/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4977.cost.json b/eval/dataset/arms/copilot_v7_goal_r1/pr4977.cost.json new file mode 100644 index 000000000..7eb32b47b --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 392.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r1/runs/pr4977/run-20260812-172351-3cfd44", + "recorded_at": "2026-08-12T17:30:22", + "attempts": [ + { + "invocation_id": "inv-2561ed34", + "outcome": "run:run-20260812-172351-3cfd44", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-172351-3cfd44", + "llm_calls": 45, + "input_tokens": 180770, + "output_tokens": 60382, + "usd": 0.142043, + "source": "spans" + } + ], + "attempt_usd_total": 0.142043, + "llm_calls": 45, + "input_tokens": 180770, + "output_tokens": 60382, + "usd": 0.142043, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-172351-3cfd44", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.142, + "minutes": 6.51, + "ci_minutes": 0, + "input_tokens": 180770, + "output_tokens": 60382, + "tool_calls": 81, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.288, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 45, + "by_role": { + "agent": { + "usd": 0.13437, + "input_tokens": 175380, + "output_tokens": 54748, + "calls": 44 + }, + "reducer": { + "usd": 0.007673, + "input_tokens": 5390, + "output_tokens": 5634, + "calls": 1 + } + }, + "cache_read_tokens": 993152, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.27, + "gate": 2.36, + "review": 384.98, + "report": 0.06 + }, + "time_to_first_result_s": 390.65 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 20943 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4977.md b/eval/dataset/arms/copilot_v7_goal_r1/pr4977.md new file mode 100644 index 000000000..cea81e805 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr4977.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] branch `version is not None` at flash_attn_hub.py:33 — all arms (version=1, version=2, version=None) can occur; the guard correctly avoids passing `version=None` to `get_kernel`, and the loop falls through properly. No dead branch. +- [sweep] branch `repo_id not in _hub_modules` at flash_attn_hub.py:44 — standard cache-miss check; both arms can occur (first call loads, subsequent calls reuse). No dead branch. +- [sweep] only changed file is flash_attn_hub.py; no other files touched. No rebase/merge damage, no duplicated code. +- [validated] `_load_hub_module` error path at flash_attn_hub.py:38-39: all three attempts fail → RuntimeError raised with `from last_error`. No fabricated fallback value. +- [validated] Thread safety: `_hub_lock` context manager guarantees lock release on exception; failed `repo_id` is NOT cached (retry on next call), preserving original retry semantics. No deadlock risk. +- [validated] `from kernels import get_kernel` remains a lazy import inside `_load_hub_module` (line 27), same as the original per-constructor import pattern. `kernels` is listed as `kernels<0.15.0` in pyproject.toml dev extras — optional, not a hard runtime dep. +- [validated] `_get_hub_module` is called from two sites only: `FlashAttentionHubImpl.__init__` (line 119) and `FlashAttention3HubImpl.__init__` (line 284), both with hardcoded repo_id strings. No external callers. +- [sweep] Branch at flash_attn_hub.py:33 `if version is not None:` — both arms reachable (version=1,2 → True; version=None → False). ✓ + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The diff introduces a module‑level `_hub_modules` cache and `threading.Lock` so that each HuggingFace Hub repo is loaded only once per process. However, there is **no test** that constructs two `FlashAttentionHubImpl` instances and verifies that `get_kernel` is called exactly once, nor any test that exercises concurrent construction. The existing `test_kernels_hub_execution` constructs only a single instance per backend and never exercises the cache‑hit path (line 44‑45). Add a test (e.g., mock `get_kernel` and assert call count, or verify that the returned `flash_attn_func` objects are identical) to validate both the caching and the lock‑based thread safety. (evidence: Read tests/diffusion/attention/test_kernels_hub.py: the test monkeypatches `sys.modules['kernels']` to test fallback routing, never instantiates a FlashAttentionHubImpl or invokes `_get_hub_module`. The cache path at flash_attn_hub.py:44-45 has no coverage.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The test `tests/diffusion/attention/test_kernels_hub.py::test_kernels_hub_execution` – the only test that exercises the new `_get_hub_module` caching path – is **never selected by any CI pipeline**. The `.buildkite/test-ready.yml` uses a non‑recursive glob (`tests/diffusion/*.py`) and the CPU step excludes hardware‑tagged tests. Consequently, the caching and lock logic introduced by this PR receives **zero CI coverage**. Add `tests/diffusion/attention/` (or the specific test file) to a CUDA CI step (e.g., the `Diffusion · Other Test` step) so that regressions will be caught. (evidence: .buildkite/test-ready.yml shows `tests/diffusion/*.py` (non-recursive glob) for the GPU step; `core_model and cpu` step excludes hardware-tagged tests.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:26` [minor] — New helper `_load_hub_module` (introduced at line 22) lacks a docstring. Document its version‑fallback loop, the fact that the caller must already hold `_hub_lock` (since this function is not self‑synchronising), and that failures are not cached (retries on every call). (evidence: Read flash_attn_hub.py lines 22-39: the function body shows the version loop and `from kernels import get_kernel` but no docstring. The lock is held by the sole caller `_get_hub_module` at line 43 (`with _hub_lock:`), but a future caller could bypass it.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [minor] — New helper `_get_hub_module` (line 42) lacks a docstring. Document its central contract: thread‑safe, load‑once‑per‑repo_id, process‑lifetime caching, and that the single `_hub_lock` serialises loads for *all* repo_ids (e.g., flash‑attn2 blocks flash‑attn3). (evidence: Read flash_attn_hub.py lines 42-45: the function body is 4 lines with no docstring. The lock scope and caching semantics are implicit.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:1` [minor] — The PR description still states "adding `trust_remote_code=True` for more reliable Hub access", but the final commit (9947f414) explicitly dropped that argument and the diff contains no `trust_remote_code` kwarg. Update the PR description to accurately reflect the delivered change so reviewers and future readers are not misled. (evidence: PR body read via gh_pr_view(4977) says 'adding `trust_remote_code=True` for more reliable Hub access'; diff in `flash_attn_hub.py` has no `trust_remote_code`; commit timeline shows 'Drop trust_remote_code from hub kernel loading' as the last commit.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [minor] — In `test_kernels_hub_execution` (lines 66‑67), the test clones the query tensor for key and value: `k = q.clone()` / `v = q.clone()`, making q, k, v numerically identical. This masks potential dimension‑swap or layout bugs in the attention kernel. Use distinct random tensors for q, k, v to improve test sensitivity. (evidence: tests/diffusion/attention/test_kernels_hub.py lines 66-67: `k = q.clone()` / `v = q.clone()`.) + +`None:None` [minor] — The CI check `buildkite/vllm-omni-npu-ci` is **failing** on this PR head (per gate_report). While the NPU platform (`npu/platform.py:113‑118`) falls back from `FLASH_ATTN_HUB` to `FLASH_ATTN` and does not use the changed code, please investigate the failure and either fix it or confirm it is a pre‑existing flake so that the check does not block merging. (evidence: gh_ci_read result shows `buildkite/vllm-omni-npu-ci` state: FAILURE.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [nit] — _load_hub_module catches bare `except Exception` (line 36 in diff). This is unchanged from the removed inlined code and does not risk swallowing `KeyboardInterrupt`/`SystemExit`, but it could catch `asyncio.CancelledError` in async contexts (Python ≥3.9). Consider scoping to expected `get_kernel` errors or adding a comment noting the intent; this is not a regression and is low-risk. (evidence: Diff shows `except Exception as exc:` on line 36; original removed code at old lines 95-103 used the same bare-except pattern. Not a regression but worth noting given the new caching layer extends the lifetime of the exception handler.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r1/pr4977.trace.json.gz new file mode 100644 index 000000000..1d28b3ebe Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r1/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr5009.cost.json b/eval/dataset/arms/copilot_v7_goal_r1/pr5009.cost.json new file mode 100644 index 000000000..98c7c0c57 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 496.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r1/runs/pr5009/run-20260812-172234-277093", + "recorded_at": "2026-08-12T17:30:49", + "attempts": [ + { + "invocation_id": "inv-2eb3d233", + "outcome": "run:run-20260812-172234-277093", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-172234-277093", + "llm_calls": 44, + "input_tokens": 526965, + "output_tokens": 64919, + "usd": 0.261564, + "source": "spans" + } + ], + "attempt_usd_total": 0.261564, + "llm_calls": 44, + "input_tokens": 526965, + "output_tokens": 64919, + "usd": 0.261564, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-172234-277093", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2616, + "minutes": 8.24, + "ci_minutes": 0, + "input_tokens": 526965, + "output_tokens": 64919, + "tool_calls": 83, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3881, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.248858, + "input_tokens": 517659, + "output_tokens": 55652, + "calls": 43 + }, + "reducer": { + "usd": 0.012706, + "input_tokens": 9306, + "output_tokens": 9267, + "calls": 1 + } + }, + "cache_read_tokens": 1773056, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.13, + "gate": 2.09, + "review": 487.75, + "report": 0.08 + }, + "time_to_first_result_s": 494.02 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 25234 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr5009.md b/eval/dataset/arms/copilot_v7_goal_r1/pr5009.md new file mode 100644 index 000000000..af951ebd8 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r1/pr5009.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] Diff hunk: platform.py: removed using_inductor branch (lines 251-262), restoring always-["vllm_c","native"] default — matches pre-#4709 behavior per PR description. +- [validated] Diff hunk: qwen_image_transformer.py: removed `from vllm.model_executor.layers.layernorm import RMSNorm` import; 5 call sites changed to `nn.RMSNorm` — grep confirms no remaining vllm RMSNorm references in qwen_image/ directory. +- [validated] Diff hunk: new test_cuda_ir_op_priority.py — parametrizes NONE/VLLM_COMPILE/STOCK_TORCH_COMPILE modes and OINK env var override; asserts priority.rms_norm == ["vllm_c","native"] and priority.fused_add_rms_norm same. +- [sweep] Grep for get_default_ir_op_priority callers: diffusion_worker.py:175 is the sole production consumer — reads platform default, then applies per-model ir_op_priority_func override if registered. +- [sweep] Grep for ir_op_priority_func registry: only Cosmos3 registers a per-model override (get_cosmos3_ir_op_priority_func) that forces ["native"] — Cosmos3 is unaffected by the platform default change. +- [sweep] RocmOmniPlatform.get_default_ir_op_priority (rocm/platform.py:173-196): still has `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE; default = ["native"] if using_inductor else ["vllm_c", "native"]` — the inductor-aware branch was NOT removed here. +- [sweep] XPUOmniPlatform.get_default_ir_op_priority (xpu/platform.py:107-119): still has the inductor-aware branch with `default = ["native"] if using_inductor else ["xpu_kernels", "native"]`. +- [sweep] Other diffusion models using vllm RMSNorm (grep shows flux2, sd3, hunyuan_video, ernie_image, etc.) — all inherit the new CUDA platform default; PR validates only Qwen-Image. Cosmos3 has override; others rely on the default. + +`vllm_omni/platforms/cuda/platform.py:?` [blocker] — The PR head has three failing non-CUDA CI checks: `buildkite/vllm-omni-amd-ci`, `buildkite/vllm-omni-intel-ci`, `buildkite/vllm-omni-npu-ci`. Since this PR changes Qwen-Image's RMSNorm (which runs on those platforms), these could be regressions. Please investigate: if the failures are pre-existing, confirm by referencing a recent main-branch run where they also fail and note it in the PR description; otherwise, fix the regression before merging. (evidence: gate_report evidence lists FAILING CHECKS: buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, buildkite/vllm-omni-npu-ci) + +`vllm_omni/platforms/cuda/platform.py:?` [major] — The CUDA platform default `IrOpPriorityConfig` now always returns `[\"vllm_c\", \"native\"]` regardless of inductor compilation. This affects every CUDA diffusion model that doesn't register a per-model `ir_op_priority_func`. Currently only Cosmos3 has an override (see `_DIFFUSION_IR_OP_PRIORITY_FUNCS` in `vllm_omni/diffusion/registry.py` line 549). All other models (e.g., FLUX, Wan, SD3, LTX2, HunyuanVideo) will silently switch from native to `vllm_c` RMSNorm. The PR validates only Qwen-Image on H200. To avoid unintended regressions, either (a) add a per-model `ir_op_priority_func` override for Qwen-Image to scope this change to that model family only, or (b) provide smoke-test evidence (at minimum a compile+dry-run forward pass) for the other affected model families. (evidence: grep of vllm_omni/diffusion/registry.py:545 shows _DIFFUSION_IR_OP_PRIORITY_FUNCS = {'Cosmos3OmniDiffusersPipeline': 'get_cosmos3_ir_op_priority_func'} — only Cosmos3 has an override; grep of _DIFFUSION_MODELS shows 40+ pipeline entries all inheriting the platform default) + +`tests/platforms/test_cuda_ir_op_priority.py:?` [minor] — The test uses `DeviceConfig(device=\"cpu\")` and only asserts the returned priority list, not actual kernel dispatch. While this covers configuration plumbing, the real performance validation relies on out-of-band GPU profiling. Consider adding a docstring note or a comment explaining that the test guards logic only; runtime RMSNorm dispatch correctness must be verified on a GPU lane. (evidence: read_file of tests/platforms/test_cuda_ir_op_priority.py:13-28 shows DeviceConfig(device='cpu') and priority list assertions only) + +`tests/platforms/test_cuda_ir_op_priority.py:23` [minor] — The new regression test `tests/platforms/test_cuda_ir_op_priority.py` covers only `CudaOmniPlatform`. However, `RocmOmniPlatform` and `XPUOmniPlatform` still contain the inductor-aware branch (see `rocm/platform.py:173-178`, `xpu/platform.py:107-116`). If that branch is intentionally kept, those platforms need their own tests pinning the expected priority lists under each `CompilationMode` to prevent future refactors from silently changing their dispatch behavior. Consider adding a platform-parametrized test or separate tests for ROCm/XPU. (evidence: Read tests/platforms/test_cuda_ir_op_priority.py (full file); grep for get_default_ir_op_priority shows no test file references RocmOmniPlatform or XPUOmniPlatform variants.) + +`vllm_omni/platforms/rocm/platform.py:?` [minor] — In `vllm_omni/platforms/rocm/platform.py` (lines 173-178), `get_default_ir_op_priority` still contains the inductor-aware branch: `default = ["native"] if using_inductor else ["vllm_c", "native"]`. The same logic is present in the XPU platform (`vllm_omni/platforms/xpu/platform.py` around line 107-116). This PR removes that branch from the CUDA platform because `vllm_c` kernels are always faster for diffusion RMSNorm. If the same reasoning applies to ROCm/XPU, remove the branch; otherwise, add a comment explaining why those platforms still prefer native decomposition under inductor. Either way, document the divergence so future maintainers don't assume it was an oversight. (evidence: Read rocm/platform.py:173-196; confimed the `using_inductor` branch and `["native"]` fallback are still present. PR description states scope is CUDA only but does not explain ROCm/XPU divergence.) + +`tests/platforms/test_cuda_ir_op_priority.py:9` [minor] — The new test file `tests/platforms/test_cuda_ir_op_priority.py` is marked with `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, but a grep of `.buildkite/` YAML for `core_model.*cpu` or `pytest.*-m.*core_model` returned no matches. The test may never be selected by any CI lane and would silently pass without running. Please add a CI lane that selects these markers (or adjust the markers) so the regression test actually executes. (evidence: grep of .buildkite/ for 'core_model.*cpu' or 'pytest.*-m.*core_model' returned no matches; the test's pytestmark may not be selected by any CI pipeline) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — In `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py`, the five `nn.RMSNorm` replacements (lines 557, 558, 593, 594, 1006) lack a comment explaining the revert. The PR description notes this undoes an accidental regression from #3588 that reverted #4074, but future readers won't see that rationale. Add a brief inline comment, e.g., `# Use torch.nn.RMSNorm instead of vLLM's RMSNorm to avoid IR op priority dispatch overhead for small head_dim norms; see #4074 and PR discussion`. (evidence: read_file of qwen_image_transformer.py lines 554-558, 592-596, 1006-1009 — no comment explaining the RMSNorm choice) + +`tests/platforms/test_cuda_ir_op_priority.py:30` [nit] — The oink test only covers `CompilationMode.VLLM_COMPILE`. Since the oink path in `get_default_ir_op_priority` is mode-independent, parametrizing `mode` with `NONE`, `STOCK_TORCH_COMPILE`, and `VLLM_COMPILE` would provide full coverage at negligible cost. (evidence: read_file of tests/platforms/test_cuda_ir_op_priority.py lines 28-37) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r1/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r1/pr5009.trace.json.gz new file mode 100644 index 000000000..32b715fd0 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r1/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r2/manifest.json b/eval/dataset/arms/copilot_v7_goal_r2/manifest.json new file mode 100644 index 000000000..aa2b26b03 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v7_goal_r2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4804.cost.json b/eval/dataset/arms/copilot_v7_goal_r2/pr4804.cost.json new file mode 100644 index 000000000..762e0e3cc --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 476.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r2/runs/pr4804/run-20260812-174944-66616a", + "recorded_at": "2026-08-12T17:57:38", + "attempts": [ + { + "invocation_id": "inv-e96a270e", + "outcome": "run:run-20260812-174944-66616a", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-174944-66616a", + "llm_calls": 43, + "input_tokens": 820082, + "output_tokens": 60514, + "usd": 0.403601, + "source": "spans" + } + ], + "attempt_usd_total": 0.403601, + "llm_calls": 43, + "input_tokens": 820082, + "output_tokens": 60514, + "usd": 0.403601, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-174944-66616a", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4036, + "minutes": 7.91, + "ci_minutes": 0, + "input_tokens": 820082, + "output_tokens": 60514, + "tool_calls": 79, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4375, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.379898, + "input_tokens": 764887, + "output_tokens": 52514, + "calls": 42 + }, + "reducer": { + "usd": 0.023703, + "input_tokens": 55195, + "output_tokens": 8000, + "calls": 1 + } + }, + "cache_read_tokens": 4281984, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.97, + "gate": 1.8, + "review": 468.41, + "report": 0.06 + }, + "time_to_first_result_s": 474.24 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 63002 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4804.md b/eval/dataset/arms/copilot_v7_goal_r2/pr4804.md new file mode 100644 index 000000000..f43d1c51b --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4804.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [sweep] seed_tts_eval.py: all new branches guarded (output_dir is None, audio_path, save_dir, failed); error paths thread audio_path through all result rows — no dead branches. +- [sweep] patch.py _pcm_s16le_to_seed_tts_wer_bytes: handles empty pcm, channels>1, sample_rate!=24000; except branch returns raw bytes — semantically type-valid (bytes) but could be wrong sample rate; warning logged so observable. +- [sweep] serving_speech.py L3708-3715: removed qwen3_tts model-type gate; always deepcopies sampling_params_list and always sets tts_local_seed. Grep confirms zero remaining qwen3_tts_request_seed references. +- [sweep] audio_tokenizer_v2.py: all indexed accesses (next(iter(...)), self.cache[0], modules[0], in_projs[0], wav_list[0], codes_list[0], etc.) have pre-conditions; no off-by-one found in RingKVCache positions or scatter logic. +- [sweep] modeling_moss_tts_codec.py _build_codec: try/v2-except/v1 fallback pattern — v2 config class name differs from v1, so risk of misload is low, but explicit version gating would be safer (checklist item). +- [sweep] modeling_moss_tts_codec.py _connector_int: extra_cfg fallback `connector_cfg.get('extra', connector_cfg)` returns whole dict if no 'extra' key — could read wrong value; but all current callers use 'extra' sub-dict per YAML convention. +- [sweep] modeling_moss_tts_local_depth.py: generator kwarg added to _sample_token and generate_frame, passed through to torch.multinomial — correct and necessary for deterministic seeding. +- [sweep] modeling_moss_tts_talker.py MossTTSLocalTalkerForGeneration.talker_mtp: audio sampling params changed to match model-card defaults (temp 1.7, top_k 25, top_p 0.8) — intentional per commit message. + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1494` [major] — This new `talker_mtp` method (~100 lines) contains the core local depth transformer frame generation with hardcoded audio sampling defaults and stop-condition logic. The only existing test coverage comes from `DummyTalkerMTP`/`CaptureTalkerMTP` stubs which are fake modules that don't exercise the real `generate_frame` call. The new defaults match upstream v1.5 but the method is untested end‑to‑end. Add a unit test that constructs a `MossTTSLocalTalkerForGeneration` with real checkpoint config, runs `talker_mtp` for at least 3 decode steps, and verifies both the output embeddings and the `audio_codes` buffer state. (evidence: Grep for `talker_mtp` in tests/: only `test_omni_gpu_model_runner.py` references it, and only via `DummyTalkerMTP`/`CaptureTalkerMTP` stubs. The `DummyTalkerMTP` returns `req_embeds + 1.0` and `torch.arange(bsz)` — it never calls `local_transformer.generate_frame`.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:225` [major] — The new ~110-line async chunk processor `talker2codec_raw_async_chunk` handles buffering, slot-based chunk emission, finish sentinels, and cleanup. Zero test references exist in the test tree. The processor has multiple exit branches and mutates shared `transfer_manager` state — all untested. Add a unit test that simulates: (a) accumulating frames below threshold and returning `None`, (b) hitting threshold and receiving the expected chunk shape, (c) finishing with buffered frames and getting the final chunk + sentinel, and (d) abort cleanup via `transfer_manager` state removal. (evidence: Grep for `talker2codec_raw_async_chunk` in tests/: 0 matches. The function is referenced only in `pipeline.py` as a config string and in `modeling_moss_tts_talker.py` as a docstring mention.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:37` [major] — `_MossCodecStreamSession` manages GPU-resident streaming state for the vendored v2 codec, with critical resource cleanup on abort. There are zero tests for this class. Add a unit test that: (a) acquires/releases slots, (b) runs `step` with multiple slots, (c) verifies `reset_slots` correctly zeros per-slot state, and (d) verifies `close` properly tears down the codec streaming context without leaking GPU memory. (evidence: Grep for `_MossCodecStreamSession` in tests/: 0 matches. The `on_requests_finished` hook is called from `gpu_generation_model_runner.py` (line 81-82) which handles `finished_req_ids` from the scheduler output; confirm unscheduled aborts also reach this path.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [minor] — The new vendored file `audio_tokenizer_v2.py` (and its config companion `configuration_moss_audio_tokenizer_v2.py`) lack the required 'Vendored from @ ' provenance header. Repo norms (see existing `audio_tokenizer.py`) demand this header on new vendored files. Add a header identifying the upstream source (e.g., `VoiceAgentGroup/moss_audio_tokenizer`) and commit reference. (evidence: Read audio_tokenizer.py which has the vendored-from header; audio_tokenizer_v2.py does not.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3711` [minor] — The `tts_local_seed` extra_arg is now set unconditionally for all TTS models (previously only `qwen3_tts`). This is intentional: the seed is consumed only by `talker_mtp`, which is enabled for models with a talker_mtp. To avoid future confusion, add a comment in `serving_speech.py` noting that injecting `tts_local_seed` is safe for all TTS models because the downstream gate (`has_talker_mtp`) ensures it only affects the intended talker path. (evidence: Grep confirms zero remaining references to `qwen3_tts_request_seed`; `tts_local_seed` is read in gpu_model_runner.py L1826 and understood by MossTTSLocalTalkerForGeneration.talker_mtp.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:728` [minor] — `_build_codec` uses try-v2-except-v1 exception-based dispatch. Both v1 and v2 configs share `model_type = 'moss-audio-tokenizer'`, so a v2 instantiation failure could silently load a v1 codec with a v2 checkpoint. Gate selection on an explicit config marker (e.g., a `version` field) instead of exception handling. (evidence: Read modeling_moss_tts_codec.py _build_codec method; grep confirms no config-version gating before the try/except.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/stage_input_processors/moss_tts.py:275` [minor] The finish sentinel sends a dummy `tensor([0])` as the audio code payload, relying on `code_flat_numel=0` in meta to prevent actual decoding. If the meta field were ever dropped or misrouted by the chunk transfer adapter +- `vllm_omni/worker/gpu_model_runner.py:1855` [minor] When `decode_batch_size > 1` with multiple explicitly-seeded requests, only `first_req_id`'s seed is used to create a single `torch.Generator` shared across all requests in the batch. In CUDA graph mode, this generator's +- `vllm_omni/model_executor/models/moss_tts/pipeline.py:18` [minor] The header comment on line 18 still says 'mono waveform chunks', but the Local-v1.5 variant (and Realtime via the same `talker2codec_raw_async_chunk` path) now produces stereo output through the v2 vendored codec (`n_cha +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:131` [minor] The `MossTTSCodecDecoder` class docstring still claims 'mono waveform', but the v2 vendored codec supports stereo output for Local-v1.5 (n_channels can be 2). Update the docstring to say '24 kHz waveform' or note that ch + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r2/pr4804.trace.json.gz new file mode 100644 index 000000000..74414947d Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r2/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4870.cost.json b/eval/dataset/arms/copilot_v7_goal_r2/pr4870.cost.json new file mode 100644 index 000000000..4e75b2415 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 509.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r2/runs/pr4870/run-20260812-180536-9d87f1", + "recorded_at": "2026-08-12T18:14:03", + "attempts": [ + { + "invocation_id": "inv-39adfcc6", + "outcome": "run:run-20260812-180536-9d87f1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-180536-9d87f1", + "llm_calls": 44, + "input_tokens": 945907, + "output_tokens": 61377, + "usd": 0.39682, + "source": "spans" + } + ], + "attempt_usd_total": 0.39682, + "llm_calls": 44, + "input_tokens": 945907, + "output_tokens": 61377, + "usd": 0.39682, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-180536-9d87f1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3968, + "minutes": 8.44, + "ci_minutes": 0, + "input_tokens": 945907, + "output_tokens": 61377, + "tool_calls": 71, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4495, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.383459, + "input_tokens": 937361, + "output_tokens": 51351, + "calls": 43 + }, + "reducer": { + "usd": 0.01336, + "input_tokens": 8546, + "output_tokens": 10026, + "calls": 1 + } + }, + "cache_read_tokens": 2737408, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.09, + "gate": 2.43, + "review": 499.84, + "report": 0.08 + }, + "time_to_first_result_s": 506.41 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 24482 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4870.md b/eval/dataset/arms/copilot_v7_goal_r2/pr4870.md new file mode 100644 index 000000000..ba2c2c85d --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4870.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] gpu_ar_model_runner.py:1734 — mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) replaces old seq_len=hidden_states.shape[0]; total_num_scheduled_tokens is the correct batch-aligned length for slicing the codes tensor; verified via to_payload_element at mm_outputs.py:108 which slices when element.shape[0]==seq_len +- [validated] gpu_ar_model_runner.py:1679 — removed seq_len parameter from _build_omni_model_runner_output_from_snapshot signature; grep confirms only one caller at line 2027 in same class, no external consumers +- [validated] gpu_ar_model_runner.py:1876 — removed seq_len = hidden_states.shape[0] from sample_tokens; this local variable was only consumed by the now-removed parameter, no other uses +- [validated] serving_speech.py:3507-3509 — qwen3_full_payload gate correctly scopes FINAL_ONLY to qwen3_tts + async_chunk=False; getattr default of True matches StageConfig.async_chunk default; test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3 verifies voxtral keeps DELTA +- [validated] tests/worker/test_gpu_ar_model_runner.py:437-478 — new test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only exercises tail-only hidden + 3-request batch with codes shape [3,16] and verifies [0:1],[1:2],[2:3] slices; test correctly sets _async_chunk=False, omni_pooler_payload_include_hidden=False, and validates per-request splitting +- [validated] tests/worker/test_gpu_ar_model_runner.py — all 5 existing test calls to _build_omni_model_runner_output_from_snapshot had seq_len=... keyword removed; new internal mm_seq_len derived from scheduler_output fills the same role +- [upstream-verify] serving_speech.py:3505 — async_chunk default getattr(..., True) matches StageConfig async_chunk default True (stage_config.py:416); missing attr on model_config would not trigger the gate, preserving old delta behavior +- [sweep] Checked NPU runner (npu_ar_model_runner.py:113) — uses _async_chunk independently; _build_omni_model_runner_output_from_snapshot is GPU-only, NPU not affected + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — The new test `test_prepare_speech_generation_no_async_chunk_stream_uses_final_only` sets `qwen3_tts_server.engine_client.model_config.async_chunk = False`, but the production code in `serving_speech.py:3505` reads `getattr(self.model_config, "async_chunk", True)`. If `self.model_config` and `engine_client.model_config` are not the same object (e.g., a proxy), the test exercises a different attribute than production. Verify they are the same reference or change the getattr to `self.engine_client.model_config` so the test and code agree. (evidence: Test at test_serving_speech.py:3458 sets engine_client.model_config.async_chunk=False; serving_speech.py:3505 reads self.model_config. If they differ, the test is testing a different code path than production.) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [minor] — In `_build_omni_model_runner_output_from_snapshot`, `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` is used to split multimodal outputs. The `to_payload_element` function slices only when `element.shape[0] == seq_len`; otherwise it treats the data as non-sliceable, which would silently reintroduce cross-request payload bleeding. Add an assert `assert multimodal_outputs["codes"]["audio"].shape[0] == mm_seq_len` when multimodal outputs are present and `omni_pooler_payload_include_hidden` is False, to catch mismatches at the point of failure. (evidence: to_payload_element at mm_outputs.py:128: slicing only fires when element.shape[0] == seq_len; if they don't match, element is treated as non-sliceable request-invariant data. The old seq_len=hidden_states.shape[0] suffered from this mismatch silently; the new mm_seq_len fixes the known case but the invariant is still implicit.) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [minor] — The change from `hidden_states.shape[0]` to `total_num_scheduled_tokens` is correct for Qwen3-TTS tail-only hidden states, but the code path with `needs_pooler_payload=True`, `omni_pooler_payload_include_hidden=False`, `async_chunk=False` is shared by all AR models. Only Qwen3-TTS was validated in testing. Add a comment listing which models are known to enter this path, or confirm no other model reaches it, to avoid silent regressions for other models. (evidence: Read gpu_ar_model_runner.py:1731-1787, to_payload_element at mm_outputs.py:108-140, test at test_gpu_ar_model_runner.py:438-483.) + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — The diff adds `async_chunk = getattr(self.model_config, "async_chunk", True)` in `serving_speech.py:3505`, defaulting to `True`. However, `GPUARModelRunner.__init__` defaults `_async_chunk` to `False` via `getattr(self.model_config, 'async_chunk', False)`, and `OmniModelConfig.async_chunk` factory default is `False`. In practice `model_config` always carries this attribute, but if it were missing the serving and runner layers would disagree. Align the default to `False` to match the runner and config, or add a comment explaining why the inverted fallback is intentional. (evidence: Read serving_speech.py:3507, gpu_ar_model_runner.py init ~line 1813; grep for async_chunk defaults confirms GPUARModelRunner uses False.) + +`tests/worker/test_gpu_ar_model_runner.py:?` [minor] — The new test `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` uses three requests each with 1 token (`num_scheduled_tokens={'r1':1,'r2':1,'r3':1}`) and verifies correct slicing. However, per-request slicing relies on boundaries from `num_scheduled_tokens_np` and `query_start_loc_cpu`; uniform counts cannot catch off-by-one errors in the start/end calculations when requests have different token lengths. Add a second scenario with non-uniform counts (e.g., r1=1, r2=2, r3=1) to exercise mixed-length slicing and guard against boundary bugs. (evidence: Diff shows `total_num_scheduled_tokens=3, num_scheduled_tokens={'r1':1,'r2':1,'r3':1}` — all requests have identical token counts. With `codes.shape=(3,16)`, each slice is exactly 1 row. If a request had 2 tokens, the `end = start + num_scheduled` calculation would be tested differently.) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [nit] — The scheduler's `total_num_scheduled_tokens` can be 0, but the execution model returns early at line ~2000 when `num_scheduled_tokens == 0`, so `mm_seq_len` will never be 0 before reaching the slicing code. Add a brief comment noting this invariant (`# total_num_scheduled_tokens > 0 by early return`) to reassure readers and future modifications. (evidence: Read gpu_ar_model_runner.py execute_model early return at num_scheduled_tokens==0 (~line 2000) and _build_omni_model_runner_output_from_snapshot at line 1734; to_payload_element at mm_outputs.py:108.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r2/pr4870.trace.json.gz new file mode 100644 index 000000000..d16764685 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r2/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4923.cost.json b/eval/dataset/arms/copilot_v7_goal_r2/pr4923.cost.json new file mode 100644 index 000000000..b393e4ce7 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 474.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r2/runs/pr4923/run-20260812-175741-7bb27e", + "recorded_at": "2026-08-12T18:05:33", + "attempts": [ + { + "invocation_id": "inv-6d49d713", + "outcome": "run:run-20260812-175741-7bb27e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-175741-7bb27e", + "llm_calls": 44, + "input_tokens": 698271, + "output_tokens": 65430, + "usd": 0.315052, + "source": "spans" + } + ], + "attempt_usd_total": 0.315052, + "llm_calls": 44, + "input_tokens": 698271, + "output_tokens": 65430, + "usd": 0.315052, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-175741-7bb27e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3151, + "minutes": 7.87, + "ci_minutes": 0, + "input_tokens": 698271, + "output_tokens": 65430, + "tool_calls": 96, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4011, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.303513, + "input_tokens": 690452, + "output_tokens": 56881, + "calls": 43 + }, + "reducer": { + "usd": 0.011539, + "input_tokens": 7819, + "output_tokens": 8549, + "calls": 1 + } + }, + "cache_read_tokens": 2020224, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.72, + "gate": 1.72, + "review": 466.66, + "report": 0.06 + }, + "time_to_first_result_s": 472.16 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 23186 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4923.md b/eval/dataset/arms/copilot_v7_goal_r2/pr4923.md new file mode 100644 index 000000000..4d30e38b3 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4923.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] qwen3_tts_code_predictor_vllm.py:39 — use_cuda_graphs=current_omni_platform.is_npu() mirrors Qwen3-Omni code predictor pattern (qwen3_omni_moe_code_predictor_mtp.py:18), GPU path gets use_cuda_graphs=False, NPU gets True. No unreachable branches. +- [validated] qwen3_tts_talker.py:327 — talker_mtp_graph_safe=True enables outer CUDAGraphWrapper in runner _init_talker_mtp (gpu_model_runner.py:208-209). Runner and model read same cudagraph_mode from shared VllmConfig; no consistency bug. +- [validated] qwen3_tts_talker.py:344-350 — cudagraph_mode gating for talker_mtp_accepts_per_row_generators: when cudagraph_mode is None (getattr default) or not FULL, per-row generators enabled; when FULL, disabled. Guard handles edge case of model construction before runner finalizes config. Runner at gpu_model_runner.py:1854 reads the flag with getattr default=False, so fallback is safe. +- [validated] qwen3_tts_talker.py:351-353 — use_async_omni_output / eager_omni_postprocess_before_async_output / omni_pooler_payload_include_hidden match Qwen3-Omni talker (qwen3_omni.py:162-164). Runner gating at gpu_ar_model_runner.py:1537-1639 already exercised by Qwen3-Omni. +- [validated] qwen3_tts.yaml NPU PIECEWISE config — stage 0 cudagraph_mode=PIECEWISE means has_full_cudagraphs() returns False, so runner does NOT wrap talker_mtp, avoiding nested graphs while code predictor's internal NPU graphs (use_cuda_graphs=True on NPU) still capture. Consistent with PR description's NPU fix rationale. +- [sweep] Grep for talker_mtp_graph_safe across repo — FishSpeech (fish_speech_slow_ar.py:198) has talker_mtp_graph_safe=True but does not use talker_mtp_accepts_per_row_generators; unaffected by this PR. +- [sweep] Grep for omni_pooler_payload_include_hidden — set at line 317 (pre-existing) and again at line 353 (new). Duplicate but same value (False); harmless. +- [sweep] YAML comment at line ~38 ('Prefix caching on by default... net throughput win') is stale now that enable_prefix_caching is false; comment was not updated with the new rationale. + +`tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py:?` [major] — In `tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py`, `test_tts_config` (line 479) hardcodes `use_cuda_graphs=True` when constructing a `CodePredictorWrapperConfig`. This PR changes `qwen3_tts_code_predictor_vllm.py` line 36 to pass `use_cuda_graphs=current_omni_platform.is_npu()`, which evaluates to `False` on GPU. The test no longer reflects the actual behavior — the model now never passes `use_cuda_graphs=True` on GPU. Update the test to assert `use_cuda_graphs=False` for the GPU scenario and add an NPU variant, or better, test the actual wrapper config produced by `Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM.__init__`. (evidence: diff qwen3_tts_code_predictor_vllm.py:36 changes use_cuda_graphs=True → use_cuda_graphs=current_omni_platform.is_npu(); test_code_predictor_dtype.py:464-488 asserts use_cuda_graphs=True) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — In `qwen3_tts_talker.py` line 350, `self.talker_mtp_accepts_per_row_generators` is now set based on `cudagraph_mode.has_full_cudagraphs()`. No test exercises this gating — the existing test in `tests/worker/test_omni_gpu_model_runner.py:374` sets the attribute directly on a mock, bypassing `__init__`. Please add a test (CPU or mock) that constructs `Qwen3TTSTalkerForConditionalGeneration` with a `VllmConfig` whose `compilation_config.cudagraph_mode` is FULL vs PIECEWISE vs None and asserts the correct value of `talker_mtp_accepts_per_row_generators`. (evidence: Grep for talker_mtp_accepts_per_row_generators shows only the assignment at line 350, the runner read at gpu_model_runner.py:1854, and the mock test at test_omni_gpu_model_runner.py:374; no test exercises the cudagraph_mode-dependent computation.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The diff at line 43 changes `enable_prefix_caching` from `true` to `false`, but the comment block starting at line 38 still claims 'Prefix caching on by default' and explains the throughput benefits of enabling it. The PR description states that disabling avoids prefix-cache bookkeeping interfering with the full-graph MTP path. Update the comment to reflect the new default and its rationale. (evidence: Read the full qwen3_tts.yaml; the comment block at lines 38-43 was not updated when enable_prefix_caching changed from true to false.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The diff at line 49 changes `max_num_batched_tokens` from 512 to 32768, but the header comment at line 6 still refers to 'Stage 0 (talker) sets max_num_batched_tokens=512'. Update the comment to 32768 to match the actual value. (evidence: Diff shows `max_num_batched_tokens: 32768` at line 49 of the new file; read_file confirms the comment at line ~6 is unchanged.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [nit] — `self.omni_pooler_payload_include_hidden = False` is already set at line 317 (unchanged) and is redundantly assigned again at line 353. This duplicate is confusing; remove one of the assignments and keep the flag with the other async-output settings (line 351-353). (evidence: qwen3_tts_talker.py:317 sets omni_pooler_payload_include_hidden=False; :353 sets it again) + +`vllm_omni/deploy/qwen3_tts.yaml:100` [nit] — The `buildkite/vllm-omni-npu-ci` check is failing (gate report). This diff adds `cudagraph_mode: PIECEWISE` for the NPU platform in `qwen3_tts.yaml` and changes `use_cuda_graphs` to `current_omni_platform.is_npu()`. Could you confirm whether the NPU CI failure is pre-existing on `main`? If it is pre-existing, please note it in the PR description to avoid blocking. (evidence: gh_ci_read for PR #4923 shows buildkite/vllm-omni-npu-ci is FAILURE; the diff changes NPU platform config and code predictor behavior on NPU.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r2/pr4923.trace.json.gz new file mode 100644 index 000000000..e08df4e8d Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r2/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4926.cost.json b/eval/dataset/arms/copilot_v7_goal_r2/pr4926.cost.json new file mode 100644 index 000000000..d0c211f6a --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 575.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r2/runs/pr4926/run-20260812-180625-3996d4", + "recorded_at": "2026-08-12T18:15:58", + "attempts": [ + { + "invocation_id": "inv-f47f45d0", + "outcome": "run:run-20260812-180625-3996d4", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-180625-3996d4", + "llm_calls": 44, + "input_tokens": 305921, + "output_tokens": 94882, + "usd": 0.231244, + "source": "spans" + } + ], + "attempt_usd_total": 0.231244, + "llm_calls": 44, + "input_tokens": 305921, + "output_tokens": 94882, + "usd": 0.231244, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-180625-3996d4", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2312, + "minutes": 9.55, + "ci_minutes": 0, + "input_tokens": 305921, + "output_tokens": 94882, + "tool_calls": 97, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4078, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.217585, + "input_tokens": 289144, + "output_tokens": 86605, + "calls": 43 + }, + "reducer": { + "usd": 0.013659, + "input_tokens": 16777, + "output_tokens": 8277, + "calls": 1 + } + }, + "cache_read_tokens": 1639808, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.4, + "gate": 1.63, + "review": 567.6, + "report": 0.1 + }, + "time_to_first_result_s": 572.95 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 30983 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4926.md b/eval/dataset/arms/copilot_v7_goal_r2/pr4926.md new file mode 100644 index 000000000..6109ada66 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4926.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [validated] flash_attn_hub.py:46 isinstance-check guards `out[0]` — safe indexed access +- [validated] flash_attn_hub.py:114 return out[0] guarded by isinstance(tuple) — safe +- [validated] flash_attn_hub.py:293 same _unwrap_flash_output pattern — safe +- [validated] flash_attn_hub.py:36-37 cu_seqlens_q arithmetic: arange(0, (B+1)*L, step=L) produces B+1 elements — correct for varlen API +- [validated] flash_attn_hub.py:107-108 RuntimeError when both flash_attn_func and flash_attn_varlen_func are None — correct failure, no fabrication +- [validated] flash_attn_hub.py:178-225 forward_cuda dispatch: piecewise → masked → dense_func → dense_varlen matches existing FlashAttentionImpl pattern +- [validated] platform.py:121-149 CUDA fallback chain: kernels missing → FLASH_ATTN; FLASH_ATTN_3_HUB on pre-Hopper → FLASH_ATTN_HUB; both correct +- [validated] musa/npu/rocm/xpu platform.py: all five non-CUDA platforms add identical 'not supported → FLASH_ATTN' gate for HUB backends — correct + +`tests/diffusion/attention/test_kernels_hub.py:70` [major] — The execution test creates K and V by cloning Q (lines 75-76: `k = q.clone(); v = q.clone()`). With Q == K == V, the attention output is a scaled copy of Q that masks many kernel bugs such as incorrect indexing of K or V. This degenerate input cannot verify the correctness of the Hub kernel paths. Use distinct random tensors for Q, K, V. (evidence: test_kernels_hub.py:78-79: `k = q.clone(); v = q.clone()` — all three tensors are identical, masking K/V indexing errors.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [major] — In `FlashAttentionHubImpl.__init__` (line 108), the guard `if self.flash_attn_func is None and self.flash_attn_varlen_func is None: raise` allows partial availability: if only `flash_attn_varlen_func` is None, the masked-attention path `_forward_varlen_masked` (line 159) crashes with `TypeError` because it calls `self.flash_attn_varlen_func(...)` unconditionally. The same bug exists in `FlashAttention3HubImpl` (line 287 init, line 338 masked path). Fix: either require both functions in init (`or` instead of `and`), or guard each call site with a fallback. (evidence: Read flash_attn_hub.py lines 106-109 (init guard), lines 147-160 (_forward_varlen_masked calls self.flash_attn_varlen_func directly), lines 170-176 (_forward_varlen_dense same). Grep confirmed no None guard before these calls.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [minor] — `test_kernels_hub_execution` only tests the dense no-mask forward path (line 81 uses `attn_mask=None`). The masked-attention path `_forward_varlen_masked` (`flash_attn_hub.py` line 205 for FA2, line 384 for FA3) is never exercised — this path is the one most vulnerable to the None-guard bug (see major finding). Add a test case with a non-trivial attention mask. (evidence: Read test_kernels_hub.py — attn_metadata_sdpa = AttentionMetadata(attn_mask=None), no full_attn_spans. flash_attn_hub.py:205 the masked path is only entered when attention_mask has any False entries.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:230` [minor] — This diff adds two nearly identical attention implementation classes: `FlashAttentionHubImpl` (lines 72-226) and `FlashAttention3HubImpl` (lines 241-406), differing only in the kernel name strings `flash-attn2` vs `flash-attn3`. The 135 lines of copy-paste will diverge over time. Factor the shared logic into a parameterized base class or common methods to reduce duplication. (evidence: Read flash_attn_hub.py: compared FlashAttentionHubImpl (lines 72-226) with FlashAttention3HubImpl (lines 235-406) — every method body is identical modulo the kernel identifier string.) + +`pyproject.toml:?` [minor] — The `FLASH_ATTN_HUB` backends added in `flash_attn_hub.py` call `get_kernel(..., version=1)` (line 99). The dependency in `pyproject.toml` line 70 declares only `kernels<0.15.0` with no lower bound. An older `kernels` release lacking the `version` kwarg would cause a confusing `TypeError` at runtime. Add a minimum version, e.g., `kernels>=0.14.1,<0.15.0`. (evidence: Read pyproject.toml line 70: `"kernels<0.15.0"`; read flash_attn_hub.py lines 92-99: calls `get_kernel(..., version=1)` with version kwarg.) + +`tests/diffusion/attention/test_kernels_hub.py:56` [minor] — The test function `test_kernels_hub_execution` (line 56) is decorated with `@pytest.mark.core_model` and `@hardware_test` but lacks `@pytest.mark.diffusion`. CI lanes that filter on `-m 'core_model and ... and diffusion and ...'` will skip this test, defeating the hardware test. Add `@pytest.mark.diffusion`. (evidence: Read test-ready.yml showing marker filters and glob patterns. Grep for 'test_kernels' in .buildkite/ returned no matches. Read tests/helpers/mark.py — hardware_test adds cuda/L4/gpu marks but not diffusion.) + +**Additional observations (beyond the comment budget):** +- `.buildkite/test-ready.yml:168` [minor] The new GPU execution test in `tests/diffusion/attention/test_kernels_hub.py` is not exercised by existing GPU CI lanes. The primary GPU lane runs `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'` — the shell gl +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:50` [minor] The five new public symbols in `flash_attn_hub.py` — `_run_varlen_dense`, `FlashAttentionHubBackend`, `FlashAttentionHubImpl`, `FlashAttention3HubBackend`, `FlashAttention3HubImpl` — lack docstrings. Since these backends + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r2/pr4926.trace.json.gz new file mode 100644 index 000000000..bb69f1b3b Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r2/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4977.cost.json b/eval/dataset/arms/copilot_v7_goal_r2/pr4977.cost.json new file mode 100644 index 000000000..d371c9037 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 430.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r2/runs/pr4977/run-20260812-175914-044a56", + "recorded_at": "2026-08-12T18:06:23", + "attempts": [ + { + "invocation_id": "inv-ceef1bc4", + "outcome": "run:run-20260812-175914-044a56", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-175914-044a56", + "llm_calls": 43, + "input_tokens": 229223, + "output_tokens": 67271, + "usd": 0.163249, + "source": "spans" + } + ], + "attempt_usd_total": 0.163249, + "llm_calls": 43, + "input_tokens": 229223, + "output_tokens": 67271, + "usd": 0.163249, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-175914-044a56", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1632, + "minutes": 7.14, + "ci_minutes": 0, + "input_tokens": 229223, + "output_tokens": 67271, + "tool_calls": 73, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3151, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.154758, + "input_tokens": 222914, + "output_tokens": 61122, + "calls": 42 + }, + "reducer": { + "usd": 0.008492, + "input_tokens": 6309, + "output_tokens": 6149, + "calls": 1 + } + }, + "cache_read_tokens": 1013376, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.4, + "gate": 1.79, + "review": 423.18, + "report": 0.06 + }, + "time_to_first_result_s": 428.42 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 19255 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4977.md b/eval/dataset/arms/copilot_v7_goal_r2/pr4977.md new file mode 100644 index 000000000..5228f6f48 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr4977.md @@ -0,0 +1,34 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] `_load_hub_module` branch `if version is not None` at line 33: both arms reachable (versions 1,2 → True; None → False); loop iterable is `(1, 2, None)` +- [validated] `_get_hub_module` branch `if repo_id not in _hub_modules` at line 44: first-call vs cache-hit both reachable; lock-protected check-then-set pattern +- [validated] `_load_hub_module` error path: all three iterations can fail; `last_error` tracks the most recent exception; `raise RuntimeError(...) from last_error` correctly chains — no swallowed exceptions +- [validated] `FlashAttentionHubImpl.__init__` and `FlashAttention3HubImpl.__init__`: `getattr(hub_module, attr, None)` defaults to None; `if self.flash_attn_func is None and self.flash_attn_varlen_func is None` guard correctly catches both-None case; read full class bodies in flash_attn_hub.py +- [validated] Thread safety: `_hub_lock` serializes cache access; `_load_hub_module` called inside the lock; on exception, cache entry NOT added → subsequent calls retry; no cache poisoning +- [validated] No rebase/merge damage: no duplicated code, no moved/renamed symbols; full file read confirmed clean extraction +- [sweep] Dependency: `kernels.get_kernel(repo_id, version=version)` signature was already used pre-PR; `kernels<0.15.0` range is pre-existing in pyproject.toml; no new dependency risk +- [sweep] Test coverage: `test_kernels_hub_platform_fallback` tests platform fallback (not caching); `test_kernels_hub_execution` exercises `FlashAttentionHubImpl` construction (hits cache path) but does not assert single-load semantics + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — This PR introduces a module-level cache (`_hub_modules`, line 20) and a loader (`_get_hub_module`) so each HuggingFace Hub kernel is loaded only once per process. The existing test `test_kernels_hub_execution` (tests/diffusion/attention/test_kernels_hub.py line 58) constructs only one `FlashAttentionHubImpl` and one `FlashAttention3HubImpl`, exercising the path but never verifying that a second construction reuses the cached result. Add a test that constructs two instances of the same class and asserts that `kernels.get_kernel` is called exactly once per `repo_id` (e.g., via `unittest.mock` on `get_kernel` or by inspecting `_hub_modules`). A regression here would reintroduce redundant Hub calls, the overhead this PR is designed to eliminate. (evidence: Read tests/diffusion/attention/test_kernels_hub.py: only one FlashAttentionHubImpl() call at line 81 and one FlashAttention3HubImpl() call at line 97; no second construction to exercise cache reuse.) + +`.buildkite/test-ready.yml:?` [major] — The CI pipeline in `.buildkite/test-ready.yml` line 145 runs `pytest -sv tests/diffusion/*.py -m 'core_model and cuda'`, which does not recurse into `tests/diffusion/attention/`. The only GPU test that exercises the new caching code (`test_kernels_hub_execution`, marked `cuda`+`L4`) lives in that subdirectory and is never executed in CI. Add `tests/diffusion/attention/` to the CUDA step or adjust the glob so this test runs. Without it, the caching logic—the primary behavior change—is untested in automation. (evidence: Grep of .buildkite/ for 'test_kernels_hub' and 'tests/diffusion/attention' returned zero matches across all pipeline YAML files.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:23` [minor] — A `threading.Lock` (`_hub_lock`) is introduced to guard `_hub_modules` (line 20), but no test exercises concurrent access to `_get_hub_module`. A TOCTOU bug or missed release could slip through. Add a concurrent test that spawns multiple threads calling `_get_hub_module` for the same `repo_id` and verify the kernel is loaded exactly once (e.g., by asserting a mock on `kernels.get_kernel` is called once). (evidence: Grep of tests/ for _hub_lock, _get_hub_module, and threading.Lock returned no matches.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r2/pr4977.trace.json.gz new file mode 100644 index 000000000..ede321b5e Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r2/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr5009.cost.json b/eval/dataset/arms/copilot_v7_goal_r2/pr5009.cost.json new file mode 100644 index 000000000..6c22c8ca4 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 567.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r2/runs/pr5009/run-20260812-174946-fa25c2", + "recorded_at": "2026-08-12T17:59:12", + "attempts": [ + { + "invocation_id": "inv-801a2d71", + "outcome": "run:run-20260812-174946-fa25c2", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-174946-fa25c2", + "llm_calls": 45, + "input_tokens": 654226, + "output_tokens": 73985, + "usd": 0.319144, + "source": "spans" + } + ], + "attempt_usd_total": 0.319144, + "llm_calls": 45, + "input_tokens": 654226, + "output_tokens": 73985, + "usd": 0.319144, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-174946-fa25c2", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3191, + "minutes": 9.43, + "ci_minutes": 0, + "input_tokens": 654226, + "output_tokens": 73985, + "tool_calls": 91, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4436, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 45, + "by_role": { + "agent": { + "usd": 0.308013, + "input_tokens": 646182, + "output_tokens": 65853, + "calls": 44 + }, + "reducer": { + "usd": 0.011131, + "input_tokens": 8044, + "output_tokens": 8132, + "calls": 1 + } + }, + "cache_read_tokens": 2263680, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.21, + "gate": 1.76, + "review": 559.94, + "report": 0.05 + }, + "time_to_first_result_s": 565.95 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 21930 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr5009.md b/eval/dataset/arms/copilot_v7_goal_r2/pr5009.md new file mode 100644 index 000000000..a582063ed --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r2/pr5009.md @@ -0,0 +1,32 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] platform.py: removed `CompilationMode` import and inductor-aware branch; grep confirms zero remaining `CompilationMode` references in the file. +- [validated] qwen_image_transformer.py: all 5 `RMSNorm`→`nn.RMSNorm` sites consistent; grep confirms no residual `from vllm.model_executor.layers.layernorm import RMSNorm` in the file. +- [validated] test_cuda_ir_op_priority.py: has `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, so it will be selected by CI. +- [sweep] Blast radius: `get_default_ir_op_priority` called from `diffusion_worker.py:175`; Cosmos3 overrides it via `get_cosmos3_ir_op_priority_func` → `["native"]` — protected. Other CUDA diffusion models (FLUX.2, SD3, Ovis-Image, Z-Image, HunyuanVideo, ErnieImage, etc.) inherit the new `["vllm_c", "native"]` default without per-model validation. +- [sweep] ROCm (`rocm/platform.py:183-196`) and XPU (`xpu/platform.py:114-119`) `get_default_ir_op_priority` still use inductor-aware `["native"]` logic — not touched by this PR. Inconsistency across platforms. +- [sweep] Test `_vllm_config` helper always passes `backend="inductor"`; test name says "when_inductor_backend" but only `mode` is parametrized — minor coverage gap for non-inductor backends. +- [sweep] CI failures (amd-ci, intel-ci, npu-ci) unlikely related to CUDA-only changes; not investigated further per gate guidance. +- [validated] Part 1 diff: vllm_omni/platforms/cuda/platform.py:252 — removes inductor-aware branch, always returns ["vllm_c", "native"] + +`vllm_omni/platforms/rocm/platform.py:?` [minor] — The PR removes the inductor-aware `default` logic from `cuda/platform.py`, but `vllm_omni/platforms/rocm/platform.py` (around line 187) and `vllm_omni/platforms/xpu/platform.py` (around line 119) still contain `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` followed by `default = ["native"] if using_inductor else ...`. This creates a cross-platform inconsistency for diffusion IR ops. Please file a tracking issue for ROCm/XPU parity, or add a comment explaining the deliberate divergence. (evidence: grep of rocm/platform.py:187 and xpu/platform.py:119 shows `using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE` followed by `default = ["native"] if using_inductor else [...]` — identical to the logic this PR removes from CUDA.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:?` [minor] — The PR restores `nn.RMSNorm` (torch.nn) in `qwen_image_transformer.py` after it was accidentally reverted in #3588 (see diff lines 17-23 and 554-557). No regression test guards against this class of import drift. Add a CPU-level test that instantiates `QwenImageCrossAttention` and `QwenImageTransformer2DModel` and asserts `isinstance(attn.norm_q, nn.RMSNorm)` and `isinstance(model.txt_norm, nn.RMSNorm)` — this would catch future accidental reverts in CI (the existing accuracy test is H100-only and not per-PR). (evidence: grep confirms no test covers nn.RMSNorm usage in qwen_image_transformer.py; only existing test is accuracy match to diffusers (not in diff)) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r2/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r2/pr5009.trace.json.gz new file mode 100644 index 000000000..0d284babd Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r2/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r3/manifest.json b/eval/dataset/arms/copilot_v7_goal_r3/manifest.json new file mode 100644 index 000000000..9e9c677bd --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v7_goal_r3", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4804.cost.json b/eval/dataset/arms/copilot_v7_goal_r3/pr4804.cost.json new file mode 100644 index 000000000..514beb7e9 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 479.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r3/runs/pr4804/run-20260812-181946-689651", + "recorded_at": "2026-08-12T18:27:43", + "attempts": [ + { + "invocation_id": "inv-4681b946", + "outcome": "run:run-20260812-181946-689651", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-181946-689651", + "llm_calls": 45, + "input_tokens": 656674, + "output_tokens": 67872, + "usd": 0.35646, + "source": "spans" + } + ], + "attempt_usd_total": 0.35646, + "llm_calls": 45, + "input_tokens": 656674, + "output_tokens": 67872, + "usd": 0.35646, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-181946-689651", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3565, + "minutes": 7.94, + "ci_minutes": 0, + "input_tokens": 656674, + "output_tokens": 67872, + "tool_calls": 76, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4199, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 45, + "by_role": { + "agent": { + "usd": 0.329122, + "input_tokens": 599725, + "output_tokens": 57007, + "calls": 44 + }, + "reducer": { + "usd": 0.027338, + "input_tokens": 56949, + "output_tokens": 10865, + "calls": 1 + } + }, + "cache_read_tokens": 3870336, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.58, + "gate": 1.85, + "review": 470.96, + "report": 0.06 + }, + "time_to_first_result_s": 476.45 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 64928 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4804.md b/eval/dataset/arms/copilot_v7_goal_r3/pr4804.md new file mode 100644 index 000000000..3dca9a549 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4804.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 5 finding(s) below | + +**Validated:** +- [validated] tests/worker/test_omni_gpu_model_runner.py:278,322,326 — test seed keys renamed from qwen3_tts_request_seed to tts_local_seed, consistent with runner and serving changes. +- [validated] grep for qwen3_tts_request_seed — zero matches; rename is complete across the entire repo. +- [validated] grep for talker2codec_async_chunk — zero matches; old function removed, all callers updated to talker2codec_delay_async_chunk or talker2codec_raw_async_chunk in pipeline.py. +- [validated] grep for on_requests_finished — called from gpu_generation_model_runner.py:81-82, gpu_ar_model_runner.py:942-943, npu_ar_model_runner.py:419-420; all three runner locations have the guard `hasattr(self.model, 'on_requests_finished')` before calling. +- [sweep] stage_input_processors/moss_tts.py talker2codec_raw_async_chunk — `put_req_chunk` defaultdict initialized at line ~257 but NEVER incremented; `emitted_any` (line ~287) is always False, so the `initial_chunk_frames` threshold is applied on every chunk forever instead of only the first chunk. This defeats the intended first-chunk-fast / steady-chunk-larger batching. +- [sweep] entrypoints/openai/serving_speech.py:3708-3715 — unconditional `copy.deepcopy(sampling_params_list)` replaces the old guarded deepcopy (only when `is self.engine_client.default_sampling_params_list`). This adds overhead for every speech request, not just those that would alias the default. +- [sweep] entrypoints/openai/serving_speech.py:3712-3715 — the old code had `if self._tts_model_type == 'qwen3_tts':` guarding the extra_args seed injection. The new code removes the model-type guard and always sets `extra_args['tts_local_seed']` for every TTS model (moss_tts, fish_tts, cosyvoice3, etc.) — a behavior change for all non-Qwen3-TTS models sharing this code path. +- [sweep] modeling_moss_tts_codec.py _build_codec — try/except broadly catches Exception; if v2 config.from_pretrained succeeds on a v1 checkpoint (both may share enough config fields to parse), the v2 model class is instantiated with v1 weight shapes, leading to shape mismatches caught later in load_weights. The fallback should be gated on an explicit config marker, not try-instantiate-and-fallback. + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:287` [major] — In `talker2codec_raw_async_chunk` (moss_tts.py ~287), `emitted_any` reads `transfer_manager.put_req_chunk.get(req_id, 0)` but the counter is never incremented. Consequently `emitted_any` is always `False`, keeping the initial chunk threshold forever instead of graduating to the steady-state `chunk_frames`. Add `transfer_manager.put_req_chunk[req_id] += 1` after emitting a chunk (after `del pending_frames[:emit_frames]`). (evidence: Read the full talker2codec_raw_async_chunk function — put_req_chunk is only read via .get(), never assigned. Grep for 'put_req_chunk' in the file confirms no assignment.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:728` [major] — In `_build_codec` (modeling_moss_tts_codec.py ~727), the try-except fallback attempts v2 config first and on any exception falls back to legacy v1 codec. Both v1 and v2 checkpoints share `model_type="moss-audio-tokenizer"` and overlapping field names, so a v1 checkpoint can be parsed as v2, instantiating a v2 model that receives v1 weights, causing shape mismatches. Replace the exception-based fallback with an explicit config version check. (evidence: Read _build_codec in modeling_moss_tts_codec.py. The try/except catches all Exception types. Both v1 and v2 configs have model_type='moss-audio-tokenizer' and overlapping field names. The checklist trigger: 'Legacy vs v2 codec/tokenizer selection: legacy checkpoints can share model_type with v2 — selection must be gated by an explicit config marker, never try-instantiate-and-fallback'.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:?` [major] — `make_omni_output` for the Local variant now returns `codes.audio` as a per-request list of tensors instead of a single accumulated tensor. The sync `talker2codec` processor (stage_input_processors/moss_tts.py, `_extract_audio_codes`) expects a single `torch.Tensor` and returns `None` for a list, causing silent empty output when `async_chunk=False`. Update `_extract_audio_codes` to handle a list, or ensure `async_chunk=True` is always enforced for this variant. (evidence: Read stage_input_processors/moss_tts.py _extract_audio_codes (isinstance check for torch.Tensor), pipeline.py sync_process_input_func, and modeling_moss_tts_talker.py make_omni_output (returns list)) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1493` [major] — The new ~150‑line `talker_mtp` method (modeling_moss_tts_talker.py ~1462) has zero dedicated unit tests; existing tests use mocks. Add a CPU test driving `talker_mtp` with synthetic hidden states to verify emission, stopping, frame accumulation, and `max_new_frames` behavior. (evidence: Grep of tests/ for 'talker_mtp' returned no matches outside test_omni_gpu_model_runner.py which only uses fake modules. Grep of tests/ for 'moss_tts_local' returned zero matches.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:225` [major] — `talker2codec_raw_async_chunk` (~120 lines, moss_tts.py ~220) has no tests. Add a unit test verifying chunk buffering below threshold, emission at `codec_chunk_frames`, finish sentinel with `code_flat_numel=0`, and `initial_codec_chunk_frames` behavior. (evidence: Grep of tests/ for 'talker2codec_raw_async_chunk' returned zero matches.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:37` [major] — The `_MossCodecStreamSession` class (~90 lines, modeling_moss_tts_codec.py ~37) managing streaming codec state has no tests. Add a test that verifies slot acquire/release, `step()` returns audio only for active slots, `reset_slots()` isolation, and `close()` cleanup. (evidence: Grep of tests/ for 'codec_stream_slots', 'codec_streaming', 'codec_chunk_frames', 'initial_codec_chunk_frames', '_MossCodecStreamSession' returned zero matches.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/entrypoints/openai/serving_speech.py:3708` [minor] In `_prepare_speech_generation` (serving_speech.py ~3708) the guard `if sampling_params_list is self.engine_client.default_sampling_params_list:` was removed, so every seeded request deep-copies the list unconditionally. +- `vllm_omni/entrypoints/openai/serving_speech.py:3712` [minor] The diff removes the `if self._tts_model_type == "qwen3_tts":` guard (serving_speech.py ~3712) and now sets `tts_local_seed` in `stage0_params.extra_args` for every TTS model that shares this code path. Currently only MO +- `vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [minor] New vendored file `audio_tokenizer_v2.py` lacks the `# Vendored from @ ` header required by repo convention (compare `audio_tokenizer.py`). Add a header identifying the upstream source and version. +- `vllm_omni/benchmarks/patch/patch.py:1247` [minor] In `_pcm_s16le_to_seed_tts_wer_bytes` (patch.py ~1247), when resampling/normalization fails the code falls back to raw `bytes(pcm_capture)`, which may have incorrect sample rate or channels, corrupting downstream WER sco + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r3/pr4804.trace.json.gz new file mode 100644 index 000000000..48d8645f7 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r3/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4870.cost.json b/eval/dataset/arms/copilot_v7_goal_r3/pr4870.cost.json new file mode 100644 index 000000000..f55a018c0 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 333.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r3/runs/pr4870/run-20260812-183333-2b005b", + "recorded_at": "2026-08-12T18:39:05", + "attempts": [ + { + "invocation_id": "inv-68639c23", + "outcome": "run:run-20260812-183333-2b005b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-183333-2b005b", + "llm_calls": 42, + "input_tokens": 1231708, + "output_tokens": 47146, + "usd": 0.46993, + "source": "spans" + } + ], + "attempt_usd_total": 0.46993, + "llm_calls": 42, + "input_tokens": 1231708, + "output_tokens": 47146, + "usd": 0.46993, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-183333-2b005b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4699, + "minutes": 5.53, + "ci_minutes": 0, + "input_tokens": 1231708, + "output_tokens": 47146, + "tool_calls": 90, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3905, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 42, + "by_role": { + "agent": { + "usd": 0.460217, + "input_tokens": 1222193, + "output_tokens": 40673, + "calls": 41 + }, + "reducer": { + "usd": 0.009714, + "input_tokens": 9515, + "output_tokens": 6473, + "calls": 1 + } + }, + "cache_read_tokens": 3166976, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.01, + "gate": 2.39, + "review": 325.36, + "report": 0.07 + }, + "time_to_first_result_s": 331.81 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 25263 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4870.md b/eval/dataset/arms/copilot_v7_goal_r3/pr4870.md new file mode 100644 index 000000000..d373402c1 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4870.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] gpu_ar_model_runner.py:1734 — mm_seq_len correctly derived from scheduler_output.total_num_scheduled_tokens, which matches the dimension-0 layout of full-payload multimodal tensors (codes.audio etc.). Read to_payload_element in vllm_omni/utils/mm_outputs.py:108-151 to confirm seq_len drives per-request tensor slicing. +- [validated] gpu_ar_model_runner.py:1876 (removed) — seq_len = hidden_states.shape[0] was indeed the root cause; for tail-only hidden states shape[0] < total_num_scheduled_tokens, causing cross-request payload mixing. +- [validated] gpu_ar_model_runner.py:1662-2042 — only one call site for _build_omni_model_runner_output_from_snapshot (line 2027). Removed parameter is not used elsewhere. +- [validated] serving_speech.py:3507-3509 — async_chunk gate defaults to True via getattr, matching OmniModelConfig default. qwen3_full_payload only True for qwen3_tts + async_chunk=False, correctly scoped. +- [validated] serving_speech.py:3509 — is_streaming_request = request.is_streaming() and not qwen3_full_payload; for non-streaming requests is_streaming() returns False regardless, so behavior is unchanged for stream=False. +- [validated] test_gpu_ar_model_runner.py:435-482 — new regression test exercises exact tail-only hidden + batched codes.audio scenario with async_chunk=False, assert per-request slicing correct. +- [validated] test_serving_speech.py:3453-3469 — qwen3 test exercises stream=True + async_chunk=False, assert coerce called with False (FINAL_ONLY). +- [validated] test_serving_speech.py:3471-3490 — voxtral test exercises async_chunk=False, assert coerce called with True (DELTA), confirming non-qwen3 models are unaffected. + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [minor] — The fix correctly uses `total_num_scheduled_tokens` as the split length because full-payload multimodal tensors (e.g., `codes.audio`) are always laid out with dim 0 equal to the scheduled token count. To prevent future regressions, add an inline comment near `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` (line 1734) stating that the MM tensor batch dimension is aligned to `total_num_scheduled_tokens`, not `hidden_states.shape[0]`. (evidence: Read to_payload_element in vllm_omni/utils/mm_outputs.py lines 108-151: seq_len drives tensor slicing via element[start:end], where start/end come from query_start_loc_cpu (cumulative sum of per-request num_scheduled_tokens). If seq_len < total_num_scheduled_tokens, the shape match `element.shape[0] == seq_len` at line 126 fails and the per-request slice is never taken, causing each request to receive the entire batched tensor.) + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [minor] — The `_build_omni_model_runner_output_from_snapshot` function is shared by all AR model runners. The `mm_seq_len` fix has only been validated for Qwen3-TTS on H20. Either run the existing GPUARModelRunner test suite for at least one additional non-Qwen3 TTS model, or add a comment explaining why `total_num_scheduled_tokens` is correct for all models, so future maintainers understand the blast radius. (evidence: Read gpu_ar_model_runner.py _build_omni_model_runner_output_from_snapshot signature and callers; grep confirmed the function is called from output_builder closure inside sample_tokens (line ~2042), which is the universal AR output path. PR description shows only Qwen3-TTS CustomVoice E2E validated.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3509` [minor] — The PR correctly scopes FINAL_ONLY streaming to qwen3_tts, and the voxtral unit test confirms that non-qwen3 models still receive DELTA coercion. However, the change affects all TTS models with `async_chunk=False`. Verify or document that other models (moss_tts, voxcpm2, higgs_audio, indextts2, qwen2_5_omni) produce correct streaming audio with the DELTA path, or note why voxtral is representative. (evidence: The repo-specific review checklist states: 'async_chunk defaults to False (OmniModelConfig). Any gate or coercion keyed on it applies to EVERY async_chunk=false model — demand scoping or cross-model evidence.' The diff scopes correctly but only one non-qwen3 model (voxtral) is tested.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3507` [minor] — In `serving_speech.py:3507`, `getattr(self.model_config, 'async_chunk', True)` uses a default of True, while the rest of the codebase (e.g., `gpu_ar_model_runner.py` init) consistently uses False. If `model_config` ever lacks the attribute, the serving side would disable the qwen3_full_payload gate, silently reintroducing the bug. Align the fallback to False or add an assertion that the attribute exists. (evidence: Read serving_speech.py:3507 — getattr(self.model_config, 'async_chunk', True). Read gpu_ar_model_runner.py __init__ — getattr(self.model_config, 'async_chunk', False). The serving-side True default contradicts the model-config default of False.) + +`tests/worker/test_gpu_ar_model_runner.py:478` [minor] — The test sets `runner._async_chunk = False` to exercise the full-payload path but never asserts it. If a shared fixture or future refactor resets the flag to True, the test would silently test the wrong code path. Add `assert runner._async_chunk is False` before calling the snapshot builder to make the test self-verifying. (evidence: Read test at line 438-482: `runner._async_chunk = False` is set on line 440 but never asserted; the non-async-chunk path is what determines that `pooler_inter` and `pooler_client` are the same list (line ~1879 in the source), which is the condition under which per-request MM slicing matters.) + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [nit] — In `_build_omni_model_runner_output_from_snapshot`, the new `mm_seq_len` only governs multimodal tensor slicing; hidden states are sliced separately via `start`/`end` indices. Consider adding an inline comment distinguishing these roles so a future reader doesn't mistakenly reuse `seq_len` for hidden-state logic. (Optional, nit) (evidence: Read `_build_omni_model_runner_output_from_snapshot` at lines 1734-1790; `mm_seq_len` flows into `_build_omni_pooler_payload(seq_len=mm_seq_len)` → `_build_omni_mm_payload(seq_len=seq_len)` → `to_payload_element(seq_len=seq_len)`. Hidden states are sliced separately via `_resolve_req_hidden_states` using `start`/`end`.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r3/pr4870.trace.json.gz new file mode 100644 index 000000000..767d6b921 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r3/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4923.cost.json b/eval/dataset/arms/copilot_v7_goal_r3/pr4923.cost.json new file mode 100644 index 000000000..fac956ead --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 347.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r3/runs/pr4923/run-20260812-182746-4a7588", + "recorded_at": "2026-08-12T18:33:31", + "attempts": [ + { + "invocation_id": "inv-f1c52055", + "outcome": "run:run-20260812-182746-4a7588", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-182746-4a7588", + "llm_calls": 43, + "input_tokens": 707029, + "output_tokens": 51677, + "usd": 0.299596, + "source": "spans" + } + ], + "attempt_usd_total": 0.299596, + "llm_calls": 43, + "input_tokens": 707029, + "output_tokens": 51677, + "usd": 0.299596, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-182746-4a7588", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2996, + "minutes": 5.75, + "ci_minutes": 0, + "input_tokens": 707029, + "output_tokens": 51677, + "tool_calls": 86, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3335, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.291202, + "input_tokens": 699369, + "output_tokens": 45948, + "calls": 42 + }, + "reducer": { + "usd": 0.008394, + "input_tokens": 7660, + "output_tokens": 5729, + "calls": 1 + } + }, + "cache_read_tokens": 1920512, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.93, + "gate": 2.38, + "review": 338.47, + "report": 0.06 + }, + "time_to_first_result_s": 344.85 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 21788 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4923.md b/eval/dataset/arms/copilot_v7_goal_r3/pr4923.md new file mode 100644 index 000000000..3b7f11d0b --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4923.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] qwen3_tts.yaml: max_num_seqs 10→64, max_num_batched_tokens 512→32768 — throughput increase, aligns with PR goal +- [sweep] qwen3_tts.yaml: enable_prefix_caching true→false — YAML comment 'Prefix caching on by default' is now stale +- [sweep] qwen3_tts.yaml: max_num_batched_tokens comment says 'small per-step batches keep first-chunk latency low' but value is now 32768 — comment contradicts current config +- [sweep] qwen3_tts.yaml: NPU PIECEWISE cudagraph_mode added for both stages — NPU CI lane (buildkite/vllm-omni-npu-ci) is red in gate_report +- [sweep] qwen3_tts_code_predictor_vllm.py:39 use_cuda_graphs=current_omni_platform.is_npu() — correct, GPU uses torch.compile only, NPU keeps internal graphs +- [validated] qwen3_tts_talker.py:327 talker_mtp_graph_safe=True — runner _init_talker_mtp reads via getattr at gpu_model_runner.py:208 +- [validated] qwen3_tts_talker.py:348 cudagraph_mode.has_full_cudagraphs() — runner _init_talker_mtp uses identical gate at gpu_model_runner.py:209 +- [sweep] qwen3_tts_talker.py:351-353 async output flags — match Qwen3-Omni's talker pattern at qwen3_omni.py:162-164 + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [major] — The code disables per-row generators under full cudagraphs, making per-request `tts_local_seed` non-reproducible (user-visible regression). Add a note in the deploy YAML or README documenting this limitation and link a tracking issue for the follow-up refactor mentioned in the TODO. (evidence: read_file at vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:338-350 shows the gating logic that disables per-row generators under full cudagraphs; grep at gpu_model_runner.py:1854 shows the runner falls back to scalar MTP calls when `talker_mtp_accepts_per_row_generators` is False; the existing test at test_omni_gpu_model_runner.py:357-374 validates the batched path but does not cover the graph-wrapped scalar fallback.) + +`vllm_omni/deploy/qwen3_tts.yaml:46` [minor] — The comment block above `enable_prefix_caching` (lines 43-46) says 'Prefix caching on by default' and 'Disable explicitly', but the value is now `false`. Update the comment to reflect the new default (e.g., why disabled and how to enable). (evidence: Diff hunk changes `enable_prefix_caching: true` to `enable_prefix_caching: false`. Comment on line 38 reads 'Prefix caching on by default.' Verified by reading `vllm_omni/deploy/qwen3_tts.yaml`.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:353` [minor] — Line 353 adds `self.omni_pooler_payload_include_hidden = False`, but the attribute is already set to `False` earlier in `__init__` (line 317). Remove this duplicate assignment to avoid confusion. (evidence: Read `qwen3_tts_talker.py` lines 310-360. Line 317 sets `self.omni_pooler_payload_include_hidden = False` in the pre-existing code; line 353 (added by this PR) sets it again to the same value. The duplicate is harmless but redundant.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The NPU platform section adds `cudagraph_mode: PIECEWISE`, and `buildkite/vllm-omni-npu-ci` is failing. Please confirm whether this failure is pre-existing on main or introduced by this PR, as the config change is directly tested by that lane. (evidence: gh_ci_read for PR #4923 shows buildkite/vllm-omni-npu-ci: FAILURE. gate_report lists this as a failing check. Cannot verify main-branch state.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The file header comment at line 6 still says `max_num_batched_tokens=512`, but the actual value is now `32768`. Update the comment to match or remove the specific number. (evidence: YAML header line 6: `# * Stage 0 (talker) sets max_num_batched_tokens=512 for async-chunk`; actual stage 0 config at line 49: `max_num_batched_tokens: 32768`) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — The `talker_mtp_graph_safe` flag enables graph-wrapping, but there is no test verifying that the graph path produces correct output vs eager. Add a small test (e.g., compare outputs with and without full cudagraphs for a fixed seed) or confirm that the existing CI lane covers both modes. (evidence: grep for talker_mtp_graph_safe in tests/ returned no results — no test asserts anything about graph-safe MTP behavior. Runner consumers found at gpu_model_runner.py lines 208-209.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r3/pr4923.trace.json.gz new file mode 100644 index 000000000..1d8834eeb Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r3/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4926.cost.json b/eval/dataset/arms/copilot_v7_goal_r3/pr4926.cost.json new file mode 100644 index 000000000..5250473d7 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 585.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r3/runs/pr4926/run-20260812-183257-8c4006", + "recorded_at": "2026-08-12T18:42:40", + "attempts": [ + { + "invocation_id": "inv-7083dc7a", + "outcome": "run:run-20260812-183257-8c4006", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-183257-8c4006", + "llm_calls": 45, + "input_tokens": 375782, + "output_tokens": 89959, + "usd": 0.249312, + "source": "spans" + } + ], + "attempt_usd_total": 0.249312, + "llm_calls": 45, + "input_tokens": 375782, + "output_tokens": 89959, + "usd": 0.249312, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-183257-8c4006", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2493, + "minutes": 9.71, + "ci_minutes": 0, + "input_tokens": 375782, + "output_tokens": 89959, + "tool_calls": 90, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4198, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 45, + "by_role": { + "agent": { + "usd": 0.233093, + "input_tokens": 358614, + "output_tokens": 79451, + "calls": 44 + }, + "reducer": { + "usd": 0.016218, + "input_tokens": 17168, + "output_tokens": 10508, + "calls": 1 + } + }, + "cache_read_tokens": 1810944, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.72, + "gate": 2.43, + "review": 575.79, + "report": 0.44 + }, + "time_to_first_result_s": 581.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 35380 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4926.md b/eval/dataset/arms/copilot_v7_goal_r3/pr4926.md new file mode 100644 index 000000000..fd142506f --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4926.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [sweep] flash_attn_hub.py:21-49 `_run_varlen_dense`: standalone function duplicates `FlashAttentionImpl._forward_varlen_dense` logic from flash_attn.py — maintenance risk but intentionally separate due to different varlen_func source (hub vs local). +- [sweep] flash_attn_hub.py:46 `out = out[0]`: tuple unwrap duplicates `_unwrap_flash_output` semantics; correct but unreachable if varlen_func returns a plain tensor (FA2 returns tensor, FA3 returns tuple). +- [sweep] flash_attn_hub.py:93-105 init try/except chain: third bare `get_kernel(...)` call at line 103 can raise, leaving `hub_module` unbound → NameError at line 106 instead of intended RuntimeError. Same bug at lines 272-284 for FA3. +- [sweep] flash_attn_hub.py:108-109 RuntimeError guard: only fires if BOTH funcs are None. If `flash_attn_varlen_func` is None but `flash_attn_func` is available, init succeeds but `_forward_varlen_masked` (line 147) and `_forward_varlen_dense` (line 170) crash on None callable. Same at lines 287/326/349 for FA3. +- [sweep] flash_attn_hub.py:78/257 `num_kv_heads` parameter accepted but never stored or used — silently ignored. Consistent with existing FlashAttentionImpl, but the doc's GQA discussion is misleading given this backend doesn't handle GQA. +- [sweep] flash_attn_hub.py:189-198 piecewise attention path: uses `self.flash_attn_func`; if None, `_flash_wrapper` falls back to varlen (correct). But if only varlen_func exists, piecewise degrades to varlen — no correctness issue. +- [sweep] cuda/platform.py:124-138 Hub fallback: when kernels unavailable, FLASH_ATTN_3_HUB → FLASH_ATTN (not FLASH_ATTN_HUB). Second block at 140-146 only fires when kernels IS available and CC < 9 → FLASH_ATTN_HUB. Correctly avoids routing through HUB without kernels. +- [sweep] cuda/platform.py:124-138: FLASH_ATTN_HUB not gated on compute capability — on sm_75 GPUs where local FLASH_ATTN is unsupported and would fallback to SDPA, FLASH_ATTN_HUB proceeds and crashes at kernel init instead of clean fallback. + +`tests/diffusion/attention/test_kernels_hub.py:91` [major] — test_kernels_hub_execution is decorated with `hardware_test(res={'cuda': 'L4'})` (line 58). L4 GPUs have compute capability 8.9 (sm_89), so the `major < 9` guard on line 92‑93 always triggers — the FA3 path is permanently skipped in CI. FlashAttention3HubImpl.forward_cuda therefore has zero execution coverage in any CI lane. Either add an H100 lane (`res={'cuda': 'H100'}`) or restructure the test so the FA3 fallback logic is unit‑tested separately, with a clear note that full Hopper execution requires a manual gate. (evidence: Read test_kernels_hub.py: hardware_test res={'cuda': 'L4'} at line 58, skip on major < 9 at line 92-93. L4 capability is 8.9, so FA3 path always skipped.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [major] — FlashAttentionHubImpl.__init__ only raises RuntimeError when both flash_attn_func and flash_attn_varlen_func are None (line 108). However, _forward_varlen_masked (line 147) and _forward_varlen_dense (line 162) call self.flash_attn_varlen_func unconditionally. If the hub provides only flash_attn_func, init succeeds but any masked‑attention or varlen‑dense request crashes with 'NoneType' callable. Add a None guard that falls back to a _run_varlen_dense workaround or raise at init when varlen_func is missing but func is present. Same class of bug exists for FlashAttention3HubImpl (init line 287, call sites ~326/349). (evidence: Read flash_attn_hub.py: the RuntimeError gate at line 108 fires only on `None and None`; masked path at line 205 calls `_forward_varlen_masked` → line 147 calls `self.flash_attn_varlen_func` with no None check.) + +`.buildkite/test-ready.yml:?` [major] — CI step "Diffusion · Other Test" (test‑ready.yml ~line 165) uses glob `tests/diffusion/*.py`, which only matches files directly under `tests/diffusion/`, not subdirectories like `tests/diffusion/attention/`. Consequently, test_kernels_hub.py—the only test file for the new hub backends—is never collected or executed on a real GPU in CI. This leaves the entire Hub‑kernel forward path with zero CI validation. Expand the glob to `tests/diffusion/**/*.py` or add an explicit attention‑test CI step. (evidence: Read .buildkite/test-ready.yml line ~165 (`tests/diffusion/*.py` glob) vs test file location at tests/diffusion/attention/test_kernels_hub.py. Grep for `pytestmark` in tests/diffusion/attention/ returned no matches — no conftest auto-markers to cause collection by other steps.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [minor] — test_kernels_hub_execution (line 75‑77) sets `k = q.clone()` and `v = q.clone()`, making Q, K, V identical. This degenerate self‑attention input cannot catch numerical errors like scaling drift or incorrect permutation/softmax handling, and the `max_diff < 1e-2` threshold passes trivially. Use independently‑sampled random tensors for K and V to create a realistic workload that validates the kernel outputs. (evidence: Read tests/diffusion/attention/test_kernels_hub.py lines 73-78: `q = torch.randn(...)`, `k = q.clone()`, `v = q.clone()` — all three tensors are identical.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [minor] — Static method `_unwrap_flash_output` is duplicated identically in FlashAttentionHubImpl (line 114) and FlashAttention3HubImpl (line 293). The comment in the FA2 copy says "FA3 may return (out, lse), FA2 returns out", which is misleading because the varlen FA2 kernel can also return a tuple (line 47). Extract this method to a module‑level helper and fix the comment to describe the actual contract: Hub kernels can return a tensor or `(tensor, lse)` tuple. (evidence: Read flash_attn_hub.py lines 112-114 (FlashAttentionHubImpl._unwrap_flash_output) and lines 291-293 (FlashAttention3HubImpl._unwrap_flash_output) — identical code.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:113` [minor] — The docstring on `FlashAttentionHubImpl._unwrap_flash_output` (line 114) says "FA3 may return (out, lse), FA2 returns out". However, the varlen FA2 kernel (line 47) also returns a tuple `(out, lse)` in some versions, and the code already handles this with `isinstance(out, tuple)`. Fix the comment to accurately describe the Hub kernel contract: kernels may return a tensor or `(tensor, lse)`; this helper unwraps to the output tensor. (evidence: Read flash_attn_hub.py: this method is defined on `FlashAttentionHubImpl` (line 75-114), not `FlashAttention3HubImpl`. The FA2 varlen path at line 47 also unwraps `out[0]` with `isinstance(out, tuple)`, confirming FA2 can return tuples too.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/platforms/cuda/platform.py:124` [minor] In cuda/platform.py (line 124), FLASH_ATTN_HUB is only gated on the `kernels` import, not on compute capability. On pre‑sm_80 GPUs where local FLASH_ATTN is unsupported (the existing check at line 148 would fall back to +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:82` [minor] Both `FlashAttentionHubImpl.__init__` (line 82) and `FlashAttention3HubImpl.__init__` (line 261) accept `num_kv_heads` but discard it—never stored, never used. GQA/MQA requests would silently be processed with the hub ke +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:155` [minor] test_kernels_hub_execution only exercises the direct `flash_attn_func` path (no mask, flash_attn_func present). The varlen‑dense, masked‑attention, and piecewise‑attention paths through `_forward_varlen_dense`, `_forward +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:57` [minor] FlashAttentionHubImpl (lines 75‑220) and FlashAttention3HubImpl (lines 255‑400) are near‑identical copies differing only in kernel repo name and init log message. All method bodies (forward_cuda, _forward_varlen_masked, + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r3/pr4926.trace.json.gz new file mode 100644 index 000000000..1b7796ea5 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r3/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4977.cost.json b/eval/dataset/arms/copilot_v7_goal_r3/pr4977.cost.json new file mode 100644 index 000000000..dfa76028f --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 402.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r3/runs/pr4977/run-20260812-182615-3355b1", + "recorded_at": "2026-08-12T18:32:55", + "attempts": [ + { + "invocation_id": "inv-c3224e5b", + "outcome": "run:run-20260812-182615-3355b1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-182615-3355b1", + "llm_calls": 40, + "input_tokens": 167911, + "output_tokens": 62096, + "usd": 0.136918, + "source": "spans" + } + ], + "attempt_usd_total": 0.136918, + "llm_calls": 40, + "input_tokens": 167911, + "output_tokens": 62096, + "usd": 0.136918, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-182615-3355b1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1369, + "minutes": 6.67, + "ci_minutes": 0, + "input_tokens": 167911, + "output_tokens": 62096, + "tool_calls": 78, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.29, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 40, + "by_role": { + "agent": { + "usd": 0.125927, + "input_tokens": 159716, + "output_tokens": 54125, + "calls": 39 + }, + "reducer": { + "usd": 0.010991, + "input_tokens": 8195, + "output_tokens": 7971, + "calls": 1 + } + }, + "cache_read_tokens": 862080, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.61, + "gate": 1.7, + "review": 394.8, + "report": 0.06 + }, + "time_to_first_result_s": 400.16 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 22776 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4977.md b/eval/dataset/arms/copilot_v7_goal_r3/pr4977.md new file mode 100644 index 000000000..0a582b170 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr4977.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [sweep] Branch `if version is not None:` at flash_attn_hub.py:33 — all three loop values (1, 2, None) are reachable; version 1 and 2 take True, None takes False. No dead branch. +- [sweep] Branch `if repo_id not in _hub_modules:` at flash_attn_hub.py:44 — True on first call, False on subsequent calls. Both reachable. No dead branch. +- [validated] flash_attn_hub.py:27-36 — `_load_hub_module` loop iterates exactly 3 times; early return on success, `RuntimeError` on all-fail. `last_error` always bound before raise. Error path preserved vs old code. +- [validated] flash_attn_hub.py:42-46 — `_get_hub_module` uses `threading.Lock` correctly; cache check and mutation are both under the lock. `_hub_modules` dict access is serialized. +- [validated] flash_attn_hub.py:116,281 — Both `FlashAttentionHubImpl.__init__` and `FlashAttention3HubImpl.__init__` replaced inline fallback with `_get_hub_module()` call; same repo_id strings, same `getattr` attribute lookups. No behavioral change in the happy path. +- [validated] flash_attn_hub.py:1 — `import threading` added; `from functools import partial` still used in forward_cuda methods. Imports consistent. +- [sweep] Grep for `get_kernel` across vllm_omni/ — only used in flash_attn_hub.py (3 occurrences, all in `_load_hub_module`). No other callers to update. +- [sweep] Grep for `flash_attn_hub` — only registry.py references the backend classes; no other consumers need updates. + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The test `test_kernels_hub_execution` constructs only one `FlashAttentionHubImpl` (line 81) and one `FlashAttention3HubImpl` (line 97); it never creates a second instance of either class. Consequently, the core caching behaviour introduced by this PR (module loaded once and reused) has zero test coverage. Add an assertion that a second construction reuses the cached module, e.g. assert that `_hub_modules['kernels-community/flash-attn2']` is already populated after the first construction, or mock `kernels.get_kernel` and assert it is called at most once. (evidence: Read test_kernels_hub.py lines 59-101: `test_kernels_hub_execution` creates one FlashAttentionHubImpl at line 81 and one FlashAttention3HubImpl at line 97; never creates a second instance of either class, so the cache-hit path is untested.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The test `test_kernels_hub_execution` (which constructs the changed `FlashAttentionHubImpl` and `FlashAttention3HubImpl` backends) is a GPU test that is not selected by any CI lane. The “Diffusion · Other Test” lane globs `tests/diffusion/*.py` (non‑recursive, missing the `attention/` sub‑directory), and the “Simple · Diffusion” lane filters with `-m core_model and cpu` (excluding cuda tests). As a result, the PR’s core behavioural change has no CI coverage—a regression in caching could go undetected. (evidence: Grep .buildkite/ for test_kernels_hub: no matches. Read .buildkite/test-ready.yml: 'Diffusion · Other Test' runs 'tests/diffusion/*.py -m core_model and cuda' (shell glob, no subdirs). 'Simple · Diffusion & Model Executor Test' runs 'tests/diffusion -m core_model and cpu'. Read tests/helpers/mark.py: hardware_test decorator adds cuda/L4 marks but not cpu.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [minor] — The check `buildkite/vllm-omni-npu-ci` is failing. This PR only touches the CUDA‑specific `flash_attn_hub.py`, and the NPU platform’s `get_diffusion_attn_backend_cls` (`platforms/npu/platform.py` line 110‑113) has its own fallback that never imports this file. Please confirm whether this failure is pre‑existing on main (if so, it is unrelated to this change). (evidence: Gate report shows `buildkite/vllm-omni-npu-ci` FAILURE; read npu/platform.py:110 confirms NPU-specific fallback logic is unchanged by this diff.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:1` [minor] — The PR description states this change “adds `trust_remote_code=True`”, but commit 9947f414 (“Drop trust_remote_code from hub kernel loading”) removed it, and the final diff never passes that parameter. Update the description to remove the stale claim or explain why `trust_remote_code` was dropped. (evidence: PR description in pr_context: 'adding trust_remote_code=True for more reliable Hub access'. Commit timeline: '9947f414 Drop trust_remote_code from hub kernel loading'. Read flash_attn_hub.py lines 33-35: get_kernel(repo_id, version=version) and get_kernel(repo_id) — no trust_remote_code kwarg.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [minor] — In `_get_hub_module` (lines 43‑46), the global `_hub_lock` is held during `_load_hub_module`, which may perform I/O (HuggingFace Hub download). This serialises **all** hub kernel loads—even for different repositories—for the duration of the download. Consider a double‑checked locking pattern (check cache outside lock, lock only on miss) or a per‑repo lock so that concurrent loads of different repos can proceed in parallel. (evidence: Read flash_attn_hub.py lines 42-46: the single `_hub_lock` guards the entire load inside the critical section; `_load_hub_module` calls `from kernels import get_kernel` which may perform I/O.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:26` [minor] — The functions `_load_hub_module` (line 25) and `_get_hub_module` (line 42) have no docstrings. They implement a version‑fallback strategy (v1 → v2 → default) and a thread‑safe lazy cache respectively. Add a one‑line docstring to each summarising its contract; the version‑fallback rationale is particularly helpful for future readers. (evidence: Read flash_attn_hub.py lines 22-47: neither function has a docstring.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:30` [minor] When all three version attempts fail in `_load_hub_module` (lines 30‑37), only the final exception is chained to the `RuntimeError`. The earlier failures (v1, v2) are logged but lost from the exception chain, complicatin +- `pyproject.toml:70` [minor] The optional dependency `"kernels"` declared in `pyproject.toml` (line 70) is `<0.15.0` with no lower bound. The new code in `_load_hub_module` calls `get_kernel(repo_id, version=version)`, which relies on a `version` ke + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r3/pr4977.trace.json.gz new file mode 100644 index 000000000..abccafe58 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r3/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr5009.cost.json b/eval/dataset/arms/copilot_v7_goal_r3/pr5009.cost.json new file mode 100644 index 000000000..3f04e083a --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 386.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r3/runs/pr5009/run-20260812-181948-f9fa04", + "recorded_at": "2026-08-12T18:26:12", + "attempts": [ + { + "invocation_id": "inv-4c65d301", + "outcome": "run:run-20260812-181948-f9fa04", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-181948-f9fa04", + "llm_calls": 43, + "input_tokens": 572566, + "output_tokens": 65463, + "usd": 0.280875, + "source": "spans" + } + ], + "attempt_usd_total": 0.280875, + "llm_calls": 43, + "input_tokens": 572566, + "output_tokens": 65463, + "usd": 0.280875, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-181948-f9fa04", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2809, + "minutes": 6.4, + "ci_minutes": 0, + "input_tokens": 572566, + "output_tokens": 65463, + "tool_calls": 88, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3456, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.273544, + "input_tokens": 563599, + "output_tokens": 60999, + "calls": 42 + }, + "reducer": { + "usd": 0.007331, + "input_tokens": 8967, + "output_tokens": 4464, + "calls": 1 + } + }, + "cache_read_tokens": 2010112, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.11, + "gate": 1.89, + "review": 378.1, + "report": 0.08 + }, + "time_to_first_result_s": 384.16 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 24417 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr5009.md b/eval/dataset/arms/copilot_v7_goal_r3/pr5009.md new file mode 100644 index 000000000..bffef7281 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r3/pr5009.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] Part 1 hunk (platform.py:249-262): removed the `using_inductor` conditional branch that set `default=["native"]` when inductor active; new code always returns `["vllm_c", "native"]`. This is a correct simplification that restores pre-#4709 behavior. +- [validated] Part 1 oink path (platform.py:256-259): `VLLM_USE_OINK_OPS` still prepends `"oink"` to the priority list. No regression in oink behavior. +- [validated] Part 1 consumer chain (diffusion_worker.py:174-179): `_resolve_ir_op_priority` calls `get_default_ir_op_priority` then applies per-model `ir_op_priority_func` override. Cosmos3 overrides exist at pipeline_cosmos3.py:612-624, confirming the PR's claim that models needing `native`-only have per-model overrides. +- [validated] Part 2 hunk (qwen_image_transformer.py:17): removed `from vllm.model_executor.layers.layernorm import RMSNorm` import. No remaining uses in file — confirmed by reading the full file. +- [validated] Part 2 hunks (qwen_image_transformer.py:554-555, 592-593, 1006): all five `RMSNorm(...)` → `nn.RMSNorm(...)` replacements are consistent and use the same `eps` values. `norm_q`/`norm_k` use the `qk_norm` conditional; `norm_added_q`/`norm_added_k` and `txt_norm` are unconditional. All correct. +- [validated] Test file (test_cuda_ir_op_priority.py): pytestmark `core_model, cpu` is a valid marker that CI lanes select (confirmed via `.buildkite/test-merge.yml` using `-m 'core_model and cpu'`). Test parametrizes all three CompilationMode values and validates the regression fix. +- [validated] Oink test (test_cuda_ir_op_priority.py:30-37): correctly monkeypatches `VLLM_USE_OINK_OPS` and asserts `["oink", "vllm_c", "native"]` priority. Correct. +- [sweep] Docstring updated in platform.py:250: new docstring accurately describes the simplified behavior. No stale comments remain. + +`vllm_omni/platforms/cuda/platform.py:251` [minor] — Part 1 removes the `using_inductor` branch in `get_default_ir_op_priority()`, making `["vllm_c", "native"]` unconditional for all CUDA diffusion models (restoring pre-#4709 behavior). The blast radius covers every CUDA diffusion model except Cosmos3 (which has a per-model `native` override). The PR validates only Qwen-Image. For the other affected families — Flux2, SD3, HunyuanVideo, Wan2.2, Z-Image, Ovis, LongCat, ERNIE, OmniGen2, LTX2 — please either provide A/B evidence that `vllm_c` priority does not regress, or explicitly document that the pre-#4709 state was already safe. (evidence: Grep confirmed `get_default_ir_op_priority` is called from `diffusion_worker.py:175` for all diffusion models on CUDA platform. Read registry.py to confirm per-model `ir_op_priority_func` mechanism exists (registry.py:546-631). Read cosmos3 pipeline_cosmos3.py:612-624 to confirm Cosmos3 override.) + +`vllm_omni/platforms/cuda/platform.py:252` [minor] — Consider adding a Qwen-Image-specific `ir_op_priority_func` override (the pattern Cosmos3 uses at `pipeline_cosmos3.py:612-624`) as defense-in-depth, so Qwen-Image remains insulated from future platform default changes even after the global default is changed. (evidence: Cosmos3's per-model override at vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:612-624 registered in _DIFFUSION_IR_OP_PRIORITY_FUNCS at registry.py:549. Qwen-Image has no such entry.) + +`tests/platforms/test_cuda_ir_op_priority.py:23` [minor] — The test function name `test_cuda_default_ir_op_priority_prefers_vllm_c_when_inductor_backend` suggests inductor dependency. After this PR the priority is always `["vllm_c", "native"]` regardless of inductor. Rename to `test_cuda_default_ir_op_priority_always_prefers_vllm_c` (or similar) to match the new behavior. (evidence: Read the full test file; the test itself correctly validates the new behavior for all three CompilationMode values.) + +`vllm_omni/platforms/rocm/platform.py:?` [minor] — ROCm (`rocm/platform.py:180`) and XPU (`xpu/platform.py:117`) platforms still keep the inductor-aware branch this PR removes from CUDA. If the fix applies equally, file a tracking issue to apply it on ROCm/XPU. If the platforms genuinely differ, add a comment explaining why the branch remains. (evidence: Read rocm/platform.py:173-196 and xpu/platform.py:107-119; both still have the `using_inductor` conditional that this PR removes from CUDA.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — The `RMSNorm` → `torch.nn.RMSNorm` switch touches only Qwen-Image. However, 13 other diffusion model files still import `vllm.model_executor.layers.layernorm.RMSNorm`. Clarify whether those models need the same switch or if they are on unaffected code paths — file a follow-up issue if needed. (evidence: Grep for `from vllm.model_executor.layers.layernorm import RMSNorm` under vllm_omni/diffusion/models/ returns 13 matches in files not touched by this PR.) + +`tests/platforms/test_cuda_ir_op_priority.py:25` [minor] — The gate report shows 3 non-CUDA CI lanes failing (`amd`, `intel`, `npu`). The PR touches only CUDA code. Please confirm these are pre-existing failures before merging. (evidence: Gate report evidence shows 3 failing checks on non-CUDA platforms. PR diff touches only CUDA platform and CUDA-specific model code.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] The `nn.RMSNorm` revert (Part 2) has no L2 unit test in this diff; the accuracy test `test_qwen_image_matches_diffusers` is not part of the merge CI suite. Provide a link to the CI run confirming this exact head passes t +- `tests/platforms/test_cuda_ir_op_priority.py:9` [minor] The new test file uses markers `pytest.mark.core_model` and `pytest.mark.cpu`, but grep of `.buildkite/test-ready.yml` found no CI lane selecting `"core_model and cpu"`. The test will be silently skipped. Add a CI lane t + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r3/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r3/pr5009.trace.json.gz new file mode 100644 index 000000000..be009db2e Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r3/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r4/manifest.json b/eval/dataset/arms/copilot_v7_goal_r4/manifest.json new file mode 100644 index 000000000..ca7479505 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v7_goal_r4", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4804.cost.json b/eval/dataset/arms/copilot_v7_goal_r4/pr4804.cost.json new file mode 100644 index 000000000..6aed9c005 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 452.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r4/runs/pr4804/run-20260812-185117-a24999", + "recorded_at": "2026-08-12T18:58:47", + "attempts": [ + { + "invocation_id": "inv-484a8a0f", + "outcome": "run:run-20260812-185117-a24999", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-185117-a24999", + "llm_calls": 43, + "input_tokens": 739098, + "output_tokens": 63237, + "usd": 0.372935, + "source": "spans" + } + ], + "attempt_usd_total": 0.372935, + "llm_calls": 43, + "input_tokens": 739098, + "output_tokens": 63237, + "usd": 0.372935, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-185117-a24999", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3729, + "minutes": 7.5, + "ci_minutes": 0, + "input_tokens": 739098, + "output_tokens": 63237, + "tool_calls": 75, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4142, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.349872, + "input_tokens": 683311, + "output_tokens": 55985, + "calls": 42 + }, + "reducer": { + "usd": 0.023064, + "input_tokens": 55787, + "output_tokens": 7252, + "calls": 1 + } + }, + "cache_read_tokens": 3845120, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 10.89, + "gate": 1.7, + "review": 437.37, + "report": 0.06 + }, + "time_to_first_result_s": 450.01 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 63345 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4804.md b/eval/dataset/arms/copilot_v7_goal_r4/pr4804.md new file mode 100644 index 000000000..4582fabac --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4804.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] serving_speech.py:3705-3715 — seed handling: removed `_tts_model_type == 'qwen3_tts'` gate, now always deep-copies sampling_params_list and sets `tts_local_seed` for all models. Grep confirms `qwen3_tts_request_seed` has 0 remaining callers. Runner reads `tts_local_seed` only via `_explicit_talker_seed` in `_talker_mtp_forward`, which is gated on `has_talker_mtp`; non-MOSS models ignore the key. Blast radius: harmless. +- [sweep] modeling_moss_tts_codec.py:726-733 — `_build_codec` uses bare `except Exception` to fall back from v2 to v1 codec. The repo checklist prohibits try-instantiate-and-fallback for codec selection. Transient errors (OOM, import failure) would silently load v1 with v2 weights → wrong decode layout. Mitigation: `logger.exception` provides observability, and v1 config load would likely also fail on v2 checkpoint. Still a sloppy catch. +- [sweep] stage_input_processors/moss_tts.py:277-278 — `int(cfg.get('codec_chunk_frames', 15) or 15)`: when config explicitly sets `codec_chunk_frames: 0`, `cfg.get` returns `0`, `0 or 15` = `15`, so the explicit zero is silently overridden. The downstream `<= 0` check never fires. `initial_codec_chunk_frames` has the same `or` pattern but with default 0, which is harmless. +- [sweep] modeling_moss_tts_codec.py:355-365 — `_finish_empty_streaming_requests` always returns `outputs = {}` (never populates it). Both callers iterate `.items()` which is a no-op. Intentional (method only does cleanup), but misleading for readers. +- [sweep] test_omni_gpu_model_runner.py:270,314 — test function names still reference `qwen3_tts_subtalker_sampling_params` but now test `tts_local_seed`. Name is stale. +- [sweep] audio_tokenizer_v2.py:1 — new vendored file lacks 'Vendored from @ ' header required by repo convention (see existing `audio_tokenizer.py`). +- [validated] modeling_moss_tts_codec.py:656-698 — weight remap for v2 checkpoint names (`.in_proj.` → `.in_projs.0.`, `.ffn.0.` → `.linear1.`, etc.) correctly maps upstream v2 naming to vendored module internals. Shape mismatch + missing weight detection now raises RuntimeError (was silent skip). +- [validated] gpu_generation_model_runner.py:79-81 — `on_requests_finished` hook called before `remove_request`, ensuring codec stream slots are released on abort/disconnect. Resource lifecycle symmetry is correct. + +`vllm_omni/entrypoints/openai/serving_speech.py:3710` [major] — The seed-injection path now unconditionally deep-copies `sampling_params_list` and sets `extra_args['tts_local_seed']` for ALL TTS model types, removing the old `qwen3_tts` gate. This is a cross-model behavior change that is not validated for other models (only MOSS-TTS-Local-v1.5 tested). The unconditional deep-copy also adds a per-request allocation on the serving hot path for all models. Either gate the extra args back to `moss_tts` (matching the old pattern) or document the expanded scope with evidence that other models are unaffected, and consider restoring the copy guard. (evidence: grep for `_tts_model_type` in serving_speech.py shows this is the shared speech generation path for all ~15 TTS model types; only moss_tts_local was tested per PR description.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1641` [major] — `make_omni_output` now reads audio codes from `info['audio_codes']['current']` with an `emit` flag, instead of running the local transformer inline. The producer is `talker_mtp` on `MossTTSLocalTalkerForGeneration` (line 1459+). The MOSS-TTS Realtime variant (`MossTTSRealtimeTalkerForGeneration`) is defined in the same file — if it inherits this `make_omni_output` without having its own `talker_mtp` that sets `audio_codes.current` + `emit`, it will silently emit empty codes on every step. Please verify that the Realtime variant either (a) has its own `make_omni_output` override, or (b) has `talker_mtp_accepts_req_infos = True` and a `talker_mtp` that populates the buffer, or (c) add a dedicated `make_omni_output` for the Realtime class that preserves the old inline-generation behavior. (evidence: Diff shows `make_omni_output` changed at line 1636 in the shared talker base; `talker_mtp` and `talker_mtp_accepts_req_infos = True` are only added to MossTTSLocalTalkerForGeneration (line 1320-1323); pipeline.py shows both local and realtime stages use raw/delay async chunk processors respectively, so Realtime is still expected to produce codes.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [minor] — New vendored files `audio_tokenizer_v2.py` and `configuration_moss_audio_tokenizer_v2.py` lack the 'Vendored from @ ' header required by repo conventions. The existing `audio_tokenizer.py` line 9 has: `# Vendored from OpenMOSS-Team/MOSS-Audio-Tokenizer (configuration_moss_audio_tokenizer.py)`. Add a header identifying the upstream source (OpenMOSS MOSS-Audio-Tokenizer-v2) and the commit/tag it was vendored from. (evidence: grep for 'Vendored from' in repo shows audio_tokenizer.py:9 has the header; audio_tokenizer_v2.py does not.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:728` [minor] — Bare `except Exception` in `_build_codec` catches all errors (OOM, import failures, etc.) and silently falls back to legacy v1 codec. This can mask genuine errors; if v2 config loads but model init fails for non-transient reasons, the fallback will attempt to load v2 weights with the legacy codec, eventually hitting the strict shape check (line 683) with a confusing error. Narrow the catch to expected failures (e.g., `ValueError`, `KeyError`) or add an explicit version marker check. (evidence: Grep confirms the vendored v1 audio_tokenizer.py header states model_type='moss-audio-tokenizer'; v2 config also sets model_type='moss-audio-tokenizer' (configuration_moss_audio_tokenizer_v2.py:76). Both share the same model_type, making the try/except the only discriminator. The repo checklist at review/guides/model-adaptation-guardrails.md explicitly prohibits this pattern.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:272` [minor] — `int(cfg.get('codec_chunk_frames', 15) or 15)` silently overrides an explicit config value of `0` to `15` because `0 or 15` evaluates to `15`. The downstream `<= 0` ValueError check can never fire for an explicit zero. Use an explicit `if` check: extract the value, default to 15 only when the key is absent, and let the `<= 0` guard catch explicit zeros. (evidence: Read the surrounding code: the `if chunk_frames <= 0: raise ValueError(...)` guard at line 279 is unreachable when `codec_chunk_frames=0` is explicitly set in config because the `or 15` masks the zero.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:?` [minor] — Class docstring says '24 kHz mono waveform' but the v2 tokenizer now supports stereo (2-channel via `enable_channel_interleave`). Update to '24 kHz mono or stereo waveform' to match actual capability. (evidence: Read the full `MossTTSCodecDecoder` docstring at modeling_moss_tts_codec.py:129-142; the `_build_codec` method now instantiates the v2 tokenizer which uses `number_channels=2` (see configuration_moss_audio_tokenizer_v2.py:113 default).) + +`vllm_omni/model_executor/models/moss_tts/configuration_moss_audio_tokenizer_v2.py:?` [minor] — Docstring says `sampling_rate` defaults to 24000 and `downsample_rate` to 1920, but the actual `__init__` signature defaults are 48000 and 3840 (line 103-104). Update the docstring to match the code. (evidence: Read the class docstring at configuration_moss_audio_tokenizer_v2.py:29-32 vs the `__init__` signature at lines 103-104.) + +`vllm_omni/model_executor/models/moss_tts/configuration_moss_audio_tokenizer_v2.py:?` [minor] — `codebook_size` property's fallback is 4096, but the default `quantizer_kwargs` dict (line 317-327) sets `codebook_size=1024`. The fallback should be 1024 to match the actual default; 4096 would silently produce a 4× larger vocabulary if the key is ever missing from a checkpoint config. (evidence: Read the property at line 354-356 and the default quantizer_kwargs at line 317-327.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1523` [minor] The audio-channel sampling parameters for the local depth transformer are hardcoded in `talker_mtp`. The docstring correctly notes that stage SamplingParams should not retune these, but there is no mechanism to configure + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r4/pr4804.trace.json.gz new file mode 100644 index 000000000..654dd0be8 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r4/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4870.cost.json b/eval/dataset/arms/copilot_v7_goal_r4/pr4870.cost.json new file mode 100644 index 000000000..fe3a38a66 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 320.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r4/runs/pr4870/run-20260812-190443-3ae19e", + "recorded_at": "2026-08-12T19:10:01", + "attempts": [ + { + "invocation_id": "inv-b235de23", + "outcome": "run:run-20260812-190443-3ae19e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-190443-3ae19e", + "llm_calls": 42, + "input_tokens": 1040032, + "output_tokens": 44823, + "usd": 0.410666, + "source": "spans" + } + ], + "attempt_usd_total": 0.410666, + "llm_calls": 42, + "input_tokens": 1040032, + "output_tokens": 44823, + "usd": 0.410666, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-190443-3ae19e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4107, + "minutes": 5.3, + "ci_minutes": 0, + "input_tokens": 1040032, + "output_tokens": 44823, + "tool_calls": 85, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3617, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 42, + "by_role": { + "agent": { + "usd": 0.401117, + "input_tokens": 1030548, + "output_tokens": 38492, + "calls": 41 + }, + "reducer": { + "usd": 0.009549, + "input_tokens": 9484, + "output_tokens": 6331, + "calls": 1 + } + }, + "cache_read_tokens": 2983424, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.83, + "gate": 1.88, + "review": 312.15, + "report": 0.06 + }, + "time_to_first_result_s": 317.92 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 22854 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4870.md b/eval/dataset/arms/copilot_v7_goal_r4/pr4870.md new file mode 100644 index 000000000..3ba08602c --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4870.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] serving_speech.py:3495-3510 — `async_chunk` gate correctly scoped to `qwen3_tts` only; non-qwen3 path tested in `test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3` +- [validated] gpu_ar_model_runner.py:1676 — `seq_len` parameter removed from `_build_omni_model_runner_output_from_snapshot` signature; all 5 existing test call sites updated +- [validated] gpu_ar_model_runner.py:1734 — `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` computed from scheduler, matching `codes.audio` first dimension +- [validated] gpu_ar_model_runner.py:1873 — removed `seq_len = hidden_states.shape[0]`; no other consumers of the old variable remain in `sample_tokens` +- [validated] gpu_ar_model_runner.py:2039 — `seq_len=seq_len` kwarg removed from `output_builder` call; no other callers of `_build_omni_model_runner_output_from_snapshot` exist outside this file (grep confirmed) +- [validated] test_gpu_ar_model_runner.py:438 — new test `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` correctly sets up tail-only hidden ([1]) with full-batch codes (3,16) and verifies per-request slicing +- [validated] test_serving_speech.py:3453 — new test verifies `coerce_param_message_types(..., False)` for qwen3_tts async_chunk=False stream=True +- [validated] test_serving_speech.py:3473 — new test verifies `coerce_param_message_types(..., True)` for voxtral async_chunk=False (non-qwen3 keeps DELTA) + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — The fallback default `True` in `getattr(self.model_config, "async_chunk", True)` is inconsistent with `GPUARModelRunner.__init__` which uses `getattr(self.model_config, "async_chunk", False)`. If the attribute were absent, the serving layer would assume async streaming while the runner would assume non-streaming, causing a mismatch in DELTA vs FINAL_ONLY coercion. Use `False` as the fallback or assert the attribute exists. (evidence: Review checklist: 'async_chunk defaults to False (OmniModelConfig)'. The getattr default of True contradicts the real default.) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [minor] — The split length change from `hidden_states.shape[0]` to `scheduler_output.total_num_scheduled_tokens` affects a shared code path (`_build_omni_model_runner_output_from_snapshot`) used by all AR models. While Qwen3-TTS is validated, other models (Voxtral, Fish, etc.) may have tail-only hidden states and could be impacted. Add a brief comment documenting which models rely on this path and whether the new behavior is safe for them. (evidence: Diff shows `seq_len` removed from signature at line 1679 and `mm_seq_len` computed at line 1734. grep confirms `_build_omni_model_runner_output_from_snapshot` is shared across all AR runner models via `GPUARModelRunner.sample_tokens`.) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [minor] — `mm_seq_len` is misnamed: it holds `total_num_scheduled_tokens` (a count), not a sequence length. Rename to `mm_total_tokens` to avoid confusion with the removed `seq_len` parameter. (evidence: read_file at gpu_ar_model_runner.py:1730-1736 confirms mm_seq_len = total_num_scheduled_tokens; PR description confirms the old code used hidden_states.shape[0].) + +`tests/worker/test_gpu_ar_model_runner.py:438` [minor] — The new test only covers uniform token counts (each request has 1 scheduled token). The slicing logic depends on cumulative sums; non-uniform scheduling (e.g., {"r1": 2, "r2": 1, "r3": 3}) would more thoroughly validate the `start`/`end` slicing. Add a parametrized heterogeneous case. (evidence: Test uses `num_scheduled_tokens={'r1': 1, 'r2': 1, 'r3': 1}` — all equal. The `to_payload_element` start/end computation depends on cumulative sums; unequal token counts would stress that path.) + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [nit] — `not bool(async_chunk)` is redundant; `not async_chunk` already coerces. Simplify to `not async_chunk`. (evidence: Python truthiness: `not x` already calls `bool(x)` implicitly.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r4/pr4870.trace.json.gz new file mode 100644 index 000000000..6cbf61c03 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r4/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4923.cost.json b/eval/dataset/arms/copilot_v7_goal_r4/pr4923.cost.json new file mode 100644 index 000000000..6826a349b --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 353.5, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r4/runs/pr4923/run-20260812-185849-0a2226", + "recorded_at": "2026-08-12T19:04:41", + "attempts": [ + { + "invocation_id": "inv-9bac5419", + "outcome": "run:run-20260812-185849-0a2226", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-185849-0a2226", + "llm_calls": 41, + "input_tokens": 723880, + "output_tokens": 43656, + "usd": 0.292358, + "source": "spans" + } + ], + "attempt_usd_total": 0.292358, + "llm_calls": 41, + "input_tokens": 723880, + "output_tokens": 43656, + "usd": 0.292358, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-185849-0a2226", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2924, + "minutes": 5.85, + "ci_minutes": 0, + "input_tokens": 723880, + "output_tokens": 43656, + "tool_calls": 77, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3338, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 41, + "by_role": { + "agent": { + "usd": 0.281065, + "input_tokens": 715243, + "output_tokens": 35532, + "calls": 40 + }, + "reducer": { + "usd": 0.011293, + "input_tokens": 8637, + "output_tokens": 8124, + "calls": 1 + } + }, + "cache_read_tokens": 1810688, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.17, + "gate": 1.76, + "review": 345.15, + "report": 0.06 + }, + "time_to_first_result_s": 351.13 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 23488 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4923.md b/eval/dataset/arms/copilot_v7_goal_r4/pr4923.md new file mode 100644 index 000000000..38c989ffe --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4923.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] qwen3_tts_code_predictor_vllm.py:39 - `use_cuda_graphs=current_omni_platform.is_npu()` mirrors qwen3_omni_moe_code_predictor_mtp.py:21 exactly; downstream CodePredictorWrapper correctly handles both True/False paths (grep: common/qwen3_code_predictor.py:477,532,552) +- [validated] qwen3_tts_talker.py:327 - `talker_mtp_graph_safe=True` is consumed by OmniGPUModelRunner._init_talker_mtp at gpu_model_runner.py:209; Fish Speech already uses the same pattern (fish_speech_slow_ar.py:198) +- [validated] qwen3_tts_talker.py:344-349 - cudagraph_mode gating: `getattr(..., None)` safe for missing attr; `cudagraph_mode is not None` guard prevents None.has_full_cudagraphs(); PIECEWISE→False, FULL→True, None→False — all branches reachable and correct +- [validated] qwen3_tts_talker.py:351-353 - async output flags (`use_async_omni_output`, `eager_omni_postprocess_before_async_output`, `omni_pooler_payload_include_hidden`) match Qwen3-Omni talker at qwen3_omni.py:162-164; consumed by GPUARModelRunner at gpu_ar_model_runner.py:1554,1639 +- [sweep] qwen3_tts_talker.py:317 vs 353 - `omni_pooler_payload_include_hidden=False` is set twice: first at line 317 (pre-existing) then redundantly at line 353 (new); benign but unnecessary +- [sweep] deploy/qwen3_tts.yaml:43 - comment says 'Prefix caching on by default' but value changed from true to false; comment is now stale +- [sweep] No test files touched in diff; existing test_code_predictor_dtype.py tests hardcode `use_cuda_graphs=False/True` on `_wrapper_config` directly, bypassing the new `current_omni_platform.is_npu()` gating — the new platform-conditional path is untested +- [sweep] CI gates red: buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci — cannot determine from diff alone whether failures are pre-existing on main or introduced here; NPU lane is relevant because YAML adds PIECEWISE compilation_config for NPU + +`vllm_omni/deploy/qwen3_tts.yaml:41` [minor] — The diff changes `enable_prefix_caching` from `true` to `false` (line 46), but the comment above (lines 38-44) still describes prefix-caching benefits and states "Prefix caching on by default." Update the comment to explain why prefix caching is now disabled (e.g. the hidden-state payload is no longer included due to `omni_pooler_payload_include_hidden=False`, and the larger `max_num_seqs=64` may require memory headroom). (evidence: Diff hunk at qwen3_tts.yaml line 43: `enable_prefix_caching` changed from `true` to `false` but the comment above still says 'Prefix caching on by default.') + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — With full cudagraphs, `talker_mtp_accepts_per_row_generators` is set to `False` (line 350), disabling per-row generators and falling back to a single scalar `talker_mtp` call per row. The existing test `test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models` only exercises the `True` path (gpu_model_runner.py:357). Add a unit test that sets `talker_mtp_accepts_per_row_generators=False` and validates that seeded requests produce a single batched call with a shared generator rather than per-row calls. (evidence: Grep for `talker_mtp_accepts_per_row_generators` shows the runner reads it at gpu_model_runner.py:1854; test at test_omni_gpu_model_runner.py:374 sets it to `True` only.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The diff bumps `max_num_seqs` (10→64) and `max_num_batched_tokens` (512→32768) in qwen3_tts.yaml (lines 35, 67, and 43's token value). The PR description focuses on graph/async changes but does not explain these concurrency/throughput bumps. Add a sentence describing the motivation and any memory trade-offs, or split the bumps into a separate follow-up if they are not required by the graph changes. (evidence: Diff shows `max_num_seqs: 10→64` and `max_num_batched_tokens: 512→32768` in qwen3_tts.yaml; PR description's 'Changes' section focuses on graph/async flags, not these config bumps.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:351` [minor] — This PR enables async omni output (`use_async_omni_output=True` and `eager_omni_postprocess_before_async_output=True` at lines 351-352) and graph-safe talker_mtp (`talker_mtp_graph_safe=True` at line 327). No test validates that talker_mtp runs under a full CUDA graph without errors or that async output produces correct audio codes end-to-end. Add a targeted test (e.g., a single request forward with full cudagraphs and async output) or reference an existing benchmark run that confirms correctness. (evidence: sweep_targets shows no test files touched; PR description mentions profile numbers but links no test command or artifact.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The gate report shows `buildkite/vllm-omni-npu-ci` failing. This PR adds NPU-specific `compilation_config` with `cudagraph_mode: PIECEWISE` (YAML lines 95-103). Please confirm whether the NPU CI failure is pre-existing on main, and if introduced by this change, verify that PIECEWISE mode is valid for Qwen3-TTS on NPU and fix the config accordingly. (evidence: gate_report shows buildkite/vllm-omni-npu-ci failing; PR adds `platforms.npu.stages[].compilation_config.cudagraph_mode: PIECEWISE` in yaml.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:39` [minor] — The diff changes `use_cuda_graphs` from `True` to `current_omni_platform.is_npu()` (code_predictor_vllm.py line 39). The existing tests in `test_code_predictor_dtype.py` set `use_cuda_graphs` directly on wrapper configs, bypassing the new platform gating. Add a test that instantiates `Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM` and asserts `use_cuda_graphs` matches the current platform, to guard against regressions in the gating logic. (evidence: Grep for tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py shows lines 291, 356, 466, 472, 481, 487, 533, 542 all set `use_cuda_graphs` directly on `CodePredictorWrapperConfig` or `_wrapper_config`, never through the Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM constructor.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The header comment at the top of qwen3_tts.yaml (line 5) still references `max_num_batched_tokens=512`, but the PR changes the stage-0 value to 32768. Update the comment to reflect the new value. (evidence: Diff changes max_num_batched_tokens from 512 to 32768; header comment was not updated.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — The diff sets `talker_mtp_graph_safe = True` (line 327), enabling the runner to wrap `talker_mtp` in a full `CUDAGraphWrapper`. No existing test exercises this path; `test_omni_gpu_model_runner.py` does not reference this flag. Add a test that verifies talker_mtp runs under full cudagraphs without errors (even a single-step forward with a mock request) or confirm that the existing e2e test `online_serving/test_qwen3_tts_base.py` already covers this configuration. (evidence: Grep for talker_mtp_graph_safe: only set in qwen3_tts_talker.py and fish_speech_slow_ar.py; consumer at gpu_model_runner.py:208-209 gates CUDAGraphWrapper wrapping. No test in tests/ references this flag.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r4/pr4923.trace.json.gz new file mode 100644 index 000000000..a91b38385 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r4/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4926.cost.json b/eval/dataset/arms/copilot_v7_goal_r4/pr4926.cost.json new file mode 100644 index 000000000..5f1d7d33c --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 470.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r4/runs/pr4926/run-20260812-190640-e42beb", + "recorded_at": "2026-08-12T19:14:28", + "attempts": [ + { + "invocation_id": "inv-1dd75bc4", + "outcome": "run:run-20260812-190640-e42beb", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-190640-e42beb", + "llm_calls": 44, + "input_tokens": 300220, + "output_tokens": 75223, + "usd": 0.209324, + "source": "spans" + } + ], + "attempt_usd_total": 0.209324, + "llm_calls": 44, + "input_tokens": 300220, + "output_tokens": 75223, + "usd": 0.209324, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-190640-e42beb", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2093, + "minutes": 7.8, + "ci_minutes": 0, + "input_tokens": 300220, + "output_tokens": 75223, + "tool_calls": 79, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3536, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.193489, + "input_tokens": 281121, + "output_tokens": 65541, + "calls": 43 + }, + "reducer": { + "usd": 0.015835, + "input_tokens": 19099, + "output_tokens": 9682, + "calls": 1 + } + }, + "cache_read_tokens": 1685888, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.82, + "gate": 2.19, + "review": 461.85, + "report": 0.07 + }, + "time_to_first_result_s": 467.92 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 33322 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4926.md b/eval/dataset/arms/copilot_v7_goal_r4/pr4926.md new file mode 100644 index 000000000..83f402311 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4926.md @@ -0,0 +1,50 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [sweep] Read vllm_omni/diffusion/attention/backends/flash_attn_hub.py (1-406) — all hunks verified in situ +- [sweep] Read tests/diffusion/attention/test_kernels_hub.py (1-104) — test integrity checked +- [sweep] Read vllm_omni/diffusion/attention/backends/registry.py (1-120) — enum entries L65-L66 verified +- [sweep] Read vllm_omni/platforms/cuda/platform.py L100-L220 — platform fallback logic verified +- [sweep] Read vllm_omni/platforms/{musa,npu,rocm,xpu}/platform.py — non-CUDA fallback blocks consistent across 4 platforms +- [sweep] Read vllm_omni/diffusion/attention/backends/flash_attn.py — existing FlashAttentionImpl pattern compared; Hub impls follow same structure +- [sweep] Read vllm_omni/diffusion/attention/backends/abstract.py — AttentionBackend/AttentionImpl contracts verified +- [sweep] Read vllm_omni/diffusion/attention/backends/utils/piecewise_attn.py — piecewise_attn contract verified; Hub impls call correctly + +`tests/diffusion/attention/test_kernels_hub.py:69` [major] — In `test_kernels_hub_execution` (test_kernels_hub.py lines 74-75) the key and value tensors are clones of the query tensor (`k = q.clone(); v = q.clone()`), making Q==K==V. This degenerate self-attention input cannot detect a kernel that mishandles cross-attention or ignores key/value. Use independent random tensors for k and v to properly exercise the attention computation. (evidence: tests/diffusion/attention/test_kernels_hub.py lines 68-73: q is created from torch.randn, then k = q.clone() and v = q.clone()) + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The `_forward_varlen_masked` path (flash_attn_hub.py lines 132-170 for FA2, lines 311-349 for FA3) is never executed in the test suite because both test cases pass `attn_mask=None`. Add a test case that supplies a 2D boolean attention mask with some `False` values and verify the masked output against an SDPA reference. (evidence: Read test_kernels_hub.py lines 74-76, flash_attn_hub.py lines 132-170 and 311-349 — confirmed attn_mask is always None in tests, so the masked path is never executed) + +`vllm_omni/platforms/cuda/platform.py:125` [major] — In `vllm_omni/platforms/cuda/platform.py` line 126, the import guard only checks `importlib.import_module("kernels")`, not that `get_kernel` is importable. If an older `kernels` version without `get_kernel` is installed, the guard passes and `FlashAttentionHubImpl.__init__` (flash_attn_hub.py:92) crashes with `ImportError`. Strengthen the guard to additionally verify `from kernels import get_kernel` or catch ImportError in __init__. (evidence: Read cuda/platform.py L124-138; pyproject.toml L70 shows 'kernels<0.15.0' with no lower bound.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:103` [major] — In `FlashAttentionHubImpl.__init__` (flash_attn_hub.py line 102), the third `get_kernel("kernels-community/flash-attn2")` call (no version parameter) is not wrapped in try/except, unlike the two preceding attempts. If all three fail, the exception propagates and crashes the server. Wrap it in its own try/except or restructure the three attempts under a single outer try/except. The identical pattern exists in FlashAttention3HubImpl at line 281. (evidence: Read flash_attn_hub.py lines 95-103 (FlashAttentionHubImpl) and lines 274-282 (FlashAttention3HubImpl).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:108` [major] — In `FlashAttentionHubImpl.__init__` (flash_attn_hub.py line 108), the guard `if self.flash_attn_func is None and self.flash_attn_varlen_func is None` allows execution when only `flash_attn_func` is present and `flash_attn_varlen_func` is None. Subsequent calls in `_forward_varlen_masked` (line 147) and `_forward_varlen_dense` (line 169) dereference `self.flash_attn_varlen_func` without a None check, causing a `TypeError` crash. Tighten the constructor guard to require both functions or add None guards in the varlen paths. The identical bug exists in FlashAttention3HubImpl (line 287). (evidence: Read flash_attn_hub.py lines 108-109: guard uses AND, permits flash_attn_varlen_func=None when flash_attn_func is present. Read lines 147-157: _forward_varlen_masked calls self.flash_attn_varlen_func without None check. Read line 169: _forward_varlen_dense does the same. Read lines 287-288: identical pattern in FlashAttention3HubImpl.) + +`tests/diffusion/attention/test_kernels_hub.py:57` [major] — The file `tests/diffusion/attention/test_kernels_hub.py` is not selected by any CI step. No pipeline lane matches `tests/diffusion/attention/` with the required `core_model and cuda` markers; the 'Simple' CPU step deselects it because the hardware_test decorator adds a `cuda` mark. As a result, the Hub backends' GPU execution paths receive zero automated testing. Add a dedicated CI step or move the file under an existing glob that captures it. (evidence: Read .buildkite/test-ready.yml lines 13-145, .buildkite/test-merge.yml, .buildkite/test-amd-ready.yaml — confirmed no step matches `tests/diffusion/attention/` with `core_model and cuda`) + +`pyproject.toml:?` [minor] — `pyproject.toml` line 70 declares `kernels<0.15.0` with no lower bound. The backend code calls `get_kernel(..., version=1)` and `get_kernel(..., version=2)` (flash_attn_hub.py:96-99) which may not exist in old `kernels` releases. The docs recommend `kernels==0.14.1`. Add a lower bound, e.g., `kernels>=0.14.0,<0.15.0`, to prevent installation of incompatible versions. (evidence: Read pyproject.toml L70; read flash_attn_hub.py L92-102; read docs/user_guide/diffusion/attention_backends.md L228.) + +`tests/diffusion/attention/test_kernels_hub.py:92` [minor] — In `tests/diffusion/attention/test_kernels_hub.py` line 92, the FA3 branch is skipped when compute capability < 9. CI runs on L4 (sm_89), so this branch never executes. The FA3 Hub backend therefore has zero automated CI coverage. Either add an H100 lane or document that FA3 execution testing is manual-only. (evidence: Read test_kernels_hub.py L58-103; grepped .buildkite/ for 'test_kernels_hub' — no entry found; grepped .buildkite/ for 'core_model' to confirm L4 is the CUDA CI hardware.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/platforms/cuda/platform.py:124` [minor] The docstring of `CudaOmniPlatform.get_diffusion_attn_backend_cls` (method starting around line 121 in the diff) does not mention the new FLASH_ATTN_HUB/FLASH_ATTN_3_HUB selection logic added at lines 124-148. Update the +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:249` [minor] FlashAttentionHubImpl (lines 71-220) and FlashAttention3HubImpl (lines 249-398) are near-identical except for the kernel repo name and __init__ details. The _flash_wrapper, _forward_varlen_masked, _forward_varlen_dense, +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:117` [minor] The `_run_varlen_dense` fallback at flash_attn_hub.py line 119 is dead code: `attn_func` is never None when called from `forward_cuda` (line 193), because the caller always passes a `partial` object wrapping the non-None +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:189` [minor] The `piecewise_attn` path (`full_attn_spans is not None`, flash_attn_hub.py line 189 for FA2, line 368 for FA3) is not covered by any test. Add a test that sets `full_attn_spans` to exercise the mixed causal/full mask lo + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r4/pr4926.trace.json.gz new file mode 100644 index 000000000..7a10777e2 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r4/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4977.cost.json b/eval/dataset/arms/copilot_v7_goal_r4/pr4977.cost.json new file mode 100644 index 000000000..bef29a3d6 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 399.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r4/runs/pr4977/run-20260812-190000-524174", + "recorded_at": "2026-08-12T19:06:37", + "attempts": [ + { + "invocation_id": "inv-73975c46", + "outcome": "run:run-20260812-190000-524174", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-190000-524174", + "llm_calls": 43, + "input_tokens": 193823, + "output_tokens": 63862, + "usd": 0.14905, + "source": "spans" + } + ], + "attempt_usd_total": 0.14905, + "llm_calls": 43, + "input_tokens": 193823, + "output_tokens": 63862, + "usd": 0.14905, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-190000-524174", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.149, + "minutes": 6.62, + "ci_minutes": 0, + "input_tokens": 193823, + "output_tokens": 63862, + "tool_calls": 70, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2944, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.140604, + "input_tokens": 187660, + "output_tokens": 57722, + "calls": 42 + }, + "reducer": { + "usd": 0.008446, + "input_tokens": 6163, + "output_tokens": 6140, + "calls": 1 + } + }, + "cache_read_tokens": 980352, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.39, + "gate": 2.37, + "review": 391.51, + "report": 0.06 + }, + "time_to_first_result_s": 397.32 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 19306 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4977.md b/eval/dataset/arms/copilot_v7_goal_r4/pr4977.md new file mode 100644 index 000000000..525763e12 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr4977.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] flash_attn_hub.py:22-38 `_load_hub_module` — traced the `for version in (1, 2, None)` loop; all three arms are reachable and the fallback order matches the old nested try/except exactly. `last_error` is always set before the final `raise`. +- [sweep] flash_attn_hub.py:42-46 `_get_hub_module` — the lock-check-load-return pattern is correct under `threading.Lock()`; failures are not cached, allowing retry on next call. +- [sweep] flash_attn_hub.py:119 `FlashAttentionHubImpl.__init__` — replaced inline try/except with `_get_hub_module("kernels-community/flash-attn2")`; same repo_id, same fallback semantics. +- [sweep] flash_attn_hub.py:284 `FlashAttention3HubImpl.__init__` — same delegation to `_get_hub_module("kernels-community/flash-attn3")`; correct. +- [validated] flash_attn_hub.py:33 `if version is not None:` — checked the else branch calls `get_kernel(repo_id)` without `version=`, matching the old default path. +- [validated] flash_attn_hub.py:44 `if repo_id not in _hub_modules:` — standard lazy-init under lock; no double-load race. +- [sweep] Repo grep for `_get_hub_module` / `_load_hub_module` consumers — only the same file references them; no stale external callers. +- [sweep] pyproject.toml:70 `kernels<0.15.0` — the `get_kernel(repo_id, version=version)` API is unchanged from the pre-PR code; no new dependency risk. + +`.buildkite/test-ready.yml:?` [major] — The CI pipeline only globs `tests/diffusion/*.py` and does not include `tests/diffusion/attention/`. Consequently, `tests/diffusion/attention/test_kernels_hub.py` is never executed in CI, leaving the PR’s core caching change completely untested. Add an explicit entry for `tests/diffusion/attention` in `.buildkite/test-ready.yml` (or the appropriate CI config) so the cache‑reuse tests run on every PR. (evidence: grep 'tests/diffusion/attention' in .buildkite/ returned zero matches; CI pipeline files only reference tests/diffusion/*.py, tests/diffusion/batching, tests/diffusion/cache, tests/diffusion/distributed, tests/diffusion/models/, tests/diffusion/offloader) + +`tests/diffusion/attention/test_kernels_hub.py:?` [minor] — The core behavioral change is the new `_get_hub_module` cache. `test_kernels_hub_execution` constructs only one `FlashAttentionHubImpl` (line 79-89) and never exercises the cache‑hit path. Add a test that constructs two `FlashAttentionHubImpl` instances and asserts they share the same `flash_attn_func` object (e.g., `assert impl1.flash_attn_func is impl2.flash_attn_func`), confirming the cached module is reused and the Hub is contacted only once. (evidence: Read tests/diffusion/attention/test_kernels_hub.py: test_kernels_hub_execution creates one FlashAttentionHubImpl; no test verifies the cache-hit path (two constructions sharing one module).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:26` [minor] — The NPU CI lane (`buildkite/vllm-omni-npu-ci`) is failing. NPU’s `get_diffusion_attn_backend_cls` falls back to local FLASH_ATTN before reaching `FlashAttentionHubImpl`, so this diff should not exercise NPU code. Please confirm the failure is pre‑existing on main (not introduced by this PR). (evidence: gh_ci_read(4977) — buildkite/vllm-omni-npu-ci FAILURE; npu/platform.py:110-115 shows fallback to FLASH_ATTN without instantiating FlashAttentionHubImpl) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:22` [minor] — New module‑level globals `_hub_modules` and `_hub_lock` lack docstrings that explain the caching contract (single‑load‑per‑process, lock‑guarded, never evicted, failures are not cached). Add a one‑line comment above `_hub_modules` and short docstrings for `_load_hub_module` and `_get_hub_module`. (evidence: Read full file — `_hub_modules` (line 22), `_hub_lock` (line 23), `_load_hub_module` (line 26), `_get_hub_module` (line 42) all lack docstrings.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [minor] — The `_get_hub_module` path for `FlashAttention3HubImpl` is never exercised in CI because `test_kernels_hub_execution` skips FA3 on GPUs with compute capability <9.0, and current CI lanes use L4 (SM 8.9). Verify whether an Hopper‑class lane covers this test; if not, document that the FA3 cache path is validated only manually. (evidence: read test_kernels_hub.py line 92, read .buildkite/test-ready.yml diffusion lanes all on gpu_1_queue (L4)) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [nit] — The PR description claims that `trust_remote_code=True` was added for more reliable HuggingFace Hub access. However, commit `9947f414` explicitly dropped `trust_remote_code` from the final diff, and the diff contains no `trust_remote_code` anywhere. Update the PR description to match the actual change, or document why `trust_remote_code` was removed (e.g., the `kernels` library does not support it). (evidence: PR description claims `trust_remote_code=True`; commit timeline shows it was dropped in 9947f414; diff contains no `trust_remote_code` anywhere.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r4/pr4977.trace.json.gz new file mode 100644 index 000000000..09a0c7cc5 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r4/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr5009.cost.json b/eval/dataset/arms/copilot_v7_goal_r4/pr5009.cost.json new file mode 100644 index 000000000..cb89c1170 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 519.9, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r4/runs/pr5009/run-20260812-185119-a56ac9", + "recorded_at": "2026-08-12T18:59:57", + "attempts": [ + { + "invocation_id": "inv-d0492df9", + "outcome": "run:run-20260812-185119-a56ac9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-185119-a56ac9", + "llm_calls": 42, + "input_tokens": 552993, + "output_tokens": 68057, + "usd": 0.281717, + "source": "spans" + } + ], + "attempt_usd_total": 0.281717, + "llm_calls": 42, + "input_tokens": 552993, + "output_tokens": 68057, + "usd": 0.281717, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-185119-a56ac9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2817, + "minutes": 8.63, + "ci_minutes": 0, + "input_tokens": 552993, + "output_tokens": 68057, + "tool_calls": 86, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4072, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 42, + "by_role": { + "agent": { + "usd": 0.274532, + "input_tokens": 544776, + "output_tokens": 63564, + "calls": 41 + }, + "reducer": { + "usd": 0.007185, + "input_tokens": 8217, + "output_tokens": 4493, + "calls": 1 + } + }, + "cache_read_tokens": 2131328, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.22, + "gate": 1.68, + "review": 511.93, + "report": 0.06 + }, + "time_to_first_result_s": 517.88 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 21906 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr5009.md b/eval/dataset/arms/copilot_v7_goal_r4/pr5009.md new file mode 100644 index 000000000..726930446 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r4/pr5009.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] platform.py:251-252 — removed inductor-aware branch and CompilationMode import; the only consumer of vllm_config.compilation_config in the removed code; no residual dead code +- [validated] platform.py:251-252 — new default=['vllm_c','native'] matches pre-#4709 behavior verified by PR description timeline +- [validated] platform.py:259 — oink env-var path preserved unchanged; VLLM_USE_OINK_OPS still prepends 'oink' +- [validated] qwen_image_transformer.py:17 — RMSNorm import removed cleanly; grep confirms no other references to vllm.model_executor.layers.layernorm.RMSNorm remain in this file +- [validated] qwen_image_transformer.py:554-555,592-593,1007 — all five nn.RMSNorm instantiations use (dim, eps=eps) matching torch.nn.RMSNorm signature; no elementwise_affine needed (defaults to True in both) +- [validated] qwen_image_transformer.py:554-555 — norm_q/norm_k normalize per-head (head_dim); TP shards across heads, not within a head, so nn.RMSNorm(head_dim) is TP-safe +- [validated] qwen_image_transformer.py:1007 — txt_norm input (encoder_hidden_states) feeds into ReplicatedLinear (txt_in), so it is replicated (not column-parallel sharded); nn.RMSNorm is TP-safe here too +- [sweep] blast-radius — get_default_ir_op_priority is called from diffusion_worker.py:175 for all diffusion models; the revert restores pre-#4709 behavior when all models worked correctly; Cosmos3 has its own per-model override (pipeline_cosmos3.py:617-622) forcing native-only; no other model in _DIFFUSION_IR_OP_PRIORITY_FUNCS registry declares an override + +`vllm_omni/platforms/cuda/platform.py:250` [minor] — This change flips the default IR op priority for ALL CUDA diffusion models (Qwen-Image, FLUX, SD3, Wan, etc.) from inductor-conditional to always `['vllm_c','native']`. Only Qwen-Image was validated on H200. Cosmos3 is protected by a per-model override (`_DIFFUSION_IR_OP_PRIORITY_FUNCS` in registry.py:549), but no audit confirms every other model is compatible with the restored default. Consider either a cross-model smoke or an explicit comment documenting the assumption that all non-Cosmos3 models are safe because pre-rebase behavior was identical. (evidence: Read _DIFFUSION_IR_OP_PRIORITY_FUNCS in registry.py:546-549 — only 'Cosmos3OmniDiffusersPipeline' has a registered ir_op_priority_func. All other ~40+ models in _DIFFUSION_MODELS inherit the platform default. Read the full get_default_ir_op_priority in platform.py:251-262 confirming the inductor branch is removed.) + +`vllm_omni/platforms/cuda/platform.py:252` [nit] — The docstring now accurately describes the new behavior, but it lost the explanation of WHY we intentionally diverge from upstream's inductor-aware default. The old docstring explained this divergence; consider adding a brief note that upstream prefers native under inductor for LLM op-fusion but diffusion denoise loops are RMSNorm-heavy and benefit from vllm_c kernels, so we override the upstream default intentionally. (evidence: Read platform.py:251 — the docstring was changed from "Copied from upstream CudaPlatformBase with inductor-aware logic. When inductor is active (compiling) use native as the default; otherwise prefer vllm_c kernels where available." to the current one-liner. A future reader seeing this code won't know why we differ from upstream vLLM's CudaPlatformBase.) + +`vllm_omni/platforms/cuda/platform.py:?` [nit] — Gate report lists failing AMD, Intel, NPU CI checks; this PR only touches CUDA and CPU test. Please confirm these failures are pre-existing on main (or investigate if unrelated) — no actionable defect in diff. (evidence: Gate report lists 3 failing checks: buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, buildkite/vllm-omni-npu-ci. The diff touches only vllm_omni/platforms/cuda/platform.py (CUDA), qwen_image_transformer.py (model code), and tests/platforms/test_cuda_ir_op_priority.py (CPU test). None of these files feed into AMD/ROCm, Intel/XPU, or NPU code paths.) + +`vllm_omni/platforms/cuda/platform.py:252` [nit] — Consider adding a docstring note in `get_default_ir_op_priority` that models needing a different priority (e.g. Cosmos3 forces native-only) should register a per-model `ir_op_priority_func` in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`. (evidence: Read vllm_omni/platforms/cuda/platform.py:251; verified Cosmos3 override at vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:612-624 via read_file; registry at vllm_omni/diffusion/registry.py:546-549 via grep.) + +`tests/platforms/test_cuda_ir_op_priority.py:23` [nit] — The function name says "when_inductor_backend" but the fix makes priority unconditional across all compilation modes (including NONE). Consider renaming to e.g. `test_cuda_default_ir_op_priority_always_prefers_vllm_c`. (evidence: Read test_cuda_ir_op_priority.py:23-27 — the test is parametrized over CompilationMode.NONE, VLLM_COMPILE, and STOCK_TORCH_COMPILE, and asserts `["vllm_c", "native"]` for all three. The name "when_inductor_backend" doesn't apply to CompilationMode.NONE.) + +`tests/platforms/test_cuda_ir_op_priority.py:21` [nit] — The parametrization covers CompilationMode values 0 (NONE), 1 (VLLM_COMPILE), and 3 (STOCK_TORCH_COMPILE), but skips value 2. If CompilationMode has a member at index 2 (e.g. PIECEWISE), add it to ensure all modes produce the expected priority (though the current unconditional behavior should pass). Verify and expand if appropriate. (evidence: Read tests/platforms/test_cuda_ir_op_priority.py:18-20; pytest cache nodeids at .pytest_cache/v/cache/nodeids show parametrized test IDs [0], [1], [3] with no [2].) + +`tests/platforms/test_cuda_ir_op_priority.py:13` [nit] — Test fixture uses DeviceConfig(device="cpu") for a CudaOmniPlatform test. The method under test only reads compilation_config, so it's harmless, but confusing. Consider changing to device="cuda" or adding a comment. (evidence: Read tests/platforms/test_cuda_ir_op_priority.py:12-16; read vllm_omni/platforms/cuda/platform.py:251-262 showing the method only accesses vllm_config.compilation_config.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r4/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r4/pr5009.trace.json.gz new file mode 100644 index 000000000..b045cd6bf Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r4/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r5/manifest.json b/eval/dataset/arms/copilot_v7_goal_r5/manifest.json new file mode 100644 index 000000000..c41ec3170 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v7_goal_r5", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4804.cost.json b/eval/dataset/arms/copilot_v7_goal_r5/pr4804.cost.json new file mode 100644 index 000000000..0af8cf74a --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 484.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r5/runs/pr4804/run-20260812-192147-2b6c8d", + "recorded_at": "2026-08-12T19:29:50", + "attempts": [ + { + "invocation_id": "inv-2d29d891", + "outcome": "run:run-20260812-192147-2b6c8d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-192147-2b6c8d", + "llm_calls": 43, + "input_tokens": 971515, + "output_tokens": 63307, + "usd": 0.453978, + "source": "spans" + } + ], + "attempt_usd_total": 0.453978, + "llm_calls": 43, + "input_tokens": 971515, + "output_tokens": 63307, + "usd": 0.453978, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-192147-2b6c8d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.454, + "minutes": 8.04, + "ci_minutes": 0, + "input_tokens": 971515, + "output_tokens": 63307, + "tool_calls": 71, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4605, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.425667, + "input_tokens": 915135, + "output_tokens": 51430, + "calls": 42 + }, + "reducer": { + "usd": 0.028311, + "input_tokens": 56380, + "output_tokens": 11877, + "calls": 1 + } + }, + "cache_read_tokens": 4519680, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.13, + "gate": 2.35, + "review": 475.98, + "report": 0.06 + }, + "time_to_first_result_s": 482.51 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 64522 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4804.md b/eval/dataset/arms/copilot_v7_goal_r5/pr4804.md new file mode 100644 index 000000000..b16b3cdd8 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4804.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [sweep] serving_speech.py:3705-3715 — seed injection now always runs `deepcopy` and injects `tts_local_seed` regardless of model type. Old code gated on `self._tts_model_type == 'qwen3_tts'`. The runner's `_explicit_talker_seed` checks for `tts_local_seed` so non-MOSS models silently carry unused extra_args, but this widens blast radius unnecessarily. +- [sweep] audio_tokenizer_v2.py:1-7 — new vendored file (~1959 lines) lacks 'Vendored from' header that existing vendored file `audio_tokenizer.py` carries at line 9. Repo checklist requires this for all vendored model files. +- [sweep] modeling_moss_tts_codec.py:726-740 `_build_codec` — tries v2 vendored classes first, catches Exception and falls back to legacy vendored classes. This try-instantiate-and-fallback pattern (repo checklist: 'selection must be gated by an explicit config marker, never try-instantiate-and-fallback') risks silently decoding a legacy checkpoint with the legacy codec after a v2 config parse error, producing wrong audio. +- [sweep] stage_input_processors/moss_tts.py: talker2codec_raw_async_chunk finish sentinel (lines ~270-284) — sets `codes=CodesStruct(audio=torch.tensor([0], dtype=torch.long))` with `code_flat_numel=0` in meta. The tensor carries a real codebook index `0` but the meta claims zero elements. The codec's `_finish_empty_streaming_requests` checks `code_flat_numel` but this sentinel has `code_flat_numel=0` with a non-empty codes tensor — inconsistent. +- [sweep] modeling_moss_tts_talker.py: talker_mtp (new method, ~lines 1459-1558) — hardcodes audio sampling parameters temperature=1.7, top_k=25, top_p=0.8. The same model's old make_omni_output used temperature=0.8, top_k=30, top_p=0.6. Comment says 'Match MOSS-TTS Local v1.5's model-card/SGLang defaults' but this is a behavioral change worth explicit justification. +- [sweep] modeling_moss_tts_codec.py: load_weights now raises RuntimeError on shape mismatches/missing weights (old code silently skipped). Stricter is good; verify on all MOSS-TTS checkpoints (delay model has n_vq={16,32}, different audio_vocab_size values). +- [sweep] modeling_moss_tts_codec.py: forward now requires `seq_token_counts` (raises RuntimeError if missing). `_dummy_run` in gpu_generation_model_runner.py now provides it. Check other codec callers. +- [sweep] pipeline.py: delay variant changed from `talker2codec_async_chunk` to `talker2codec_delay_async_chunk`; local variant to `talker2codec_raw_async_chunk`. The Realtime variant (line 62) also changed to `talker2codec_raw_async_chunk`. The old function was renamed — verified no stale references remain via grep. + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:729` [major] — The `_build_codec` method (`modeling_moss_tts_codec.py:726-740`) tries the vendored v2 classes first and falls back to the legacy vendored codec on **any** `Exception`. A genuine v2 checkpoint that fails instantiation for a transient reason (e.g. a config key mismatch or an OOM) would silently load as v1 — producing audio with the wrong decoder architecture and no error. Repo checklist explicitly requires version selection via an explicit config marker, not a try-instantiate-and-fallback pattern. Gate on the `version` field from `MossAudioTokenizerV2Config` and raise a clear error when a v2-intended checkpoint fails. (evidence: read_file of modeling_moss_tts_codec.py lines 726-740 shows the try/except fallback pattern. The MossAudioTokenizerV2Config has a `version` field that could serve as an explicit gate.) + +`vllm_omni/worker/gpu_model_runner.py:1874` [major] — The new code path in `_talker_mtp_forward` (gpu_model_runner.py:1874-1878) passes `req_ids` and `req_infos` to `talker_mtp` when `talker_mtp_accepts_req_infos` is True. This flag is set only on `MossTTSLocalTalkerForGeneration`, yet the existing test fakes (`CaptureTalkerMTP`, `DummyTalkerMTP`) do not accept these kwargs, and `_make_runner` creates a model without the flag. No test exercises the Local‑v1.5 streaming path through the runner. Add a unit or integration test that sets `talker_mtp_accepts_req_infos` and verifies that `req_ids`/`req_infos` reach `talker_mtp`. (evidence: Read tests/worker/test_omni_gpu_model_runner.py lines 141-190: `_make_runner` creates `runner.model = SimpleNamespace(...)` without `talker_mtp_accepts_req_infos`. `CaptureTalkerMTP.forward` signature at line 300-318 accepts `generator` but not `req_ids` or `req_infos`. Grep for `talker_mtp_accepts_req_infos` confirms it is only set on `MossTTSLocalTalkerForGeneration` (modeling_moss_tts_talker.py:1324), never in tests.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [minor] — New vendored file `audio_tokenizer_v2.py` is missing the 'Vendored from @ ' header line. The existing vendored file `audio_tokenizer.py` carries this at line 9. Repo checklist requires vendored model files to carry this attribution. Add a line identifying the upstream source (e.g. `# Vendored from VoiceAgentGroup/moss_audio_tokenizer @ `). (evidence: read_file of audio_tokenizer.py:9 shows 'Vendored from OpenMOSS-Team/MOSS-Audio-Tokenizer'; grep 'Vendored from' on audio_tokenizer_v2.py returns no matches.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3715` [minor] — The diff removes the `self._tts_model_type == "qwen3_tts"` guard at `serving_speech.py:3710` and now unconditionally sets `extra_args["tts_local_seed"]` for every TTS model when `request.seed` is not None. Previously only Qwen3-TTS models received the seed; now all models in `_TTS_MODEL_STAGES` will carry this extra_arg. The runner's `_explicit_talker_seed` (gpu_model_runner.py) reads `tts_local_seed` only when the model sets `talker_mtp_accepts_req_infos` (currently only `MossTTSLocalTalkerForGeneration`), so the key is harmless for other models. However, the wider blast radius is unnecessary noise. Either gate the injection back to moss_tts models or document that `tts_local_seed` is now a universal extra_arg that all TTS models may receive. (evidence: grep for tts_local_seed shows only gpu_model_runner.py:1826 and tests consume it; grep for qwen3_tts_request_seed returns no matches (fully removed). The serving_speech.py code path is reached by _TTS_MODEL_STAGES which includes ~15 model families.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:297` [minor] — The finish sentinel emitted by `talker2codec_raw_async_chunk` (stage_input_processors/moss_tts.py:270-290) passes `codes=CodesStruct(audio=torch.tensor([0], dtype=torch.long))` with `meta.code_flat_numel=0`. The codec's `forward` correctly checks `code_flat_numel==0` for streaming requests and calls `_finish_empty_streaming_requests`. However, a future non-streaming consumer that reads `codes.audio` directly would see a valid codebook index. To harden the sentinel, either use an empty tensor (`CodesStruct(audio=torch.tensor([], dtype=torch.long))`) or assert/check `code_flat_numel == 0` before touching `codes.audio` in all downstream consumers. The current streaming path is safe. (evidence: read_file of stage_input_processors/moss_tts.py lines 270-290; read_file of modeling_moss_tts_codec.py `_finish_empty_streaming_requests` and `forward` shows the `code_flat_numel=0` check but only in the streaming path.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1524` [minor] — The new `talker_mtp` method in `MossTTSLocalTalkerForGeneration` (modeling_moss_tts_talker.py:1514) hardcodes audio sampling parameters (temperature=1.7, top_k=25, top_p=0.8) with an inline comment `# Match MOSS-TTS Local v1.5's model-card/SGLang defaults`. This is a deliberate behavioral change from the old Local `make_omni_output` (0.8/30/0.6). The inline comment already explains the intent, but consider adding a reference to the upstream SGLang implementation or model card so future maintainers can verify correctness without guesswork. (evidence: read_file of modeling_moss_tts_talker.py talker_mtp method shows hardcoded values; old make_omni_output for Realtime variant uses 0.8/30/0.6 (line ~49539 area of the file).) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:?` [minor] — The class docstring for `MossTTSCodecDecoder` (modeling_moss_tts_codec.py:133) still says "24 kHz mono waveform", but the diff adds stereo support (48 kHz stereo for v2 codec). Update the docstring to describe that output format depends on the codec checkpoint (mono 24 kHz for v1, stereo 48 kHz for v2). The class constant `_OUTPUT_SAMPLE_RATE = 24000` (line 172) is overwritten after `load_weights`; consider removing it or documenting it as a default. (evidence: load_weights() at line 698 reads codec_cfg.sampling_rate and number_channels; line 172 sets _OUTPUT_SAMPLE_RATE=24000 which is overwritten later.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1462` [minor] — In `MossTTSLocalTalkerForGeneration.preprocess` (modeling_moss_tts_talker.py:1462), `max_new_frames = info_dict.get("max_new_frames", [-1])[0]` assumes the value is a list with at least one element. An empty list would raise `IndexError`. Use a safer access like `(info_dict.get("max_new_frames", [-1]) or [-1])[0]`, matching the defensive checks in the Delay talker's equivalent code (line 472-477). (evidence: Compare with Delay talker preprocess at line 472: `if isinstance(max_new_frames_req, (list, tuple)) and max_new_frames_req: max_new_frames_req = max_new_frames_req[0]`) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:38` [minor] `_MossCodecStreamSession.close()` (modeling_moss_tts_codec.py:87) is defined but never called. The session lives for the server process lifetime, which is intentional for normal operation. However, in test/CI environment + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r5/pr4804.trace.json.gz new file mode 100644 index 000000000..bcad6582e Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r5/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4870.cost.json b/eval/dataset/arms/copilot_v7_goal_r5/pr4870.cost.json new file mode 100644 index 000000000..2c975610f --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 382.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r5/runs/pr4870/run-20260812-193627-12c6a0", + "recorded_at": "2026-08-12T19:42:48", + "attempts": [ + { + "invocation_id": "inv-756b3e4b", + "outcome": "run:run-20260812-193627-12c6a0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-193627-12c6a0", + "llm_calls": 44, + "input_tokens": 887778, + "output_tokens": 60853, + "usd": 0.384595, + "source": "spans" + } + ], + "attempt_usd_total": 0.384595, + "llm_calls": 44, + "input_tokens": 887778, + "output_tokens": 60853, + "usd": 0.384595, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-193627-12c6a0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3846, + "minutes": 6.35, + "ci_minutes": 0, + "input_tokens": 887778, + "output_tokens": 60853, + "tool_calls": 69, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.385, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.377967, + "input_tokens": 878267, + "output_tokens": 57184, + "calls": 43 + }, + "reducer": { + "usd": 0.006628, + "input_tokens": 9511, + "output_tokens": 3669, + "calls": 1 + } + }, + "cache_read_tokens": 2887296, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.78, + "gate": 1.77, + "review": 375.2, + "report": 0.06 + }, + "time_to_first_result_s": 380.81 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 24493 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4870.md b/eval/dataset/arms/copilot_v7_goal_r5/pr4870.md new file mode 100644 index 000000000..3d06f294b --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4870.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] Hunk 1 (_build_omni_model_runner_output_from_snapshot): removed seq_len param from signature, added mm_seq_len=int(scheduler_output.total_num_scheduled_tokens) inside needs_pooler_payload guard — correct alignment with multimodal tensor dims. Read vllm_omni/worker/gpu_ar_model_runner.py:1662-1795. +- [validated] Hunk 2 (sample_tokens): removed seq_len = hidden_states.shape[0] computation and seq_len=seq_len kwarg from output_builder call — clean removal, no residual references. Read vllm_omni/worker/gpu_ar_model_runner.py:1873-2042. +- [validated] Hunk 3 (serving_speech.py): new async_chunk gate correctly scopes FINAL_ONLY to qwen3_tts + async_chunk=False; non-qwen3 models keep DELTA coercion. Read vllm_omni/entrypoints/openai/serving_speech.py:3495-3507. +- [validated] Test hunk (test_gpu_ar_model_runner.py): removed seq_len kwarg from 5 existing test calls, all have matching total_num_scheduled_tokens in their SimpleNamespace — consistent cleanup. +- [validated] New test test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only: sets async_chunk=False, tail-only hidden (omni_pooler_payload_include_hidden=False), 3 requests × 1 token, codes.audio shape (3,16). Asserts output.inter_stage_outputs[i]["codes.audio"] == codes[i:i+1] — directly exercises the fix. Read tests/worker/test_gpu_ar_model_runner.py:438-479. +- [validated] New test test_prepare_speech_generation_no_async_chunk_stream_uses_final_only: qwen3_tts + async_chunk=False → coerce_param_message_types called with False (FINAL_ONLY). Read tests/entrypoints/openai_api/test_serving_speech.py:3453-3470. +- [validated] New test test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3: voxtral + async_chunk=False → coerce_param_message_types called with True (DELTA). Read tests/entrypoints/openai_api/test_serving_speech.py:3473-3495. +- [sweep] No other callers of _build_omni_model_runner_output_from_snapshot outside gpu_ar_model_runner.py — grep confirmed single definition + single internal call at line 2027. + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — The new gate at line 3507 reads `async_chunk = getattr(self.model_config, "async_chunk", True)`. `OmniModelConfig.async_chunk` defaults to `False` (config/model.py:121). If the attribute were ever removed from `model_config`, this fallback would silently invert the guard, causing Qwen3-TTS to leave FINAL_ONLY mode on streaming requests and potentially regress #4851. Align the fallback with the config class: `getattr(self.model_config, "async_chunk", False)`. (evidence: Review checklist states `async_chunk` defaults to False in `OmniModelConfig`; the getattr default of True contradicts that.) + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — The Qwen3-TTS full-payload gate at line 3508 uses `self._tts_model_type == "qwen3_tts"`. The same file already defines `_QWEN3_TTS_MODEL_STAGES` (a set of valid stage strings) at lines ~3478-3484. If a future Qwen3-TTS variant registers under a different `_tts_model_type`, this string comparison silently excludes it, falling back to the old delta-coercion path. Use a stage-set membership check (e.g., `self._tts_stage.engine_args.model_stage in _QWEN3_TTS_MODEL_STAGES`) to keep the gate aligned with the classification constants. (evidence: serving_speech.py lines 3478-3484 define _QWEN3_TTS_MODEL_STAGES; the new gate at line 3508 compares _tts_model_type string instead of using the existing stage set.) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [minor] — `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` at line 1734 assumes the first dimension of multimodal tensors (e.g., `codes.audio.shape[0]`) equals `total_num_scheduled_tokens`. This holds for Qwen3-TTS, but the function is shared across all AR models. In `to_payload_element` (mm_outputs.py:108-117) a mismatched `shape[0]` causes a batch-wide `element.clone()` — the exact corruption this PR fixes. Add an assertion (e.g., `assert any(mt.shape[0] == total_num_scheduled_tokens for mt in multimodal_outputs['codes'].values())`) and a comment documenting the invariant, so a future model that produces sparse multimodal outputs fails early and clearly at the slicing site. (evidence: Read vllm_omni/utils/mm_outputs.py:108-117 — `to_payload_element` checks `element.shape[0] == seq_len` and falls through to `element.clone()` on mismatch (whole-batch leak). Read vllm_omni/worker/gpu_ar_model_runner.py:1734-1790 for the mm_seq_len → seq_len flow.) + +`tests/worker/test_gpu_ar_model_runner.py:438` [minor] — The new test `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` covers the non-prefix-cache path. The same `mm_seq_len` fix also applies to the prefix-cache branch through `_prepare_prefix_cache_pooler_payload_sources` → `to_payload_element(seq_len=mm_seq_len)` (gpu_ar_model_runner.py around lines 1740-1760). Add a companion test or extend the existing one to validate per-request slicing when `omni_prefix_cache is not None` with tail-only hidden states, to prevent future regression on the cached path. (evidence: gpu_ar_model_runner.py lines 1740-1760 show the prefix-cache branch uses the same mm_seq_len; test only covers the non-prefix-cache path.) + +`vllm_omni/entrypoints/openai/serving_speech.py:3507` [nit] — The new `async_chunk` lookup at line 3507 uses `self.model_config`. Elsewhere in the same file, e.g., line 3823, the same attribute is read through `self.engine_client.model_config`, and the runner init uses that path too (`gpu_ar_model_runner.py`). While functionally equivalent today, the inconsistency may confuse future readers. Normalize to `self.engine_client.model_config` to match the file's dominant access convention. (evidence: grep serving_speech.py for `self.model_config` vs `self.engine_client.model_config`; line 3823 already uses the engine_client path for the same attribute.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r5/pr4870.trace.json.gz new file mode 100644 index 000000000..1c3e8d8c1 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r5/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4923.cost.json b/eval/dataset/arms/copilot_v7_goal_r5/pr4923.cost.json new file mode 100644 index 000000000..c60ab3539 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 394.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r5/runs/pr4923/run-20260812-192952-7193d0", + "recorded_at": "2026-08-12T19:36:25", + "attempts": [ + { + "invocation_id": "inv-92909e4f", + "outcome": "run:run-20260812-192952-7193d0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-192952-7193d0", + "llm_calls": 42, + "input_tokens": 621704, + "output_tokens": 51277, + "usd": 0.270983, + "source": "spans" + } + ], + "attempt_usd_total": 0.270983, + "llm_calls": 42, + "input_tokens": 621704, + "output_tokens": 51277, + "usd": 0.270983, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-192952-7193d0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.271, + "minutes": 6.54, + "ci_minutes": 0, + "input_tokens": 621704, + "output_tokens": 51277, + "tool_calls": 88, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3455, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 42, + "by_role": { + "agent": { + "usd": 0.259028, + "input_tokens": 613566, + "output_tokens": 42428, + "calls": 41 + }, + "reducer": { + "usd": 0.011955, + "input_tokens": 8138, + "output_tokens": 8849, + "calls": 1 + } + }, + "cache_read_tokens": 1730304, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.75, + "gate": 2.08, + "review": 386.61, + "report": 0.06 + }, + "time_to_first_result_s": 392.48 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 23272 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4923.md b/eval/dataset/arms/copilot_v7_goal_r5/pr4923.md new file mode 100644 index 000000000..09c1d18ee --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4923.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] qwen3_tts_talker.py:327-353 — verified talker_mtp_graph_safe=True, per-row generator gating on has_full_cudagraphs(), and async output flags set; all branches logically consistent +- [sweep] qwen3_tts_code_predictor_vllm.py:36 — verified use_cuda_graphs=current_omni_platform.is_npu() prevents inner GPU graphs while keeping NPU path +- [sweep] qwen3_tts.yaml:35-96 — config changes verified: max_num_seqs 10→64, enable_prefix_caching true→false, max_num_batched_tokens 512→32768, NPU cudagraph_mode PIECEWISE +- [validated] gpu_model_runner.py:208-210 — confirmed _init_talker_mtp checks talker_mtp_graph_safe and has_full_cudagraphs() before wrapping; consistent with model declaration +- [validated] gpu_model_runner.py:1854 — confirmed runner reads talker_mtp_accepts_per_row_generators; gating logic in talker.py is the producer +- [validated] gpu_ar_model_runner.py:1547-1639 — confirmed eager_omni_postprocess_before_async_output and omni_pooler_payload_include_hidden are consumed by runner; new flags correctly opt into async path +- [sweep] grep for has_full_cudagraphs — confirmed method exists on CUDAGraphMode; getattr fallback for None handles eager/PIECEWISE safely +- [sweep] grep for fish_speech talker_mtp_graph_safe — Fish Speech also declares graph_safe=True but does NOT read cudagraph_mode for per-row gating; runner default (False) means per-row generators are disabled for Fish Speech under all modes (no regression) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [major] — The new code introduces a branch where `talker_mtp_accepts_per_row_generators = False` when full cudagraphs are enabled, switching to scalar per-request MTP calls. No existing test covers this False path; the current test (`test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models`) only verifies the True path. Add a CPU test that sets the flag to False, runs two seeded requests, and asserts they are dispatched as two separate scalar calls (batch_size=1 each) with per-call generators, mirroring the existing scalar test. (evidence: grep('talker_mtp_accepts_per_row_generators', 'tests') — only test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models sets it True; no test sets it False) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:327` [major] — This PR changes graph-capture strategy (nested → outer wrap), async output, and per-row RNG gating — all correctness-critical — but touches zero test files. The code predictor's `use_cuda_graphs` flip and the new `_capture_talker_mtp_graphs` path have no coverage. Add a CPU test that exercises talker_mtp with full cudagraphs, verifies audio output is not corrupted, and ensures seeded requests remain reproducible in eager mode. (evidence: sweep_targets reports TEST FILES TOUCHED: NONE. Grep for qwen3_tts in tests/ returned no matches. PR description mentions profiling but no test verification.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The comment block on lines 38-42 says 'Prefix caching on by default' and argues for its performance benefits, but the value is now `false` (line 43). Update the comment to explain why prefix caching is now disabled (e.g., memory headroom with higher max_num_seqs=64) or revert the change if the disable was unintentional. (evidence: Read vllm_omni/deploy/qwen3_tts.yaml lines 35-43: comment says 'Prefix caching on by default' and describes throughput benefits, but enable_prefix_caching is now false.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — The TODO at lines 340-346 acknowledges that per-request `tts_local_seed` determinism is silently dropped when decode batch > 1 under full cudagraphs, and the fix is a follow-up refactor. Please link a tracking issue (e.g., `#NNNN`) in the TODO so this regression is not lost. (evidence: Read qwen3_tts_talker.py lines 337-346: the TODO describes the determinism gap but has no linked issue. PR description confirms 'per-request tts_local_seed is not reproducible' under full cudagraphs.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — The TODO at line 345 acknowledges a layering violation (reading `cudagraph_mode` in the model). Please either implement the refactor now to move the gating into the runner, or file a follow-up issue and reference it in the TODO comment so it is not forgotten. (evidence: PR's own comment block: 'TODO(#4923): the model should not read cudagraph_mode — it is an upstream (runner) concern... Once all TTS models declare talker_mtp_graph_safe we should move this gating into the model runner.') + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The NPU CI lane (`buildkite/vllm-omni-npu-ci`) is failing. The diff adds NPU-specific `cudagraph_mode: PIECEWISE` config. Please confirm whether this failure is pre-existing on main (prior to this PR) or introduced by this change; if pre-existing, note in PR thread. (evidence: gate_report shows FAILING CHECKS: ['buildkite/vllm-omni-npu-ci']. Diff adds NPU platform config at qwen3_tts.yaml lines 95-100.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The header comment on line 6 says `max_num_batched_tokens=512` for async-chunk latency tuning, but the diff bumps the value to 32768. Update the comment to explain the new value and any latency/tradeoff implications. (evidence: Diff hunk shows `max_num_batched_tokens: 32768` at stage 0; header comment on line 6 still reads `max_num_batched_tokens=512`.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:353` [nit] — `self.omni_pooler_payload_include_hidden` is set to `False` at line 317 (pre-existing) and again at line 353 (new). The second assignment is redundant. Remove the duplicate assignment at line 353 to avoid confusion. (evidence: Read vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py lines 317 and 353: both assign self.omni_pooler_payload_include_hidden = False.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r5/pr4923.trace.json.gz new file mode 100644 index 000000000..44085cad2 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r5/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4926.cost.json b/eval/dataset/arms/copilot_v7_goal_r5/pr4926.cost.json new file mode 100644 index 000000000..304c64043 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 770.6, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r5/runs/pr4926/run-20260812-193530-07c3cf", + "recorded_at": "2026-08-12T19:48:19", + "attempts": [ + { + "invocation_id": "inv-d4107a03", + "outcome": "run:run-20260812-193530-07c3cf", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-193530-07c3cf", + "llm_calls": 56, + "input_tokens": 379701, + "output_tokens": 103982, + "usd": 0.27814, + "source": "spans" + } + ], + "attempt_usd_total": 0.27814, + "llm_calls": 56, + "input_tokens": 379701, + "output_tokens": 103982, + "usd": 0.27814, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-193530-07c3cf", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2781, + "minutes": 12.8, + "ci_minutes": 0, + "input_tokens": 379701, + "output_tokens": 103982, + "tool_calls": 104, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5027, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 56, + "by_role": { + "agent": { + "usd": 0.264266, + "input_tokens": 359904, + "output_tokens": 96254, + "calls": 55 + }, + "reducer": { + "usd": 0.013874, + "input_tokens": 19797, + "output_tokens": 7728, + "calls": 1 + } + }, + "cache_read_tokens": 2268160, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.06, + "gate": 1.73, + "review": 762.3, + "report": 0.07 + }, + "time_to_first_result_s": 768.14 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 35994 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4926.md b/eval/dataset/arms/copilot_v7_goal_r5/pr4926.md new file mode 100644 index 000000000..8fefc6c0a --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4926.md @@ -0,0 +1,50 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] flash_attn_hub.py:46 `isinstance(out, tuple)` — correctly unwraps (out,lse) tuples from FA3-like varlen functions; consistent with existing _unwrap_flash_output pattern in flash_attn.py:77 +- [validated] flash_attn_hub.py:88 `if backend_kwargs:` — warns and ignores; same pattern as flash_attn.py:75 +- [validated] flash_attn_hub.py:108 guard uses `and` — only raises when BOTH funcs are None; asymmetric case (one present, one absent) allowed through +- [sweep] flash_attn_hub.py:149 direct call to self.flash_attn_varlen_func in _forward_varlen_masked — no None guard; crashes if only flash_attn_func is available +- [sweep] flash_attn_hub.py:169 _forward_varlen_dense calls _run_varlen_dense(self.flash_attn_varlen_func, ...) — no None guard +- [sweep] flash_attn_hub.py:225 final fallback in forward_cuda calls _forward_varlen_dense — no guard for flash_attn_varlen_func being None +- [validated] flash_attn_hub.py:114 static _unwrap_flash_output — correctly handles tuple/scalar return; consistent with flash_attn.py:77 +- [validated] flash_attn_hub.py:117 _flash_wrapper attn_func branch and varlen fallback — both reachable, correct + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [major] — The init guard at `FlashAttentionHubImpl.__init__` (line 108) and `FlashAttention3HubImpl.__init__` (line 287) uses `and` — it raises only when **both** `flash_attn_func` and `flash_attn_varlen_func` are `None`. If `flash_attn_func` is present but `flash_attn_varlen_func` is `None`, the later calls to `self.flash_attn_varlen_func(...)` in `_forward_varlen_masked` (line 149/328) and `_forward_varlen_dense` (line 169/348) will crash with `TypeError: 'NoneType' object is not callable`. Change the guard to `or` so that a missing function always raises immediately. (evidence: Read flash_attn_hub.py:108 guard `if self.flash_attn_func is None and self.flash_attn_varlen_func is None`; then read flash_attn_hub.py:149 `out_unpad = self.flash_attn_varlen_func(...)` inside _forward_varlen_masked with no preceding None check; line 169 `_run_varlen_dense(self.flash_attn_varlen_func, ...)` also unguarded. The same pattern repeats in FlashAttention3HubImpl at lines 287, 328, 348.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:71` [major] — `FlashAttentionHubImpl` (lines 71‑226) and `FlashAttention3HubImpl` (lines 250‑405) duplicate ~200 lines of identical forward logic: `_forward_varlen_masked`, `_forward_varlen_dense`, `forward_cuda`, `_flash_wrapper`, and `_unwrap_flash_output` are copy‑pasted except for the kernel name string passed to `get_kernel()` in `__init__`. This doubles the maintenance surface — a fix in the masked‑attention path would need to be manually replicated in both classes. Extract the shared methods into a single base class (e.g., `_BaseFlashAttentionHubImpl`) that both backends inherit, keeping only the kernel‑loading `__init__` distinct. (evidence: Read full flash_attn_hub.py (406 lines); diffed FlashAttentionHubImpl (lines 71–220) against FlashAttention3HubImpl (lines 250–399): identical except for 'flash-attn2'/'flash-attn3' strings in __init__.) + +`tests/diffusion/attention/test_kernels_hub.py:57` [major] — `test_kernels_hub_execution` is decorated with `@hardware_test(res={'cuda': 'L4'})` (line 58), which binds it to L4 GPUs (sm_89). The FA3 execution guard `if major < 9: pytest.skip(...)` at line 92 therefore always skips the `FlashAttention3HubImpl` path in CI — the FA3 forward code has zero automated execution coverage on any CI lane. This is a significant test‑integrity gap. Either request an H100‑based CI lane for this test (split FA3 into a separate function if needed) or explicitly document that FA3 correctness currently relies on manual verification. (evidence: Read test_kernels_hub.py (104 lines); hardware_test binds L4 per tests/helpers/mark.py:136. L4 = sm_89 (Ada), major=8 < 9. Grepped .buildkite/test-ready.yml and test-amd-ready.yaml — no H100-specific attention test lane found.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:111` [minor] — The comment on `FlashAttentionHubImpl._unwrap_flash_output` (line 113) says `# FA3 may return (out, lse), FA2 returns out`, but this class loads `kernels-community/flash-attn2`. The same comment correctly appears in `FlashAttention3HubImpl` (line 293). Change the misleading comment to a neutral description such as `# Hub kernel may return (out, lse) tuple or plain tensor`. (evidence: Read flash_attn_hub.py lines 111-114. The identical method on FlashAttention3HubImpl (line 292-294) has the same comment, where it is appropriate.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:71` [minor] — `FlashAttentionHubImpl.__init__` (line 69) and `FlashAttention3HubImpl.__init__` (line 248) execute a cascading version fallback (`get_kernel(..., version=1)` → `version=2` → no‑version default) without any docstring or inline comment explaining why three attempts are necessary and what each version provides. Add a class‑level docstring or at least inline comments documenting the version strategy. (evidence: Read flash_attn_hub.py lines 69-108 and 248-287; the try/except chain at lines 95-105 and 274-284 has no docstring or inline explanation of the version cascade.) + +`tests/diffusion/attention/test_kernels_hub.py:87` [minor] — The test at lines 78‑90 compares Hub kernel output against PyTorch SDPA (reference), but the PR's stated goal is train ↔ rollout consistency. SDPA is a third implementation, not the HuggingFace Diffusers training attention path. The current test only proves the kernel doesn't crash and is numerically close to SDPA within 1e‑2, not that it matches the training kernel. Consider adding a comparison against the actual Diffusers attention path (using `kernels`-backed flash‑attn) or tighten the tolerance and document why SDPA is an acceptable reference. (evidence: Read test_kernels_hub.py lines 78-90. PR description states 'numerical drift / sampling divergence caused by executing different local kernel versions during rollout' — the SDPA reference is a third implementation, not the training kernel.) + +`vllm_omni/platforms/rocm/platform.py:?` [minor] — Gate report shows `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` failing. This PR adds Hub‑backend rejection blocks in `rocm/platform.py:91` and `npu/platform.py:110`. Please confirm whether these failures are pre‑existing on main or introduced by this branch, as the added logic is a simple early fallback and unlikely to regress. (evidence: gate_report: 'FAILING CHECKS (2): [buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-npu-ci]'; diff adds Hub rejection blocks to rocm/platform.py:91-97 and npu/platform.py:110-117.) + +`tests/diffusion/attention/test_kernels_hub.py:70` [minor] — The test at lines 68‑69 sets `k = q.clone(); v = q.clone()`, making key and value identical to query. This degenerate self‑attention input can mask kernel bugs (e.g., transposition errors, GQA/MQA indexing mistakes) because every head sees identical projections. Use independent random tensors (`torch.randn_like(q)`) for K and V to exercise typical diffusion inputs more thoroughly. (evidence: Read test_kernels_hub.py lines 67-73. The SDPA reference and Hub backends all receive the same degenerate (q,q,q) input, so the test only validates that the kernel doesn't crash/NaN on self-attention, not that it computes correct attention on realistic inputs.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:47` [minor] `_run_varlen_dense` at line 47 unconditionally unwraps tuple output with `out = out[0]`, but there is no comment documenting that the first element is guaranteed to be the attention result. The same pattern in `_unwrap_f +- `pyproject.toml:70` [minor] The `kernels` dependency in `pyproject.toml` line 70 has only an upper bound `<0.15.0` but no lower bound. The code calls `get_kernel(..., version=1)` / `get_kernel(..., version=2)` (see `flash_attn_hub.py` lines 95‑107) +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:159` [minor] The bare `assert attention_mask.ndim == 2` at line 159 (`FlashAttentionHubImpl._forward_varlen_masked`) and line 338 (`FlashAttention3HubImpl._forward_varlen_masked`) will be stripped when Python runs with `-O`. A 3D mas +- `vllm_omni/diffusion/attention/backends/registry.py:57` [minor] The comment at line 57 (`# Common backends (available on most platforms)`) is misleading for `FLASH_ATTN_HUB` and `FLASH_ATTN_3_HUB` because all five non‑CUDA platforms (ROCm, NPU, MUSA, XPU) explicitly fall them back. E + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r5/pr4926.trace.json.gz new file mode 100644 index 000000000..51fc6457a Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r5/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4977.cost.json b/eval/dataset/arms/copilot_v7_goal_r5/pr4977.cost.json new file mode 100644 index 000000000..82e6a1785 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 394.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r5/runs/pr4977/run-20260812-192856-01bafc", + "recorded_at": "2026-08-12T19:35:28", + "attempts": [ + { + "invocation_id": "inv-538cc397", + "outcome": "run:run-20260812-192856-01bafc", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-192856-01bafc", + "llm_calls": 42, + "input_tokens": 133285, + "output_tokens": 62361, + "usd": 0.127031, + "source": "spans" + } + ], + "attempt_usd_total": 0.127031, + "llm_calls": 42, + "input_tokens": 133285, + "output_tokens": 62361, + "usd": 0.127031, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-192856-01bafc", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.127, + "minutes": 6.53, + "ci_minutes": 0, + "input_tokens": 133285, + "output_tokens": 62361, + "tool_calls": 63, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2815, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 42, + "by_role": { + "agent": { + "usd": 0.114535, + "input_tokens": 123889, + "output_tokens": 53333, + "calls": 40 + }, + "reducer": { + "usd": 0.012495, + "input_tokens": 9396, + "output_tokens": 9028, + "calls": 2 + } + }, + "cache_read_tokens": 831360, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.51, + "gate": 1.85, + "review": 386.13, + "report": 0.07 + }, + "time_to_first_result_s": 391.52 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 18708 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4977.md b/eval/dataset/arms/copilot_v7_goal_r5/pr4977.md new file mode 100644 index 000000000..0c4fa50c4 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr4977.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] Branch `if version is not None` at flash_attn_hub.py:33 — all three arms (v1, v2, None/default) reachable via loop iteration; no dead branch. +- [validated] Branch `if repo_id not in _hub_modules` at flash_attn_hub.py:44 — standard cache guard, always reachable on first call per repo_id. +- [validated] Error path at flash_attn_hub.py:38 — `raise RuntimeError(...) from last_error` always has `last_error` bound because at least first iteration's except sets it. +- [validated] Except/fallback at flash_attn_hub.py:32-36 — only logs and continues; no fabricated fallback value; final RuntimeError is raised after loop exhaustion. +- [validated] No rebase/merge damage — no duplicated code blocks, no stale symbol references across the touched file. +- [validated] Thread safety — `_get_hub_module` acquires `_hub_lock` for the full check-then-load sequence (line 43-46); no TOCTOU window. +- [sweep] Verified `kernels` dependency range at pyproject.toml:70 — `kernels<0.15.0`; the `get_kernel(repo_id, version=version)` call signature unchanged from old code, so no compatibility regression. +- [sweep] Checked all in-repo consumers via grep for `get_kernel`, `_get_hub_module`, `_load_hub_module` — only in-file references; no stale external callers. + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The test `test_kernels_hub_execution` (tests/diffusion/attention/test_kernels_hub.py) is the only test that constructs `FlashAttentionHubImpl` and `FlashAttention3HubImpl`, exercising the new `_get_hub_module` caching path. However, it is never selected by CI: the CPU step (`-m 'core_model and cpu'`) skips it because `@hardware_test(res={'cuda': 'L4'})` adds no `cpu` marker; the GPU step (`Diffusion · Other Test`) uses glob `tests/diffusion/*.py` which does not descend into `tests/diffusion/attention/`. Consequently, the new hub caching logic has zero CI coverage. Fix: either give the test a `cpu` marker if it can be adapted, or change the CI glob (e.g., `tests/diffusion/` or add a dedicated step for the attention subdirectory). (evidence: Read .buildkite/test-ready.yml (Simple · Diffusion step: `-m 'core_model and cpu'`, Diffusion · Other Test step: `tests/diffusion/*.py -m 'core_model and cuda'`); read tests/helpers/mark.py (`hardware_test` adds `cuda` and `L4` markers, not `cpu`); file test_kernels_hub.py lines 59-102 construct FlashAttentionHubImpl and FlashAttention3HubImpl.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:22` [minor] — The new module-level `_hub_modules` dict (line 19) is never cleared. If a test suite reuses the same Python process, a stale `_load_hub_module` mock or a previously loaded module could survive across test cases. Either document that the cache is process-lifetime and tests must handle it, or add a `clear_hub_modules()` helper to be called in test teardown. (evidence: Grep for _hub_modules across repo — only defined at line 19 and read/written at lines 44-46; no clear/teardown path exists.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [minor] — The diff introduces caching via `_get_hub_module` (lines 42-47), but there is no test that even calls it. The only test file, `tests/diffusion/attention/test_kernels_hub.py`, only covers platform fallback via `test_kernels_hub_platform_fallback` (which mocks the kernels module and never exercises `_get_hub_module`). Please add a small unit test that monkeypatches `_load_hub_module` and asserts it is called exactly once across multiple `_get_hub_module` invocations for the same repo_id — this guards against a regression that reinstates per-instance loading. (evidence: Read tests/diffusion/attention/test_kernels_hub.py: test_kernels_hub_platform_fallback only exercises platform fallback; test_kernels_hub_execution requires GPU and doesn't assert cache semantics.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [minor] — Both new functions `_load_hub_module` (line 26) and `_get_hub_module` (line 42) lack docstrings. Please add one-line summaries: for `_load_hub_module` describe the v1→v2→default fallback and lazy import; for `_get_hub_module` note the thread-safe caching. This is the core new logic, so its contract should be documented. (evidence: read_file flash_attn_hub.py lines 26-46) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [minor] — In the PR description: "adding `trust_remote_code=True` for more reliable Hub access" but commit 9947f414 ('Drop trust_remote_code from hub kernel loading') removed that kwarg; the final diff (flash_attn_hub.py) does not pass `trust_remote_code` to `get_kernel`. Please update the description to match the committed code. (evidence: PR description says 'adding trust_remote_code=True' but diff contains no trust_remote_code kwarg; commit timeline shows it was added then dropped.) + +`.buildkite/test-ready.yml:?` [minor] — The gate check `buildkite/vllm-omni-npu-ci` is failing. The NPU pipeline (`.buildkite/pipeline-npu.yaml` / `.buildkite/test-npu-ready.yaml`) runs only TTS e2e tests (`test_qwen3_tts_customvoice.py`), and this PR does not touch any NPU code or diffusion backend selection. This is likely a pre-existing failure on main. Please confirm whether main's NPU CI is also red; if it is new to this PR, investigate the cause. (evidence: Read gate_report (failing check: buildkite/vllm-omni-npu-ci); read .buildkite/pipeline-npu.yaml and .buildkite/test-npu-ready.yaml (NPU steps cover only TTS Qwen3-TTS, no diffusion attention).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [nit] — The fallback order is encoded by the tuple `(1, 2, None)` on line 33 with no inline comment. A brief `# try v1, then v2, then default (no version kwarg)` would make the intent immediately clear. (evidence: read_file flash_attn_hub.py line 33) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r5/pr4977.trace.json.gz new file mode 100644 index 000000000..ec8db24ce Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r5/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr5009.cost.json b/eval/dataset/arms/copilot_v7_goal_r5/pr5009.cost.json new file mode 100644 index 000000000..6a00b6f0a --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 425.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r5/runs/pr5009/run-20260812-192150-d12575", + "recorded_at": "2026-08-12T19:28:54", + "attempts": [ + { + "invocation_id": "inv-8077ddca", + "outcome": "run:run-20260812-192150-d12575", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-192150-d12575", + "llm_calls": 41, + "input_tokens": 531300, + "output_tokens": 60511, + "usd": 0.263066, + "source": "spans" + } + ], + "attempt_usd_total": 0.263066, + "llm_calls": 41, + "input_tokens": 531300, + "output_tokens": 60511, + "usd": 0.263066, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-192150-d12575", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2631, + "minutes": 7.06, + "ci_minutes": 0, + "input_tokens": 531300, + "output_tokens": 60511, + "tool_calls": 82, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.357, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 41, + "by_role": { + "agent": { + "usd": 0.250844, + "input_tokens": 520490, + "output_tokens": 52075, + "calls": 40 + }, + "reducer": { + "usd": 0.012222, + "input_tokens": 10810, + "output_tokens": 8436, + "calls": 1 + } + }, + "cache_read_tokens": 1964928, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 5.49, + "gate": 1.78, + "review": 416.36, + "report": 0.06 + }, + "time_to_first_result_s": 423.67 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 28260 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr5009.md b/eval/dataset/arms/copilot_v7_goal_r5/pr5009.md new file mode 100644 index 000000000..1043b8af7 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r5/pr5009.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] platform.py:249-262 — `using_inductor` branch removed cleanly; no CompilationMode import remains needed; oink path unchanged; Cosmos3 per-model override (registry.py:547-549, pipeline_cosmos3.py:612-624) confirmed to force ['native'] independently of platform default +- [validated] qwen_image_transformer.py:17 — import `from vllm.model_executor.layers.layernorm import RMSNorm` removed; grep confirms zero remaining vllm RMSNorm references in qwen_image/ directory +- [validated] qwen_image_transformer.py:554-555 — `self.norm_q / self.norm_k = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; conditional preserved, eps wired through +- [validated] qwen_image_transformer.py:592-593 — `self.norm_added_q / self.norm_added_k = nn.RMSNorm(head_dim, eps=eps)`; eps=1e-6 preserved from class signature default +- [validated] qwen_image_transformer.py:1006 — `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)`; entry-norm change consistent with Part 2 scope +- [validated] test_cuda_ir_op_priority.py:18-28 — parametrized over {NONE, VLLM_COMPILE, STOCK_TORCH_COMPILE}, each asserting ['vllm_c', 'native'] for both rms_norm and fused_add_rms_norm; test properly marked with pytestmark=[core_model, cpu] +- [validated] test_cuda_ir_op_priority.py:30-38 — oink test monkeypatches VLLM_USE_OINK_OPS and asserts ['oink', 'vllm_c', 'native']; matches platform code path +- [sweep] rocm/platform.py:184-185 — RocmOmniPlatform.get_default_ir_op_priority still has `using_inductor = cc.backend == 'inductor' and cc.mode != CompilationMode.NONE; default = ['native'] if using_inductor else ['vllm_c', 'native']` — same inductor-aware branch removed from CUDA but not from ROCm + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:?` [major] — This change fixes an accuracy regression (SSIM 0.94→0.98) by switching Qwen‑Image's five attention norm layers to `nn.RMSNorm`. No automated test guards this path — searching `tests/` for `norm_q`, `norm_added_q`, or `txt_norm` yields zero results. The same regression happened before in #3588. Add a test that asserts `isinstance(attn.norm_q, nn.RMSNorm)` for a constructed `QwenImageCrossAttention`, or extend the existing end‑to‑end accuracy test to assert SSIM ≥ 0.97 to catch future norm‑dispatch regressions. (evidence: Grep `tests/` for `norm_q\|norm_added_q\|txt_norm` returned zero results. Read `tests/diffusion/models/qwen_image/` — listed files: test_qwen_image_edit_plus.py, test_qwen_image_max_sequence_length.py, test_qwen_image_rope_table_failure_case.py, test_qwen_image_rope_utils.py, test_qwen_image_size_utils.py — none cover QwenImageCrossAttention.__init__.) + +`vllm_omni/platforms/cuda/platform.py:251` [minor] — The PR changes `CudaOmniPlatform.get_default_ir_op_priority` to always return `['vllm_c', 'native']` for all diffusion models, reverting to the pre‑#4709 behavior. While the revert should be safe because it restores a previously validated priority, the PR only retested Qwen‑Image; no evidence is provided that other models (e.g., Flux, SD3, HunyuanVideo) still work correctly with this default. Consider adding a comment in the code or PR description that pre‑#4709 CI validated this priority for all models, or verifying a representative sample of other models before merging. (evidence: Read registry.py:545-549 — only Cosmos3OmniDiffusersPipeline has an entry in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`. Read diffusion_worker.py:175 — `_resolve_ir_op_priority` calls `current_omni_platform.get_default_ir_op_priority(vllm_config)` for every model, then applies any per-model override. The PR description says 'Models that need native-only already have per-model overrides (e.g. Cosmos3)' but registry.py shows Cosmos3 is the sole override.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — In `QwenImageCrossAttention.__init__`, `qk_norm` defaults to `True` and no caller passes `False`, so the `nn.Identity()` fallback branches for `norm_q` and `norm_k` are dead code. Consider removing the dead branches or adding a test that exercises `qk_norm=False` if that configuration is supported. (evidence: Read qwen_image_transformer.py:510-557. `qk_norm` parameter defaults to `True` at line 527 and is unconditionally passed as `True` from `QwenImageTransformerBlock.__init__` at line 673 (`QwenImageCrossAttention(..., qk_norm=True not even passed — uses default)` — actually the call at lines 673-680 passes no `qk_norm` kwarg, so it defaults to `True`). No caller in the codebase passes `qk_norm=False`.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:?` [minor] — The five `nn.RMSNorm` layers (norm_q, norm_k, norm_added_q, norm_added_k, txt_norm) were changed from vLLM's `RMSNorm` to `nn.RMSNorm` to fix an accuracy regression (SSIM). There is no comment explaining this history. Add a comment near the norm declarations (e.g., '# torch.nn.RMSNorm bypasses vLLM IR dispatch; restores #4074 fix, do not switch back to vllm.RMSNorm — see #3588') to prevent accidental reversion. (evidence: Read qwen_image_transformer.py:554-558, 592-596, 1006-1009. No comment near any of the 5 nn.RMSNorm sites. PR description Parts 2 documents the #3933→#4074→#3588 history.) + +`vllm_omni/platforms/rocm/platform.py:?` [minor] — The PR removed the inductor‑aware native‑only fallback from `CudaOmniPlatform`, but `RocmOmniPlatform` at line 185 and `XPUOmniPlatform` at line 116 retain the same branch. If Qwen‑Image or other models use these platforms with inductor compilation, they may still hit the native‑only path that this PR fixes for CUDA. Either extend the fix or add a comment documenting why these platforms are intentionally excluded. (evidence: Read rocm/platform.py:173-195 — `using_inductor` branch still present. Read xpu/platform.py:107-119 — same pattern. The CUDA fix at cuda/platform.py:251-262 removes this branch entirely. Grep for `using_inductor` in vllm_omni/ confirms only rocm/ and xpu/ retain it.) + +`tests/platforms/test_cuda_ir_op_priority.py:25` [minor] — The test `test_cuda_default_ir_op_priority_prefers_vllm_c_when_inductor_backend` asserts only `rms_norm` and `fused_add_rms_norm` fields of `IrOpPriorityConfig`. The `default` field set via `with_default(default, ...)` is not checked. Asserting `priority.default == ['vllm_c', 'native']` would guard against a future regression that changes the default independently of the named op priorities. (evidence: Read platform.py:262 — `IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)`. The `default` field is set from the first argument. Read test_cuda_ir_op_priority.py:25-28 — only `rms_norm` and `fused_add_rms_norm` are asserted.) + +`vllm_omni/platforms/rocm/platform.py:?` [minor] — The docstring of `RocmOmniPlatform.get_default_ir_op_priority` still claims 'Copied from upstream RocmPlatform with inductor-aware logic', but this PR deliberately diverged the CUDA platform by removing that logic. Update the docstring to explain why ROCm retains the inductor branch (e.g., ROCm diffusion models need native‑only under inductor) or add a TODO to align with CUDA after validation. (evidence: Read rocm/platform.py:170-190 — still has `using_inductor` branch. Read cuda/platform.py:251-253 — branch removed. The platforms now disagree on the semantics of inductor + IR op priority, and the ROCm docstring cites the CUDA platform's old behavior as its rationale.) + +`tests/platforms/test_cuda_ir_op_priority.py:9` [minor] — The test `test_cuda_default_ir_op_priority_prefers_vllm_c` runs on CPU and only checks the returned priority lists, not the actual kernel dispatch. The regression (#4964) involved real GPU dispatch from `aten::mean` to `_C::rms_norm`. While this test validates the contract, consider adding a comment noting that it does not verify dispatch, or add a GPU test that checks kernel names for at least one model to catch future dispatch regressions. (evidence: Read test_cuda_ir_op_priority.py — test only asserts `priority.rms_norm == ['vllm_c', 'native']` on CPU. The PR description's profiler table shows the real dispatch verification (aten::mean count dropping from 2042 to 114) was done manually on H200.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r5/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r5/pr5009.trace.json.gz new file mode 100644 index 000000000..e0c34a69b Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r5/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r6/manifest.json b/eval/dataset/arms/copilot_v7_goal_r6/manifest.json new file mode 100644 index 000000000..294f169a5 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v7_goal_r6", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4804.cost.json b/eval/dataset/arms/copilot_v7_goal_r6/pr4804.cost.json new file mode 100644 index 000000000..e33ecaeab --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 439.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r6/runs/pr4804/run-20260812-194829-a4c3e9", + "recorded_at": "2026-08-12T19:55:46", + "attempts": [ + { + "invocation_id": "inv-d5c42529", + "outcome": "run:run-20260812-194829-a4c3e9", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-194829-a4c3e9", + "llm_calls": 45, + "input_tokens": 776250, + "output_tokens": 63952, + "usd": 0.397567, + "source": "spans" + } + ], + "attempt_usd_total": 0.397567, + "llm_calls": 45, + "input_tokens": 776250, + "output_tokens": 63952, + "usd": 0.397567, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-194829-a4c3e9", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3976, + "minutes": 7.28, + "ci_minutes": 0, + "input_tokens": 776250, + "output_tokens": 63952, + "tool_calls": 80, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4175, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 45, + "by_role": { + "agent": { + "usd": 0.368398, + "input_tokens": 714816, + "output_tokens": 52543, + "calls": 43 + }, + "reducer": { + "usd": 0.029168, + "input_tokens": 61434, + "output_tokens": 11409, + "calls": 2 + } + }, + "cache_read_tokens": 4356736, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.87, + "gate": 1.89, + "review": 430.95, + "report": 0.07 + }, + "time_to_first_result_s": 436.76 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 67779 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4804.md b/eval/dataset/arms/copilot_v7_goal_r6/pr4804.md new file mode 100644 index 000000000..904c286a6 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4804.md @@ -0,0 +1,50 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | 1 finding(s) below | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 4 finding(s) below | + +**Validated:** +- [sweep] talker2codec_raw_async_chunk put_req_chunk: initialized at line 246 as defaultdict(int), read at line 287 for emitted_any check, but never incremented anywhere in the diff — initial_chunk_frames optimization is dead code +- [sweep] patch.py:1249 fallback: except branch fabricates bytes(pcm_capture) which may be stereo 48kHz — not semantically valid for 24kHz mono WER evaluation +- [sweep] audio_tokenizer_v2.py: missing 'Vendored from @ ' header required by repo convention (cf. audio_tokenizer.py line 9) +- [sweep] audio_tokenizer_v2.py MossAudioTokenizerPatchedPretransform.encode: comment says 'ceil division' at line ~905 but code is floor (input_lengths // self.patch_size) +- [sweep] modeling_moss_tts_codec.py:703 inferred_channels uses 'v2' in codec_path.lower() — fragile string heuristic that could silently misdetect stereo/mono +- [sweep] serving_speech.py: seeds now set for ALL TTS models (was gated on _tts_model_type=='qwen3_tts'); PR validates only MOSS-TTS-Local-v1.5 +- [sweep] serving_speech.py: always deep-copies sampling_params_list (was guarded on is default_sampling_params_list); allocation overhead on every request +- [validated] tests/worker/test_omni_gpu_model_runner.py: qwen3_tts_request_seed → tts_local_seed consistent with serving_speech.py and gpu_model_runner.py + +`vllm_omni/entrypoints/openai/serving_speech.py:3708` [major] — Shared TTS serving code change removes model‑type guard and affects all 15+ model types. In `serving_speech.py` lines 3708‑3715 the diff (a) unconditionally deep‑copies `sampling_params_list` (previously gated), (b) unconditionally injects `tts_local_seed` into `extra_args`, and (c) removes the `if self._tts_model_type == "qwen3_tts"` guard. The PR only validates MOSS‑TTS‑Local‑v1.5. The consumer of `tts_local_seed` is `_explicit_talker_seed` in `gpu_model_runner.py:1826`, which is called only for models with a `talker_mtp`. Scope the injection to model types that actually consume this key (e.g. by checking an attribute) or document why injecting it into every TTS model’s `extra_args` is harmless. + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:?` [major] — Put‑req chunk counter never increased — steady‑state threshold unreachable. In `talker2codec_raw_async_chunk` (line 287) `emitted_any` reads `transfer_manager.put_req_chunk.get(req_id, 0)` but the value is only read, never written. The threshold logic `threshold = initial_chunk_frames if initial_chunk_frames > 0 and not emitted_any else chunk_frames` therefore always selects the initial size, defeating the intended two‑phase (small first chunk, then larger steady chunks) design. Add `transfer_manager.put_req_chunk[req_id] += 1` after emitting a chunk so `emitted_any` reflects actual sends. (evidence: Grepped for `put_req_chunk` across the file — only `__setattr__` guard and the `.get()` read exist; no `.put_req_chunk[...] =` assignment.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [minor] — New vendored file `audio_tokenizer_v2.py` lacks the provenance header required by the repo convention. The existing `audio_tokenizer.py` carries `# Vendored from OpenMOSS-Team/MOSS-Audio-Tokenizer (configuration_moss_audio_tokenizer.py` at line 9. Add a similar header, e.g. `# Vendored from VoiceAgentGroup/moss_audio_tokenizer @ ` to aid future maintenance. (evidence: Compared with `audio_tokenizer.py` line 9 which has the vendor header; the two new v2 files lack it.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:37` [minor] — `_MossCodecStreamSession.close()` is never called, risking resource leaks. The `__init__` enters a streaming context via `ExitStack`, but `MossTTSCodecDecoder` never invokes `.close()`. If `load_weights` is re‑invoked (model reload), a new session is created while the old one’s `ExitStack` and streaming state leak. Add an explicit `close()` call in a shutdown or re‑init path, or document that the session is intentionally single‑use. (evidence: Read _MossCodecStreamSession.__init__ at line 55-57 (enters streaming context). Searched MossTTSCodecDecoder for calls to .close() — none found. The session reference _stream_session is only set, never cleared or closed.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:37` [minor] — New `_MossCodecStreamSession` (~60 lines, stateful streaming resource pool) has zero test coverage. This class manages slot acquisition, reset, step, and close; bugs here silently corrupt streaming decode or leak slots. Add a CPU‑unit test that exercises: acquire‑until‑exhaustion, step with mixed slots, release and re‑acquire, close, and the abort path via `on_requests_finished`. Grep confirms no existing test covers this class. (evidence: grep tests/ for _MossCodecStreamSession returned no matches; grep for on_requests_finished only hits voxcpm2 tests, not MOSS-TTS) + +`vllm_omni/benchmarks/patch/patch.py:?` [minor] — Fallback in benchmark PCM normalisation silently feeds wrong audio format to WER evaluator. In `async_request_openai_audio_speech` (patch.py:1243‑1251) when `_pcm_s16le_to_seed_tts_wer_bytes` raises, raw `bytes(pcm_capture)` is used. The raw PCM may be stereo 48 kHz (as configured for MOSS‑TTS‑Local‑v1.5), but downstream `compute_seed_tts_wer_metrics` assumes 24 kHz mono. This produces garbage WER scores with no warning. Set `output.tts_output_pcm_bytes = None` on failure, or add a flag so the evaluator can skip the sample instead of silently computing incorrect metrics. (evidence: Read `_pcm_s16le_to_seed_tts_wer_bytes` which normalizes to 24 kHz mono; `async_request_openai_audio_speech` at line 1243-1251 captures PCM with try/except around the normalize call.) + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:?` [minor] — Comment claims ceil division but the code uses floor division. `MossAudioTokenizerPatchedPretransform.encode` (audio_tokenizer_v2.py:905) has `# Use a ceil division … output_lengths = input_lengths // self.patch_size`. In Python `//` is floor division. If upstream padding ensures `input_lengths` is already a multiple of `patch_size`, the floor is correct and the comment is stale. Align the comment with the actual arithmetic or document the upstream padding contract. (evidence: Read `MossAudioTokenizerPatchedPretransform.encode()` at the cited location; `//` is floor division in Python, not ceil.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:?` [minor] — Channel‑count heuristic based on checkpoint path substring is fragile. In `modeling_moss_tts_codec.py` line 701, `inferred_channels = 2 if "v2" in codec_path.lower() else 1` is used as a fallback. The codec config already carries an authoritative `number_channels` (or `num_channels`) field; if the config lacks both, a renamed or symlinked checkpoint could silently flip the channel count. Prefer reading the config value directly and raise an error when it is missing, rather than guessing from the path. (evidence: Read `load_weights` method; `codec_cfg` already carries `number_channels` or `num_channels`.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:726` [minor] Vendored‑v2‑first fallback in `_build_codec` relies on exception to detect v1 layout. At `modeling_moss_tts_codec.py:726‑735` the code tries vendored v2 first and catches `Exception`. If a checkpoint has a v2‑style confi +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1139` [minor] Emission contract changed for `MossTTSRealtimeTalkerForGeneration.make_omni_output` from accumulated codes to per‑frame emission (line 1139: `per_req_codes[i] = new_codes.unsqueeze(0)`). The delay variant’s processor `ta +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:260` [minor] The codec forward now raises `RuntimeError` when `seq_token_counts` is missing, removing the old `num_scheduled_tokens` fallback to a single‑request. GPU and NPU runners pass this key, but non‑runner callers (e.g. direct +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:145` [minor] Class docstring outdated. `MossTTSCodecDecoder` (modeling_moss_tts_codec.py:145) states it “decodes … to a 24 kHz mono waveform”. The codec now supports stereo output and its sample rate is read from the config. Update t + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r6/pr4804.trace.json.gz new file mode 100644 index 000000000..ee3bb0d33 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r6/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4870.cost.json b/eval/dataset/arms/copilot_v7_goal_r6/pr4870.cost.json new file mode 100644 index 000000000..ddab59f41 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 684.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r6/runs/pr4870/run-20260812-200213-143f80", + "recorded_at": "2026-08-12T20:13:35", + "attempts": [ + { + "invocation_id": "inv-36ed84c8", + "outcome": "run:run-20260812-200213-143f80", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-200213-143f80", + "llm_calls": 57, + "input_tokens": 1310242, + "output_tokens": 77688, + "usd": 0.552665, + "source": "spans" + } + ], + "attempt_usd_total": 0.552665, + "llm_calls": 57, + "input_tokens": 1310242, + "output_tokens": 77688, + "usd": 0.552665, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-200213-143f80", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5527, + "minutes": 11.36, + "ci_minutes": 0, + "input_tokens": 1310242, + "output_tokens": 77688, + "tool_calls": 107, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5837, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 57, + "by_role": { + "agent": { + "usd": 0.540286, + "input_tokens": 1297564, + "output_tokens": 69574, + "calls": 55 + }, + "reducer": { + "usd": 0.01238, + "input_tokens": 12678, + "output_tokens": 8114, + "calls": 2 + } + }, + "cache_read_tokens": 4201600, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.76, + "gate": 1.75, + "review": 675.04, + "report": 0.06 + }, + "time_to_first_result_s": 681.6 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 25158 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4870.md b/eval/dataset/arms/copilot_v7_goal_r6/pr4870.md new file mode 100644 index 000000000..1839e5f84 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4870.md @@ -0,0 +1,36 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] gpu_ar_model_runner.py:1733 — `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` correctly replaces `seq_len`; read the full `_build_omni_model_runner_output_from_snapshot` and traced the value through `_build_omni_pooler_payload` -> `_build_omni_mm_payload` -> `to_payload_element` where `seq_len` gates per-request tensor slicing +- [validated] gpu_ar_model_runner.py:1876 — removed `seq_len = hidden_states.shape[0]`; confirmed the variable was only used in the `output_builder` closure which also had its `seq_len=` kwarg removed +- [validated] gpu_ar_model_runner.py:2042 — `output_builder` no longer passes `seq_len`; the only consumer was `_build_omni_model_runner_output_from_snapshot` which now derives it internally +- [validated] serving_speech.py:3507-3510 — `async_chunk = getattr(self.model_config, 'async_chunk', True)` with default True is correct (defensive: missing attr → treat as async-enabled → DELTA coercion); `qwen3_full_payload` gate scoped to qwen3_tts + async_chunk=False +- [validated] test_gpu_ar_model_runner.py:438-478 — new test `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` creates hidden_states shape [1,1] with total_num_scheduled_tokens=3 and codes shape [3,16], asserts per-request slicing; this exercises the exact bug scenario +- [validated] test_serving_speech.py:3453,3473 — two new tests confirm qwen3_tts async_chunk=False gets FINAL_ONLY and voxtral async_chunk=False keeps DELTA; read the test bodies and verified the mock assertions match the coercion logic +- [validated] test_gpu_ar_model_runner.py:266-668 — five existing test call sites had `seq_len=` kwargs removed; each test's hidden_states shape matched the removed seq_len value, so the internal derivation from `scheduler_output.total_num_scheduled_tokens` produces the same result +- [validated] mm_outputs.py:108-147 — read `to_payload_element`; confirmed the `seq_len` gate at line ~129 correctly slices tensors whose `shape[0] == seq_len`, and the dict recursion correctly carries `seq_len` down to nested tensors + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — The diff introduces `getattr(self.model_config, 'async_chunk', True)` in serving_speech.py line 3507. In `gpu_ar_model_runner.py`, the same attribute fallback is `False`. Since `OmniModelConfig` always sets `async_chunk` and its default is `False`, the divergent `True` default in the serving path is a latent hazard if the attribute ever becomes absent. Align the fallback to `False` to match the runner and the config, or add a comment explaining the intentional difference. (evidence: grep of `async_chunk` in gpu_ar_model_runner.py shows `getattr(self.model_config, "async_chunk", False)` at init; repo review checklist states `async_chunk` defaults to `False` in `OmniModelConfig`.) + +`tests/worker/test_gpu_ar_model_runner.py:438` [minor] — The new test `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` assigns `num_scheduled_tokens={'r1':1,'r2':1,'r3':1}`, which only exercises the case where every request has the same number of scheduled tokens. The fix uses `total_num_scheduled_tokens` as the split length, and the slices are derived from the per-request `num_scheduled_tokens`. To guard against off-by-one errors when token counts are unequal, add a parameterized variant with non-uniform counts (e.g., `{'r1':1,'r2':2,'r3':3}`) and verify the resulting slices match the request token boundaries. (evidence: PR description: debug logs showed 'each request accumulating full-batch codes.audio: (3, 16) payloads' — the real serving failure was concurrent requests with batch=3; the test covers the mismatch case (hidden_states shape 1 vs total=3) but only with uniform 1-token requests. The existing `test_build_omni_output_copies_hidden_for_partial_downstream_batch` tests varying token counts but not in the tail-only path.) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [minor] — The diff removes the `seq_len` parameter from `_build_omni_model_runner_output_from_snapshot` and now computes `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` (line 1734). This function is the central output builder that used to rely on `hidden_states.shape[0]` as the split length, which caused the bug. Add a docstring documenting that multimodal payload splitting is now driven by `scheduler_output.total_num_scheduled_tokens`, not by the shape of `hidden_states`, to prevent future misunderstandings. (evidence: Read gpu_ar_model_runner.py:1662-1740 — no docstring on the function; the new `mm_seq_len` computation at line 1734 is a critical behavioral change.) + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [nit] — The diff computes `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` at line 1734 and passes it to `to_payload_element` for slicing. The correctness relies on the invariant that `codes.audio.shape[0]` equals `total_num_scheduled_tokens`. Add an inline comment near `mm_seq_len` noting this invariant (e.g., “full-payload multimodal tensors are dimensioned to the total scheduled token count, so this is the correct split length”). (evidence: Traced the `mm_seq_len` -> `_build_omni_pooler_payload` -> `_build_omni_mm_payload` -> `to_payload_element` chain; read `to_payload_element` in mm_outputs.py:108-147 where `seq_len` gates slicing: `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len: return element[start:end].contiguous()`.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r6/pr4870.trace.json.gz new file mode 100644 index 000000000..f23a73232 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r6/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4923.cost.json b/eval/dataset/arms/copilot_v7_goal_r6/pr4923.cost.json new file mode 100644 index 000000000..59f6c969f --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 384.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r6/runs/pr4923/run-20260812-195548-92b5e4", + "recorded_at": "2026-08-12T20:02:11", + "attempts": [ + { + "invocation_id": "inv-0523ce54", + "outcome": "run:run-20260812-195548-92b5e4", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-195548-92b5e4", + "llm_calls": 44, + "input_tokens": 685395, + "output_tokens": 50946, + "usd": 0.287674, + "source": "spans" + } + ], + "attempt_usd_total": 0.287674, + "llm_calls": 44, + "input_tokens": 685395, + "output_tokens": 50946, + "usd": 0.287674, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-195548-92b5e4", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2877, + "minutes": 6.37, + "ci_minutes": 0, + "input_tokens": 685395, + "output_tokens": 50946, + "tool_calls": 88, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3473, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.275964, + "input_tokens": 673630, + "output_tokens": 43217, + "calls": 42 + }, + "reducer": { + "usd": 0.01171, + "input_tokens": 11765, + "output_tokens": 7729, + "calls": 2 + } + }, + "cache_read_tokens": 1725056, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.33, + "gate": 2.25, + "review": 376.41, + "report": 0.06 + }, + "time_to_first_result_s": 382.04 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 21908 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4923.md b/eval/dataset/arms/copilot_v7_goal_r6/pr4923.md new file mode 100644 index 000000000..b65403ec1 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4923.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] qwen3_tts_code_predictor_vllm.py:39 — `use_cuda_graphs=current_omni_platform.is_npu()` matches Qwen3-Omni pattern (`qwen3_omni_moe_code_predictor_mtp.py:21` uses identical guard); GPU path gets `False` (torch.compile only), NPU keeps internal graphs. +- [validated] qwen3_tts_talker.py:327 — `talker_mtp_graph_safe=True` enables runner-side outer `CUDAGraphWrapper` via `gpu_model_runner.py:209` (`has_full_cudagraphs() and talker_mtp_graph_safe`). +- [validated] qwen3_tts_talker.py:346 — `cudagraph_mode is not None` guard prevents AttributeError when `compilation_config.cudagraph_mode` is absent; `getattr(..., None)` is a safe fallback. +- [validated] qwen3_tts_talker.py:350-353 — `use_async_omni_output`, `eager_omni_postprocess_before_async_output`, `omni_pooler_payload_include_hidden` all match the Qwen3-Omni talker pattern (`qwen3_omni.py:158-164`). +- [sweep] Duplicate assignment detected: `omni_pooler_payload_include_hidden = False` already exists at line 317 (pre-existing); the diff adds an identical second assignment at line 353 — merge artifact from `3b2d87c9 Merge branch 'main'`. +- [sweep] YAML file header comment at line 7 still says `max_num_batched_tokens=512` but value changed to 32768 at line 48. +- [sweep] YAML inline comment block at lines 38-43 says 'Prefix caching on by default' and justifies its benefits, but `enable_prefix_caching` was changed to `false` at line 43. +- [sweep] Gate report: `buildkite/vllm-omni-npu-ci` is FAILING; diff modifies NPU platform section and `current_omni_platform.is_npu()` gate — cannot determine pre-existing vs. introduced. + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:353` [minor] — This is a duplicate assignment. `self.omni_pooler_payload_include_hidden = False` is already set at line 317 (pre-existing code above the talker_mtp block). The diff adds an identical second assignment — likely a merge artifact from `3b2d87c9 Merge branch 'main'`. Remove the duplicate; keeping both is harmless but misleading. (evidence: read_file vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py lines 315-355 shows line 317 already has `self.omni_pooler_payload_include_hidden = False`. The diff adds the same line again at new line 353.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — This PR changes the per-row generator gating, async output flags, and code-predictor graph mode without any test modifications. The existing test at `tests/worker/test_omni_gpu_model_runner.py:357` mocks `talker_mtp_accepts_per_row_generators=True` but does not exercise the new `cudagraph_mode`-dependent path. Add a test that verifies the flag is `False` when `cudagraph_mode` is `FULL` and `True` otherwise, or confirm that existing Qwen3-TTS model tests pass with the new behavior. (evidence: Read tests/worker/test_omni_gpu_model_runner.py lines 357-374 — mocks the flag as True unconditionally; no test verifies the new gating logic.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — This comment block still says 'Prefix caching on by default' and argues it's a net throughput win, but the value was changed from `true` to `false`. The comment now contradicts the config. Update it to reflect the rationale from the PR description: prefix caching is disabled to avoid bookkeeping interference with the full-graph MTP path at high concurrency (and note the follow-up planned for re-enabling). (evidence: Read deploy/qwen3_tts.yaml diff hunk at lines 38-46; the PR description table says rationale is 'Avoid prefix-cache bookkeeping interfering with full-graph MTP path at high concurrency (see follow-up below)'.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — Stale header comment: `max_num_batched_tokens` was changed from 512 to 32768 (line 48), but this header comment still claims 512 and the rationale of 'small per-step batches keep first-chunk latency low'. Update or remove the stale value and rationale to match the new 32768 setting. (evidence: read_file vllm_omni/deploy/qwen3_tts.yaml: header comment at line 7 says `max_num_batched_tokens=512`; actual value at line 48 is `max_num_batched_tokens: 32768`.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — The TODO correctly identifies that reading `cudagraph_mode` in model code duplicates the runner's decision in `_init_talker_mtp` at `gpu_model_runner.py:209`. However, the TODO references the current PR (#4923), which is circular. File a separate follow-up issue for the refactor and reference it here so this TODO does not become permanent tech debt. (evidence: Read qwen3_tts_talker.py lines 340-349 and gpu_model_runner.py lines 208-210; both independently derive the same wrap decision from cudagraph_mode.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:?` [minor] — The comment documents that `tts_local_seed` determinism is silently lost under full cudagraphs. This is a user-visible semantic weakening — any caller relying on seeded TTS output reproducibility will get different results after this change with no warning. Either link a tracking issue for the per-row-generator cudagraph-safe refactor mentioned in the TODO, or add a logged warning when `tts_local_seed` is provided and full cudagraphs are active. (evidence: Read qwen3_tts_talker.py lines 333-339 documenting the determinism loss; no corresponding test or user-facing warning exists.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r6/pr4923.trace.json.gz new file mode 100644 index 000000000..c78e7e67f Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r6/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4926.cost.json b/eval/dataset/arms/copilot_v7_goal_r6/pr4926.cost.json new file mode 100644 index 000000000..936e1fab8 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 635.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r6/runs/pr4926/run-20260812-200905-db89c1", + "recorded_at": "2026-08-12T20:19:38", + "attempts": [ + { + "invocation_id": "inv-c4f75a20", + "outcome": "run:run-20260812-200905-db89c1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-200905-db89c1", + "llm_calls": 45, + "input_tokens": 276031, + "output_tokens": 110241, + "usd": 0.243113, + "source": "spans" + } + ], + "attempt_usd_total": 0.243113, + "llm_calls": 45, + "input_tokens": 276031, + "output_tokens": 110241, + "usd": 0.243113, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-200905-db89c1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2431, + "minutes": 10.56, + "ci_minutes": 0, + "input_tokens": 276031, + "output_tokens": 110241, + "tool_calls": 85, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4371, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 45, + "by_role": { + "agent": { + "usd": 0.217245, + "input_tokens": 251364, + "output_tokens": 92811, + "calls": 43 + }, + "reducer": { + "usd": 0.025868, + "input_tokens": 24667, + "output_tokens": 17430, + "calls": 2 + } + }, + "cache_read_tokens": 1752576, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.38, + "gate": 1.73, + "review": 628.24, + "report": 0.06 + }, + "time_to_first_result_s": 633.4 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 33743 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4926.md b/eval/dataset/arms/copilot_v7_goal_r6/pr4926.md new file mode 100644 index 000000000..da820dd65 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4926.md @@ -0,0 +1,47 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] flash_attn_hub.py:30-35 — `cu_seqlens_q`/`cu_seqlens_k` arange math is correct for batch_size+1 elements +- [validated] flash_attn_hub.py:46-47 — `isinstance(out, tuple)` unwrap in `_run_varlen_dense` matches documented FA3 (out,lse) convention +- [validated] flash_attn_hub.py:108 — init RuntimeError gate fires only when both funcs are None, consistent with docstring intent +- [validated] flash_attn_hub.py:189,368 — `forward_cuda` dispatch order (piecewise→masked→dense→varlen) matches existing FlashAttentionImpl pattern +- [validated] platforms/{musa,npu,rocm,xpu}/platform.py — non-CUDA fallbacks all route HUB→FLASH_ATTN with consistent warning log +- [validated] registry.py:65-66 — enum additions correctly point to new backend class paths +- [sweep] flash_attn_hub.py:130-157 — `_forward_varlen_masked` calls `self.flash_attn_varlen_func(...)` without None guard; init allows flash_attn_varlen_func=None when flash_attn_func is present; masked path would crash with TypeError +- [sweep] flash_attn_hub.py:163-177 — `_forward_varlen_dense` same vulnerability: calls `self.flash_attn_varlen_func` without None check + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The test `test_kernels_hub_execution` (line 58) is decorated with `@hardware_test(res={'cuda': 'L4'})`, so it runs on L4 GPUs (sm_89). The FA3 path is guarded by `if major < 9: pytest.skip(...)` (line 92), which is always true on L4. The CI pipeline has no Hopper (sm_90+) lane for diffusion attention tests, so `FlashAttention3HubImpl.forward` is never exercised in CI. Either split the test (FA2 on L4, FA3 on H100) or document the gap and provide a manual verification procedure. (evidence: Read test_kernels_hub.py lines 58-103, tests/helpers/mark.py line 136 for hardware_test decorator, grepped .buildkite/*.yml and .buildkite/*.yaml for H100/diffusion/attention test lanes — none found.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — `test_kernels_hub_execution` (lines 64–103) uses `k = q.clone()` and `v = q.clone()` (identity Q=K=V), `batch_size=1`, `num_heads=8`, no `num_kv_heads` (so no GQA), `causal=False`, and `attn_metadata.attn_mask=None`. It also accepts a loose tolerance `max_diff < 1e-2` (line 87). This does not exercise GQA, causal masking, attention masks, or the `piecewise_attn` path — all present in the implementation. Use independent random tensors for k/v, add GQA (e.g., `num_heads=8, num_kv_heads=2`), add `causal=True` and a non-trivial attention mask test, and tighten or document the tolerance rationale. (evidence: Read test_kernels_hub.py lines 64-103; FlashAttentionHubImpl __init__ signature accepts num_kv_heads (line 79) but test never passes it; AttentionMetadata at line 76 has attn_mask=None.) + +`tests/diffusion/attention/test_kernels_hub.py:58` [major] — `test_kernels_hub_execution` only tests the dense forward path (`attn_metadata.attn_mask=None`, `full_attn_spans=None`). The `_forward_varlen_masked` path (flash_attn_hub.py lines 130–157 for FA2, 312–339 for FA3) and the `piecewise_attn` path (lines 189–201 and 368–380) are completely untested. A bug in these branches — wrong kwargs, missing unwrap, shape mismatch — would not be caught. Add test cases that set a non-trivial attention mask and `full_attn_spans` list to exercise these paths against an SDPA reference. (evidence: Read test_kernels_hub.py lines 59-104: only creates AttentionMetadata(attn_mask=None); read flash_attn_hub.py lines 189-221: forward_cuda has three distinct paths (piecewise, masked, dense) but only dense is tested.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [major] — In `FlashAttentionHubImpl.__init__` (line 108), the gate `if self.flash_attn_func is None and self.flash_attn_varlen_func is None` allows initialisation when only `flash_attn_func` is available and `flash_attn_varlen_func` is `None`. However `_forward_varlen_masked` (line 150) and `_forward_varlen_dense` (line 169) call `self.flash_attn_varlen_func` unconditionally, causing a `TypeError` on any request with a non-trivial attention mask. The same issue exists in `FlashAttention3HubImpl` (init gate line 287, calls at lines 328 and 348). Fix: raise `RuntimeError` in `__init__` if `self.flash_attn_varlen_func is None`, or guard the calls with a fallback / descriptive error. (evidence: Read flash_attn_hub.py lines 104-108: gate is `if self.flash_attn_func is None and self.flash_attn_varlen_func is None`; read lines 130-157 and 163-177: both call `self.flash_attn_varlen_func(...)` without None check.) + +`.buildkite/test-ready.yml:?` [major] — The CUDA CI lane in `.buildkite/test-ready.yml` (line 145) runs `pytest tests/diffusion/*.py` — a non-recursive glob that does not collect `tests/diffusion/attention/test_kernels_hub.py`. The new Hub backend execution test is therefore never executed on real GPU hardware in CI, even on L4. Change the glob to `tests/diffusion/` (recursive) or add a separate step for `tests/diffusion/attention/`. (evidence: grep of .buildkite/ found zero references to tests/diffusion/attention or test_kernels_hub; test-ready.yml:145 is the only CUDA lane covering tests/diffusion/ and uses a non-recursive *.py glob.) + +`pyproject.toml:?` [major] — `pyproject.toml` line 70 declares only `kernels<0.15.0` with no lower bound. The code calls `get_kernel(..., version=1)` and `version=2` (flash_attn_hub.py lines 96, 101, 275, 280). If an older `kernels` release that lacks the `version` keyword argument is installed, the call fails with a misleading `TypeError`. Add a lower bound (e.g., `>=0.14.0`, matching the documented version `0.14.1`) to prevent runtime failures on incompatible versions. (evidence: Read pyproject.toml line 70; read flash_attn_hub.py lines 94-102 and 273-281 which call get_kernel(..., version=1/2).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:113` [minor] — In `FlashAttentionHubImpl._unwrap_flash_output` (line 113), the comment says 'FA3 may return (out, lse), FA2 returns out'. This is inside the FA2 Hub implementation — the FA3 reference is wrong. The comment was likely copied from `FlashAttentionImpl` and not updated. Reword to: 'Hub kernel may return (out, lse) or a plain tensor; unwrap if needed.' (evidence: Read flash_attn_hub.py:113-114 and compared with flash_attn.py:76-77 — the identical comment appears on the FA2 local impl too, but in the Hub FA2 context the 'FA3' reference is misleading.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:250` [minor] — `FlashAttentionHubImpl` (lines 71–221) and `FlashAttention3HubImpl` (lines 250–400) duplicate ~150 lines of forward logic: `_flash_wrapper`, `_forward_varlen_masked`, `_forward_varlen_dense`, `forward_cuda`, and `_unwrap_flash_output` are identical except for the kernel repo string ('flash-attn2' vs 'flash-attn3') and log messages. Extract a shared mixin or base class (e.g., `_HubFlashAttentionImpl`) to avoid drift — every bug fix to one must be mirrored to the other. (evidence: Read flash_attn_hub.py in full; both impl classes share _forward_varlen_masked (lines 123-160 vs 302-339), _forward_varlen_dense (lines 162-175 vs 341-354), forward_cuda (lines 177-221 vs 356-400), _flash_wrapper (lines 115-126 vs 294-305), _unwrap_flash_output (lines 112-114 vs 291-293).) + +**Additional observations (beyond the comment budget):** +- `docs/user_guide/diffusion/attention_backends.md:236` [minor] The docs (attention_backends.md line 236) claim 'perfect numerical consistency' and 'eliminates numerical drift', but the only evidence in this PR is a single-shape smoke test (batch_size=1, seq_len=32, identical Q=K=V, + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r6/pr4926.trace.json.gz new file mode 100644 index 000000000..85766b6e8 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r6/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4977.cost.json b/eval/dataset/arms/copilot_v7_goal_r6/pr4977.cost.json new file mode 100644 index 000000000..d89b52c35 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 673.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r6/runs/pr4977/run-20260812-195751-4e36be", + "recorded_at": "2026-08-12T20:09:02", + "attempts": [ + { + "invocation_id": "inv-12b88965", + "outcome": "run:run-20260812-195751-4e36be", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-195751-4e36be", + "llm_calls": 55, + "input_tokens": 181493, + "output_tokens": 81761, + "usd": 0.169661, + "source": "spans" + } + ], + "attempt_usd_total": 0.169661, + "llm_calls": 55, + "input_tokens": 181493, + "output_tokens": 81761, + "usd": 0.169661, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-195751-4e36be", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1697, + "minutes": 11.18, + "ci_minutes": 0, + "input_tokens": 181493, + "output_tokens": 81761, + "tool_calls": 97, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4163, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 55, + "by_role": { + "agent": { + "usd": 0.159634, + "input_tokens": 171590, + "output_tokens": 75108, + "calls": 53 + }, + "reducer": { + "usd": 0.010027, + "input_tokens": 9903, + "output_tokens": 6653, + "calls": 2 + } + }, + "cache_read_tokens": 1137792, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.33, + "gate": 1.96, + "review": 665.63, + "report": 0.07 + }, + "time_to_first_result_s": 670.98 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 20476 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4977.md b/eval/dataset/arms/copilot_v7_goal_r6/pr4977.md new file mode 100644 index 000000000..48afbf1ad --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr4977.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [validated] flash_attn_hub.py:26-39 _load_hub_module: version-fallback loop (1, 2, None) — all three branches reachable; last_error chains correctly into RuntimeError on total failure; no fabricated fallback values. +- [validated] flash_attn_hub.py:42-45 _get_hub_module: lock+dict cache; _load_hub_module called inside lock → no TOCTOU; failed loads not cached (assignment skipped on exception) → retry on next call; correct. +- [validated] flash_attn_hub.py:119,284: call sites in FlashAttentionHubImpl.__init__ and FlashAttention3HubImpl.__init__ replaced inline get_kernel with _get_hub_module — behaviorally equivalent; downstream getattr+None-check unchanged. +- [sweep] flash_attn_hub.py:33 `if version is not None:` — both branches reachable (version 1,2 vs None); no dead code. +- [sweep] flash_attn_hub.py:44 `if repo_id not in _hub_modules:` — first call loads, subsequent calls hit cache; branch semantics correct. +- [sweep] grep for get_kernel repo-wide: only caller is _load_hub_module in flash_attn_hub.py; no other direct get_kernel sites to update. +- [sweep] grep for FlashAttentionHubImpl/FlashAttention3HubImpl: only referenced within flash_attn_hub.py and registry.py; no consumer left stale. +- [sweep] pyproject.toml: kernels<0.15.0 in dev extras; no lower bound — pre-existing, not introduced by this PR. The old code already used get_kernel(version=...) API. + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The test `tests/diffusion/attention/test_kernels_hub.py::test_kernels_hub_execution` is never selected by any CI lane. No lane targets `tests/diffusion/attention/`; the existing `tests/diffusion/*.py` shell glob in "Diffusion · Other Test" does not match subdirectories, and the `-m 'core_model and cpu'` lane deselects it because the test carries `cuda` and `L4` markers. Consequently the entire `_get_hub_module` → `FlashAttentionHubImpl.__init__` path introduced by this PR has zero CI coverage. Fix: add `tests/diffusion/attention/` to a CUDA CI lane, or add a `@pytest.mark.cpu` marker and ensure the kernel load works in CPU-only mode. (evidence: Grep of .buildkite/test-ready.yml shows no lane targeting tests/diffusion/attention/; the closest is 'tests/diffusion/*.py' (shell glob, no subdirs) and 'tests/diffusion' (recursive but with -m 'core_model and cpu' which misses this test's cuda+L4 marks).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [major] — The PR introduces a process-level cache (`_hub_modules`) to avoid redundant kernel loads, but the existing test `test_kernels_hub_execution` constructs only one `FlashAttentionHubImpl` and one `FlashAttention3HubImpl` — it cannot distinguish a cached load from a fresh load. Add a test that constructs two instances of the same backend (e.g., two `FlashAttentionHubImpl` objects) and asserts they share the same kernel object (e.g., `fa_impl1.flash_attn_func is fa_impl2.flash_attn_func` or verify that `len(_hub_modules)` does not increase). This is essential to validate the PR's core claim. Evidence: test file lines 54–93 show a single instantiation per backend. (evidence: Read tests/diffusion/attention/test_kernels_hub.py lines 79-93: only one `FlashAttentionHubImpl(...)` instantiation, no second construction, no cache-hit assertion.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:22` [minor] — The four new module-level items (`_hub_modules`, `_hub_lock`, `_load_hub_module`, `_get_hub_module`) have zero docstrings. Document the caching contract: process-lifetime, thread-safe single-load, and the version-fallback order (v1 → v2 → default). A future reader investigating kernel-update behaviour would benefit from knowing the cache is never invalidated and the fallback order is intentional. Evidence: reading `flash_attn_hub.py` lines 19–47 confirms no docstrings exist. (evidence: Read flash_attn_hub.py lines 19–47; grep for `_hub_modules|_hub_lock|_load_hub_module|_get_hub_module` in file confirms no docstrings exist on any of the four new items.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:43` [minor] — `_load_hub_module` performs network/Hub I/O while holding the global `_hub_lock`. This unnecessarily serializes loads of different repo_ids (e.g., `flash-attn2` and `flash-attn3`). Consider using per-repo locks or a load-then-insert pattern so that distinct repos can be loaded concurrently, improving startup parallelism in multi-worker scenarios. Evidence: `_hub_lock` is a single `threading.Lock()` and `_load_hub_module` calls `get_kernel` which may download/load from the network. (evidence: `_hub_lock` is a single `threading.Lock()` guarding the entire `_hub_modules` dict; `_load_hub_module` calls `get_kernel(repo_id, ...)` which may download/load from network. All repo_ids contend on the same lock.) + +`.github/PULL_REQUEST_TEMPLATE.md:?` [minor] — The PR description states this PR adds `trust_remote_code=True`, but commit 9947f414 ("Drop trust_remote_code from hub kernel loading") removed it from the final diff. Update the description to reflect the actual change: the final diff is a lock-protected process-level cache with version fallback and no `trust_remote_code` argument anywhere. Evidence: commit timeline shows the drop; diff grep for `trust_remote_code` returns zero matches. (evidence: commit timeline: 9947f414 'Drop trust_remote_code from hub kernel loading'; diff grep for trust_remote_code returns zero matches) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [minor] — The `buildkite/vllm-omni-npu-ci` check is failing on this PR. The NPU platform (`npu/platform.py` lines 108-116) force-redirects `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` to `FLASH_ATTN` before the hub-loading code changed here is ever reached. This failure is almost certainly pre-existing on main; please confirm the same lane is also red on main before merging so the failure is not incorrectly attributed to this PR. Evidence: gate report shows FAILING; npu/platform.py falls back hub backends to local FA with a warning. (evidence: Gate report shows buildkite/vllm-omni-npu-ci FAILING; npu/platform.py:110 force-falls back FLASH_ATTN_HUB/FLASH_ATTN_3_HUB before hub loading code is reached.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r6/pr4977.trace.json.gz new file mode 100644 index 000000000..7c73584d0 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r6/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr5009.cost.json b/eval/dataset/arms/copilot_v7_goal_r6/pr5009.cost.json new file mode 100644 index 000000000..4d84eafda --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 559.6, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r6/runs/pr5009/run-20260812-194831-80423e", + "recorded_at": "2026-08-12T19:57:49", + "attempts": [ + { + "invocation_id": "inv-a40360b4", + "outcome": "run:run-20260812-194831-80423e", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-194831-80423e", + "llm_calls": 57, + "input_tokens": 730428, + "output_tokens": 76498, + "usd": 0.354472, + "source": "spans" + } + ], + "attempt_usd_total": 0.354472, + "llm_calls": 57, + "input_tokens": 730428, + "output_tokens": 76498, + "usd": 0.354472, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-194831-80423e", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3545, + "minutes": 9.3, + "ci_minutes": 0, + "input_tokens": 730428, + "output_tokens": 76498, + "tool_calls": 114, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4549, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 57, + "by_role": { + "agent": { + "usd": 0.341873, + "input_tokens": 717550, + "output_tokens": 68234, + "calls": 55 + }, + "reducer": { + "usd": 0.012599, + "input_tokens": 12878, + "output_tokens": 8264, + "calls": 2 + } + }, + "cache_read_tokens": 2707712, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.78, + "gate": 2.23, + "review": 551.65, + "report": 0.07 + }, + "time_to_first_result_s": 557.71 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 25847 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr5009.md b/eval/dataset/arms/copilot_v7_goal_r6/pr5009.md new file mode 100644 index 000000000..d8678f3b9 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r6/pr5009.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 5 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] platform.py:251-262 — `get_default_ir_op_priority` now always returns `["vllm_c", "native"]`, removing the `using_inductor` branch; read full method, confirmed oink path unchanged +- [validated] qwen_image_transformer.py:17 — removed `from vllm.model_executor.layers.layernorm import RMSNorm`; grep confirmed no remaining references to this import in qwen_image/ +- [validated] qwen_image_transformer.py:554-558 — `self.norm_q` and `self.norm_k` now use `nn.RMSNorm(head_dim, eps=eps)` instead of vLLM `RMSNorm`; read in situ +- [validated] qwen_image_transformer.py:592-596 — `self.norm_added_q` and `self.norm_added_k` now use `nn.RMSNorm(head_dim, eps=eps)`; read in situ +- [validated] qwen_image_transformer.py:1006 — `self.txt_norm` now uses `nn.RMSNorm(joint_attention_dim, eps=1e-6)`; read in situ +- [validated] test_cuda_ir_op_priority.py:23-28 — parametrized test asserts `["vllm_c", "native"]` for all three CompilationMode values, correctly validating the fix +- [validated] test_cuda_ir_op_priority.py:30-38 — oink monkeypatch test asserts `["oink", "vllm_c", "native"]`, preserving existing env-var behavior +- [sweep] rocm/platform.py:173-196 — ROCm platform still uses inductor-aware `default = ["native"] if using_inductor else ["vllm_c", "native"]`; not touched by this CUDA-scoped PR + +`vllm_omni/platforms/cuda/platform.py:252` [minor] — The docstring describes the new default but does not mention that per-model overrides (e.g., Cosmos3's `ir_op_priority_func` in registry.py) can override these settings. Consider adding a note linking to the registry or the resolution chain, so maintainers understand how to override for models that require native-only. (evidence: Read `vllm_omni/diffusion/registry.py:546-549` showing only Cosmos3 registers an override; read `vllm_omni/diffusion/worker/diffusion_worker.py:174-178` showing the two-step resolution: platform default → per-model merge.) + +`vllm_omni/platforms/cuda/platform.py:251` [minor] — The unconditional default `["vllm_c", "native"]` now applies to all ~40 CUDA diffusion models; only Cosmos3 has an `ir_op_priority_func` override (registry.py:546-549). The PR validated only Qwen-Image. To avoid regressions on other models (e.g., Wan, Flux), either validate at least one additional model family with inductor active or scope the change to Qwen-Image via a model-specific override. A suggested sweep: `pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level 'core_model' -k 'not cosmos3'`. (evidence: Read vllm_omni/platforms/cuda/platform.py:249-270 (new `get_default_ir_op_priority`); confirmed Cosmos3 override at pipeline_cosmos3.py:612-624; grep confirmed no other per-model ir_op_priority_func overrides in registry.py:546-549. PR description acknowledges scope is 'all diffusion models using vLLM RMSNorm under inductor' but only provides Qwen-Image profiler/serving evidence.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — The replacement of vLLM `RMSNorm` with `nn.RMSNorm` restores accuracy (SSIM/PSNR recovery) but lacks a CI guard against future reversions. Add a regression assertion in an existing Qwen-Image test: either enforce `isinstance(norm, torch.nn.RMSNorm)` or set a minimum SSIM/PSNR threshold that would fail on a revert, to prevent issues like #3588. (evidence: Read qwen_image_transformer.py: lines 554-558, 592-596, 1006-1009 — all 5 RMSNorm usages are now `nn.RMSNorm`. PR description shows manual accuracy recovery. Grep of tests/ found no test asserting nn.RMSNorm type or guarding the PSNR threshold.) + +`tests/platforms/test_cuda_ir_op_priority.py:9` [minor] — The new test file uses `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`. If the CI pipeline’s marker expression does not match both `core_model` and `cpu`, this test will be skipped without detection, leaving the regression unguarded. Verify the Buildkite CUDA CI pipeline marker selection and adjust the test markers or CI config so that this test actually executes in CI. (evidence: Read test_cuda_ir_op_priority.py:9; searched for other test files using similar markers (tests/platforms/npu/* uses the same pattern). The main vllm-omni CI lane is green, but whether this specific test executed is not evident from the CI summary alone.) + +`tests/platforms/test_cuda_ir_op_priority.py:19` [minor] — The parametrization covers CompilationMode.NONE, VLLM_COMPILE, STOCK_TORCH_COMPILE but omits PIECEWISE_COMPILE (mode=2), which is also inductor-active. The old inductor-aware branch would have switched to native-only for PIECEWISE_COMPILE, so the test should verify the fix for that mode as well. Add PIECEWISE_COMPILE to the parametrization to ensure complete coverage. (evidence: Test parametrizes [NONE(0), VLLM_COMPILE(1), STOCK_TORCH_COMPILE(3)]; PIECEWISE_COMPILE(2) is inductor-active and not covered.) + +`.buildkite/test-nightly.yml:?` [minor] — The gate report shows `buildkite/vllm-omni-amd-ci`, `buildkite/vllm-omni-intel-ci`, `buildkite/vllm-omni-npu-ci` failing. This PR touches only CUDA platform code and Qwen-Image. Confirm that the same failures exist on the main branch. If they are new, the merge of main (commit 7314f7e) may have introduced a regression; otherwise they are pre-existing and non-blocking. (evidence: Gate report lists 3 failing checks. Diff touches only CUDA platform.py and CUDA-only Qwen-Image transformer. PR commit timeline shows '7314f7e8 Merge branch main' — if main recently broke AMD/Intel/NPU, that merge would carry the breakage.) + +`vllm_omni/platforms/cuda/platform.py:252` [nit] — The old docstring (removed in this diff) explained the inductor-aware priority logic. Now that the branch is removed, the docstring is minimal. For future maintainers, add a brief note why vllm_c is preferred: diffusion denoise loops are RMSNorm-heavy and vllm_c fused kernels outperform native decomposition even under inductor (e.g., `# Always prefer vllm_c; diffusion denoise loops are RMSNorm-heavy.`). (evidence: Read platform.py:251-262; the old docstring (visible in the diff's removal lines) explained the inductor-aware logic; the replacement is minimal.) + +`vllm_omni/platforms/cuda/platform.py:251` [nit] — The `vllm_config` parameter is now unused because the function returns a constant default. It is kept for interface compatibility with the `OmniPlatform` base class. Add a brief comment (e.g., `# vllm_config kept for interface compatibility; this implementation is mode-independent.`) to clarify. (evidence: Read the full function body at `platform.py:251-260` — `vllm_config` is accepted but never dereferenced after the removal of the `using_inductor` branch.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r6/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r6/pr5009.trace.json.gz new file mode 100644 index 000000000..15a552908 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r6/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/manifest.json b/eval/dataset/arms/copilot_v7_goal_r7/manifest.json new file mode 100644 index 000000000..b4df31121 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v7_goal_r7", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4804.cost.json b/eval/dataset/arms/copilot_v7_goal_r7/pr4804.cost.json new file mode 100644 index 000000000..d842dcec4 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4804.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 417.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r7/runs/pr4804/run-20260812-204055-6438b0", + "recorded_at": "2026-08-12T20:47:50", + "attempts": [ + { + "invocation_id": "inv-d1aa150f", + "outcome": "run:run-20260812-204055-6438b0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-204055-6438b0", + "llm_calls": 42, + "input_tokens": 782680, + "output_tokens": 52131, + "usd": 0.3848, + "source": "spans" + } + ], + "attempt_usd_total": 0.3848, + "llm_calls": 42, + "input_tokens": 782680, + "output_tokens": 52131, + "usd": 0.3848, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-204055-6438b0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3848, + "minutes": 6.92, + "ci_minutes": 0, + "input_tokens": 782680, + "output_tokens": 52131, + "tool_calls": 84, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.402, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 42, + "by_role": { + "agent": { + "usd": 0.355398, + "input_tokens": 721479, + "output_tokens": 40456, + "calls": 40 + }, + "reducer": { + "usd": 0.029401, + "input_tokens": 61201, + "output_tokens": 11675, + "calls": 2 + } + }, + "cache_read_tokens": 4301184, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.55, + "gate": 1.77, + "review": 408.61, + "report": 0.06 + }, + "time_to_first_result_s": 414.98 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 64315 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4804.md b/eval/dataset/arms/copilot_v7_goal_r7/pr4804.md new file mode 100644 index 000000000..a7532031d --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4804.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4804, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 6 finding(s) below | +| Behavior / compatibility | 4 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [validated] modeling_moss_tts_codec.py L726-732: New `_build_codec` tries vendored v2 first, falls back to legacy on exception. Shape mismatch check in load_weights provides partial safety, but repo checklist requires explicit config-marker gating for codec selection. Not a blocker since mismatched weights will fail the new shape check, but the try-except pattern violates the stated design rule. +- [validated] modeling_moss_tts_codec.py L34-130: _MossCodecStreamSession acquires streaming context in __init__ and releases in close(). Resource lifecycle is symmetric with on_requests_finished() covering the abort/disconnect path. gpu_generation_model_runner.py L79 correctly calls model.on_requests_finished() before input_batch.remove_request() so the model can clean up slots while req_id is still known. +- [validated] stage_input_processors/moss_tts.py L217-302: talker2codec_raw_async_chunk correctly handles the finish-sentinel path (pending<=0 && is_finished) with meta.code_flat_numel=0, which the codec forward() at L293 checks to skip decode. +- [validated] serving_speech.py L3708: `copy.deepcopy(sampling_params_list)` is now unconditional. Previously guarded by `is self.engine_client.default_sampling_params_list` identity check. The old guard prevented mutating the shared default; removing it avoids mutation of the canonical default but pays a copy cost on every request including non-TTS ones that reach this path. +- [sweep] Diff hunk patch.py L137-160: _pcm_s16le_to_seed_tts_wer_bytes correctly handles empty input, multi-channel downmix, and resampling. AudioResampler import is inside the conditional block, avoiding import on the fast path. The except fallback at L1251 returns raw bytes — semantically type-valid but may cause downstream WER mismatch if normalization fails. Acceptable since the warning is logged. +- [sweep] audio_tokenizer_v2.py L1959 lines: Vendored from upstream MOSS-Audio-Tokenizer. No vendoring header found — repo checklist requires 'Vendored from @ ' header. Checked against existing audio_tokenizer.py which has such a header. +- [sweep] pipeline.py L31-94: MOSS_TTS_PIPELINE now uses talker2codec_delay_async_chunk, MOSS_TTS_REALTIME_PIPELINE and MOSS_TTS_LOCAL_PIPELINE use talker2codec_raw_async_chunk. The sync fallback talker2codec is unchanged. Verified renamed function exists as talker2codec_delay_async_chunk in moss_tts.py (previously talker2codec_async_chunk). +- [sweep] gpu_generation_model_runner.py L76-80: on_requests_finished call added before remove_request. Correct ordering — model needs req_id to know which slot to release. + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:?` [major] — In `_build_codec`, the bare `except Exception` catches *all* errors during v2 instantiation (including transient errors like missing attributes or OSError) and silently falls back to the v1 codec, which has a different decoder layout and could produce structurally wrong audio for a v2 checkpoint. Although `load_weights` now checks shape consistency, a mis-loaded codec could still pass that check. Narrow the catch to specific exception types (e.g., `ValueError`, `KeyError`) that indicate missing v2 support, and re-raise unexpected errors. (evidence: Read modeling_moss_tts_codec.py lines 726-736; the bare `except Exception` catches all errors including ImportError, OSError, etc.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:?` [major] — In `talker2codec_raw_async_chunk`, `transfer_manager.put_req_chunk` is initialised as `defaultdict(int)` but never incremented. Consequently `emitted_any = int(transfer_manager.put_req_chunk.get(req_id, 0)) > 0` is always `False`, so the chunk threshold stays at `initial_codec_chunk_frames` (1) and never graduates to `codec_chunk_frames` (15). Add `transfer_manager.put_req_chunk[req_id] = transfer_manager.put_req_chunk.get(req_id, 0) + 1` after emitting a chunk (after `del pending_frames[:emit_frames]`). (evidence: Grep for `put_req_chunk` across vllm_omni shows all other stage input processors increment it after emitting. The moss_tts raw processor only reads it at line 287 but never writes to it — the `if not hasattr(transfer_manager, 'put_req_chunk'): transfer_manager.put_req_chunk = defaultdict(int)` at line 245-246 creates it, but no subsequent `transfer_manager.put_req_chunk[req_id] += 1` or assignment exists.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:248` [major] — `talker2codec_raw_async_chunk` stores per-request pending frames in `transfer_manager.code_prompt_token_ids[req_id]` and only cleans them when `is_finished=True`. If a request is aborted (client disconnect, engine abort), that cleanup never executes, causing a memory leak. Wire a cleanup hook to the stage connector's request-finished callback that releases the entry. (evidence: Read stage_input_processors/moss_tts.py lines 234-262; cleanup only on is_finished=True at line 297; no abort path exists.) + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:225` [major] — The new function `talker2codec_raw_async_chunk` implements critical streaming chunk logic (~120 lines) with zero test coverage. Add a unit test that exercises buffering, chunk emission with initial/steady thresholds, and the finish-sentinel path. (evidence: grep for talker2codec_raw_async_chunk in tests/ returned no matches. grep for moss_tts in .buildkite/ returned no matches.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:37` [major] — The `_MossCodecStreamSession` class, which manages RingKVCache streaming state for the v2 codec, has no test coverage. Add a test that exercises the acquire/release lifecycle, including slot reuse and the reset path on abort, to prevent slot leaks. (evidence: grep for _MossCodecStreamSession or MossCodecStreamSession in tests/ returned no matches.) + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — The diff removes the `if self._tts_model_type == "qwen3_tts":` guard and the identity check for the default sampling_params_list. Now `copy.deepcopy(sampling_params_list)` runs on every request with a seed, and `tts_local_seed` is set for all TTS models. While `tts_local_seed` is only consumed in the `talker_mtp` path (`gpu_model_runner.py:1826`) and is harmless elsewhere, the unconditional deepcopy adds per-request allocation overhead. Either restore a guard for models that don't need `talker_mtp` seed, or add a comment explaining the intentional scope expansion. (evidence: Grep for `tts_local_seed` shows only two references in the repo: the writer here and the reader in gpu_model_runner.py:1826 `extra_args.get('tts_local_seed')`. The old key `qwen3_tts_request_seed` has zero remaining references — the rename is complete but the scope expansion from qwen3_tts-only to all-models is unverified.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:?` [minor] — The expression `info_dict.get('max_new_frames', [-1])[0]` does not guard against an empty list. The old code used `if isinstance(..., (list, tuple)) and max_new_frames_req: max_new_frames_req = max_new_frames_req[0]`. An empty `max_new_frames` list would cause an `IndexError`. Use a length guard such as `info_dict.get('max_new_frames') or [-1]` to keep the effective default. (evidence: Diff of modeling_moss_tts_talker.py shows the old code had `max_new_frames_req = info_dict.get('max_new_frames')` followed by `if isinstance(max_new_frames_req, (list, tuple)) and max_new_frames_req: max_new_frames_req = max_new_frames_req[0]` then `try: max_new_frames = int(max_new_frames_req) ...`. The new one-liner `info_dict.get('max_new_frames', [-1])[0]` drops the empty-list guard.) + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:?` [minor] — The `seq_token_counts` kwarg is now mandatory — a missing value raises a `RuntimeError`. The dummy/profiling path in `gpu_generation_model_runner.py` was updated, but any other internal caller that relied on the old fallback will now crash quickly. Document this requirement in the method docstring. (evidence: diff has 'if token_counts is None: raise RuntimeError(...)' replacing old 'if isinstance(num_scheduled_tokens, ...) else offsets = [0, int(ids_flat.shape[0])]') + +`vllm_omni/model_executor/models/moss_tts/audio_tokenizer_v2.py:1` [minor] — Missing vendoring header required by repo checklist (cf. the existing 'Vendored from' header in audio_tokenizer_v2.py). Add a header line like '# Vendored from @ '. (evidence: [sweep] new file audio_tokenizer_v2.py:1-15 lacks 'Vendored from OpenMOSS-Team/...' header present in audio_tokenizer.py:9 — minor doc compliance) + +`vllm_omni/model_executor/models/moss_tts/configuration_moss_audio_tokenizer_v2.py:28` [minor] — Docstring states sampling_rate defaults to 24000, but the actual default in the signature (line 97) is 48000. Update the docstring to match the code. (evidence: [validated] configuration_moss_audio_tokenizer_v2.py:28-29 — docstring says sampling_rate defaults to 24000; read signature line 97 which shows default=48000; docstring and code disagree.) + +`vllm_omni/model_executor/models/moss_tts/configuration_moss_audio_tokenizer_v2.py:345` [minor] — The codebook_size property default says 4096, but the quantizer_kwargs default codebook_size (line 334) is 1024. This discrepancy may mislead users about the actual token codebook size. (evidence: [validated] configuration_moss_audio_tokenizer_v2.py:345 — codebook_size property default says 4096 but quantizer_kwargs default codebook_size is 1024; read line 334.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/model_executor/models/moss_tts/configuration_moss_audio_tokenizer_v2.py:28` [minor] The class docstring claims `sampling_rate` defaults to 24000, but the `__init__` signature (line 97) defaults to 48000. Update the docstring to match the actual default (48000) used by the v2 tokenizer. +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:137` [minor] The `MossTTSCodecDecoder` class docstring says it decodes to “24 kHz mono waveform”, but with the added v2 codec the output can be 48 kHz stereo. Update the docstring to reflect that the format depends on the codec versi +- `vllm_omni/model_executor/models/moss_tts/configuration_moss_audio_tokenizer_v2.py:334` [minor] The `codebook_size` property fallback is 4096, but the `quantizer_kwargs` default in `__init__` is 1024. If the key is missing from `quantizer_kwargs`, the wrong value will be returned. Align the fallback with the actual + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4804.trace.json.gz new file mode 100644 index 000000000..134b016d4 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4810.cost.json b/eval/dataset/arms/copilot_v7_goal_r7/pr4810.cost.json new file mode 100644 index 000000000..0d810f9a6 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4810.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 437.2, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r7/runs/pr4810/run-20260812-211133-2fbff3", + "recorded_at": "2026-08-12T21:18:48", + "attempts": [ + { + "invocation_id": "inv-2adf78ee", + "outcome": "run:run-20260812-211133-2fbff3", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-211133-2fbff3", + "llm_calls": 43, + "input_tokens": 730065, + "output_tokens": 64096, + "usd": 0.331601, + "source": "spans" + } + ], + "attempt_usd_total": 0.331601, + "llm_calls": 43, + "input_tokens": 730065, + "output_tokens": 64096, + "usd": 0.331601, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-211133-2fbff3", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3316, + "minutes": 7.24, + "ci_minutes": 0, + "input_tokens": 730065, + "output_tokens": 64096, + "tool_calls": 66, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3905, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.318472, + "input_tokens": 718527, + "output_tokens": 55015, + "calls": 41 + }, + "reducer": { + "usd": 0.013129, + "input_tokens": 11538, + "output_tokens": 9081, + "calls": 2 + } + }, + "cache_read_tokens": 2369536, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 9.8, + "gate": 1.62, + "review": 423.16, + "report": 0.06 + }, + "time_to_first_result_s": 434.63 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 25992 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4810.md b/eval/dataset/arms/copilot_v7_goal_r7/pr4810.md new file mode 100644 index 000000000..b7ee0c670 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4810.md @@ -0,0 +1,48 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 8 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] hunyuan_image3.py:222 — removed get_cache_scale branch; vLLM DefaultModelLoader applies cache_scale_mapper before calling load_weights, and the fallthrough `maybe_remap_kv_scale_name` in the `else` block handles any remaining unmapped names. Logic correct. +- [validated] mammoth_moda2.py:486-488 — new `get_cache_scale_mapper().apply(weights)` before the loop; both None guards (quant_config, mapper) are needed and correctly ordered. Removed old per-weight get_cache_scale call inside the loop. +- [validated] mimo_audio_llm.py:1156-1158 — same mapper pattern as MammothModa2; mapper runs before the name.startswith('model.') prefix logic, which is the correct ordering. +- [validated] qwen2_old.py:338 — removed getattr-guarded get_cache_scale fallback; Qwen2ForCausalLM.load_weights uses AutoWeightsLoader which runs the mapper upstream, so the inner Qwen2Model.load_weights never sees unmapped cache-scale names. Removal is correct. +- [sweep] Checked all four new branch conditions (sweep_targets): mammoth_moda2.py:486 (quant_config is not None) and :488 (cache_scale_mapper is not None) — both arms can occur independently; mimo_audio_llm.py:1156 and :1158 same. No dead branches. +- [sweep] Verified none of the removed get_cache_scale calls were the only path for loading any weight — each loader's fallthrough (maybe_remap_kv_scale_name or default_weight_loader) handles the renamed scale names. +- [sweep] Read hunyuan_image3.py offset 200-400: verified removal of get_cache_scale branch at ~line 225, replacement with comment about AutoWeightsLoader +- [sweep] Read mammoth_moda2.py offset 470-540: verified new get_cache_scale_mapper() application at lines 486-489 + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:?` [blocker] — The PR fixes four AR loader files but misses the HunyuanImage3 diffusion (DiT) transformer loader at `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238`, which still calls `self.quant_config.get_cache_scale(name)`. A repo‑wide grep at the PR head confirms this is the only remaining `get_cache_scale` caller. Quantized DiT checkpoints will hit the same `AttributeError` this PR aims to fix. The PR’s own test output on main showed this file FAILING the `test_model_loaders_do_not_call_removed_get_cache_scale` check, yet it was dropped from the final `_STALE_API_FILES`. Either apply the same mapper pattern as MammothModa2/MiMoAudio in this loader, or add a `# TODO` comment and a tracking issue if the diffusion fix is deferred. Merging without addressing this leaves a known regression. (evidence: grep -rn 'get_cache_scale(' vllm_omni/ at PR head f9665cc2: only match is vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238. The four AR loaders in the diff are clean.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:78` [major] — The `_STALE_API_FILES` list in `test_model_loaders_do_not_call_removed_get_cache_scale` omits `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py`, which still calls `.get_cache_scale(` at line 2238. The PR’s own test output on main showed this file FAILING that check, yet it was excluded. If the diffusion loader is intentionally out of scope, add a comment explaining why and a tracking issue, but the test must include this file to document the known remaining caller and prevent silent regression. If it should have been fixed, add it to the list and fix the loader. (evidence: PR description test output shows this file fails the grep check on main (it contains `.get_cache_scale(`). The current `_STALE_API_FILES` omits it, so the test won't catch regressions in this file.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:15` [minor] — The `_STALE_API_FILES` list is static; if a new model file copies the old `get_cache_scale` pattern, the test won't catch it unless manually updated. Replace the list with a dynamic check (e.g., `grep -rl 'get_cache_scale(' vllm_omni/model_executor/models/`) and assert that the result equals the expected fixed‑file list (or empty). This makes the guard robust against future drift. (evidence: test_model_loaders_do_not_call_removed_get_cache_scale only checks the four explicit paths; a repo-wide grep (not run but suggested) would be more robust against future drift.) + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:?` [minor] — The new mapper application block at mammoth_moda2.py:486 lacks a comment explaining why it is needed: this loader is called directly, not through an outer `AutoWeightsLoader`, so it must perform the mapper step itself. The corresponding deletion in hunyuan_image3.py left an explanatory comment; add a similar note for clarity. (evidence: Diff adds these lines at mammoth_moda2.py:486-489 without a comment; compare with hunyuan_image3.py:224-226 which has an explanatory comment about the outer AutoWeightsLoader) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:80` [minor] — The static check `'.get_cache_scale(' not in source` only catches direct attribute access. The old `qwen2_old.py` code used `getattr(self.quant_config, "get_cache_scale", None)`, which escapes this check. Also search for the string `"get_cache_scale"` or a `getattr` pattern to make the guard comprehensive. (evidence: Read tests/model_executor/models/test_kv_cache_scale_mapper.py line 78; diff of qwen2_old.py shows removed getattr pattern) + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:486` [minor] — Verify whether `MammothModa2ARForConditionalGeneration.load_weights` (inherited from `Qwen2_5_VLForConditionalGeneration`) already applies the cache‑scale mapper via `AutoWeightsLoader` before delegating to `MammothModa2Qwen2ForCausalLM.load_weights`. If so, the new mapper application at line 486 is redundant (though harmless). Confirm the inheritance chain and add a comment if redundancy exists. (evidence: Read mammoth_moda2.py load_weights and outer MammothModa2ForConditionalGeneration.load_weights; could not verify Qwen2_5_VLForConditionalGeneration.load_weights chain) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:112` [minor] — In `test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale`, the test feeds pre-mapped names (`_MAPPED_SCALE_NAME`) directly to `HunyuanModel.load_weights` and `Qwen2Model.load_weights`. This proves the deleted `get_cache_scale` path is not needed for already-mapped names. However, the fallthrough `else` branch in these loaders still calls `maybe_remap_kv_scale_name` as a safety net. Add a parametrized case that feeds the unmapped `_SOURCE_SCALE_NAME` to verify the fallthrough handles it correctly, ensuring the safety net of the removed `get_cache_scale` path still works. (evidence: Read hunyuan_image3.py:222-230 — removed get_cache_scale branch falls through to maybe_remap_kv_scale_name in the final else; the test never exercises that path for HunyuanModel/Qwen2Model.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:3` [minor] — The PR description suggests removing this test after maintainer validation. It is the only regression coverage for the migration from `get_cache_scale` to `get_cache_scale_mapper` across the four fixed loaders. Without it, a future rebase could silently reintroduce the removed API. Keep this test permanently to guard the fix. (evidence: PR description text: 'I added a small CPU-based regression test for the API change as demonstration; Will remove it from the PR once this is validated by a maintainer as I don't think this coverage is necessary!') + +**Additional observations (beyond the comment budget):** +- `tests/model_executor/models/test_kv_cache_scale_mapper.py:26` [minor] `_RecordingCacheScaleMapper.apply` is a generator that re‑yields from the input `weights` list each time it is called. Real compressed‑tensors mappers typically consume the input iterator, so calling `apply` twice would +- `tests/model_executor/models/test_kv_cache_scale_mapper.py:108` [minor] `test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale` calls `HunyuanModel.load_weights` directly, bypassing the outer `AutoWeightsLoader` that the fix’s comment (line 225) relies on. The test verifies th + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4810.trace.json.gz new file mode 100644 index 000000000..eea8c7815 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4816.trace.json.gz new file mode 100644 index 000000000..cbde4a478 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4825.trace.json.gz new file mode 100644 index 000000000..0885a2ff8 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4837.trace.json.gz new file mode 100644 index 000000000..81357ee0f Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4870.cost.json b/eval/dataset/arms/copilot_v7_goal_r7/pr4870.cost.json new file mode 100644 index 000000000..3c960904a --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 318.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r7/runs/pr4870/run-20260812-205429-886cea", + "recorded_at": "2026-08-12T20:59:45", + "attempts": [ + { + "invocation_id": "inv-85bd54cc", + "outcome": "run:run-20260812-205429-886cea", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-205429-886cea", + "llm_calls": 43, + "input_tokens": 1132561, + "output_tokens": 45363, + "usd": 0.441908, + "source": "spans" + } + ], + "attempt_usd_total": 0.441908, + "llm_calls": 43, + "input_tokens": 1132561, + "output_tokens": 45363, + "usd": 0.441908, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-205429-886cea", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4419, + "minutes": 5.27, + "ci_minutes": 0, + "input_tokens": 1132561, + "output_tokens": 45363, + "tool_calls": 86, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3721, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.429125, + "input_tokens": 1117742, + "output_tokens": 37408, + "calls": 41 + }, + "reducer": { + "usd": 0.012783, + "input_tokens": 14819, + "output_tokens": 7955, + "calls": 2 + } + }, + "cache_read_tokens": 3193216, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.17, + "gate": 1.69, + "review": 310.48, + "report": 0.07 + }, + "time_to_first_result_s": 316.39 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 25931 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4870.md b/eval/dataset/arms/copilot_v7_goal_r7/pr4870.md new file mode 100644 index 000000000..6c64edc55 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4870.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] serving_speech.py:3507-3509 — async_chunk read from model_config with qwen3_tts scoping gate; non-qwen3 models preserve old is_streaming_request behavior +- [validated] gpu_ar_model_runner.py:1734 — mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) replaces removed seq_len from hidden_states.shape[0]; mm_seq_len is always available inside the needs_pooler_payload block +- [validated] gpu_ar_model_runner.py:1876 — removed seq_len = hidden_states.shape[0] and the corresponding seq_len=seq_len arg at line 2042; consistent with method signature change +- [validated] test_gpu_ar_model_runner.py:438-477 — new test sets omni_pooler_payload_include_hidden=False, _async_chunk=False, 3 requests x 1 token, and verifies codes.audio (3,16) is split into three (1,16) per-request slices +- [validated] test_serving_speech.py:3453-3470 — qwen3_tts test verifies coerce_param_message_types called with False (FINAL_ONLY) when async_chunk=False +- [validated] test_serving_speech.py:3473-3494 — voxtral test verifies coerce_param_message_types called with True (DELTA) when async_chunk=False, confirming non-qwen3 models unchanged +- [sweep] gpu_ar_model_runner.py — checked _build_omni_model_runner_output_from_snapshot has no other callers beyond sample_tokens and tests; all test call sites updated to remove seq_len arg +- [sweep] grep for seq_len=seq_len — confirmed no remaining references to the removed parameter in gpu_ar_model_runner.py + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — The `qwen3_full_payload` gate (line 3508) correctly scopes FINAL_ONLY to `qwen3_tts`, and the PR already comments that other async_chunk=False models keep DELTA coercion. Add the specific unaffected model names (e.g., moss_tts, voxcpm2, indextts2) to that comment so future readers see the intentional exclusion list without external lookup. (evidence: read repo checklist Streaming/TTS serving section: async_chunk defaults to False and applies to many models) + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [minor] — The new `mm_seq_len` at line 1734 uses `scheduler_output.total_num_scheduled_tokens` instead of the removed `hidden_states.shape[0]`. Add a brief comment explaining why the scheduler total is needed: when `_model_needs_full_prefix_hidden_states()` returns False, hidden_states may be a tail-only slice (e.g., shape (1,)) while multimodal outputs span the full batch, as described in #4851. This prevents a future maintainer from reintroducing the old bug by using `hidden_states.shape[0]` again. (evidence: Read gpu_ar_model_runner.py offset 1720-1790; the previous `seq_len` (hidden_states.shape[0]) was the root cause of cross-request codes.audio contamination as documented in PR description and linked issue #4851.) + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [minor] — The `mm_seq_len` at line 1734 relies on the invariant that every batched multimodal tensor's first dimension equals `total_num_scheduled_tokens`. Add an assert or comment documenting this assumption — a mismatch would silently produce wrong payloads through `to_payload_element` (mm_outputs.py:108-145). (evidence: Read gpu_ar_model_runner.py:1734; traced the value through _build_omni_pooler_payload → _build_omni_mm_payload → to_payload_element (mm_outputs.py:108-145).) + +`tests/worker/test_gpu_ar_model_runner.py:438` [minor] — The new test `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` (line 438) uses uniform `num_scheduled_tokens={'r1':1,'r2':1,'r3':1}` and uniform `query_start_loc_cpu`. Add a variant with unequal per-request token counts (e.g., `{'r1':1,'r2':2,'r3':1}` and `query_start_loc_cpu=[0,1,3]`) to validate that `partition_payload_list` correctly slices multimodal tensors when request sizes differ — a realistic batch scenario that could mask index-calculation bugs. (evidence: Read test at test_gpu_ar_model_runner.py offset 435-485; all three requests have `num_scheduled_tokens=1`. Unequal token counts would exercise `to_payload_element` with `start != end-1`.) + +`tests/entrypoints/openai_api/test_serving_speech.py:3473` [minor] — The gate report shows `buildkite/vllm-omni-amd-ci` failing. This PR touches shared runner and serving code but no AMD-specific paths. Confirm whether the AMD lane failure is pre-existing on main (unrelated) or was introduced by this change — if introduced, it may signal a platform-specific regression that should be fixed before merge. (evidence: gh_ci_read(4870) shows 9 passing checks + 1 FAILURE: `buildkite/vllm-omni-amd-ci`. Read diff confirms no AMD-specific platform code is changed.) + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [nit] — The `getattr(..., 'async_chunk', True)` in serving_speech.py (line 3507) defaults to `True`, but `GPUARModelRunner.__init__` (gpu_ar_model_runner.py) defaults to `False` for the same config attribute, and `OmniModelConfig` defines `async_chunk: bool = False`. In practice the attribute is always set, so the fallback is harmless. Align the serving_speech default to `False` (matching runner and config) or add a comment explaining the choice to avoid confusing readers. (evidence: read gpu_ar_model_runner.py __init__: `self._async_chunk = getattr(self.model_config, "async_chunk", False)`; read serving_speech.py:3507 showing True default) + +`vllm_omni/worker/gpu_ar_model_runner.py:?` [nit] — Removing `seq_len` from the signature of `_build_omni_model_runner_output_from_snapshot` is correct — it's a private method and all in-repo callers have been updated. Out-of-tree monkey-patching that passes `seq_len=` would break, but that is expected for an underscore-prefixed parameter removal. (evidence: Grep'd repo for `_build_omni_model_runner_output_from_snapshot`; only callers are in sample_tokens (via output_builder) and tests — all updated.) + +`vllm_omni/utils/mm_outputs.py:?` [nit] — The docstring of `to_payload_element` (mm_outputs.py line 122) still describes `seq_len` as 'dim 0 of hidden states', but the PR now passes `total_num_scheduled_tokens`. Update the docstring to describe both valid sources, e.g., '(e.g., dim 0 of hidden states or total_num_scheduled_tokens)', to match the new usage. (evidence: Read vllm_omni/utils/mm_outputs.py:108-145 — the docstring at line 122 says '(i.e., dim 0 of hidden states)' but the new caller at gpu_ar_model_runner.py:1734 passes `total_num_scheduled_tokens`.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4870.trace.json.gz new file mode 100644 index 000000000..a694fb552 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4893.trace.json.gz new file mode 100644 index 000000000..e8169533d Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4923.cost.json b/eval/dataset/arms/copilot_v7_goal_r7/pr4923.cost.json new file mode 100644 index 000000000..c24360798 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 396.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r7/runs/pr4923/run-20260812-204752-db9a44", + "recorded_at": "2026-08-12T20:54:26", + "attempts": [ + { + "invocation_id": "inv-b4575ac7", + "outcome": "run:run-20260812-204752-db9a44", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-204752-db9a44", + "llm_calls": 44, + "input_tokens": 823618, + "output_tokens": 58381, + "usd": 0.346423, + "source": "spans" + } + ], + "attempt_usd_total": 0.346423, + "llm_calls": 44, + "input_tokens": 823618, + "output_tokens": 58381, + "usd": 0.346423, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-204752-db9a44", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3464, + "minutes": 6.56, + "ci_minutes": 0, + "input_tokens": 823618, + "output_tokens": 58381, + "tool_calls": 91, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3766, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.332148, + "input_tokens": 812334, + "output_tokens": 48202, + "calls": 42 + }, + "reducer": { + "usd": 0.014275, + "input_tokens": 11284, + "output_tokens": 10179, + "calls": 2 + } + }, + "cache_read_tokens": 2215808, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.98, + "gate": 1.83, + "review": 387.79, + "report": 0.06 + }, + "time_to_first_result_s": 393.64 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 21875 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4923.md b/eval/dataset/arms/copilot_v7_goal_r7/pr4923.md new file mode 100644 index 000000000..ce5c1b098 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4923.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] qwen3_tts_code_predictor_vllm.py:39: use_cuda_graphs=current_omni_platform.is_npu() correctly gates to NPU-only; CodePredictorWrapper._setup_compile (common/qwen3_code_predictor.py:477,532,552) on GPU skips both NPU graphs and CUDA graphs, using torch.compile only. +- [validated] qwen3_tts_talker.py:346-349: talker_mtp_accepts_per_row_generators gating on cudagraph_mode.has_full_cudagraphs() correctly disables per-row generators when outer graph wraps; runner _init_talker_mtp (gpu_model_runner.py:208-209) derives same condition from talker_mtp_graph_safe + cudagraph_mode. +- [validated] qwen3_tts_talker.py:348: cudagraph_mode is not None guard before .has_full_cudagraphs() — safe under eager mode where cudagraph_mode may be None. +- [validated] qwen3_tts_talker.py:351-353: use_async_omni_output, eager_omni_postprocess_before_async_output, omni_pooler_payload_include_hidden consumed by runner at gpu_ar_model_runner.py:1537-1554, 1625-1639, 1534-1535. +- [validated] qwen3_tts_talker.py:327: talker_mtp_graph_safe = True consumed by runner at gpu_model_runner.py:209 and gpu_ar_model_runner.py:576. +- [validated] qwen3_omni_moe_code_predictor_mtp.py:21: Qwen3-Omni already uses same use_cuda_graphs=current_omni_platform.is_npu() pattern — this PR aligns Qwen3-TTS. +- [sweep] fish_speech_slow_ar.py:198: FishSpeechSlowAR also sets talker_mtp_graph_safe=True but does NOT gate per-row generators on cudagraph_mode — pre-existing, not introduced here. +- [sweep] deploy/qwen3_tts.yaml:95-100: NPU platform section adds PIECEWISE cudagraph_mode for both stage 0 and stage 1 — correct for NPU graph stack. + +`vllm_omni/deploy/qwen3_tts.yaml:41` [minor] — The diff changes `enable_prefix_caching` from `true` to `false`. The existing comment block (lines 39-42) says it is on by default and only disabled when memory is tight, but the new default is off to avoid prefix-cache bookkeeping interfering with the full-graph MTP path (PR rationale). Update the comment to match the new default and the MTP-interference rationale. (evidence: Diff hunk at line 43: enable_prefix_caching: false; inline comment at lines 39-42 describes benefits and says 'Disable explicitly if memory headroom is tight' but it is already disabled.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:346` [minor] — The talker model now decides `talker_mtp_accepts_per_row_generators` based on `cudagraph_mode` (lines 343-350). The runner independently decides to wrap `talker_mtp` using the same flag (gpu_model_runner.py:208-209). If the model misjudges, per-row generators may be used inside a graph, causing silent failures. Add an assert in the runner that the model's `talker_mtp_accepts_per_row_generators` is consistent with whether the runner actually wrapped `talker_mtp`, to guard against drift. (evidence: Read gpu_model_runner.py:208-209 (`_init_talker_mtp`) and talker.py L343-350 (model-side gating). Both check `cudagraph_mode.has_full_cudagraphs()` independently.) + +`vllm_omni/deploy/qwen3_tts.yaml:46` [minor] — The gate report shows both `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` failing. This PR changes the code-predictor graph layout (GPU: no inner device graphs; NPU: `use_cuda_graphs=True` via `current_omni_platform.is_npu()`) and adds NPU-specific `cudagraph_mode: PIECEWISE` in the deploy YAML. Verify whether these CI failures are pre-existing on main. If they are new, the graph configuration or platform gating may need adjustment. (evidence: Gate report lists failing checks: buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-npu-ci. The NPU platform section (lines 95-101) was added in this PR.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The diff changes stage 0 `max_num_batched_tokens` from 512 to 32768. The header comment at line 6 still says 512 and describes the old rationale for small batches. Update the comment to reflect the new value and the throughput rationale under full cudagraphs. (evidence: Diff hunk at line 48: max_num_batched_tokens: 32768; header comment at line 6 still reads 512.) + +`tests/.../test_omni_gpu_model_runner.py:357` [minor] — The existing test `test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models` only covers `talker_mtp_accepts_per_row_generators=True`. No test exercises the new `False` path introduced for full cudagraph mode. Should a test be added to assert correct behavior (e.g., scalar dispatch, no per-row generators, no seeding) when `cudagraph_mode.has_full_cudagraphs()` is `True`? (evidence: [sweep] No test files touched — test_talker_mtp_forward_batches_seeded_requests_for_opted_in_models at test_omni_gpu_model_runner.py:357 tests talker_mtp_accepts_per_row_generators=True only. No test covers the False (cudagraph) path added here.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:353` [nit] — The new code at line 353 sets `self.omni_pooler_payload_include_hidden = False`. This flag is already set to `False` at line 317 (pre-existing, with a comment about skipping hidden-state D2H). Remove the redundant assignment or consolidate into a single location. (evidence: Line 317 (pre-existing, outside diff): self.omni_pooler_payload_include_hidden = False; line 353 (diff-added): same assignment.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4923.trace.json.gz new file mode 100644 index 000000000..fa88ca479 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4926.cost.json b/eval/dataset/arms/copilot_v7_goal_r7/pr4926.cost.json new file mode 100644 index 000000000..b65bc312a --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4926.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 584.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r7/runs/pr4926/run-20260812-205723-6f0b2d", + "recorded_at": "2026-08-12T21:07:05", + "attempts": [ + { + "invocation_id": "inv-94818d8f", + "outcome": "run:run-20260812-205723-6f0b2d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-205723-6f0b2d", + "llm_calls": 42, + "input_tokens": 308047, + "output_tokens": 77083, + "usd": 0.212778, + "source": "spans" + } + ], + "attempt_usd_total": 0.212778, + "llm_calls": 42, + "input_tokens": 308047, + "output_tokens": 77083, + "usd": 0.212778, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-205723-6f0b2d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2128, + "minutes": 9.7, + "ci_minutes": 0, + "input_tokens": 308047, + "output_tokens": 77083, + "tool_calls": 83, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4028, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 42, + "by_role": { + "agent": { + "usd": 0.187845, + "input_tokens": 283950, + "output_tokens": 60363, + "calls": 40 + }, + "reducer": { + "usd": 0.024933, + "input_tokens": 24097, + "output_tokens": 16720, + "calls": 2 + } + }, + "cache_read_tokens": 1659776, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.28, + "gate": 1.94, + "review": 575.35, + "report": 0.06 + }, + "time_to_first_result_s": 581.63 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 34009 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4926.md b/eval/dataset/arms/copilot_v7_goal_r7/pr4926.md new file mode 100644 index 000000000..4ea1bffae --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4926.md @@ -0,0 +1,55 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4926, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 4 finding(s) below | + +**Validated:** +- [validated] flash_attn_hub.py:108-109 — constructor correctly guards `flash_attn_func is None and flash_attn_varlen_func is None` → RuntimeError; at least one func is always available after init. +- [validated] flash_attn_hub.py:46-47 — `_run_varlen_dense` checks `isinstance(out, tuple)` before indexing `out[0]`; safe. +- [validated] cuda/platform.py:124-148 — HUB fallback chain: kernels import → FLASH_ATTN fallback → Hopper gate → FLASH_ATTN_HUB fallback; all arms reachable and logically consistent. +- [validated] musa/npu/rocm/xpu platform.py hunks — identical pattern warns and falls back to FLASH_ATTN; `DiffusionAttentionBackendEnum[backend_upper]` always resolves to a valid enum member after fallback. +- [validated] flash_attn_hub.py:189-203 — piecewise attention dispatches to `_flash_wrapper` with `partial(attn_func=self.flash_attn_func)`; the `attn_func is not None` branch in `_flash_wrapper` handles the fallback to varlen correctly. +- [sweep] flash_attn_hub.py:150 — `_forward_varlen_masked` calls `self.flash_attn_varlen_func(...)` with no None check; if only `flash_attn_func` loaded from Hub, crashes on masked path. +- [sweep] flash_attn_hub.py:329 — same issue in `FlashAttention3HubImpl._forward_varlen_masked`. +- [sweep] flash_attn_hub.py:70-220 vs 249-399 — `FlashAttentionHubImpl` and `FlashAttention3HubImpl` are near-identical (~170 lines each); only `__init__` kernel name differs. Same for the Backend classes (lines 51-67 vs 230-246). + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [major] — `FlashAttentionHubImpl.__init__` (line 73) and `FlashAttention3HubImpl.__init__` (line 252) accept `num_kv_heads: int | None = None` but never store it or pass it to the Hub kernels. In GQA/MQA scenarios (`num_kv_heads < num_heads`), key/value tensors have fewer heads, but the backend receives mismatched shapes without any GQA configuration. The local `FlashAttentionImpl` handles GQA explicitly; the Hub backends silently ignore the parameter. At minimum, log a warning when `num_kv_heads is not None and num_kv_heads != num_heads`. Ideally, pass the GQA configuration to the Hub kernel if supported. (evidence: Read flash_attn_hub.py lines 71-86 (FlashAttentionHubImpl.__init__) and lines 250-265 (FlashAttention3HubImpl.__init__); num_kv_heads is accepted but not stored as self.num_kv_heads, nor passed to flash_attn_func/varlen_func.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:?` [major] — `_forward_varlen_masked` at line 150 directly calls `self.flash_attn_varlen_func(...)` without a None check. The constructor (line 108) only guarantees not both are None, so one could be None while the other is set. If a request has an attention mask and the Hub kernel provides only `flash_attn_func` (no varlen variant), this will cause `TypeError: 'NoneType' object is not callable`. Same issue in `FlashAttention3HubImpl._forward_varlen_masked` (line ~329). Guard: raise `RuntimeError('flash_attn_varlen_func is required for attention masking')` or fall back by decomposing the mask. (evidence: Read flash_attn_hub.py: constructor sets `self.flash_attn_varlen_func = getattr(hub_module, 'flash_attn_varlen_func', None)` (line 105); only checks `if self.flash_attn_func is None and self.flash_attn_varlen_func is None` (line 108). So one may be None while the other is set. The masked path at line 150 directly calls `self.flash_attn_varlen_func(...)` with no None check.) + +`tests/diffusion/attention/test_kernels_hub.py:70` [minor] — The test uses `k = q.clone()`, `v = q.clone()` making Q, K, V identical. Under self-attention with Q=K=V, attention weights become uniform (1/seq_len), so the `max_diff < 1e-2` assertion is trivially satisfied and cannot catch kernel bugs such as shape mismatches or key/value axis confusion. Use distinct random tensors for K and V. Additionally, the test only exercises the dense (no-mask) path; add a variant with an attention mask to cover `_forward_varlen_masked` (line 150) and the piecewise attention path (line 189). (evidence: Read test_kernels_hub.py:70-72 — `q = torch.randn(...); k = q.clone(); v = q.clone()`; attn_metadata is `AttentionMetadata(attn_mask=None)` (line 77), so the `forward_cuda` method takes the dense path at line 213 (`if self.flash_attn_func is not None`), never entering the masked or piecewise branches.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:113` [minor] — The comment at line 113 says 'FA3 may return (out, lse), FA2 returns out' but this is inside `FlashAttentionHubImpl` (FA2 Hub backend). The code correctly handles both tuple and tensor via `isinstance`, but the comment misattributes tuple returns to FA3. Also, the FA3 variant at line 293 has '# FA3 returns (out, lse)' implying always-tuple, while the code also handles both. Change comments to reflect the actual Hub behavior: 'The Hub kernel may return (out, lse) or a plain tensor' for both locations. (evidence: Read flash_attn_hub.py:113 — comment says 'FA3 may return (out, lse), FA2 returns out' but this is inside class FlashAttentionHubImpl which loads flash-attn2 from Hub. Compare with flash_attn.py:81 — same comment on `FlashAttentionImpl._unwrap_flash_output` where it's also potentially misleading (local FA could be FA2 or FA3 depending on what's installed).) + +`docs/user_guide/diffusion/attention_backends.md:228` [minor] — The documentation at line 228 instructs users to `pip install kernels==0.14.1`, but `pyproject.toml` line 70 declares `kernels<0.15.0` without a hard pin. This doc pin is more restrictive than the actual dependency constraint and could prevent users from getting critical fixes in 0.14.x releases. Align the doc with the dependency spec: either use the range `kernels<0.15.0` or remove the version pin so `pyproject.toml` remains the single source of truth. (evidence: Read docs/user_guide/diffusion/attention_backends.md:228 — `pip install kernels==0.14.1`. Read pyproject.toml:70 — `"kernels<0.15.0"` under dev dependencies. The doc exact-pin is inconsistent with the range constraint.) + +`docs/user_guide/diffusion/attention_backends.md:?` [minor] — The documentation describes two fallback triggers (kernels not installed, pre-Hopper GPU for FA3) but omits a third: on non-CUDA platforms (MUSA, NPU, ROCm, XPU), the Hub backends unconditionally fall back to local FLASH_ATTN even when `kernels` IS installed. This is implemented in `platform.py` for each platform (e.g., rocm/platform.py:91, npu/platform.py:110). Add a note to the 'HuggingFace Kernels Hub Backends' section that on non-CUDA platforms the Hub backends are always redirected to FLASH_ATTN. (evidence: Read the four non-CUDA platform files (musa, npu, rocm, xpu platform.py) — each has identical unconditional fallback from FLASH_ATTN_HUB/FLASH_ATTN_3_HUB to FLASH_ATTN.) + +`pyproject.toml:?` [minor] — The `kernels` dependency in `pyproject.toml` line 70 is `"kernels<0.15.0"` with no lower bound. The documentation (attention_backends.md:228) pins `kernels==0.14.1`. An older kernels release may lack the `get_kernel` API or the required repo/version combination. Add a lower bound matching the documented known-good version: `"kernels>=0.14.0,<0.15.0"`. (evidence: pyproject.toml line 70 shows 'kernels<0.15.0' with no minimum; docs/user_guide/diffusion/attention_backends.md says 'pip install kernels==0.14.1'.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [minor] — The FA3 Hub execution test (line 92) skips when `major < 9`, but the CI hardware annotation `@hardware_test(res={'cuda': 'L4'}, num_cards=1)` provides only an L4 GPU (sm_89, major=8). Therefore the FA3 path is always skipped in CI, providing zero coverage for `FlashAttention3HubImpl`. This means a regression in that code will only surface on Hopper GPUs outside CI. Either add a Hopper CI lane for this test or document that FA3 Hub coverage is manual-only. (evidence: Read tests/diffusion/attention/test_kernels_hub.py lines 58 and 92; grep of .buildkite/ for attention test patterns showed no Hopper-specific lane running this test file.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:69` [minor] — FlashAttentionHubImpl (line 69-205) and FlashAttention3HubImpl (line 248-384) are near-identical copies, duplicating ~200 lines including _forward_varlen_masked, _forward_varlen_dense, forward_cuda, _flash_wrapper, and _unwrap_flash_output. A bugfix applied to one but not the other is a real risk. Is there a plan to deduplicate or ensure they stay in sync? (evidence: - [concern] flash_attn_hub.py:69-205 vs 248-384 — FlashAttentionHubImpl and FlashAttention3HubImpl are near-clones (~200 lines each). _forward_varlen_masked, _forward_varlen_dense, forward_cuda, _flash_wrapper, _unwrap_flash_output all duplicated. A bugfix applied to one but not the other is a real risk) + +`vllm_omni/platforms/rocm/platform.py:91` [minor] — The buildkite/AMD and NPU CI lanes are red. This PR modifies rocm/platform.py and npu/platform.py (adding fallback logic). Could you verify whether the failures are pre-existing on main and not introduced by these changes? (evidence: - [sweep] gate_report — buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are red; ROCm/NPU platform changes are minimal fallback-only additions, likely pre-existing on main.) + +`tests/diffusion/attention/test_kernels_hub.py:19` [minor] — No test covers the non-CUDA platform fallback paths (musa, npu, rocm, xpu) that redirect FLASH_ATTN_HUB/FLASH_ATTN_3_HUB to FLASH_ATTN. The existing test_kernels_hub_platform_fallback only covers CUDA. Should tests be added for these platform-specific fallback paths? (evidence: - [sweep] Read vllm_omni/platforms/{musa,npu,rocm,xpu}/platform.py — each adds identical fallback for FLASH_ATTN_HUB/FLASH_ATTN_3_HUB → FLASH_ATTN. No test covers any non-CUDA platform fallback.) + +**Additional observations (beyond the comment budget):** +- `tests/diffusion/attention/test_kernels_hub.py:58` [minor] The GPU execution test `test_kernels_hub_execution` (line 58) is decorated with `@hardware_test(res={'cuda': 'L4'}, num_cards=1)` and `@pytest.mark.core_model`, but the CI pipeline 'Diffusion · Other Test' runs `pytest - +- `tests/diffusion/attention/test_kernels_hub.py:19` [minor] The fallback test `test_kernels_hub_platform_fallback` only exercises `CudaOmniPlatform`. The same fallback logic (warning + redirect to FLASH_ATTN) is added to MUSA, NPU, ROCm, and XPU platforms, but none of these paths +- `vllm_omni/diffusion/attention/backends/flash_attn_hub.py:189` [minor] The execution test `test_kernels_hub_execution` only exercises the simplest forward path (dense, no-mask, no piecewise attention). The `piecewise_attn` branch (line 189) and the `_forward_varlen_masked` path (line 205) i + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4926.trace.json.gz new file mode 100644 index 000000000..76f464651 Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4977.cost.json b/eval/dataset/arms/copilot_v7_goal_r7/pr4977.cost.json new file mode 100644 index 000000000..ef245dca1 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4977.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 515.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r7/runs/pr4977/run-20260812-204847-ea66b4", + "recorded_at": "2026-08-12T20:57:21", + "attempts": [ + { + "invocation_id": "inv-082299ac", + "outcome": "run:run-20260812-204847-ea66b4", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-204847-ea66b4", + "llm_calls": 44, + "input_tokens": 199347, + "output_tokens": 68632, + "usd": 0.155951, + "source": "spans" + } + ], + "attempt_usd_total": 0.155951, + "llm_calls": 44, + "input_tokens": 199347, + "output_tokens": 68632, + "usd": 0.155951, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-204847-ea66b4", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.156, + "minutes": 8.56, + "ci_minutes": 0, + "input_tokens": 199347, + "output_tokens": 68632, + "tool_calls": 71, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3484, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.13846, + "input_tokens": 189814, + "output_tokens": 55103, + "calls": 42 + }, + "reducer": { + "usd": 0.01749, + "input_tokens": 9533, + "output_tokens": 13529, + "calls": 2 + } + }, + "cache_read_tokens": 986368, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.48, + "gate": 1.8, + "review": 508.09, + "report": 0.06 + }, + "time_to_first_result_s": 513.43 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 19305 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4977.md b/eval/dataset/arms/copilot_v7_goal_r7/pr4977.md new file mode 100644 index 000000000..0b74bbe97 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr4977.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4977, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 5 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] `_load_hub_module` loop (v1→v2→default) all branches reachable; `last_error` chains correctly into final RuntimeError — flash_attn_hub.py:26-38 +- [validated] `_get_hub_module` lock-guarded dict access thread-safe; repo-level cache prevents redundant loads on subsequent `FlashAttention*HubImpl` constructions — flash_attn_hub.py:42-45 +- [validated] Both `FlashAttentionHubImpl.__init__` (line 119) and `FlashAttention3HubImpl.__init__` (line 284) correctly delegate to `_get_hub_module`; `getattr` fallback and RuntimeError guard preserved identically +- [validated] No external callers of `_get_hub_module` or `_load_hub_module` — repo-wide grep confirms only internal references within flash_attn_hub.py +- [sweep] Branch `if version is not None:` (line 33) — reachable for versions 1 and 2; `None` branch reachable on third loop iteration; no dead arms +- [sweep] Branch `if repo_id not in _hub_modules:` (line 44) — reachable on first call, false on subsequent; no dead arms +- [sweep] Failed loads are not cached — subsequent constructions retry; same behavior as old code, no regression +- [sweep] No rebase/merge damage — no duplicated code, no stale symbol references, no dropped hunks + +`tests/diffusion/attention/test_kernels_hub.py:?` [major] — The caching logic introduced by this diff (`_get_hub_module` / `_load_hub_module`) has no CI coverage. The sole hub-load test (`test_kernels_hub_execution` in `tests/diffusion/attention/test_kernels_hub.py`) is gated with `@hardware_test(res={'cuda': 'L4'})` and is excluded from the standard CI filter (`core_model and cpu` in `.buildkite/test-merge.yml`). That test also constructs each impl only once, so the cache-hit path is never verified. Add a CI-level smoke test that constructs two instances of the same impl and verifies `get_kernel` is called once (e.g., by patching `_load_hub_module`), and confirm the L4-gated test passes with this change. (evidence: Read tests/diffusion/attention/test_kernels_hub.py:19-57 (platform_fallback monkeypatches kernels to None). Read tests/diffusion/attention/test_kernels_hub.py:58-105 (hardware_test on L4, not cpu). Read .buildkite/test-merge.yml:13 (CI filter `core_model and cpu` excludes L4-gated tests).) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:26` [minor] — The new helper `_load_hub_module` (line 26) implements a version-fallback loop (v1 → v2 → default), but it has no docstring. Add a docstring explaining the fallback strategy and the returned kernel module type. This replaces the now-removed inline comment `# Lazily get the kernel from the Hub...`. (evidence: Read flash_attn_hub.py lines 19-38; the old code at line ~91 had the comment, which is deleted in the diff with no replacement.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:42` [minor] — The new helper `_get_hub_module` (line 42) provides thread-safe, load-once caching via `_hub_lock` and `_hub_modules`. It has no docstring. Add a docstring documenting the caching and thread-safety guarantees. (evidence: Read flash_attn_hub.py lines 42-45; no docstring present.) + +`tests/diffusion/attention/test_kernels_hub.py:?` [minor] — In `test_kernels_hub_execution` (line 70), `k` is set to `q.clone()`, making key identical to query — a degenerate attention pattern. While pre-existing, this weakens the test's ability to catch regressions in the hub kernel path. Use independent random tensors for `q`, `k`, `v` to better exercise the attention function. (evidence: Read tests/diffusion/attention/test_kernels_hub.py:70 — k is a clone of q, not an independent random tensor.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:119` [minor] — The PR description states the diff adds `trust_remote_code=True` for more reliable Hub access, but the final commit (`9947f414`) removed that kwarg and the diff has no `trust_remote_code` calls. Please update the description to remove the `trust_remote_code=True` claim, or document why it was dropped. (Evidence: the `_load_hub_module` body calls `get_kernel(repo_id)` with no `trust_remote_code` keyword argument.) + +`vllm_omni/platforms/npu/platform.py:110` [minor] — The NPU CI (`buildkite/vllm-omni-npu-ci`) is currently red. While the diff is unlikely to be the cause (NPU platform falls back without instantiating the hub backends), please confirm the failure is pre-existing and not introduced by this change. (evidence: NPU CI (`buildkite/vllm-omni-npu-ci`) is red; NPU platform's `get_diffusion_attn_backend_cls` falls back from FLASH_ATTN_HUB/FLASH_ATTN_3_HUB at the platform level without instantiating these backends, so this diff is unlikely causal) + +`tests/diffusion/attention/test_kernels_hub.py:95` [minor] — The FA3 caching path (`_get_hub_module("kernels-community/flash-attn3")`) has no CI coverage because `test_kernels_hub_execution` skips it on L4 hardware due to `major < 9`. Consider adding a test on capable hardware (e.g., H100) or explicitly noting this coverage gap. (evidence: FA3 path in test_kernels_hub_execution skips on L4 (major < 9); no CI hardware exercises the _get_hub_module("kernels-community/flash-attn3") caching path) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:22` [nit] — The new module-level cache `_hub_modules` is typed `dict[str, object]` (line 22), but the values are kernel modules with known attributes (`flash_attn_func`, `flash_attn_varlen_func`). To make the code clearer, replace `dict[str, object]` with a more descriptive type such as `dict[str, Any]` or a `Protocol`, or add a comment noting that values are the return type of `kernels.get_kernel()`. (evidence: Read flash_attn_hub.py lines 22-23: `_hub_modules` is typed `dict[str, object]` but consumers call `getattr(hub_module, "flash_attn_func", None)` — values are modules with at least those two attributes.) + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:22` [nit] — The process-wide cache `_hub_modules` (line 22) is never cleared; there is no invalidation path if a kernel needs to be reloaded during a process lifetime. Add a brief comment noting this is an intentional process-lifetime cache to avoid misleading future maintainers. + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr4977.trace.json.gz new file mode 100644 index 000000000..d197a30ae Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr5009.cost.json b/eval/dataset/arms/copilot_v7_goal_r7/pr5009.cost.json new file mode 100644 index 000000000..5440f3071 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 471.3, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v7_goal_r7/runs/pr5009/run-20260812-204056-35271f", + "recorded_at": "2026-08-12T20:48:45", + "attempts": [ + { + "invocation_id": "inv-dde9d84e", + "outcome": "run:run-20260812-204056-35271f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-204056-35271f", + "llm_calls": 43, + "input_tokens": 433842, + "output_tokens": 65279, + "usd": 0.243079, + "source": "spans" + } + ], + "attempt_usd_total": 0.243079, + "llm_calls": 43, + "input_tokens": 433842, + "output_tokens": 65279, + "usd": 0.243079, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-204056-35271f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2431, + "minutes": 7.82, + "ci_minutes": 0, + "input_tokens": 433842, + "output_tokens": 65279, + "tool_calls": 85, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3691, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.227332, + "input_tokens": 420654, + "output_tokens": 54229, + "calls": 41 + }, + "reducer": { + "usd": 0.015747, + "input_tokens": 13188, + "output_tokens": 11050, + "calls": 2 + } + }, + "cache_read_tokens": 2004992, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.02, + "gate": 1.67, + "review": 463.42, + "report": 0.06 + }, + "time_to_first_result_s": 469.15 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 24921 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr5009.md b/eval/dataset/arms/copilot_v7_goal_r7/pr5009.md new file mode 100644 index 000000000..0da5a1eb9 --- /dev/null +++ b/eval/dataset/arms/copilot_v7_goal_r7/pr5009.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [validated] vllm_omni/platforms/cuda/platform.py:251-256 — `using_inductor` branch correctly removed; default is now always `["vllm_c", "native"]`; oink override preserved intact (lines 257-261). +- [validated] vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:17 — import `from vllm.model_executor.layers.layernorm import RMSNorm` removed; grep confirms no other file at that path references it; the import removal is clean. +- [validated] vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:554-558,592-596,1006-1009 — all 5 `RMSNorm`→`nn.RMSNorm` substitutions are mechanically correct; eps defaults are preserved; no new branches or error paths introduced. +- [validated] tests/platforms/test_cuda_ir_op_priority.py:10 — pytestmark includes `core_model` and `cpu` markers; test will be selected by CI (meets checklist requirement). +- [validated] tests/platforms/test_cuda_ir_op_priority.py:23-28 — test parametrizes all three CompilationMode values and asserts `["vllm_c", "native"]`; correctly exercises the regression fix. +- [validated] vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:612-625 — Cosmos3's `ir_op_priority_func` override forces `["native"]` and is untouched by this PR; the override path in `_resolve_ir_op_priority` (diffusion_worker.py:175-178) still works correctly. +- [sweep] vllm_omni/platforms/rocm/platform.py:173-195 — RocmOmniPlatform still has the `using_inductor` branch (`["native"]` when inductor active); this PR does not touch ROCm but the PR description says the fix is CUDA-only. Gate report shows AMD CI failing — may be pre-existing. +- [sweep] vllm_omni/platforms/xpu/platform.py:107-118 — XPUOmniPlatform still has the `using_inductor` branch; same inconsistency. Gate report shows Intel CI failing — may be pre-existing. + +`vllm_omni/platforms/cuda/platform.py:253` [minor] — This PR restores the pre-#4709 default IR op priority of `["vllm_c", "native"]` for all CUDA diffusion models. The PR description notes that models needing native-only (e.g. Cosmos3) have per-model overrides in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`. To close the validation gap, consider adding a smoke test (or L2 accuracy check) on one additional model family (e.g. FLUX or SD3) to guard against unforeseen interactions with the restored default. (evidence: Read vllm_omni/diffusion/registry.py:546-549 — only Cosmos3OmniDiffusersPipeline is in _DIFFUSION_IR_OP_PRIORITY_FUNCS. Grep for `from vllm.model_executor.layers.layernorm import RMSNorm` in vllm_omni/diffusion/models/ returns 14 files (flux, flux2, sd3, wan2_2, ltx2, hunyuan_video, longcat_image, ovis_image, ernie_image, omnigen2, mistral_encoder, t5_encoder, ming_flash_omni) — all inheriting the new default.) + +`vllm_omni/platforms/cuda/platform.py:252` [minor] — Add a brief note in the docstring explaining *why* diffusion denoise loops are an exception to the inductor-native preference (RMSNorm-heavy, vllm_c faster even under `torch.compile`) to prevent a future rebase from re-adding the inductor branch. (evidence: PR body: 'diffusion denoise loops are RMSNorm-heavy and vllm_c is faster'; upstream vLLM 0.24 added the inductor-aware branch for LLM AR workloads.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:?` [minor] — Add a brief comment at the `norm_q`/`norm_k` assignment (line 554) noting that `nn.RMSNorm` (not vllm) is intentional — see #4074 accuracy/latency fix and #3588 regression — to prevent another accidental switchback. (evidence: Grep for '#4074' and '#3588' in vllm_omni/diffusion/models/qwen_image/ returned zero results. The PR description documents this history but the code does not.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:1009` [minor] — Part 2 switches `txt_norm` from `vllm.model_executor.layers.layernorm.RMSNorm` to `nn.RMSNorm` (line 1006). Since this is a deliberate restoration of #4074 (not a mistake), add a brief comment here referencing #4074 to prevent future accidental reverts like #3588. (evidence: Read qwen_image_transformer.py line 1009 (diff context); Part 1 restores vllm_c priority in platform.py line 252. The txt_norm dim=3584 is set at __init__ time (line 1009 `joint_attention_dim`).) + +`vllm_omni/platforms/rocm/platform.py:?` [minor] — ROCm and XPU platforms still have the inductor-aware `["native"]`-only branch that this PR removes from CUDA. While this PR is CUDA-only, the same regression could affect ROCm/XPU diffusion workloads (AMD/Intel CI is failing, though possibly pre-existing). Please open a tracking issue for ROCm/XPU evaluation, or add a comment stating that the fix is intentionally CUDA-only because vllm_c kernels are not available on those backends. (evidence: Read vllm_omni/platforms/rocm/platform.py:173-195 and vllm_omni/platforms/xpu/platform.py:107-118 — both still have the inductor-aware branch. Gate report from evidence shows AMD/Intel CI failing.) + +`tests/e2e/accuracy/test_qwen_image.py:197` [minor] — Part 2's `nn.RMSNorm` changes currently only have L4 full-model accuracy test coverage (H100 gate). Would a faster L2 regression test for these specific norm layers help guard against accidental reverts like #3588? (evidence: [risk] tests/e2e/accuracy/test_qwen_image.py:197-217 — accuracy tests that validate the nn.RMSNorm change are L4 (full_model marker, H100 hardware gate); no L2 regression test covers Part 2.) + +`tests/platforms/test_cuda_ir_op_priority.py:25` [nit] — The new `test_cuda_default_ir_op_priority` test only asserts the returned priority list; it does not exercise actual kernel dispatch. Adding a comment that the existing L4 accuracy tests on Qwen-Image (or a future perf smoke test on another model) cover the end-to-end path would suffice. No test changes required. (evidence: Read tests/platforms/test_cuda_ir_op_priority.py:23-28. Test asserts priority list values but doesn't exercise the downstream consumer chain: priority → IrOpPriorityConfig → actual kernel dispatch.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:?` [nit] — The switch to `nn.RMSNorm` is only verified by L4 accuracy tests (H100 gate). Instead of a fragile L2 instance-type assertion, consider adding a comment in the class docstring or at the import site noting why vLLM RMSNorm must NOT be used here (referencing #4074 and #3588). (evidence: Read tests/e2e/accuracy/test_qwen_image.py:197-217 — both accuracy tests use @pytest.mark.full_model and @hardware_test(res={'cuda':'H100'}); these are L4, not L2. Read .buildkite/test-ready.yml — L2 'Simple · Diffusion & Model Executor Test' runs `-m 'core_model and cpu'`, which cannot run the nn.RMSNorm validation.) + +`tests/platforms/test_cuda_ir_op_priority.py:23` [nit] — Rename `test_cuda_default_ir_op_priority_prefers_vllm_c_when_inductor_backend` to `test_cuda_default_ir_op_priority_prefers_vllm_c` or add a comment that `CompilationMode.NONE` is included for completeness. (evidence: Read tests/platforms/test_cuda_ir_op_priority.py:18-21 — parametrize includes CompilationMode.NONE alongside VLLM_COMPILE and STOCK_TORCH_COMPILE.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v7_goal_r7/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v7_goal_r7/pr5009.trace.json.gz new file mode 100644 index 000000000..3e115aa1d Binary files /dev/null and b/eval/dataset/arms/copilot_v7_goal_r7/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v8_official_r1/manifest.json b/eval/dataset/arms/copilot_v8_official_r1/manifest.json new file mode 100644 index 000000000..c4e2dc7e1 --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v8_official_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4810.cost.json b/eval/dataset/arms/copilot_v8_official_r1/pr4810.cost.json new file mode 100644 index 000000000..61a5f5a8e --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4810.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1467.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v8_official_r1/runs/pr4810/run-20260813-093038-72498d", + "recorded_at": "2026-08-13T09:55:04", + "attempts": [ + { + "invocation_id": "inv-31a8b002", + "outcome": "run:run-20260813-093038-72498d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-093038-72498d", + "llm_calls": 61, + "input_tokens": 911343, + "output_tokens": 175684, + "usd": 0.53467, + "source": "spans" + } + ], + "attempt_usd_total": 0.53467, + "llm_calls": 61, + "input_tokens": 911343, + "output_tokens": 175684, + "usd": 0.53467, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-093038-72498d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5347, + "minutes": 24.42, + "ci_minutes": 0, + "input_tokens": 911343, + "output_tokens": 175684, + "tool_calls": 126, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8227, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 61, + "by_role": { + "agent": { + "usd": 0.507024, + "input_tokens": 893042, + "output_tokens": 155044, + "calls": 59 + }, + "reducer": { + "usd": 0.027645, + "input_tokens": 18301, + "output_tokens": 20640, + "calls": 2 + } + }, + "cache_read_tokens": 3531648, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.23, + "gate": 1.59, + "review": 1459.43, + "report": 0.07 + }, + "time_to_first_result_s": 1465.3 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 28565 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4810.md b/eval/dataset/arms/copilot_v8_official_r1/pr4810.md new file mode 100644 index 000000000..a92babcb7 --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4810.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] grep `get_cache_scale` repo-wide: 4 model_executor files are now clean; 1 remaining hit at vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 (not touched by this diff). +- [sweep] mammoth_moda2.py hunks 483-490 read in situ: `if self.quant_config is not None:` and `if cache_scale_mapper is not None:` are both reachable and not redundant; mapper.apply is gated correctly. +- [sweep] mimo_audio_llm.py hunks 1145-1162 read in situ: same two guards are reachable; mapper is applied before the pre-existing `model.` prefix handling (the prefix-doubling is unchanged by this diff). +- [sweep] hunyuan_image3.py hunk 222 read in situ: removed manual branch; verified parent HunyuanImage3ForConditionalGeneration.load_weights uses AutoWeightsLoader, so the delegation claim is consistent. +- [sweep] qwen2_5_omni/qwen2_old.py hunk 335 read in situ: removed `getattr(self.quant_config, 'get_cache_scale', None)` fallback; no remaining quant branch means quant_config=None is safe. +- [sweep] No arithmetic hunks, no off-by-one risk, no except/fallback fabrication, no external dependency changes, no rebase/merge duplication in the squashed diff. +- [validated] tests/model_executor/models/test_kv_cache_scale_mapper.py: `_STALE_API_FILES` omits the diffusion transformer that the PR body's own main-branch run listed and showed failing (7 failed included hunyuan_image3_transformer.py) — coverage was narrowed to make the test green. +- [validated] test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale feeds `_MAPPED_SCALE_NAME` directly; it never drives `_SOURCE_SCALE_NAME` through HunyuanImage3ForConditionalGeneration.load_weights / the Qwen2.5 parent, so it does not verify the outer AutoWeightsLoader actually applies the mapper. + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:15` [major] — This diff adds `_STALE_API_FILES` in tests/model_executor/models/test_kv_cache_scale_mapper.py with only the four `vllm_omni/model_executor/models/*` paths, but the PR description's own 'on current main' pytest output shows `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py` FAILING `test_model_loaders_do_not_call_removed_get_cache_scale`. That file still contains the removed `.get_cache_scale(` call at line 2238 and is not fixed in this diff, so the regression test is green only because coverage was narrowed; a quantized HunyuanImage3 DiT checkpoint would still raise `AttributeError` instead of fully resolving #4809. Migrate `HunyuanImage3Model.load_weights` to `get_cache_scale_mapper()`/`AutoWeightsLoader` and add that path back to `_STALE_API_FILES`, or explicitly split it into a tracked follow-up and stop claiming `Resolves #4809`. (evidence: grep for 'get_cache_scale(' over the repo returns vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 as a remaining caller; the PR body (pr_context evidence) shows that same file failing test_model_loaders_do_not_call_removed_get_cache_scale on main, while the committed diff's _STALE_API_FILES omits it.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:12` [minor] — This diff adds `tests/model_executor/models/test_kv_cache_scale_mapper.py` with `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, and `gate_report` lists `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` as failing. The AMD merge lane selects `tests/model_executor -m 'core_model and cpu'` (test-amd-merge.yml:32), so this new test is exercised there; before treating those lanes as a merge gate, verify whether they are already red on main or diagnose the lane log if this PR introduced the failure. (evidence: gate_report evidence lists the two failing checks; grep of .buildkite confirms the amd merge lane runs `pytest -sv tests/model_executor -m 'core_model and cpu'` (test-amd-merge.yml:32), which would select this new test file.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:112` [minor] — This diff adds `test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale`, but it calls `HunyuanModel.load_weights` / `Qwen2Model.load_weights` directly with the already-renamed `_MAPPED_SCALE_NAME` (`layers.0.self_attn.attn.k_scale`), so it never constructs the outer `AutoWeightsLoader` that the hunyuan_image3.py and qwen2_old.py changes depend on for renaming. Add a case that drives `_SOURCE_SCALE_NAME` through the outer entry point (for example `HunyuanImage3ForConditionalGeneration.load_weights` / the Qwen2.5-Omni parent loader) with the fake quant config and asserts `_MAPPED_SCALE_NAME` is returned/loaded; otherwise the regression test stays green even if the outer mapper stops applying. (evidence: Read tests/model_executor/models/test_kv_cache_scale_mapper.py lines 100-116: the test passes `_MAPPED_SCALE_NAME` directly to the delegated loader and uses a fake quant_config that only implements get_cache_scale_mapper; it does not exercise AutoWeightsLoader. Read hunyuan_image3.py:2169-2180 and qwen2_old.py:444 showing the outer AutoWeightsLoader construction the PR relies on.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:144` [minor] — This diff removes the old `loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]` / `loaded_weight[0]` shape normalization from the four loaders, but every new test feeds 0-dim `torch.ones(())`, the shape that never needed that normalization. Real compressed-tensors `output_scale` weights are the 1-element tensors the removed `[0]` guarded; add a `torch.ones(1)` case for the direct loaders and run `pytest -q tests/model_executor/models/test_kv_cache_scale_mapper.py` to prove the mapped path does not fail with a shape mismatch, restoring scalar extraction in the mapped path if it does. (evidence: diff removes `loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]` from mammoth_moda2.py and mimo_audio_llm.py and `loaded_weight = loaded_weight[0]` from hunyuan_image3.py; diffusion transformer retains the same `loaded_weight[0]` pattern, confirming 1-element scales are the real input shape; test uses `torch.ones(())` at lines 112, 121, 144.) + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:487` [minor] — The raw sweep notes that the real `get_cache_scale_mapper()` implementation and `AutoWeightsLoader` live in vllm (not vendored in this checkout), so the mapping semantics could not be read here; the ASSUMPTIONS section also says vLLM 0.24 quant_config objects expose `get_cache_scale_mapper()` (returning None or a mapper with `apply()`), but this is not directly verified. Could you add or point to a dependency-version/CI check against the assumed vLLM 0.24 API that validates the two new production `quant_config.get_cache_scale_mapper()` call sites and the `AutoWeightsLoader` dispatch order? (evidence: sweep] `get_cache_scale_mapper` grep: only mammoth_moda2.py:487, mimo_audio_llm.py:1157 and the test fake reference it in-repo; the real mapper implementation and `AutoWeightsLoader` live in vllm (not vendored here), so its mapping semantics could not be read from this checkout. ASSUMPTIONS: vLLM 0.24 quant_config objects expose get_cache_scale_mapper() (returning None or a mapper with apply()), as the PR and the two new production call sites assume.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:111` [minor] — The raw sweep notes the new tests use fake quant configs and `torch.ones()` and therefore do not exercise a real compressed-tensors `quant_config.get_cache_scale_mapper()` return, so real quantized-checkpoint load evidence is still missing (L2 plumbing only). Could you add an integration test with a real compressed-tensors quant config, or explicitly document that this PR covers only the L2 name-mapping plumbing and real compressed-tensors coverage is tracked separately? (evidence: sweep] New tests use fake quant configs and torch.ones(); they do not exercise a real compressed-tensors quant_config.get_cache_scale_mapper() return, so real quantized-checkpoint load evidence is still missing (L2 plumbing only).) + +`vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:?` [nit] — This diff changes MiMoAudioLLMForConditionalGeneration.load_weights to call cache_scale_mapper.apply(weights) before the loop, while the removed get_cache_scale(name) branch ran after `if name.startswith("model."): name = "model." + name`. Add a direct-loader test with a `model.`-prefixed source scale such as `model.layers.0.self_attn.k_proj.output_scale` (or verify the mapper is specified to match raw checkpoint names) so this reordering cannot silently leave prefixed KV-cache scale names unmapped. (evidence: diff hunk for mimo_audio_llm.py shows the mapper block inserted before the loop and the removed `get_cache_scale(name)` call positioned after `name = "model." + name` in the old code.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v8_official_r1/pr4810.trace.json.gz new file mode 100644 index 000000000..f01b7b93c Binary files /dev/null and b/eval/dataset/arms/copilot_v8_official_r1/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4816.cost.json b/eval/dataset/arms/copilot_v8_official_r1/pr4816.cost.json new file mode 100644 index 000000000..f45accd0a --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4816.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 1366.1, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v8_official_r1/runs/pr4816/run-20260813-095507-2aefd1", + "recorded_at": "2026-08-13T10:17:50", + "attempts": [ + { + "invocation_id": "inv-5884efaf", + "outcome": "run:run-20260813-095507-2aefd1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-095507-2aefd1", + "llm_calls": 35, + "input_tokens": 437375, + "output_tokens": 98061, + "usd": 0.261617, + "source": "spans" + } + ], + "attempt_usd_total": 0.261617, + "llm_calls": 35, + "input_tokens": 437375, + "output_tokens": 98061, + "usd": 0.261617, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-095507-2aefd1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2616, + "minutes": 22.72, + "ci_minutes": 0, + "input_tokens": 437375, + "output_tokens": 98061, + "tool_calls": 44, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6677, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 35, + "by_role": { + "planner": { + "usd": 0.000515, + "input_tokens": 87, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.220095, + "input_tokens": 419907, + "output_tokens": 64680, + "calls": 31 + }, + "reducer": { + "usd": 0.041007, + "input_tokens": 17381, + "output_tokens": 32981, + "calls": 3 + } + }, + "cache_read_tokens": 1320704, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.23, + "gate": 1.59, + "review": 1358.19, + "report": 0.07 + }, + "time_to_first_result_s": 1363.06 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 15845 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4816.md b/eval/dataset/arms/copilot_v8_official_r1/pr4816.md new file mode 100644 index 000000000..501458341 --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4816.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] Repo-wide grep for `openai_serving_tokenization` returned 0 matches; grep for `serving_tokenization` returned 8 consistent matches (3 in tests/entrypoints/openai_api/test_serving_speech.py, 5 in vllm_omni/entrypoints/openai/api_server.py). No stale in-repo consumer left behind. +- [validated] api_server.py:754 pure-diffusion branch now sets `state.serving_tokenization = None`; the adjacent comment 'tokenization endpoints are not supported in pure diffusion mode' remains true. +- [validated] api_server.py:1017 standard branch assigns a real `ServingTokenization` unconditionally, so multi-stage error routes that call `_create_speech_error_json_response` now resolve `base(raw_request)` correctly. +- [validated] api_server.py:1157/1248/1286/1329 getattr fallbacks all renamed to `serving_tokenization` with the None guard branch preserved. +- [validated] tests/entrypoints/openai_api/test_serving_speech.py:2533/2546/2615 set `app.state.serving_tokenization = fake_base`, matching the attribute the production getattr now reads; a mismatched name would fail the 404 assertions. +- [sweep] Residual: `_create_speech_error_json_response` (api_server.py:369-376) calls `base(raw_request).create_error_response(...)` with no None guard. In pure-diffusion mode `state.serving_tokenization` is None (api_server.py:754), so `base()` returns None and error paths in `delete_voice`/`upload_voice` (api_server.py:1453-1486) still raise `AttributeError: 'NoneType' object has no attribute 'create_error_response'` -> 500. Pre-existing failure mode, not introduced by this diff. +- [sweep] Fallback-fabrication check: module-level `create_error_response` is already imported at api_server.py:76 and could serve as a semantically valid fallback when `base(raw_request)` is None. +- [sweep] Rebase/merge damage: the four hunks show no duplicated code or moved/renamed symbols; the `base` import line is unchanged. + +`vllm_omni/entrypoints/openai/api_server.py:754` [minor] — This hunk renames the pure-diffusion app state to `state.serving_tokenization = None`. Because `_create_speech_error_json_response` (api_server.py:369-376) calls upstream `base(raw_request)` — which now returns `state.serving_tokenization` — this branch makes `base()` return `None`, so pure-diffusion speech/voice error paths (e.g. `upload_voice` ValueError and `delete_voice` not-found, api_server.py:1453-1486) still crash with `AttributeError: 'NoneType' object has no attribute 'create_error_response'` and return 500 instead of the intended 4xx. Add a None guard that falls back to the already-imported module-level `create_error_response` (api_server.py:76), and add a regression test driving an error through `_create_speech_error_json_response` with `serving_tokenization=None`. Evidence: read api_server.py:369-376, 751-754, 1453-1486; repo grep shows no other writer of `serving_tokenization` in pure-diffusion mode. (evidence: read vllm_omni/entrypoints/openai/api_server.py (lines 369-377, 751-754, 1453-1486); grep 'serving_tokenization' and 'openai_serving_tokenization' repo-wide.) + +`vllm_omni/entrypoints/openai/api_server.py:?` [minor] — This diff renames the state attribute that upstream `base()` now reads, but the PR's actual bug path has no regression test. `_create_speech_error_json_response` (api_server.py:376) calls `base(raw_request).create_error_response(...)`, which is what previously raised `AttributeError: 'State' object has no attribute 'serving_tokenization'`; the tests changed here (test_serving_speech.py:2533/2546/2615) only drive the `handler is None` 404 branches of `create_speech`/`create_speech_batch`/`create_audio_generate`, not the `base()` path. Add a test that calls `_create_speech_error_json_response` (or POSTs a sad `/v1/audio/voices` request) with the real upstream `base` unpatched after `omni_init_app_state`, asserting a 400 JSON error rather than 500. (evidence: Read api_server.py:376 (`_create_speech_error_json_response` -> `base(raw_request).create_error_response`); read diff test hunks at tests/entrypoints/openai_api/test_serving_speech.py:2530-2615 (only create_speech/create_speech_batch/create_audio_generate are exercised); grep shows `test_api_server_delete_voice_exception_returns_500` uses `_patch_api_server_base` (test:2515), which mocks the `base` seam and so cannot catch the rename regression.) + +`pyproject.toml:1` [minor] — The project has no vllm version pin, so the oldest allowed upstream version cannot be verified to include PR 46022 (where `base()` reads `state.serving_tokenization`). Please confirm/document the minimum vllm version that provides this behavior. (evidence: [sweep] Dependency compatibility: grep in pyproject.toml, setup.py, and requirements/* found no vllm version pin, so I could not verify the oldest allowed upstream includes PR 46022 (`base()` reading `state.serving_tokenization`).) + +`vllm_omni/entrypoints/openai/api_server.py:1248` [minor] — The missing-attr fallback at api_server.py:1248-1251 raises HTTPException(404), so the updated tests' status_code==404 assertion may not distinguish the fake-base path from the missing-attr fallback. Please add an assertion that specifically verifies the fake_base/renamed attribute is actually consumed, or test the absent-attr fallback separately. (evidence: [sweep] Test integrity: api_server.py:1248-1251 raises HTTPException(404) when the attr is absent, so a status_code==404 assertion may not distinguish the fake-base path from the missing-attr fallback.) + +`vllm_omni/entrypoints/openai/api_server.py:1157` [minor] — The `create_chat_completion` handler-None branch (api_server.py:1157) was also renamed to use `serving_tokenization`, but no touched test covers that branch. Please add a regression test or confirm existing coverage. (evidence: [sweep] `create_chat_completion` handler-None branch (api_server.py:1157) was also renamed but the touched test file covers only speech/audio endpoints; no diff test exercises that branch.) + +`vllm_omni/entrypoints/openai/api_server.py:754` [nit] — Gate reports buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-intel-ci, and buildkite/vllm-omni-npu-ci failing. This diff touches the shared serving entrypoint (`api_server.py`) that these lanes exercise, so please confirm whether the failures are pre-existing on main or introduced by this rename; do not merge while they are red unless pre-existing is confirmed. (evidence: gate_report lists the three failing checks; the diff changes `vllm_omni/entrypoints/openai/api_server.py` serving initialization.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v8_official_r1/pr4816.trace.json.gz new file mode 100644 index 000000000..036c93d22 Binary files /dev/null and b/eval/dataset/arms/copilot_v8_official_r1/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4825.cost.json b/eval/dataset/arms/copilot_v8_official_r1/pr4825.cost.json new file mode 100644 index 000000000..fd2f82549 --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4825.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 709.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v8_official_r1/runs/pr4825/run-20260813-101753-4b19c0", + "recorded_at": "2026-08-13T10:29:40", + "attempts": [ + { + "invocation_id": "inv-f4834048", + "outcome": "run:run-20260813-101753-4b19c0", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-101753-4b19c0", + "llm_calls": 44, + "input_tokens": 413841, + "output_tokens": 101541, + "usd": 0.264095, + "source": "spans" + } + ], + "attempt_usd_total": 0.264095, + "llm_calls": 44, + "input_tokens": 413841, + "output_tokens": 101541, + "usd": 0.264095, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-101753-4b19c0", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2641, + "minutes": 11.77, + "ci_minutes": 0, + "input_tokens": 413841, + "output_tokens": 101541, + "tool_calls": 82, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.474, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 44, + "by_role": { + "agent": { + "usd": 0.238661, + "input_tokens": 403972, + "output_tokens": 80873, + "calls": 42 + }, + "reducer": { + "usd": 0.025434, + "input_tokens": 9869, + "output_tokens": 20668, + "calls": 2 + } + }, + "cache_read_tokens": 1506048, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.42, + "gate": 2.26, + "review": 700.46, + "report": 0.06 + }, + "time_to_first_result_s": 706.18 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 19423 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4825.md b/eval/dataset/arms/copilot_v8_official_r1/pr4825.md new file mode 100644 index 000000000..add1c7233 --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4825.md @@ -0,0 +1,36 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4825, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac3a2ce5a9b9) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] manager.py:368-380 hunk read in situ; `unet` is added to `default_components` and the loop's `hasattr` + `isinstance(component, nn.Module)` guards are unchanged and correct. +- [sweep] tests/diffusion/lora/test_lora_manager.py:587 branch `if isinstance(layer, _FakeLinearBase):` — both arms occur: container modules (`unet`, `down_block`) return `layer` unchanged while `_FakeLinearBase` returns the wrapper; not dead/redundant. +- [validated] New test drives the real entry point `_replace_layers_with_lora` and asserts `down_block.proj` in replace_calls, `unet.down_block.proj` in `_lora_modules`, and in-tree replacement via isinstance — it exercises the named discovery behavior. +- [sweep] `_matches_target` consumers at manager.py:408 and :415 read; this component-list change does not alter target matching semantics. +- [sweep] `_fake_from_layer_diffusion` references at test_lora_manager.py:110/150/544/586 are all monkeypatched test-local fixtures; no production callers. +- [sweep] `_lora_components` consumers: only sensenova_u1/pipeline_sensenova_u1.py:543 defines it; no other model opts in, so the default list addition is the only change. +- [sweep] Blast radius: grep `self\.unet =` in vllm_omni/diffusion/models found only sdxl/pipeline_sdxl.py:110 (top-level pipeline) and soulx_singer rmvpe.py:236 (nested preprocess module, not the pipeline passed to DiffusionLoRAManager); current inheritance is SDXL-only. +- [validated] SDXL declares `_dit_modules = ["unet"]` (pipeline_sdxl.py:46), sets `self.unet` (line 110), and registry.py:427 `_apply_sequence_parallel_if_enabled` already includes `unet`, so the new default is consistent with component discovery. + +`vllm_omni/diffusion/lora/manager.py:375` [minor] — This diff hardcodes `"unet"` into `default_components`, duplicating existing knowledge: `SupportsComponentDiscovery._dit_modules` (vllm_omni/diffusion/models/interface.py:86-93) and SDXL `_dit_modules = ["unet"]` (pipeline_sdxl.py:46) already identify the denoiser component, and `registry.py:427` has its own `transformer_attrs = ["transformer", "transformer_2", "dit", "unet"]`. Derive components from the pipeline's `_dit_modules` when it implements the protocol, falling back to the current defaults otherwise, so SDXL and future UNet-style pipelines do not require another manager edit and these lists cannot drift. (evidence: Read manager.py hunk; read vllm_omni/diffusion/models/interface.py:86-93, pipeline_sdxl.py:46, registry.py:427.) + +`vllm_omni/diffusion/lora/manager.py:380` [minor] — This diff adds `"unet"` to `default_components`, and the new test in `tests/diffusion/lora/test_lora_manager.py` carries `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, so the AMD ready lane (`.buildkite/test-amd-merge.yml:21`, `pytest -sv tests/diffusion -m 'core_model and cpu'`) selects it. `gh_ci_read` for PR #4825 shows `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` failing; please confirm on the merge-base whether these are pre-existing on `main` or introduced by this PR/test, and attach the failing job log if the new test or default-component change caused the red. (evidence: gh_ci_read(4825) shows amd-ci and npu-ci FAILURE; .buildkite/test-amd-merge.yml:21 runs tests/diffusion with -m 'core_model and cpu'; tests/diffusion/lora/test_lora_manager.py has pytestmark=[pytest.mark.core_model, pytest.mark.cpu].) + +`vllm_omni/diffusion/lora/manager.py:380` [minor] — This diff adds `"unet"` to the shared `default_components`, so every pipeline exposing a top-level `unet` is now scanned, not only SDXL; grep in `vllm_omni/diffusion` shows `pipeline_ovis_image.py:590` and `pipeline_flux2_klein.py:800` also reference `self.unet`. Confirm OvisImage/Flux2Klein LoRA behavior under the new default is intended (`from_layer_diffusion` returns non-replaceable layers unchanged, so it may be benign) or scope the new default to SDXL-style pipelines so unrelated models do not silently change discovery behavior. (evidence: grep 'unet' vllm_omni/diffusion -> pipeline_ovis_image.py:590, pipeline_flux2_klein.py:800, pipeline_sdxl.py:46/110; vllm_omni/diffusion/lora/utils.py from_layer_diffusion returns layer unchanged when no class can replace it.) + +`tests/diffusion/lora/test_lora_manager.py:587` [minor] — The diff adds `test_lora_manager_discovers_unet_component`, but the test uses `pipeline.unet.down_block.proj = _FakeLinearBase()` with a monkeypatched `from_layer_diffusion` that wraps only `_FakeLinearBase`, and `peft_helper = type("_PH", (), {"r": 1})()` has no `target_modules`, so `_matches_target` returns True for every module (manager.py:355). It therefore proves only that the `unet` component is scanned, not that real SDXL layers can be replaced or that SDXL target-module names match. Add a CPU test that constructs the real `SDXLUNet2DConditionModel` (mock/empty weights) and passes an SDXL `target_modules` list (e.g. `["to_k", "to_q", "to_v", "to_out.0"]`), asserting that e.g. `unet.down_blocks.1.attentions.0.transformer_blocks.0.attn1.to_qkv` is wrapped and only intended modules are replaced; the commit timeline shows a `to_out.0` normalization was tried and reverted, so target-name matching is the residual risk. (evidence: Read vllm_omni/diffusion/models/sdxl/sdxl_unet.py:151/175 (SDXL `to_out` is `nn.ModuleList([RowParallelLinear(...)])`, a shape the fake pipeline does not model) and manager.py:415-431 (the real `from_layer_diffusion` call).) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v8_official_r1/pr4825.trace.json.gz new file mode 100644 index 000000000..e7c13cc8a Binary files /dev/null and b/eval/dataset/arms/copilot_v8_official_r1/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4837.cost.json b/eval/dataset/arms/copilot_v8_official_r1/pr4837.cost.json new file mode 100644 index 000000000..2a344bfc0 --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4837.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 769.9, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v8_official_r1/runs/pr4837/run-20260813-093040-302be1", + "recorded_at": "2026-08-13T09:43:28", + "attempts": [ + { + "invocation_id": "inv-a11afb77", + "outcome": "run:run-20260813-093040-302be1", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-093040-302be1", + "llm_calls": 43, + "input_tokens": 459634, + "output_tokens": 109156, + "usd": 0.290286, + "source": "spans" + } + ], + "attempt_usd_total": 0.290286, + "llm_calls": 43, + "input_tokens": 459634, + "output_tokens": 109156, + "usd": 0.290286, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-093040-302be1", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2903, + "minutes": 12.8, + "ci_minutes": 0, + "input_tokens": 459634, + "output_tokens": 109156, + "tool_calls": 77, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5083, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.273412, + "input_tokens": 450977, + "output_tokens": 95941, + "calls": 41 + }, + "reducer": { + "usd": 0.016874, + "input_tokens": 8657, + "output_tokens": 13215, + "calls": 2 + } + }, + "cache_read_tokens": 1707904, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.87, + "gate": 1.61, + "review": 762.45, + "report": 0.06 + }, + "time_to_first_result_s": 767.97 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 17236 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4837.md b/eval/dataset/arms/copilot_v8_official_r1/pr4837.md new file mode 100644 index 000000000..fa0733006 --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4837.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4837, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] NEW BRANCH vllm_omni/engine/orchestrator.py:1290 `if len(diffusion_prompt) == 1:` — both arms live: empty list is handled earlier (orchestrator.py:1274-1288 routes a terminal error), len==1 unwraps to a dict, len>1 stays a list and is still rejected by StagePool.submit_initial (stage_pool.py:947-952). The branch is not dead; removing the `already_submitted and` guard is the intended fix. +- [sweep] DIFF HUNK vllm_omni/engine/orchestrator.py:1287 read in situ (orchestrator.py offset 49059) — confirmed the unwrap now runs before the `already_submitted` dispatch for both submit_update and submit_initial. +- [sweep] DIFF HUNK vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:446/462/472 read in situ — old `outputs[0]` + hasattr(.output/.images) fallback replaced with `raw = outputs.output` and `return outputs`. +- [validated] Orchestrator fix matches linked issue #4839: a single-element list from thinker2imagegen is no longer passed as a list to submit_initial (which raises 'Diffusion list-prompt batch requests are no longer supported' at stage_pool.py:947-952); len>1 remains intentionally rejected by the same guard, consistent with the scheduler-batching contract. +- [validated] Blast radius of orchestrator unwrap: it is shared by every diffusion stage with a custom_process_input_func (grep found ming_flash_omni, hunyuan_image3, glm_image, mammoth_moda2, dynin_omni, aura_omni, mimo_audio, qwen3_omni, minicpmo_4_5, cosyvoice3). The old behavior rejected ALL list prompts on initial submit, so no model could rely on the old rejection; the change is strictly more permissive and the cross-model regression risk is low. +- [sweep] Changed output format consumers: get_ming_image_post_process_func docstring says the post-processor reads `output_data` as the `DiffusionOutput.output` tensor; returning the original DiffusionOutput preserves `.output` and now also preserves `stage_durations`. No stale in-repo consumer found for the removed `.images`/`outputs[0]` fallbacks. +- [sweep] No test files are touched. grep tests for MingImagePipeline only finds test_config_factory.py:1344/1382 `model_arch == "MingImagePipeline"` assertions; no test exercises MingImagePipeline.forward or the orchestrator single-element unwrap branch. +- [sweep] gate_report failing: buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci — recorded as one minor comment asking whether pre-existing on main; attribution is not claimed. + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:?` [blocker] — The diff changes pipeline_ming_imagegen.py to set `raw = outputs.output` and `return outputs` (lines 449-472), relying on `super().forward(...)` returning a `DiffusionOutput`. But the touched call at line 449 still invokes `super().forward(z_req, prompt=None, height=height, width=width, num_inference_steps=..., guidance_scale=..., prompt_embeds=..., negative_prompt_embeds=..., generator=..., output_type="pt", return_dict=True)`. The current `ZImagePipeline.forward` at vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413 is `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput` with no **kwargs, so this call raises `TypeError: forward() got an unexpected keyword argument 'prompt'` before `return outputs` executes, breaking every Ming image request on this head. Update the call to match the current `ZImagePipeline.forward` contract (pass only `z_req`, reading image parameters from `req.sampling_params` as main does) or restore a kwargs-accepting forward, then re-run the PR's serve/curl reproduction on this head. (evidence: ZImagePipeline.forward signature at vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413 accepts only req; call site at pipeline_ming_imagegen.py:449-461 passes unsupported kwargs.) + +`vllm_omni/engine/orchestrator.py:?` [minor] — The diff changes orchestrator.py:1290 from `if already_submitted and len(diffusion_prompt) == 1:` to `if len(diffusion_prompt) == 1:`, so the singleton `list[dict]` unwrap now runs on the initial-submit path as well. No test in this diff exercises that branch: the in-repo `_forward_to_next_stage` test callers are `tests/engine/test_orchestrator_kv_sender_info.py` and `tests/engine/test_orchestrator_stage_input_bridge.py`, and no test references `pipeline_ming_imagegen`. Add a regression test that drives `_forward_to_next_stage` with `stage_type == 'diffusion'`, a bridge returning `[{"prompt": ...}]`, and asserts `submit_initial` receives the unwrapped dict on the `already_submitted=False` path (the case this PR fixes), plus a second case for the `already_submitted=True` update path. (evidence: grep tests for `_forward_to_next_stage` returned test_orchestrator_kv_sender_info.py and test_orchestrator_stage_input_bridge.py; grep tests for `pipeline_ming_imagegen` returned no matches.) + +`vllm_omni/engine/orchestrator.py:?` [minor] — The diff changes the guard to `if len(diffusion_prompt) == 1:` at orchestrator.py:1290 and unwraps only singleton lists; for `len > 1`, the list still goes unchanged to `submit_initial`/`submit_update`, which the PR description states reject lists for diffusion stages. A bridge that returns more than one dict (e.g. CFG companion outputs bundled by `diffusion_source_outputs`) would therefore still hit the downstream `ValueError` rather than get an explicit orchestrator-level decision. Either handle `len > 1` deliberately or raise a clear error for it. (evidence: Read orchestrator.py diffusion branch: after `if len(diffusion_prompt) == 1` there is no else/error for len>1, and the list is passed directly to `submit_update`/`submit_initial`; PR description states both submit paths reject a list for diffusion stages.) + +`buildkite/vllm-omni-amd-ci:1` [minor] — The raw gate report lists buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci as failing, but attribution to this diff is unverified. Can you confirm whether these gates are pre-existing on main or introduced by this change, and if introduced, address them before merge? (evidence: [gate] buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci failing; attribution to this diff is unverified, recorded as a minor comment only.) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:449` [minor] — The raw sweep found no test exercises MingImagePipeline.forward; MingImagePipeline appears only in tests/test_config_factory.py as model_arch assertions and never exercises forward behavior. Since this PR changes the forward return path, could you add coverage for the MingImagePipeline.forward path so this remains tested on the head? (evidence: [sweep] tests grep `pipeline_ming_imagegen`: no matches under tests/; `MingImagePipeline` appears only in tests/test_config_factory.py:1344/1381-1382 as model_arch assertions, never exercising forward behavior.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v8_official_r1/pr4837.trace.json.gz new file mode 100644 index 000000000..921404c39 Binary files /dev/null and b/eval/dataset/arms/copilot_v8_official_r1/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4893.cost.json b/eval/dataset/arms/copilot_v8_official_r1/pr4893.cost.json new file mode 100644 index 000000000..00fc85456 --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4893.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 2086.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v8_official_r1/runs/pr4893/run-20260813-094332-59938d", + "recorded_at": "2026-08-13T10:18:15", + "attempts": [ + { + "invocation_id": "inv-08f25290", + "outcome": "run:run-20260813-094332-59938d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-094332-59938d", + "llm_calls": 69, + "input_tokens": 730307, + "output_tokens": 225503, + "usd": 0.514685, + "source": "spans" + } + ], + "attempt_usd_total": 0.514685, + "llm_calls": 69, + "input_tokens": 730307, + "output_tokens": 225503, + "usd": 0.514685, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-094332-59938d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5147, + "minutes": 34.72, + "ci_minutes": 0, + "input_tokens": 730307, + "output_tokens": 225503, + "tool_calls": 133, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.9481, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 69, + "by_role": { + "agent": { + "usd": 0.463012, + "input_tokens": 707645, + "output_tokens": 184128, + "calls": 66 + }, + "reducer": { + "usd": 0.051673, + "input_tokens": 22662, + "output_tokens": 41375, + "calls": 3 + } + }, + "cache_read_tokens": 2572160, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.26, + "gate": 1.75, + "review": 2076.94, + "report": 0.06 + }, + "time_to_first_result_s": 2082.99 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 22947 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4893.md b/eval/dataset/arms/copilot_v8_official_r1/pr4893.md new file mode 100644 index 000000000..d86bf4b4e --- /dev/null +++ b/eval/dataset/arms/copilot_v8_official_r1/pr4893.md @@ -0,0 +1,53 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4893, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | 4 finding(s) below | + +**Validated:** +- [sweep] diff hunk tests/diffusion/distributed/test_expert_parallel_layout.py:19-24 read in situ; `if reduce_scatter is not None:` only fires for the fake vLLM coordinator and correctly avoids setting a fake attribute on diffusion fakes. +- [sweep] diff hunk tests/diffusion/distributed/test_expert_parallel_layout.py:59-92 read in situ; `fake_init_vllm_model_parallel_group` mirrors the real wrapper's kwargs (device_communicator + reduce_scatter) and monkeypatches the new symbol; no stale callers. +- [sweep] diff hunk tests/diffusion/distributed/test_expert_parallel_layout.py:109-190 read in situ; `_PCP is not _SP`, world sizes, device_communicator/reduce_scatter presence, and `vllm_pcp/tp/dp/ep` creation order are all asserted consistently with the new code. +- [sweep] diff hunk vllm_omni/diffusion/distributed/parallel_state.py:512-526 read in situ; `init_vllm_model_parallel_group` is a thin passthrough adding `use_device_communicator=True`; no re-derived rank math. +- [sweep] diff hunk vllm_omni/diffusion/distributed/parallel_state.py:853-904 read in situ; the three `if use_moe_parallel_mapping:` blocks cannot diverge (the flag is computed once and unchanged) and the `else` arm preserves the pre-existing non-EP diffusion `_TP`. +- [sweep] branch parallel_state.py:882 `if use_moe_parallel_mapping:` widened from `use_moe_parallel_mapping and cfg_parallel_size > 1`; this is required for the reported `tp2_sp2` (cfg=1) layout, not a regression. +- [sweep] branch hunyuan_fused_moe.py:31/34/38 read in situ; early-returns for no forward context, existing dp_metadata, and world_size<=1 are all reachable and semantically correct. +- [sweep] branch hunyuan_fused_moe.py:47 `if any(count is None ...)` is unreachable defensive code: `torch.distributed.all_gather_object` either fills every slot or raises; flagged as a nit to remove/simplify. + +`tests/diffusion/distributed/test_expert_parallel_layout.py:?` [major] — The new mapping tests assert attributes on monkeypatched/GroupCoordinator fakes that the fakes themselves inject. The production `init_vllm_model_parallel_group` path is not exercised, so regressions in real vLLM coordinator assignment would pass. Add a test that calls the real `init_vllm_model_parallel_group` (or uses real coordinators) and asserts the resulting `_vllm_ps._TP/_DP/_EP/_PCP`. (evidence: Branch at tests/diffusion/distributed/test_expert_parallel_layout.py:24; no production init_vllm_model_parallel_group test found.) + +`vllm_omni/diffusion/distributed/parallel_state.py:?` [major] — `use_moe_parallel_mapping` is set from `od_config.is_moe`, which is shared across diffusion MoE models. That means any is_moe model enabling EP inherits the new vLLM coordinators, but only HunyuanImage3 was validated. Please make the gate HunyuanImage3-specific or explicitly validate/document Wan2.2-I2V-A14B and other is_moe models. (evidence: Behavior lens item 9 blast radius: gate use_moe_parallel_mapping = bool(od_config and od_config.is_moe) is model-agnostic; Wan2.2-I2V-A14B is_moe per tests/e2e/online_serving/test_wan22_expansion.py:147.) + +`tests/diffusion/distributed/test_expert_parallel_layout.py:?` [major] — The new vLLM GroupCoordinator wiring and widened `use_moe_parallel_mapping` condition are not exercised by an automated unit test; only the remote smoke covers the tp2_sp2 cfg=1 launch path. Add a regression test for cfg_parallel_size=1 with `use_moe_parallel_mapping=True`. (evidence: Summary states no unit coverage for the cfg=1 remap the diff newly enables.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:?` [major] — The pre-hook always gathers dp_metadata. In non-EP MoE runs with data_parallel_size > 1, the DP coordinator is the diffusion `_DP`, not the vLLM cfg-dp group, so the gather may run on the wrong communicator. Guard the gather with the same EP-mapping condition (or verify `_vllm_ps._DP` is the real vLLM group) and add a non-EP DP>1 regression test. (evidence: parallel_state.py:882 DP is folded into vLLM cfg-dp only under EP; non-EP retains `_vllm_ps._DP = _DP` alias; no test covers non-EP DP>1 path.) + +`vllm_omni/diffusion/distributed/parallel_state.py:868` [major] — Buildkite vllm-omni-amd-ci and vllm-omni-npu-ci are failing, and the diff touches shared diffusion parallel state consumed by the NPU runtime. The available evidence cannot determine whether the failures are pre-existing or introduced. Could you confirm attribution and attach logs before merging? (evidence: [sweep] CI gate: buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci are failing; the diff touches shared diffusion parallel state consumed by the NPU runtime, so attribution (pre-existing vs introduced) cannot be decided from the available evidence.) + +`tests/diffusion/distributed/test_expert_parallel_layout.py:?` [minor] — The None-sentinel guard is needed only for the fake vLLM coordinator path. Consider narrowing the fake class contract rather than testing for None broadly. (evidence: Draft sweep notes the guard correctly avoids setting a fake attribute on diffusion fakes.) + +`vllm_omni/diffusion/distributed/parallel_state.py:?` [minor] — Confirm all four companion groups (_TP, _SP, _DP, _EP) should use use_device_communicator=True, and that no companion requires a CPU-only group for the existing cp_group consumers. (evidence: Summary lists confirmation that all four companion groups need use_device_communicator=True as remaining.) + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:?` [minor] — Add hook-level test coverage proving the fused MoE pre-hook populates ForwardContext.dp_metadata from the vLLM DP group, rather than only unit-testing DPMetadata construction. (evidence: Finding: '[sweep] test_sets_dp_metadata_from_vllm_dp_group constructs a real vllm.forward_context.DPMetadata and asserts num_tokens_across_dp_cpu == [1024, 1025]...' and summary lists hook-level test coverage as remaining.) + +`vllm_omni/diffusion/distributed/parallel_state.py:?` [minor] — Harden cleanup for partially-initialized companion coordinators: if an exception occurs after _PCP creation but before later assignments, destroy_distributed_env may not call destroy_model_parallel because model_parallel_is_initialized() is False, leaking the new communicator until process exit. (evidence: Finding: '[sweep] resource lifecycle on abort: a raise between _PCP creation (parallel_state.py:859) and _TP assignment ... leaves the new PCP communicator un-destroyed...') + +`vllm_omni/diffusion/distributed/parallel_state.py:868` [minor] — The review could not inspect the installed vLLM 0.24 source to confirm GroupCoordinator exposes cpu_group/world_size/device_communicator/reduce_scatter as claimed. Could you pin/verify the vLLM version and provide evidence for these APIs, especially cpu_group used by the new dp_metadata gather? (evidence: vLLM 0.24 GroupCoordinator exposes cpu_group/world_size/device_communicator/reduce_scatter as claimed by the PR; I could not inspect the installed vLLM source in this worktree.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:47` [minor] — The `if any(count is None ...)` guard appears unreachable because torch.distributed.all_gather_object either fills every slot or raises. Could you remove or simplify this defensive branch? (evidence: [sweep] branch hunyuan_fused_moe.py:47 `if any(count is None ...)` is unreachable defensive code: `torch.distributed.all_gather_object` either fills every slot or raises; flagged as a nit to remove/simplify.) + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/distributed/parallel_state.py:0` [minor] Verify the AMD and NPU CI failures observed in gate_report are pre-existing on main and not caused by this diff. + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v8_official_r1/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v8_official_r1/pr4893.trace.json.gz new file mode 100644 index 000000000..737599f3f Binary files /dev/null and b/eval/dataset/arms/copilot_v8_official_r1/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v9_official_r1/manifest.json b/eval/dataset/arms/copilot_v9_official_r1/manifest.json new file mode 100644 index 000000000..3779b512a --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "copilot_v9_official_r1", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4810.cost.json b/eval/dataset/arms/copilot_v9_official_r1/pr4810.cost.json new file mode 100644 index 000000000..ea95d4dc3 --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4810.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 866.8, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v9_official_r1/runs/pr4810/run-20260813-103518-322e84", + "recorded_at": "2026-08-13T10:49:43", + "attempts": [ + { + "invocation_id": "inv-36a18cf5", + "outcome": "run:run-20260813-103518-322e84", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-103518-322e84", + "llm_calls": 40, + "input_tokens": 641971, + "output_tokens": 148264, + "usd": 0.403348, + "source": "spans" + } + ], + "attempt_usd_total": 0.403348, + "llm_calls": 40, + "input_tokens": 641971, + "output_tokens": 148264, + "usd": 0.403348, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-103518-322e84", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4033, + "minutes": 14.41, + "ci_minutes": 0, + "input_tokens": 641971, + "output_tokens": 148264, + "tool_calls": 110, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5918, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 40, + "by_role": { + "agent": { + "usd": 0.379034, + "input_tokens": 626934, + "output_tokens": 129883, + "calls": 38 + }, + "reducer": { + "usd": 0.024314, + "input_tokens": 15037, + "output_tokens": 18381, + "calls": 2 + } + }, + "cache_read_tokens": 2478720, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.7, + "gate": 1.88, + "review": 858.11, + "report": 0.06 + }, + "time_to_first_result_s": 864.75 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 25112 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4810.md b/eval/dataset/arms/copilot_v9_official_r1/pr4810.md new file mode 100644 index 000000000..14f7a704f --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4810.md @@ -0,0 +1,42 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4810, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] mammoth_moda2.py:486 `if self.quant_config is not None:` — read hunk in situ; arm reachable for unquantized models and correctly skips mapper application. +- [sweep] mammoth_moda2.py:488 `if cache_scale_mapper is not None:` — reachable for quant configs without a cache-scale mapper; guard correct. +- [sweep] mimo_audio_llm.py:1156/1158 same two guards — read hunk in situ; same reasoning as mammoth_moda2. +- [sweep] Diff hunk locations — read hunyuan_image3.py around 222, mammoth_moda2.py around 483, mimo_audio_llm.py around 1153, and qwen2_old.py in full; every changed hunk was verified in situ. +- [validated] test file parametrization covers all four touched non-test files: hunyuan_image3 + qwen2_old via test_auto_weights_loader_delegated_loaders_accept_mapped_cache_scale, mammoth_moda2 + mimo_audio via test_direct_custom_loaders_apply_cache_scale_mapper. +- [validated] repo-wide grep for `get_cache_scale(` returns only the untouched vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 plus the new test's own literal; the four touched files are clean of the removed API. +- [sweep] Item 3 breaking-behavior: no public default/API/protocol/output-format signature changes in this diff — only internal weight-loading call paths; no in-repo consumers of old behavior were left stale. +- [sweep] Item 9 blast radius: changed values live in per-model loaders (hunyuan_image3 AR, mammoth_moda2 AR, mimo_audio LLM, qwen2.5-omni old Qwen2); no shared platform/default dispatch inherits them. + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:?` [major] — The diff removes the `get_cache_scale` branches from the four `model_executor` loaders and the new `_STALE_API_FILES` list contains only those four files, but `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` still calls `self.quant_config.get_cache_scale(name)` (the PR body's own "on current main" pytest run lists this file as FAILED). This loader is neither migrated nor guarded, so a quantized HunyuanImage3 diffusion checkpoint would still raise `AttributeError ... get_cache_scale`; migrate it to `get_cache_scale_mapper()` or keep it in `_STALE_API_FILES` and track the remaining work instead of dropping the coverage. (evidence: grep for `.get_cache_scale(` across the repo returns only this line (2238) outside the test file; the PR body's 'on current main' pytest output explicitly lists this file as FAILED, while the diff's `_STALE_API_FILES` no longer includes it.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:81` [minor] — The diff removes `get_cache_scale = getattr(self.quant_config, "get_cache_scale", None)` from `qwen2_old.py`, but the new scan test at tests/model_executor/models/test_kv_cache_scale_mapper.py:81 asserts only `".get_cache_scale(" not in source`. That literal is absent from the removed `getattr(..., "get_cache_scale", ...)` form, so a reintroduction of exactly that guarded qwen2_old pattern would pass. Add a separate assertion or parametrized guard for the `getattr` form (`"get_cache_scale"` absent as well for qwen2_old). (evidence: read the qwen2_old.py diff hunk (removed `get_cache_scale = getattr(self.quant_config, "get_cache_scale", None)`) and the test assertion at line 81.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:?` [minor] — The new source-scan guard at tests/model_executor/models/test_kv_cache_scale_mapper.py:81 is the only guard against reintroducing the removed API, but it is satisfiable by deleting files from `_STALE_API_FILES` (this diff already drops `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py`, which still has a call) and it would miss an indirect call such as `mapper = getattr(quant_config, 'get_cache_scale'); mapper(name)`. Prefer a behavioral test for the remaining loader, or at minimum keep `_STALE_API_FILES` complete until every caller is migrated. (evidence: `_STALE_API_FILES` (lines 15-20) contains only the four model_executor paths, while grep shows `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` still calls `.get_cache_scale(`.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:144` [minor] — The diff deletes `loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]` from mammoth_moda2/mimo_audio, but `test_direct_custom_loaders_apply_cache_scale_mapper` only passes scalar `torch.ones(())`, so the 1-D per-channel scale case that the old `[0]` slicing handled is no longer covered. Add a parametrized 1-D scale weight (e.g. `torch.ones(4)`, with a matching fake parameter shape) and assert the loaded value, so the new mapper path is verified for non-scalar scales. (evidence: diff of mammoth_moda2.py and mimo_audio_llm.py removes `loaded_weight = loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0]`; test file uses `torch.ones(())` in every load_weights call.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:112` [minor] — The diff removes the Hunyuan-image AR and Qwen2 old `get_cache_scale` branches on the assumption that the outer `AutoWeightsLoader` maps `k_proj.output_scale` -> `attn.k_scale`, but the new delegated-loader test calls `HunyuanModel.load_weights` / `Qwen2Model.load_weights` directly with the already-mapped `_MAPPED_SCALE_NAME` and never constructs the outer loader. It would still pass if the outer mapping stopped occurring. Drive the unmapped `_SOURCE_SCALE_NAME` through the top-level model's `load_weights` (or rename this test and add a separate outer-mapping test). (evidence: Test function lines 93-109; hunyuan_image3.py:2178 `loader = AutoWeightsLoader(...)`; qwen2_old.py parent `load_weights` returns `loader.load_weights(weights)` with `AutoWeightsLoader(self, ...)`.) + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:487` [minor] — The diff makes mammoth_moda2/mimo_audio call `self.quant_config.get_cache_scale_mapper()`, an API introduced in vLLM v0.23.0, but the repo does not declare a vLLM minimum (grep showed no vllm requirement in requirements/common.txt, and setup.py only loads platform requirement files). If vLLM is intentionally undeclared, enforce/document the >=0.23 minimum so an older vLLM environment fails with a clear version error instead of an AttributeError in these loaders. (evidence: requirements/common.txt (read) contains no vllm requirement; setup.py get_install_requires loads requirements/.txt; PR description states get_cache_scale_mapper landed in vLLM v0.23.0rc2.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:12` [minor] — The gate report shows red on buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci, and this new test file is selected by those lanes. Please confirm whether those failures are pre-existing on main or introduced by this branch, and link the baseline/head comparison before merge. (evidence: [sweep] gate_report lists buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci failing; the diff touches model_executor loaders and adds a test selected in those lanes — attribution cannot be verified from here.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v9_official_r1/pr4810.trace.json.gz new file mode 100644 index 000000000..530a04c8a Binary files /dev/null and b/eval/dataset/arms/copilot_v9_official_r1/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4816.cost.json b/eval/dataset/arms/copilot_v9_official_r1/pr4816.cost.json new file mode 100644 index 000000000..2c6d9c295 --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4816.cost.json @@ -0,0 +1,113 @@ +{ + "wall_s": 522.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v9_official_r1/runs/pr4816/run-20260813-104945-e95971", + "recorded_at": "2026-08-13T10:58:25", + "attempts": [ + { + "invocation_id": "inv-c5c3cdea", + "outcome": "run:run-20260813-104945-e95971", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-104945-e95971", + "llm_calls": 33, + "input_tokens": 556406, + "output_tokens": 70011, + "usd": 0.271714, + "source": "spans" + } + ], + "attempt_usd_total": 0.271714, + "llm_calls": 33, + "input_tokens": 556406, + "output_tokens": 70011, + "usd": 0.271714, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-104945-e95971", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2717, + "minutes": 8.66, + "ci_minutes": 0, + "input_tokens": 556406, + "output_tokens": 70011, + "tool_calls": 57, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4035, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 33, + "by_role": { + "planner": { + "usd": 0.000515, + "input_tokens": 87, + "output_tokens": 400, + "calls": 1 + }, + "agent": { + "usd": 0.251968, + "input_tokens": 546534, + "output_tokens": 54562, + "calls": 30 + }, + "reducer": { + "usd": 0.01923, + "input_tokens": 9785, + "output_tokens": 15049, + "calls": 2 + } + }, + "cache_read_tokens": 1647104, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.19, + "gate": 2.04, + "review": 514.13, + "report": 0.07 + }, + "time_to_first_result_s": 519.41 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 16895 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4816.md b/eval/dataset/arms/copilot_v9_official_r1/pr4816.md new file mode 100644 index 000000000..1208906ec --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4816.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4816, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] grep repo-wide for `openai_serving_tokenization`: 0 matches; grep `serving_tokenization`: only api_server.py (754, 1017, 1157, 1248, 1286, 1329) and test_serving_speech.py (2533, 2546, 2615) — rename is complete, no stale consumers. +- [validated] api_server.py:754 pure-diffusion `state.serving_tokenization = None` — hunk matches diff; no arithmetic/off-by-one/branch-co-occurrence issue. +- [validated] api_server.py:1017 LLM/multi-stage `state.serving_tokenization = ServingTokenization(...)` — constructor args unchanged from the old name, only the attribute name changed. +- [validated] api_server.py:1157, 1248, 1286, 1329 all read via `getattr(raw_request.app.state, "serving_tokenization", None)`, so a missing attribute degrades to the intended 404 branch instead of AttributeError. +- [validated] api_server.py:369-377 `_create_speech_error_json_response` is the only `base(raw_request)` caller (grep found one match); upstream `base()` reads `request.app.state.serving_tokenization` per PR description, so LLM mode now returns a real handler. +- [sweep] Pure-diffusion residual: `state.serving_tokenization = None` means `_create_speech_error_json_response` still does `base(raw_request).create_error_response(...)` on None, crashing speech error routes (e.g. upload_voice decode ValueError) in pure diffusion mode — pre-existing behavior, but the stated fix does not cover this launch path. +- [sweep] Tests: the three changed assertions only rename the injected attribute for existing 404 tests; none drives the actual bug path (`upload_voice` ValueError -> `_create_speech_error_json_response` -> upstream `base()`), so the fixed behavior is untested. +- [sweep] Simplifiability: no branches for cases that cannot co-occur, no hand-re-derived helper values, no duplicated code, no arithmetic hunk. + +`vllm_omni/entrypoints/openai/api_server.py:?` [major] — Diff changes the pure-diffusion branch from `state.openai_serving_tokenization = None` to `state.serving_tokenization = None`; upstream `base()` (called by `_create_speech_error_json_response` at api_server.py:376) returns `request.app.state.serving_tokenization` with no default, so a pure-diffusion voice route that reaches the error helper now evaluates `None.create_error_response(...)` and still returns 500. Guard `_create_speech_error_json_response` against a `None` base (or install a real error-response provider in pure-diffusion mode) and add a pure-diffusion `upload_voice` ValueError regression test. (evidence: Read api_server.py:369-377 (_create_speech_error_json_response), api_server.py:751-754 (pure-diffusion branch sets state.serving_tokenization=None), api_server.py:1462-1483 (delete_voice/upload_voice call the helper on ValueError/not-found); grep confirms no other `serving_tokenization` setter in the pure-diffusion path.) + +`vllm_omni/entrypoints/openai/api_server.py:754` [minor] — Diff renames `state.openai_serving_tokenization` to `state.serving_tokenization`, but the updated tests only lock the direct `getattr(..., "serving_tokenization", None)` branches in `create_speech`, `create_speech_batch`, and `create_audio_generate`; no test exercises `_create_speech_error_json_response` with `base` unmocked, so the actual fixed regression (upload_voice ValueError becoming a 500 because upstream `base()` reads `app.state.serving_tokenization`) is not covered. Add a regression test that sets `raw_request.app.state.serving_tokenization` to a fake with `create_error_response`, does not patch `api_server_module.base`, and drives `upload_voice` through a ValueError, asserting a 4xx OpenAI JSON error instead of 500 (e.g. `pytest tests/entrypoints/openai_api/test_serving_speech.py -m 'core_model and cpu' -k upload_voice`). (evidence: Read api_server.py:369-376 (`_create_speech_error_json_response` -> `base(raw_request)`); read test_serving_speech.py:2339-2348 (`_patch_api_server_base` mocks `api_server_module.base`); grep confirmed no remaining `openai_serving_tokenization` references.) + +`vllm_omni/entrypoints/openai/api_server.py:754` [minor] — The gate report lists buildkite/vllm-omni-amd-ci, -intel-ci, and -npu-ci as failing, but the raw sweep cannot attribute the failures to this PR versus main. Can you confirm whether these lanes are red on the head/main or were introduced by this diff before landing? (evidence: gate_report lists buildkite/vllm-omni-amd-ci, -intel-ci, -npu-ci failing; attribution to this PR vs main cannot be determined from the gate alone.) + +`vllm_omni/entrypoints/openai/api_server.py:754` [minor] — `state.serving_tokenization = None` is set for every pure-diffusion server, and `state.serving_tokenization = ServingTokenization(...)` is set for all LLM/multi-stage/TTS-only servers; the PR description validates only Qwen3-Omni (multi-stage). Can you explicitly validate pure-diffusion mode beyond the error-response path, or scope this change as validated only for multi-stage? (evidence: blast radius of changed values: `state.serving_tokenization = ServingTokenization(...)` (api_server.py:1017) is reached by all LLM/multi-stage and TTS-only servers; `state.serving_tokenization = None` (api_server.py:754) is reached by every pure-diffusion (single diffusion stage) server. PR description validates only Qwen3-Omni (multi-stage); pure-diffusion behavior is unvalidated.) + +`vllm_omni/entrypoints/openai/api_server.py:1157` [minor] — The changed init hunks at 754/1017 have no test asserting `state.serving_tokenization`, and the renamed `getattr` fallback in `create_chat_completion` at line 1157 has no in-repo test driving it. Can you add coverage for the initialization state and `create_chat_completion` handler-None branch, or explicitly limit testing scope to the speech handlers already updated? (evidence: changed init hunks (754/1017) have no test asserting `state.serving_tokenization`; omni_init_app_state consumers (test_verl_omni_e2e.py, test_omni_sleep_wakeup.py, test_video_server.py) do not assert the tokenization state attribute. create_chat_completion handler-None branch (1157) has no in-repo test driving the renamed getattr (listed consumers call the serving handler, not api_server_module.create_chat_completion).) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v9_official_r1/pr4816.trace.json.gz new file mode 100644 index 000000000..272b0670f Binary files /dev/null and b/eval/dataset/arms/copilot_v9_official_r1/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4825.cost.json b/eval/dataset/arms/copilot_v9_official_r1/pr4825.cost.json new file mode 100644 index 000000000..8054e37c9 --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4825.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1113.9, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v9_official_r1/runs/pr4825/run-20260813-105828-0667fb", + "recorded_at": "2026-08-13T11:16:59", + "attempts": [ + { + "invocation_id": "inv-03d7f8c5", + "outcome": "run:run-20260813-105828-0667fb", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-105828-0667fb", + "llm_calls": 52, + "input_tokens": 513690, + "output_tokens": 151903, + "usd": 0.354654, + "source": "spans" + } + ], + "attempt_usd_total": 0.354654, + "llm_calls": 52, + "input_tokens": 513690, + "output_tokens": 151903, + "usd": 0.354654, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-105828-0667fb", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3547, + "minutes": 18.52, + "ci_minutes": 0, + "input_tokens": 513690, + "output_tokens": 151903, + "tool_calls": 106, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.647, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 52, + "by_role": { + "agent": { + "usd": 0.326412, + "input_tokens": 502119, + "output_tokens": 129100, + "calls": 50 + }, + "reducer": { + "usd": 0.028242, + "input_tokens": 11571, + "output_tokens": 22803, + "calls": 2 + } + }, + "cache_read_tokens": 1809792, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.27, + "gate": 1.53, + "review": 1106.43, + "report": 0.08 + }, + "time_to_first_result_s": 1111.28 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 24059 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4825.md b/eval/dataset/arms/copilot_v9_official_r1/pr4825.md new file mode 100644 index 000000000..afc75a102 --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4825.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4825, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head ac3a2ce5a9b9) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] Read manager.py hunk in situ (read_file offset 260, lines 351-430): default_components adds 'unet'; hasattr/nn.Module guards skip absent components; non-replaceable modules return identity from from_layer_diffusion and are filtered by `lora_layer is not module`; no rewrap of base_layer. +- [sweep] New branch `if isinstance(layer, _FakeLinearBase):` at tests/diffusion/lora/test_lora_manager.py:587 CAN occur: pipeline.unet.down_block.proj is `_FakeLinearBase()`; other named modules fall through the fake returning layer unchanged — not dead/redundant. +- [validated] test_lora_manager_discovers_unet_component asserts both `unet.down_block.proj` in `_lora_modules` and `isinstance(pipeline.unet.down_block.proj, _DummyBaseLayerWithLoRA)` (tree actually replaced), so it genuinely exercises discovery rather than asserting a pre-transformed value. +- [sweep] Changed-symbol consumers: grep shows `_matches_target` only at manager.py:351/408/415 and `_fake_from_layer_diffusion` only in test files; no external caller of the changed tuple is left stale. +- [sweep] None/empty handling: `extra_components = tuple(getattr(self.pipeline, "_lora_components", ()) or ())` unchanged; when target_modules is None, `_matches_target` returns True (manager.py:351-356) and the new test relies on that short-circuit. +- [sweep] Arithmetic: no arithmetic hunk in this diff, so no off-by-one/floor-vs-ceil to check. +- [sweep] Rebase/merge damage: commit timeline shows a 'Normalize to_out.0. to .to_out.' commit later reverted; final diff touches only manager.py and the test, and grep confirms sdxl_unet.py still uses `self.to_out = nn.ModuleList([RowParallelLinear...])` — no dropped hunk or duplicated code in the net diff. +- [sweep] Blast radius: grep shows `self.unet` also on SoulXSinger preprocess modules (rmvpe.py:236, `DeepUnet0`), Flux2Klein and OvisImage pipelines; those now enter the scan only if LoRA is loaded, and from_layer_diffusion only replaces vLLM LinearBase, so nn.Linear/Conv modules are returned unchanged. + +`tests/diffusion/lora/test_lora_manager.py:580` [major] — This diff adds `unet` to `default_components` in vllm_omni/diffusion/lora/manager.py (lines 374-380) and adds test_lora_manager_discovers_unet_component (line 580), but the new test drives only a bare `_FakeLinearBase` under `unet.down_block.proj` with a `peft_helper` that has no `target_modules`, so `_matches_target` (manager.py:351-356) is never actually constrained and the real SDXL replacement surface is unverified. SDXL's denoiser resolves packed `attn1.to_qkv` via `stacked_params_mapping` and stores `to_out` as nn.ModuleList([RowParallelLinear]) with target path `attn1.to_out.0`; `stacked_params_mapping` is assigned only in `load_weights()` (sdxl_unet.py:631) while `DiffusionLoRAManager` consumes it at construction (manager.py:145-153), so constructing the manager before weights are loaded can make QKV LoRA resolve to nothing; the commit timeline also shows a `to_out.0 -> to_out` normalization was added then reverted, leaving `to_out.0` attachment unresolved. Please add a packed `to_qkv` layer with target_modules=['to_q', 'to_k', 'to_v'] plus a ModuleList-wrapped `to_out` targeted as `to_out.0`, assert replacement under `unet`, and verify manager construction happens after `load_weights()` so the packed mapping is populated. (evidence: read tests/diffusion/lora/test_lora_manager.py:580-620 (no target_modules on peft_helper), manager.py:145-153 and 351-415 (packed mapping + target matching), sdxl_unet.py load_weights (sets self.stacked_params_mapping).) + +`vllm_omni/diffusion/lora/manager.py:375` [minor] — This diff adds `unet` to the hardcoded default_components tuple (manager.py:374-380), duplicating denoiser attribute names already tracked in registry.py:427 `transformer_attrs` and pipeline_sdxl.py:46 `_dit_modules`; the lists can drift when a pipeline attr is added or renamed, silently regressing LoRA discovery. Derive this tuple from a single shared constant/source, or at minimum add cross-reference comments so updates stay in sync. (evidence: grep 'unet' in vllm_omni/diffusion; read registry.py:427 transformer_attrs and pipeline_sdxl.py:46 _dit_modules.) + +`vllm_omni/diffusion/lora/manager.py:375` [minor] — This diff makes `unet` a shared default component (manager.py:374-380), so every diffusion pipeline exposing a top-level `unet` is scanned on LoRA load, not just SDXL. Grep shows `self.unet` on StableDiffusionXLPipeline (pipeline_sdxl.py:110) and also on SoulXSinger preprocess (models/soulx_singer/modules/preprocess/rmvpe.py:236), with references in OvisImage and Flux2Klein pipelines; because from_layer_diffusion wraps vLLM LinearBase layers, any vLLM linear inside those non-SDXL components could now be wrapped. Prefer scoping the change via `_lora_components = ('unet',)` on StableDiffusionXLPipeline and leave default_components unchanged; if you keep the default, confirm the other `unet`-exposing pipelines are unaffected/intended and update the now-inaccurate '(opt-in; other pipelines unchanged)' comment above (manager.py:368-369). (evidence: Read vllm_omni/diffusion/lora/manager.py:368-390; grep `_lora_components` -> sensenova_u1/pipeline_sensenova_u1.py:543; grep `self.unet` over vllm_omni/diffusion/models -> only pipeline_sdxl.py:110 (others are docstring text).) + +`vllm_omni/diffusion/lora/manager.py:372` [minor] — This diff adds `unet` to shared default_components (manager.py:374-380), and gate_report lists buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci as failing. The AMD lane runs pytest -s -v tests/diffusion/lora/test_images_generations_lora.py (.buildkite/test-amd.yaml:23), which exercises the shared LoRA manager; please confirm whether those failures are pre-existing on main or introduced by this diff and state the attribution in the PR thread so the red lanes are not mistaken for a regression. (evidence: gate_report (FAILING CHECKS: buildkite/vllm-omni-amd-ci, buildkite/vllm-omni-npu-ci); grep lora in .buildkite -> test-amd.yaml:23 runs tests/diffusion/lora/test_images_generations_lora.py.) + +`tests/diffusion/lora/test_lora_manager.py:580` [minor] — This diff adds test_lora_manager_discovers_unet_component (line 580), but it monkeypatches from_layer_diffusion and replace_submodule, so it proves only that the attribute name `unet` is walked; it never drives SDXL's real QKV/ModuleList layers through the real replacement/activation path, and the PR's only SDXL image evidence is an attached external script. Add an in-repo SDXL base+LoRA case to tests/diffusion/lora/test_images_generations_lora.py (grep currently shows no `unet` reference there) so the new default is exercised end-to-end in CI and cannot silently regress. (evidence: Read tests/diffusion/lora/test_lora_manager.py:580-620; grep 'unet' in tests/diffusion/lora/test_images_generations_lora.py -> no matches.) + +`tests/diffusion/lora/test_lora_manager.py:600` [nit] — This diff adds test_lora_manager_discovers_unet_component (line 580), which is a near-verbatim duplicate of test_lora_manager_discovers_bagel_component (lines 545-577) with the same monkeypatches, fake layer, and assertions, differing only in component name/path. Parametrize the component name/path so the two discovery cases live in one test and avoid double maintenance. (evidence: Read tests/diffusion/lora/test_lora_manager.py:545-577 (bagel test) and 580-620 (unet test).) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v9_official_r1/pr4825.trace.json.gz new file mode 100644 index 000000000..83dc64eda Binary files /dev/null and b/eval/dataset/arms/copilot_v9_official_r1/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4837.cost.json b/eval/dataset/arms/copilot_v9_official_r1/pr4837.cost.json new file mode 100644 index 000000000..fb05583e9 --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4837.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 654.4, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v9_official_r1/runs/pr4837/run-20260813-103520-dd0f51", + "recorded_at": "2026-08-13T10:46:13", + "attempts": [ + { + "invocation_id": "inv-1f1f9016", + "outcome": "run:run-20260813-103520-dd0f51", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-103520-dd0f51", + "llm_calls": 42, + "input_tokens": 529096, + "output_tokens": 99284, + "usd": 0.300518, + "source": "spans" + } + ], + "attempt_usd_total": 0.300518, + "llm_calls": 42, + "input_tokens": 529096, + "output_tokens": 99284, + "usd": 0.300518, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-103520-dd0f51", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3005, + "minutes": 10.87, + "ci_minutes": 0, + "input_tokens": 529096, + "output_tokens": 99284, + "tool_calls": 65, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4702, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 42, + "by_role": { + "agent": { + "usd": 0.283749, + "input_tokens": 521327, + "output_tokens": 85978, + "calls": 40 + }, + "reducer": { + "usd": 0.016769, + "input_tokens": 7769, + "output_tokens": 13306, + "calls": 2 + } + }, + "cache_read_tokens": 1794432, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.71, + "gate": 1.88, + "review": 646.71, + "report": 0.06 + }, + "time_to_first_result_s": 652.35 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 16787 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4837.md b/eval/dataset/arms/copilot_v9_official_r1/pr4837.md new file mode 100644 index 000000000..a295879d9 --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4837.md @@ -0,0 +1,38 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4837, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [sweep] orchestrator.py:1290 `if len(diffusion_prompt) == 1:` — enumerated all arms: [] is handled just above with a terminal error; len==1 is unwrapped for both submit_update and submit_initial; len>1 still reaches StagePool.submit_* which raises ValueError by design (stage_pool.py submit_initial/submit_update). `already_submitted` is still used later to pick submit_update vs submit_initial, so it is not dead. +- [validated] pipeline_ming_imagegen.py:462-472 — old `outputs[0]` would TypeError because DiffusionOutput has no __getitem__ (diffusion/data.py:1180); new `outputs.output` matches the dataclass field and the non-tensor RuntimeError guard is sound. +- [sweep] Docstrings/comments in touched files — Ming forward docstring (`Returns: One DiffusionOutput with .output set to [B,3,H,W] tensor in [-1,1]`), get_ming_image_post_process_func docstring, module ASCII diagram, and _forward_to_next_stage docstring all remain true under the new behavior; none found stale. +- [sweep] Indexed/first-element access added by the diff — orchestrator.py:1290 `diffusion_prompt[0]` encodes 'single-element list from custom_process_input_func == one non-batched request'; guaranteed by the empty-list branch above, StagePool's explicit list rejection, and thinker2imagegen returning exactly [one dict] (ming_flash_omni.py:346-427). No assert/comment added, but semantics are consistent. +- [sweep] Resource lifecycle — no acquire/register/allocate added or moved; `set_forward_context_ref_latent(ref_latent)` is still released in the `finally` block on all paths (pipeline_ming_imagegen.py:465). +- [sweep] Consumer scan — _forward_to_next_stage callers at orchestrator.py:944/955/1060/1096 and tests in test_orchestrator_kv_sender_info.py / test_orchestrator_stage_input_bridge.py still hold; MingImagePipeline.forward has no other in-repo callers beyond the diffusion worker dispatch. +- [sweep] Rebase/merge damage — found the call/shim contract mismatch vs current ZImagePipeline.forward (pipeline_z_image.py:413) reported in review comment 1. +- [sweep] Read orchestrator.py hunk in situ (lines ~1245-1312): `if len(diffusion_prompt) == 1:` now unwraps before the already_submitted submit_update/submit_initial split; empty-list error path and len>1 passthrough unchanged. + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:?` [blocker] — The diff annotates the existing `super().forward(z_req, prompt=None, height=..., width=..., num_inference_steps=..., guidance_scale=..., prompt_embeds=..., negative_prompt_embeds=..., generator=..., output_type="pt", return_dict=True)` call and switches the return path to `return outputs`, but on this head `ZImagePipeline.forward` is declared as `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput` with no `**kwargs` (pipeline_z_image.py:413), so this call raises TypeError before the newly added `outputs.output` extraction runs; the shim also cannot satisfy the parent body because `_ZPipelineSamplingParams` has no `output_type`, `extra_args`, `latents`, or `cfg_normalize` that the parent reads. Reconcile this call/shim with the current parent contract (or confirm merge drift and align with base), and re-run the PR's exact `vllm serve ... --deploy-config vllm_omni/deploy/ming_flash_omni_image.yaml` + curl command on this head before merge. (evidence: read vllm_omni/diffusion/models/z_image/pipeline_z_image.py:413 (forward signature `def forward(self, req: DiffusionRequestBatch) -> DiffusionOutput`) and its accesses to req.sampling_params.output_type / extra_args / latents / cfg_normalize; read the _ZPipelineSamplingParams dataclass fields in pipeline_ming_imagegen.py.) + +`vllm_omni/engine/orchestrator.py:?` [major] — The diff removes the `already_submitted and` guard so `diffusion_prompt = diffusion_prompt[0]` now runs on the initial submit path as well. No test file is touched in this diff, and the existing `_forward_to_next_stage` test callers (`tests/engine/test_orchestrator_kv_sender_info.py:157,186,224` and `tests/engine/test_orchestrator_stage_input_bridge.py:149`) do not cover a bridge returning `[{"prompt": ...}]` with `already_submitted=False`. Add/extend a test in `tests/engine` whose fake `custom_process_input_func` returns a single-element list and assert the diffusion stage's `add_request_async` receives the unwrapped dict rather than the list; otherwise this exact #4839 regression can silently return. (evidence: changed_symbol_consumers lists those test files as the only `_forward_to_next_stage` callers; sweep_targets reports no test files touched in the diff.) + +`vllm_omni/engine/orchestrator.py:?` [minor] — The diff removes the `already_submitted and` guard so a single-element `diffusion_prompt` list is unwrapped on both submit paths, but a bridge returning 2+ dicts still falls through and passes the raw list to `submit_initial`/`submit_update` unchanged, hitting the same list rejection the PR says it is fixing. If Ming cannot produce multi-element output (`supports_request_batch = False`), assert `len(diffusion_prompt) == 1` so a future regression fails loudly; otherwise handle the batch case explicitly. (evidence: Read orchestrator.py lines 1287-1312 in situ; pipeline_ming_imagegen.py declares `supports_request_batch = False`.) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:?` [minor] — The diff replaces the `outputs[0]`/`hasattr(output, "images")` extraction with `raw = outputs.output` and returns `outputs` directly, which is correct because `ZImagePipeline.forward` returns `DiffusionOutput(output=image, stage_durations=stage_durations)`; this also stops dropping `stage_durations`. There is no automated test for `MingImagePipeline.forward` (grep under `vllm_omni/diffusion/models/ming_flash_omni` and `tests/diffusion` finds only Bagel tests), so add a CPU/mock smoke test asserting `forward()` returns a DiffusionOutput whose `.output` is the decoded tensor and `stage_durations` survives a return-value wrapping regression. (evidence: Read pipeline_ming_imagegen.py lines 462-475 and pipeline_z_image.py forward return; repo_map shows only tests/diffusion/models/bagel test files.) + +`vllm_omni/engine/orchestrator.py:?` [nit] — The diff changes the shared orchestrator submit path at `vllm_omni/engine/orchestrator.py:1290`, and the gate report lists `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` as failing. Please confirm whether both are red on main too (pre-existing) or were introduced by this change, and if introduced, address them before merge. (evidence: gate_report evidence lists FAILING CHECKS ['buildkite/vllm-omni-amd-ci','buildkite/vllm-omni-npu-ci']; the diff changes vllm_omni/engine/orchestrator.py around line 1290.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v9_official_r1/pr4837.trace.json.gz new file mode 100644 index 000000000..9fc55da07 Binary files /dev/null and b/eval/dataset/arms/copilot_v9_official_r1/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4893.cost.json b/eval/dataset/arms/copilot_v9_official_r1/pr4893.cost.json new file mode 100644 index 000000000..7b026ee88 --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4893.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1029.6, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v9_official_r1/runs/pr4893/run-20260813-104616-64af0b", + "recorded_at": "2026-08-13T11:03:22", + "attempts": [ + { + "invocation_id": "inv-9e837324", + "outcome": "run:run-20260813-104616-64af0b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-104616-64af0b", + "llm_calls": 66, + "input_tokens": 462968, + "output_tokens": 178053, + "usd": 0.384139, + "source": "spans" + } + ], + "attempt_usd_total": 0.384139, + "llm_calls": 66, + "input_tokens": 462968, + "output_tokens": 178053, + "usd": 0.384139, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-104616-64af0b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3841, + "minutes": 17.11, + "ci_minutes": 0, + "input_tokens": 462968, + "output_tokens": 178053, + "tool_calls": 134, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6355, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 66, + "by_role": { + "agent": { + "usd": 0.354471, + "input_tokens": 449309, + "output_tokens": 154466, + "calls": 64 + }, + "reducer": { + "usd": 0.029668, + "input_tokens": 13659, + "output_tokens": 23587, + "calls": 2 + } + }, + "cache_read_tokens": 2343680, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.93, + "gate": 1.67, + "review": 1021.01, + "report": 0.07 + }, + "time_to_first_result_s": 1026.65 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 26982 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4893.md b/eval/dataset/arms/copilot_v9_official_r1/pr4893.md new file mode 100644 index 000000000..b8126af76 --- /dev/null +++ b/eval/dataset/arms/copilot_v9_official_r1/pr4893.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4893, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [sweep] indexed access hunyuan_fused_moe.py:91 hidden_states.shape[0]: read hunyuan_fused_moe.py; it is guarded by the pre-existing `if hidden_states is None and args: hidden_states = args[0]` pattern, so hidden_states is a tensor whose dim0 is the token count — OK. +- [sweep] branch tests/diffusion/distributed/test_expert_parallel_layout.py:24 reduce_scatter fake guard: both arms reachable (fake_init_vllm_model_parallel_group passes reduce_scatter; fake_init_model_parallel_group does not) — OK. +- [sweep] branches parallel_state.py:868/875/882 use_moe_parallel_mapping: both arms occur; True path exercised by test_expert_parallel_layout.py:100-108, False path by :234-242 and :299-307 — OK. +- [sweep] branches hunyuan_fused_moe.py:31/34/38/47: all arms reachable; context-unavailable tested (test_noop_when_context_unavailable), dp_metadata-present tested (test_keeps_existing_dp_metadata); the world_size<=1 and RuntimeError arms have no dedicated test — noted. +- [sweep] verification of non-test touched files: parallel_state.py is covered by tests/diffusion/distributed/test_expert_parallel_layout.py; hunyuan_fused_moe.py is covered by tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py (both named in the PR test plan) — OK. +- [validated] destroy_model_parallel symmetry parallel_state.py:919-933: the new separate vLLM _DP and _PCP coordinators are destroyed when not aliased to diffusion groups — no teardown leak found. +- [validated] in-repo consumers of vllm_parallel_state._DP/_TP/_EP/_PCP: grep across the repo shows only parallel_state.py and the updated tests reference them; no stale consumer of the old aliasing behavior was found. +- [validated] blast radius: grep for enable_expert_parallel=true in deploy/config shows only HunyuanImage3 YAMLs/tests enable EP in-tree, so the use_moe_parallel_mapping path is currently HunyuanImage3-scoped; the shared helper remains generic for future MoE EP models. + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:34` [minor] — `hunyuan_fused_moe.py` now calls `_set_forward_context_dp_metadata(hidden_states.shape[0])` on every MoE pre-hook, and `_set_forward_context_dp_metadata` returns immediately when `forward_context.dp_metadata` already exists (line 34), while `_set_forward_context_num_tokens()` still overwrites `num_tokens` each time. If a single `ForwardContext` is reused across denoising steps or a scheduled batch with a different per-rank token count, MoE dispatch would use the first gathered DP counts while `num_tokens` stays current; either recompute when the gathered count no longer matches `num_tokens`, or explicitly assert/document the invariant that one `ForwardContext` has one token count. (evidence: hunyuan_fused_moe.py:18-22 always overwrites forward_context.num_tokens; hunyuan_fused_moe.py:34-36 returns without recomputing dp_metadata; tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:79-91 asserts existing metadata is kept.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:45` [minor] — `_set_forward_context_dp_metadata` now dereferences `dp_group.cpu_group` (line 42) after only checking that `dp_group` is not None and `world_size > 1`; the only branch that makes `_vllm_ps._DP` a vLLM `GroupCoordinator` is the `use_moe_parallel_mapping` path in `initialize_model_parallel`, while the non-mapping path leaves `vllm_parallel_state._DP` as the diffusion coordinator. For a HunyuanImage3 MoE forward with `data_parallel_size > 1` and `enable_expert_parallel=False`, this can raise `AttributeError` if the diffusion coordinator lacks `cpu_group`; guard the attribute (`cpu_group = getattr(dp_group, "cpu_group", None)` and return/raise if missing) or restrict the hook to the vLLM companion-coordinator path, and add a unit test for that configuration. (evidence: Read vllm_omni/diffusion/distributed/parallel_state.py (non-mapping path assigns vllm_parallel_state._DP = _DP) and hunyuan_fused_moe.py:34-42 (the only pre-check is world_size); the existing tests only exercise the vLLM-mapping fake group, not a non-EP DP>1 diffusion coordinator.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:?` [minor] — The diff adds `_num_tokens_pre_hook` calling `_set_forward_context_dp_metadata(hidden_states.shape[0])` after `_set_forward_context_num_tokens` (hunyuan_fused_moe.py:88-92), but the new `TestSetForwardContextDPMetadata` tests cover only `world_size=2` with a mocked `_vllm_ps._DP` and the already-set `dp_metadata` branch (test_hunyuan_fused_moe.py:56-89); the `dp_group is None` and `world_size <= 1` return arms and the real pre-hook wiring are never exercised. Add cases for `_vllm_ps._DP = None` and `world_size=1` to `TestSetForwardContextDPMetadata`, and a small test that constructs the Hunyuan MoE dispatcher, invokes the registered pre-hook with `hidden_states`, and asserts `forward_context.dp_metadata` is populated. (evidence: hunyuan_fused_moe.py:26-50; tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:53-89) + +`vllm_omni/diffusion/distributed/parallel_state.py:526` [minor] — `initialize_model_parallel` now builds every companion coordinator (`_PCP`, `_TP`, `_DP`, `_EP`) through `init_vllm_model_parallel_group`, which hardcodes `use_device_communicator=True` (parallel_state.py:515-527), and the gate report lists `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` failing. Before merge, confirm whether those lanes are already red on main or introduced by this shared-path change, and if introduced, that `use_device_communicator=True` is supported on the AMD and NPU backends; the NPU path reaches this expert-parallel group layout through `vllm_omni/platforms/npu/models/hunyuan_fused_moe.py` (`prepare_hunyuan_fused_moe_runtime -> get_expert_parallel_group_ranks`). (evidence: gate_report lists ['buildkite/vllm-omni-amd-ci','buildkite/vllm-omni-npu-ci']; grep for enable_expert_parallel/get_expert_parallel_group_ranks shows vllm_omni/platforms/npu/models/hunyuan_fused_moe.py consumes the shared expert-parallel group path.) + +`tests/diffusion/distributed/test_expert_parallel_layout.py:121` [minor] — The new test asserts only that the monkeypatched `fake_init_vllm_model_parallel_group` injected `device_communicator=object()` and `reduce_scatter=...` (test_expert_parallel_layout.py:62-72, 118-126), so it cannot catch a regression in the real `init_vllm_model_parallel_group` wrapper at parallel_state.py:515-527, such as `use_device_communicator=True` being removed or the vLLM coordinator no longer exposing `reduce_scatter`. Add a direct unit test for that wrapper (or mock at its real call seam) asserting the returned `GroupCoordinator` has `device_communicator`/`reduce_scatter`. (evidence: test_expert_parallel_layout.py:100-108 monkeypatches vllm_parallel_state.init_vllm_model_parallel_group; the assertion at :121 only reads back the monkeypatched device_communicator=object(), so it does not exercise the real GroupCoordinator behavior.) + +`vllm_omni/diffusion/distributed/parallel_state.py:?` [minor] — `initialize_model_parallel` now swaps `_PCP`, `_TP`, `_DP`, and `_EP` from diffusion aliases/coordinators to vLLM companion coordinators for every config where `use_moe_parallel_mapping` is true (parallel_state.py:853-901), but the PR evidence only validates HunyuanImage3 `tp2_cfgp2`/`tp2_sp2`; other `is_moe` callers share this path and could silently change their parallel mapping. Add an explicit allow-list/guard or test coverage for non-Hunyuan `is_moe` configs, or document why the generic switch is required for all of them. (evidence: parallel_state.py:850-875; PR description; tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json and test_hunyuan_image_tp2_sp2.json only cover HunyuanImage3.) + +`vllm_omni/diffusion/distributed/parallel_state.py:526` [minor] — Can you pin or document the minimum vLLM version required by the new `init_vllm_model_parallel_group(..., use_device_communicator=True)` call? The PR relies on a vLLM 0.24 API surface, but the reviewed manifest/requirements sections have no explicit vLLM lower bound (docs pin vllm==0.24.0 while apps/lvsa-omni-showcase/README.md pins vllm==0.22.0), so compatibility with the declared dependency floor is unverified. (evidence: [sweep] new external calls `use_device_communicator=True` and `_vllm_fc.DPMetadata(torch.tensor(...))`: pyproject.toml declares dynamic dependencies and requirements/common.txt has no vllm entry; docs pin vllm==0.24.0 but apps/lvsa-omni-showcase/README.md pins vllm==0.22.0, so the declared lower bound is unverified.) + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:56` [minor] — The new test verifies only `DPMetadata.num_tokens_across_dp_cpu`; the all2all dispatcher may consume other GPU-side fields of vLLM's `DPMetadata`, and without reading the installed vLLM 0.24 `forward_context` we cannot confirm the positional `DPMetadata(torch.tensor(...))` construction populates every field the dispatcher actually needs. Could you confirm the upstream field usage, or extend the test to assert the GPU-side fields that are consumed? (evidence: [unverified] DPMetadata(torch.tensor(...)) positional construction: the added test only asserts num_tokens_across_dp_cpu; without reading the installed vLLM 0.24 forward_context, could not verify the GPU-side fields the all2all dispatcher actually consumes are populated — recommend confirming upstream field usage.) + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_fused_moe.py:47` [minor] — The `any(count is None)` RuntimeError arm in `_set_forward_context_dp_metadata` has no dedicated test: the new tests cover the no-context and existing-dp_metadata branches, but not the RuntimeError path. Can you add a case where a mocked `all_gather_object` returns `None` for one rank and assert the RuntimeError/context state? (evidence: [sweep] branches hunyuan_fused_moe.py:31/34/38/47: all arms reachable; context-unavailable tested (test_noop_when_context_unavailable), dp_metadata-present tested (test_keeps_existing_dp_metadata); the world_size<=1 and RuntimeError arms have no dedicated test — noted.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v9_official_r1/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v9_official_r1/pr4893.trace.json.gz new file mode 100644 index 000000000..df76e3def Binary files /dev/null and b/eval/dataset/arms/copilot_v9_official_r1/pr4893.trace.json.gz differ diff --git a/eval/dataset/arms/cursor_composer25/manifest.json b/eval/dataset/arms/cursor_composer25/manifest.json new file mode 100644 index 000000000..c813ed1c2 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/manifest.json @@ -0,0 +1,13 @@ +{ + "arm": "cursor_composer25", + "model_requested": "composer-2.5", + "harness": "cursor-agent --print --trust --force stream-json", + "leakage_control": "DETECTIVE, not preventive: gh auth stripped (empty GH_CONFIG_DIR, tokens removed) and every shell command/read audited post-run; items with violations are failed. cursor-agent has no tool allowlist, so prevention equivalent to the claude baseline is not available.", + "pr_context": "frozen snapshot (no_discussion), byte-identical to the other arms", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "n_items": 9, + "concurrency": 3 +} diff --git a/eval/dataset/arms/cursor_composer25/pr5509.cost.json b/eval/dataset/arms/cursor_composer25/pr5509.cost.json new file mode 100644 index 000000000..4d0463309 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5509.cost.json @@ -0,0 +1,15 @@ +{ + "arm": "cursor_composer2.5", + "model_requested": "composer-2.5", + "head": "58131b8dc1cb7273362574bee8472b08ea46c91f", + "base": "0c88f8a0b662ae497a6485db6f3b91b79620608c", + "snapshot_sha256": "7fbc3a6958a884db3fcc43bbeaa8163e7a1c6458ab51ac62c6a765b9a4c4ff27", + "wall_s": 203.5, + "shell_commands": 7, + "file_reads": 13, + "cost_usd": null, + "audit_ok": true, + "audit_violations": [], + "artifact_chars": 7978, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_composer25/pr5509.md b/eval/dataset/arms/cursor_composer25/pr5509.md new file mode 100644 index 000000000..7e817c9c7 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5509.md @@ -0,0 +1,121 @@ +I'll review PR #5509 by inspecting the full diff and tracing how the new SAGE quantization hooks into TRTLLM_ATTN. +Next I'll read the tests and trace how `backend_kwargs` reach the attention backend. +## Verdict: **Approve with minor follow-ups** + +This is a well-scoped kernel integration: typed `quant` config flows cleanly from `AttentionSpec` → `backend_kwargs()` → `TrtllmAttentionImpl`, SAGE kwargs are gated so the dense path stays compatible with older FlashInfer, and the init-time resolution of `trtllm_sage_attention_quantize` is the right fix for the Dynamo graph-break issue the authors already hit. Tests cover config round-tripping, init-time failures, quantize wiring, and (when hardware allows) numerical smoke checks. The main gaps are a silent ring-SP footgun, a docs regression, and backend-specific validation that is weaker than the documentation claims. + +--- + +### Strengths + +1. **Clean config layering** — `AttnQuantSpec` in `data.py` centralizes validation; `QuantConfig` in the backend is a thin runtime view. The shared `_coerce()` refactor for `skip_softmax` / `quant` is tidy. + +2. **Backward compatibility** — `sage_kwargs` are only passed when `self.quant.enabled`, so dense `TRTLLM_ATTN` on older FlashInfer builds is unaffected (`trtllm_attn.py:292-299`). + +3. **Fail-fast on missing deps** — Init checks for both the kernel signature (`sage_attn_sfs`) and the quantize routine are clear and actionable (`trtllm_attn.py:204-216`). + +4. **Good test coverage** — Config serialization, dtype rejection, quantize call shapes, and optional GPU integration tests (`test_trtllm_attn.py:304-321`) with arch-dependent skip for int8 on non-SM100. + +5. **SAGE + Skip-Softmax composition** — Both kwargs reach `trtllm_ragged_attention_deepseek` in the same call; config coexistence is tested. + +--- + +### Issues to address + +#### 1. **Ring SP + SAGE quant can silently ignore quantization** (medium) + +`Attention._run_ring_attention` rejects configured skip-softmax because the ring path bypasses the backend impl, but there is no equivalent guard for SAGE quant: + +```319:326:vllm_omni/diffusion/attention/layer.py + def _run_ring_attention(self, query, key, value, attn_metadata): + skip = getattr(self.attention, "skip", None) + if skip is not None and getattr(skip, "configured", False): + raise NotImplementedError( + "Skip-Softmax (TRTLLM_ATTN) is not supported with ring sequence parallelism: " + ... + ) +``` + +**What to change:** In `layer.py` around line 320, mirror the skip-softmax check for `getattr(self.attention, "quant", None)` and `quant.enabled`, raising `NotImplementedError` with the same “ring bypasses backend” rationale. Add a unit test analogous to `test_ring_sp_with_skip_softmax_rejected`. + +--- + +#### 2. **Docs regression: removed sections + dangling cross-reference** (medium) + +The PR adds a good SAGE section but also deletes unrelated content: “Known Limitations” (LTX-2 / FA4), “Validation Guidance”, “Reproducing the Benchmark Table”, and “Mixed backends across roles”. Worse, a stale reference remains: + +```232:232:docs/user_guide/diffusion/attention_backends.md +- **Workaround for known issues**: see Known Limitations above. +``` + +**What to change:** Restore the removed sections (they are still accurate and useful), or at minimum fix the dangling link and move LTX-2 / validation guidance somewhere visible. The SAGE addition should not have come at the cost of unrelated doc deletions. + +--- + +#### 3. **Docs overstate FLASHINFER_ATTN `quant` support** (low) + +`attention_backends.md:185-188` says `FLASHINFER_ATTN` reads `dtype_qk` / `dtype_vo` / `flashinfer_backend`, and `AttentionSpec` allows `quant` on that backend (`data.py:1447-1450`). But `FlashInferAttentionImpl.__init__` never accepts or uses `backend_kwargs` — quant config is parsed and serialized but ignored at runtime. + +**What to change:** Either (a) mark FLASHINFER quant as “reserved / not yet wired” in docs and restrict `AttentionSpec` validation to backends that actually consume it, or (b) implement consumption in `flashinfer_attn.py`. As merged, users can configure quant on `FLASHINFER_ATTN` and get a silent no-op. + +--- + +#### 4. **Backend-specific `dtype_qk` validation is late and uses the wrong exception type** (low) + +Docs say “A `dtype_qk` from the wrong set raises” (`attention_backends.md:188`), implying parse-time `ValueError` like other `AttnQuantSpec` checks. In practice, `bfloat16` / `float16` pass `AttnQuantSpec` and only fail at layer construction with `RuntimeError` (`trtllm_attn.py:199-203`). `test_quant_rejects_non_sage_dtype` covers this, but the UX is inconsistent with `skip_softmax` validation in `AttentionSpec.__post_init__`. + +**What to change:** In `AttentionSpec.__post_init__` (around `data.py:1447`), when `backend == "TRTLLM_ATTN"` and `quant.dtype_qk` is set, require `dtype_qk in {"int8", "fp8_e4m3"}` and raise `ValueError`. Reserve `RuntimeError` for environment/kernel availability only. + +--- + +#### 5. **Partial / misleading `quant` blocks silently no-op on TRTLLM** (low) + +`AttnQuantSpec.enabled` is true if *either* `dtype_qk` or `dtype_vo` is set (`data.py:1424-1425`), so `quant: {"dtype_vo": "fp8_e4m3"}` or `quant: {"k_block_size": 4}` (without `dtype_qk`) serializes into `backend_kwargs` but `QuantConfig.enabled` stays false (`trtllm_attn.py:130-131`). User thinks quant is on; attention runs dense. + +**What to change:** For `TRTLLM_ATTN`, require `dtype_qk` whenever the `quant` block is present and enabled. For `FLASHINFER_ATTN`, define the parallel rule once that backend actually consumes quant. + +--- + +#### 6. **`per_role` flatten error message is stale** (nit) + +```1557:1559:vllm_omni/diffusion/data.py + raise ValueError( + f"Invalid per_role entry for role {role!r}: cannot mix backend/skip_softmax with nested role keys." + ) +``` + +**What to change:** Include `quant` in the message now that it is a top-level spec key. + +--- + +#### 7. **`torch.compile` + per-step quantize** (follow-up, not blocking) + +Quantization runs inside `forward_cuda` on every call (`trtllm_attn.py:296-297`). The PR correctly resolves the function pointer at init to avoid the `lru_cache` graph break, but the quantize call itself may still force Dynamo recompiles or fall out of cudagraphs under regional `torch.compile` (the PR’s own benchmarks use compile). Worth a follow-up benchmark with `torch._dynamo.explain` or checking whether quantize should sit behind `@torch.compiler.disable` or be lifted to a custom op boundary — especially since the claimed 1.15–1.19× wins already include compile. + +--- + +### Minor positives in implementation details + +- `sage_attn_sfs = (q_sfs, k_sfs, None, v_sfs)` and `num_elts_per_sage_attn_blk = (q_block_size, k_block_size, 0, 1)` match the FlashInfer SAGE API contract. +- Default block sizes (`q=1`, `k=16`) match the PR examples and docs. +- `int8` arch gating is handled pragmatically via runtime kernel miss + test skip rather than brittle capability checks at import time. + +--- + +### Test plan assessment + +| Area | Covered? | +|------|----------| +| Config parse / serialize | Yes (`test_attention_config.py`) | +| Init-time env failures | Yes (mocked) | +| Quantize wiring / shapes | Yes (unit) | +| GPU numerical smoke (fp8/int8) | Yes (conditional) | +| SAGE + skip-softmax on GPU | No | +| Ring SP + SAGE rejection | No | +| `torch.compile` interaction | No | + +--- + +### Summary + +Shippable core logic with sensible defaults and good backward compatibility. Before calling this fully production-ready, I'd want the ring-SP guard (issue 1) and the docs fix (issue 2). Issues 3–5 are polish that align behavior with documentation and prevent silent misconfiguration. None of these undermine the benchmark results on the intended path (single-GPU / Ulysses SP, Wan I2V, `torch.compile`), but they matter for users combining SAGE with ring parallelism or misreading the shared `quant` schema. diff --git a/eval/dataset/arms/cursor_composer25/pr5550.cost.json b/eval/dataset/arms/cursor_composer25/pr5550.cost.json new file mode 100644 index 000000000..c3c2b5b1e --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5550.cost.json @@ -0,0 +1,15 @@ +{ + "arm": "cursor_composer2.5", + "model_requested": "composer-2.5", + "head": "20ca4b93de6e61b42544f2b4ba1101711e467986", + "base": "c4a14091c9b8fd61f4da46022f683adc1dc91550", + "snapshot_sha256": "164875457cfa48b5dbdf9910ab1823eb23590cb709fe0dcf988c6e837f562a27", + "wall_s": 217.9, + "shell_commands": 5, + "file_reads": 28, + "cost_usd": null, + "audit_ok": true, + "audit_violations": [], + "artifact_chars": 6845, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_composer25/pr5550.md b/eval/dataset/arms/cursor_composer25/pr5550.md new file mode 100644 index 000000000..981c3eefc --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5550.md @@ -0,0 +1,101 @@ +Reviewing PR #5550: I'll read the review skill, pull the full diff, and trace how the new Worker KV contracts are used at call sites. +## Verdict: **Approve** + +This PR delivers what it promises: a narrow W0 slice that defines the Scheduler→Worker allocation wire DTOs, threads them through `NewRequestData` and the multiproc RPC boundary, and validates identity/mode at the Worker entry without yet installing or consuming block tables. The scope matches the stated W0 boundary, dense-path backward compatibility is preserved (legacy 3-arg RPC when metadata is absent), and the new contract tests are targeted and pass locally (37 tests in `tests/diffusion/diffusion_kv/` plus the engine cleanup test). + +The main follow-ups belong in W1 (allocator, BlockTable installer, step-state threading), not in this transport PR. + +--- + +## Strengths + +- **Clean ownership split**: Scheduler-mutable `DiffusionKVRequest` stays off the wire; `DiffusionKVMetadata` is a one-shot allocation snapshot on `NewRequestData` (`vllm_omni/diffusion/diffusion_kv/metadata.py`, `vllm_omni/diffusion/sched/interface.py:155-178`). +- **Fail-fast identity checks on the request RPC path**: `validate_new_request_data_identity` runs before dispatch and before the RPC error wrapper (`vllm_omni/diffusion/executor/multiproc_executor.py:454-460`), so bad envelopes never become worker outputs. +- **Backward-compatible RPC shape**: metadata is an optional 4th positional arg only when present (`multiproc_executor.py:544-546`); tests lock this (`tests/diffusion/diffusion_kv/test_worker_contract.py:204-221`). +- **Mode gating is conservative**: dense rejects unexpected metadata; paged accepts but does not require it (`diffusion_model_runner.py:163-178`). + +--- + +## Findings + +### 1. W1 follow-up — metadata is validated then dropped (step path) + +**`vllm_omni/diffusion/worker/diffusion_model_runner.py:673-679`** + +`execute_stepwise` validates `new_req.diffusion_kv_metadata` at entry (`:750-755`) but `_update_states` builds `StepRequestState` without capturing it—only `prepared_layout` is carried forward. Once W1 installs BlockTables, step mode will need either a field on `StepRequestState` or a Worker-side allocation registry keyed by `(request_id, allocation_generation)`. + +**What to change (W1):** extend `StepRequestState` (or add a parallel registry) and thread metadata from the first `scheduled_new_reqs` envelope into subsequent step waves. + +--- + +### 2. Low — `execute_batch` does not validate identity at the executor + +**`vllm_omni/diffusion/executor/multiproc_executor.py:586-620`** + +`execute_request` validates every `NewRequestData` before RPC (`:459-460`). `execute_batch` only does so indirectly when `len(scheduled_new_reqs) <= 1` (delegates to `execute_request`) or on the DLO-DP path (`:605-613`). The fused multi-request `execute_model_batch` RPC (`:615-619`) ships the scheduler output without executor-level identity checks; validation happens only inside `DiffusionModelRunner.execute_model_batch` (`diffusion_model_runner.py:636-641`). + +**Impact:** Identity errors on multi-request batch waves fail on the worker and may surface through a different error path than request-mode identity failures (which are deliberately kept outside the per-request RPC `try/except`). + +**What to change:** hoist the `validate_new_request_data_identity` loop to the top of `execute_batch` (and optionally `execute_step`) for parity with `execute_request`. + +--- + +### 3. Low — DLO DP multi-concurrency silently falls back to serial RPC when any metadata is present + +**`vllm_omni/diffusion/executor/multiproc_executor.py:462-471`** + +`has_diffusion_kv_metadata` disables the fused DLO AllGather multi-request wave and forces per-request `execute_model` RPCs. That is functionally safe but is a performance regression once the Scheduler starts attaching metadata to concurrent DLO waves. + +**What to change (W1):** document this in the executor dispatch comment or RFC; plan a metadata-aware DLO batching strategy if DP concurrency under paged KV is a requirement. + +--- + +### 4. Low — custom runners/extensions will break once metadata is emitted + +**`vllm_omni/diffusion/worker/diffusion_worker.py:462-465`** + +When metadata is non-`None`, it is passed as a keyword to `model_runner.execute_model`. Custom `diffusion_model_runner_cls` implementations or worker extensions whose `execute_model` lacks `diffusion_kv_metadata` (and `**kwargs`) will raise `TypeError` once W1 starts producing metadata. + +The existing test only covers the no-metadata dense path (`tests/diffusion/diffusion_kv/test_worker_contract.py:274-290`). + +**What to change (W1):** add a migration note and a test that a runner accepting `**kwargs` survives metadata; update platform runner base signatures when the allocator lands. + +--- + +### 5. Nit — RPC routing bypasses `WorkerWrapperBase.execute_model` + +**`vllm_omni/diffusion/worker/diffusion_worker.py:1035-1036` vs `:1353-1374`** + +Worker RPC calls `WorkerWrapperBase.execute_method`, which dispatches to `DiffusionWorker.execute_model` directly (`getattr(self.worker, method)`), not `WorkerWrapperBase.execute_model`. Both layers duplicate the optional-metadata kwargs logic. Not a W0 bug (positional 4th arg works on `DiffusionWorker`), but extensions overriding only the wrapper method would miss metadata. + +**What to change:** in W1, either route RPC through the wrapper method or document that extensions must override `DiffusionWorker.execute_model`. + +--- + +### 6. Nit — unrelated but good: engine init failure cleanup + +**`vllm_omni/diffusion/diffusion_engine.py:191-208`** + +Scheduler/executor teardown on init failure is a worthwhile fix, though orthogonal to KV plumbing. Test coverage is adequate (`tests/diffusion/test_diffusion_engine_cleanup.py:126-164`). + +--- + +### 7. Test gap — no MP serialization roundtrip + +Tests mock `collective_rpc` and call runners directly. There is no test that `DiffusionKVMetadata` survives the actual multiprocessing/pickle transport used by `MessageQueue`. + +**What to change (W1):** add a small integration or executor-level test that enqueues metadata through the real RPC path. + +--- + +### 8. Nit — DTO mutability + +**`vllm_omni/diffusion/diffusion_kv/metadata.py:16-57`** + +Scheduler-side `DiffusionKVContext` is frozen (`request.py:13`); wire metadata DTOs are mutable dataclasses. Consider `frozen=True` in W1 once construction sites are fixed, to prevent accidental mutation between Scheduler packaging and Worker installation. + +--- + +## Summary + +No merge-blocking defects for the stated W0 scope. The transport contract, validation placement, and dense-path compatibility are sound. The actionable work is mostly W1: consume/store metadata (especially in step mode), align executor validation across batch/step paths, and prepare custom runner/extension signatures before the allocator starts emitting snapshots. diff --git a/eval/dataset/arms/cursor_composer25/pr5610.cost.json b/eval/dataset/arms/cursor_composer25/pr5610.cost.json new file mode 100644 index 000000000..a5897d244 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5610.cost.json @@ -0,0 +1,15 @@ +{ + "arm": "cursor_composer2.5", + "model_requested": "composer-2.5", + "head": "50126edf2f50a85e7e41b32b70da7d8e70b21aec", + "base": "900a7f0813d0482811b0e4dfd3cf7deabbe2429f", + "snapshot_sha256": "155a8a48d7ac9a99f8aa125d3ccb82c7bacd8524d84d04518b200e037950d936", + "wall_s": 124.6, + "shell_commands": 2, + "file_reads": 22, + "cost_usd": null, + "audit_ok": true, + "audit_violations": [], + "artifact_chars": 8073, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_composer25/pr5610.md b/eval/dataset/arms/cursor_composer25/pr5610.md new file mode 100644 index 000000000..be4db6198 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5610.md @@ -0,0 +1,130 @@ +I'll review PR #5610 by inspecting the diff and the documentation against the implementation it describes. +Verifying documentation claims against the implementation in the key source files. +## Verdict: **Approve** (with minor follow-up suggestions) + +This is a strong, implementation-faithful design doc. I cross-checked the new page against `gpu_ar_model_runner.py`, model opt-in flags, deploy YAMLs, NPU behavior, guard logic, tests, navigation wiring, and the vendored SVG. The architecture narrative, activation conditions, fallbacks, and performance attribution match the code at `50126edf`. For a documentation-only PR that is already merged, the remaining items are polish and discoverability—not correctness blockers. + +--- + +## What works well + +**Accurate core story.** The synchronous vs asynchronous decode-path diagrams align with `GPUARModelRunner.sample_tokens()`: when `_should_use_async_omni_output()` is true, payload work moves into a background builder while sampled-token feedback is registered immediately via `set_async_sampled_token_ids()`. + +**Guard table is correct.** Every row in the compatibility table maps to `_should_use_async_omni_output()`: + +```1595:1619:vllm_omni/worker/gpu_ar_model_runner.py + def _should_use_async_omni_output(self) -> bool: + if not self.use_async_scheduling: + return False + if self.omni_prefix_cache is not None: + return False + if self.speculative_config is not None: + return False + ... + if not bool(getattr(model_config, "async_chunk", False)): + return False + if bool(getattr(model_config, "enable_return_routed_experts", False)): + return False + ... + if not self._model_omni_flag(model, "use_async_omni_output"): + return False + if self._model_omni_flag(model, "has_postprocess") and not self._model_omni_flag( + model, "eager_omni_postprocess_before_async_output" + ): + return False + return True +``` + +**Model-specific behavior is documented correctly.** Qwen3-Omni Thinker/Talker opt-in (`use_async_omni_output`, `eager_omni_postprocess_before_async_output`, `omni_pooler_payload_include_hidden`) and Qwen3-TTS Talker settings match `qwen3_omni.py` and `qwen3_tts_talker.py`. + +**NPU distinction is accurate.** `NPUARModelRunner` builds `OmniModelRunnerOutput`, drains connector output synchronously, then wraps with upstream `AsyncGPUModelRunnerOutput`—not `OmniAsyncGPUModelRunnerOutput`. + +**Live-state / connector discussion is careful and correct.** The note that `get_omni_connector_output()` drains live runner state, and that `_lock` coordinates receive-side connector state, matches `omni_connector_model_runner_mixin.py`. + +**Full-payload accumulation caveat is right.** `should_accumulate_full_payload_output()` returns `False` when `async_chunk` is enabled, so the `self.requests` accumulation branch is unreachable on this path. + +**Examples and warnings are actionable.** `--no-async-chunk` is a real CLI flag; prefix-cache incompatibility is correctly explained; deploy YAML excerpts match bundled profiles. + +**Tests cited exist.** `tests/worker/test_gpu_ar_model_runner.py` covers guards, hidden-state opt-out, connector ordering after accumulation, snapshot request order, lazy build, and background exception propagation. + +**SVG is self-contained and accessible.** Valid XML, `title`/`desc`, and metrics consistent with the figure caption (2.8 ms → 41 µs). + +--- + +## Specific comments + +### 1. Add a reciprocal link from `async_chunk.md` +**File:** `docs/design/feature/async_chunk.md` (Related Files section, ~line 262) + +The new page links to async chunk in the overview and related files, but `async_chunk.md` does not link back. Readers landing on the async-chunk doc will miss this companion optimization. + +**Change:** Add a bullet such as: +`- [Async Omni Output Materialization](omni_async_output_materialization.md): Defers CPU payload construction off the AR decode critical path (requires async_chunk).` + +--- + +### 2. Clarify the two CUDA copy streams in Architecture +**File:** `docs/design/feature/omni_async_output_materialization.md:145-150` and `:155-168` + +The doc reads as if one dedicated stream handles all async copies. In code there are two: + +- `_omni_payload_copy_stream` — hidden/multimodal payload D2H in `_snapshot_tensor_payload_to_cpu_async()` +- `async_output_copy_stream` — sampled-token (and logprob) D2H in `OmniAsyncGPUModelRunnerOutput.__init__()` + +The lifecycle is still correct at a high level, but naming both streams would help operators debugging overlap or stream stalls. + +--- + +### 3. Tighten lifecycle step ordering slightly +**File:** `docs/design/feature/omni_async_output_materialization.md:139-153` + +Step 5 says the background thread “starts” after D2H enqueue; step 6 says sampled tokens are registered “immediately.” In practice, `OmniAsyncGPUModelRunnerOutput.__init__()` enqueues sampled-token D2H, starts the background thread, and then `sample_tokens()` calls `set_async_sampled_token_ids()` before returning. Consider reordering or noting that sampled-token registration happens after the async output object is constructed, not strictly before the background thread starts. + +--- + +### 4. Qwen3-TTS example YAML is slightly under-specified +**File:** `docs/design/feature/omni_async_output_materialization.md:300-310` + +The excerpt omits `async_scheduling: true` on Stage 1 (Code2Wav), which is present in `vllm_omni/deploy/qwen3_tts.yaml:81`. That is not wrong—the generation stage does not use async Omni output materialization—but a one-line note (“Stage 1 may still set `async_scheduling: true`; this feature applies only to the AR Talker”) would prevent readers from thinking the excerpt is incomplete or wrong. + +--- + +### 5. Performance table could name the third enabling knob explicitly +**File:** `docs/design/feature/omni_async_output_materialization.md:108-112` + +The baseline row is “CUDA Graph + async chunk.” The incremental row is “+ async output materialization.” That is fine given the blog attribution, but a footnote that the third row also assumes default Qwen3-Omni deploy settings (`async_scheduling` on AR stages, prefix caching off, model opt-in) would make the benchmark preconditions self-contained without rereading the Enabling section. + +--- + +### 6. Consider listing other opt-in models (optional) +**File:** `docs/design/feature/omni_async_output_materialization.md:89-100` or Compatibility section + +The doc correctly focuses on Qwen3-Omni and Qwen3-TTS. If the intent is to document the *mechanism* rather than the *product surface*, a short note that activation is per-model via `use_async_omni_output` (and that other models default to `False`, e.g. Moss TTS) would set expectations for contributors adding new pipelines. + +--- + +### 7. `design/index.md` remains selectively curated (pre-existing) +**File:** `docs/design/index.md:9-16` + +Adding this page is good. `async_chunk.md` and several other feature docs are still absent from the index. Not introduced by this PR, but if the index is meant to be a complete feature catalog, a follow-up pass would help. + +--- + +## Validation notes (from PR description) + +The author’s self-check results are credible: link/anchor/SVG/YAML/pre-commit/`git diff --check` validation is appropriate for this change set. `mkdocs build --strict` being blocked by an existing `qwen-asr` / `transformers` pin conflict is an environment issue, not a defect in this doc. + +--- + +## Summary + +| Area | Assessment | +|------|------------| +| Technical accuracy vs code | Excellent | +| Activation / fallback documentation | Complete and matches guards | +| Examples (Qwen3-Omni, Qwen3-TTS) | Correct; minor YAML completeness nit | +| Diagram & performance claims | Properly attributed; SVG consistent | +| Navigation (`docs/.nav.yml`, `design/index.md`) | Correctly wired | +| Test references | Accurate | + +**Recommendation:** Ship as-is (already merged). Address items 1–2 in a small follow-up if you want maximum cross-doc discoverability and debugging clarity. diff --git a/eval/dataset/arms/cursor_composer25/pr5703.cost.json b/eval/dataset/arms/cursor_composer25/pr5703.cost.json new file mode 100644 index 000000000..8f9bc4657 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5703.cost.json @@ -0,0 +1,15 @@ +{ + "arm": "cursor_composer2.5", + "model_requested": "composer-2.5", + "head": "1c6ee28f627cef5e2659b3671fb3451e718562f6", + "base": "ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783", + "snapshot_sha256": "6959d8d9cb9a0f35dcd94367f9717276b8fe358f3147b030589ab34459a7564e", + "wall_s": 177.0, + "shell_commands": 9, + "file_reads": 13, + "cost_usd": null, + "audit_ok": true, + "audit_violations": [], + "artifact_chars": 6691, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_composer25/pr5703.md b/eval/dataset/arms/cursor_composer25/pr5703.md new file mode 100644 index 000000000..d21801aa6 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5703.md @@ -0,0 +1,105 @@ +Reviewing PR #5703: I'll inspect the diff and trace how the changed RNG and VAE paths are used in callers. +## Verdict: **Approve with minor follow-ups** + +The core change is small, correctly motivated, and aligned with PyTorch’s device-module API. Replacing hard-coded `torch.cuda.*` calls in the conditioned video-VAE encode paths fixes a real MUSA bug: on MUSA, `parameter.device.type` is `"musa"`, so the old `== "cuda"` guard skipped accelerator RNG seeding entirely and broke fixed-seed conditioned encoding. The `fork_rng` + `manual_seed(42)` structure is preserved, post-encode CPU FP32 normalization is untouched, and ROCm behavior is unchanged (`device.type` remains `"cuda"`). Reported MUSA smoke (deterministic latents, RNG restore, Ref2VA serving) is credible for this diff. + +The main gaps are documentation accuracy, missing automated coverage for the RNG fix, and a slightly loose device binding — not correctness blockers for the stated CUDA/MUSA goal, but worth fixing in a follow-up. + +**CI:** `buildkite/vllm-omni-amd-ci` is failing per PR gates. This diff does not touch AMD/ROCm paths (ROCm still reports `cuda`), so treat that gate as unrelated infrastructure noise rather than evidence against this change. + +--- + +## Specific comments + +### 1. Recipe copy-paste error: Ref2VA example serves `FL2VA` — `recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76` + +The prose describes the **validated Ref2VA TP4 profile**, but the shell block sets: + +```bash +export MODEL="${MODEL_ROOT}/FL2VA" +``` + +For Ref2VA reproduction this should be `Ref2VA`, consistent with line 70 (“or to `Ref2VA` for Ref2VA”) and the validation section at lines 124–127. As written, users following the recipe will start the wrong checkpoint partition while expecting Ref2VA behavior. + +**Change:** set `MODEL="${MODEL_ROOT}/Ref2VA"` in the Ref2VA example, or split into separate FL2VA and Ref2VA server blocks like `MiniMax-H3-NPU.md`. + +--- + +### 2. Bind `device_module` to the VAE’s constructor device — `vllm_omni/diffusion/models/minimax_h3/vae.py:120` + +```120:120:vllm_omni/diffusion/models/minimax_h3/vae.py + self.device_module = torch.get_device_module() +``` + +`torch.get_device_module()` without an argument resolves the **current default accelerator**, not necessarily the `device` passed into `__init__`. On MUSA this likely works in practice because the pipeline is constructed on-device, but it is fragile if init ordering or offload ever diverges. + +**Change:** use the constructor argument: + +```python +self.device_module = torch.get_device_module(device) +``` + +Alternatively, resolve at encode time from `parameter.device` so the module always matches the tensor’s backend. + +--- + +### 3. Device allowlist is broader than the PR claims — `vllm_omni/diffusion/models/minimax_h3/vae.py:170,214` + +```170:170:vllm_omni/diffusion/models/minimax_h3/vae.py + devices = [parameter.device] if parameter.device.type != "cpu" else [] +``` + +The PR description says the allowlist is **CUDA and MUSA only**, but `!= "cpu"` also enables accelerator RNG for `"npu"`, `"xpu"`, `"mps"`, etc. Previously, only `"cuda"` (including ROCm) was seeded; NPU/Ascend encoding silently skipped accelerator seeding. + +That may be desirable on NPU, but it is **unvalidated scope expansion** relative to the PR text and the existing NPU recipe. If the intent is truly CUDA/MUSA-only, tighten the guard: + +```python +if parameter.device.type in ("cuda", "musa"): +``` + +If the broader behavior is intentional, update the PR/recipe wording and add NPU regression coverage. + +--- + +### 4. PR description overclaims test additions + +The description says this PR “adds coverage for the torchaudio, soundfile, and ffmpeg audio-loading paths,” but the diff contains **no test file changes**. Audio fallback logic lives in `reference_video.py` (`load_audio_file`, `load_video_audio`) and predates this PR; `grep` finds no tests exercising those paths under `tests/`. + +The validation command also references `tests/diffusion/models/minimax_h3/test_minimax_h3_vae.py`, which **does not exist** in the tree. I ran the three existing focused modules and got **63 passed** — close to but not matching the claimed “36 passed” count, and none of those tests assert MUSA/CUDA RNG seeding. + +**Change:** add a focused unit test that mocks `device_module.manual_seed` / `fork_rng` for a non-CPU device type (including `"musa"`), and optionally tests for `load_audio_file` fallback chains. Until then, soften the PR description to “validated manually on MUSA” rather than “adds coverage.” + +--- + +### 5. Existing contract tests bypass the RNG path — `tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py:344-353` + +`test_video_vae_encode_uses_configured_parallel_tiling` constructs a bare `MiniMaxH3VideoVAE` via `object.__new__` without setting `device_module`. It validates parallel-tiling restore and shape contracts but never exercises the new seeding logic. That is why the focused suite can pass while the MUSA bug would have remained undetected. + +**Change:** extend this test (or add a sibling) with a fake `device_module` and assert `manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED)` is invoked when `parameter.device.type == "musa"` (or any mocked non-CPU backend). + +--- + +### 6. MUSA recipe is otherwise useful; one minor consistency note — `recipes/MiniMaxAI/MiniMax-H3-MUSA.md:114-127` + +The “Start a server” block describes a **4-GPU TP4** profile, while the E2E smoke at lines 114–117 says Ref2VA image+audio was validated on **one** S5000 with CPU offload. That is not wrong if intentional (smoke vs production profile), but a short clarifying sentence would reduce confusion — e.g. “single-GPU smoke below; TP4 block is the multi-GPU Ref2VA serving profile.” + +--- + +### 7. `recipes/README.md:48-49` — looks good + +The index entry for `MiniMax-H3-MUSA.md` and the tightened `MiniMax-H3.md` task wording are appropriate and consistent with the new recipe. + +--- + +## Summary + +| Area | Assessment | +|------|------------| +| Core RNG fix (`vae.py`) | Correct direction; fixes MUSA conditioned VAE determinism | +| ROCm/CUDA regression risk | Low — ROCm still uses `"cuda"` device type | +| NPU/other backends | Medium — behavior change without stated validation | +| Recipe | Useful, but Ref2VA server example has a blocking copy-paste error | +| Tests | Gap between PR claims and actual diff | +| AMD CI | Failing gate; unrelated to this change | + +**Recommendation:** merge was reasonable given MUSA hardware validation. Follow up with (1) recipe `MODEL` path fix, (2) `get_device_module(device)`, (3) an explicit device allowlist or expanded validation docs, and (4) a small RNG unit test so this does not regress silently on the next non-CUDA backend. diff --git a/eval/dataset/arms/cursor_composer25/pr5715.cost.json b/eval/dataset/arms/cursor_composer25/pr5715.cost.json new file mode 100644 index 000000000..e78c60d08 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5715.cost.json @@ -0,0 +1,15 @@ +{ + "arm": "cursor_composer2.5", + "model_requested": "composer-2.5", + "head": "765caa2a121aec024201936c4decb1151225991c", + "base": "7a2007cc17d63ce9070e1feefd29fcca8a93cf34", + "snapshot_sha256": "3723b9e637bb5d67daa30874435afe2848c3e1ee689c5b0501634fb0eba751d1", + "wall_s": 110.6, + "shell_commands": 3, + "file_reads": 13, + "cost_usd": null, + "audit_ok": true, + "audit_violations": [], + "artifact_chars": 5393, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_composer25/pr5715.md b/eval/dataset/arms/cursor_composer25/pr5715.md new file mode 100644 index 000000000..84daa0324 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5715.md @@ -0,0 +1,64 @@ +I'll review PR #5715 by inspecting the diff and reading the changed documentation files. +Verifying referenced paths and checking for version inconsistencies across related docs. +## Verdict: **Approve with minor nits** + +Solid, scoped documentation PR for the v0.26.0 release. It does what it claims: announces 0.26.0, highlights MiniCPM-o 4.5 and MiniMax H3, marks full-duplex as experimental, consolidates older release notes, documents the vLLM 0.26.x compatibility rule, and aligns CUDA/NPU install pins with the 0.26 line. Recipe links resolve, full-duplex wording matches `vllm_omni/experimental/fullduplex/`, and the Docker `BASE_IMAGE` example now matches `docker/Dockerfile.cuda`. No code-path regressions; this is docs-only. + +The main follow-ups are editorial (readability, cross-linking, NPU branch naming), not blockers. + +--- + +## Strengths + +- **Release narrative is accurate.** MiniMax H3 is a joint video/audio diffusion model (`recipes/MiniMaxAI/MiniMax-H3.md`); MiniCPM-o 4.5 full-duplex is correctly labeled experimental and backed by `examples/online_serving/minicpmo/` and `vllm_omni/experimental/fullduplex/`. +- **Compatibility guidance is a clear win.** The installation callout and CUDA dependency text state the major/minor pairing explicitly. +- **Version alignment fixes real drift.** The Docker build-arg example was still on `v0.22.1`; it now matches the default in `docker/Dockerfile.cuda`. +- **README ↔ docs landing stay in sync** for feature bullets and supported-model examples. +- **NPU source-install path fix** (`cd vllm-omni` vs the old absolute path) is correct for the shown clone sequence. + +--- + +## Specific comments + +### 1. `README.md:19` — News bullet is hard to scan + +The 0.26.0 announcement is one long sentence covering MiniMax H3, full-duplex, layerwise offload, hardware, streaming, TTS, and quantization. Consider splitting into two bullets or using a short lead sentence plus sub-bullets so the headline features (MiniMax H3, experimental MiniCPM-o 4.5 duplex) stand out. + +### 2. `README.md:21` — Cadence rollup scope could confuse readers + +The consolidated history bullet lists releases through **0.22.0**, while **0.24.0** remains as its own item on line 20. That’s reasonable editorially (keep recent releases separate), but “continued this cadence” plus an enumerated list ending at 0.22 may read as if 0.24 is outside the cadence. Consider appending “(through 0.22.0)” or “with 0.24.0 and later releases continuing separately above.” + +### 3. `README.md:54` and `docs/README.md:55` — Experimental feature has no pointer + +“Full-duplex realtime serving … (experimental)” is accurate, but there’s no link to how to try it. Add a link to `examples/online_serving/minicpmo/README.md` or `vllm_omni/experimental/fullduplex/DESIGN.md` so readers know where to start. + +### 4. `docs/configuration/README.md:5` — Version-pinned upstream link will go stale + +Pinning to `https://docs.vllm.ai/en/v0.26.0/configuration/index.html` is correct for 0.26.0, but unlike most other docs links (`en/latest`), this will need a manual bump each release. Consider a short note (“match your installed vLLM minor version”) or a maintenance comment for release doc updates. + +### 5. `docs/getting_started/installation/npu/npu.inc.md:62` — RC branch vs stable tags elsewhere + +The “build from main” flow clones vLLM-Ascend at `releases/v0.26.0rc`, while Docker examples use `v0.26.0`. If a stable `v0.26.0` (or `releases/v0.26.0`) branch/tag exists on vllm-ascend, prefer that for consistency with the release Docker path; otherwise add a one-line note explaining why source builds track the RC branch. + +### 6. `docs/getting_started/installation/gpu/cuda.inc.md:8` — Slight redundancy with installation README + +The new dependency sentence duplicates `docs/getting_started/installation/README.md:3-4`. Not wrong—CUDA readers may never open the parent page—but you could shorten cuda.inc.md to “See the compatibility note above” if you want less repetition across the install tree. + +### 7. `README.md` vs `docs/README.md` — News section only on GitHub README + +`docs/README.md` mirrors feature/model updates but has no *Latest News* block. That’s fine if intentional, but it will drift over time unless release updates always touch both or news lives only on GitHub. Worth deciding explicitly in a follow-up. + +### 8. `README.md:21` — Paper visibility reduced in news (optional) + +Removing the standalone 0.14.0 bullet dropped the in-news link to the architecture paper. The Citation section still has it (`README.md:77-88`). If discoverability matters, add “see our paper” once in the consolidated history bullet. + +--- + +## Out of scope / correctly left alone + +- MUSA (`docs/getting_started/installation/gpu/musa.inc.md`) and quickstart were not changed, matching the stated review constraint. +- Odd-minor Omni wheels (0.21, 0.23, 0.25) in `gpu.md` coexist with the “even-minor stable cadence” wording because the latter refers to **stable** aligned releases—a distinction worth keeping in mind for future doc edits, but not introduced by this PR. + +--- + +**Bottom line:** Safe to merge (and already merged). The changes are accurate, internally consistent with Dockerfiles and the 0.26 install docs, and appropriately cautious about experimental full-duplex. Address the readability and cross-link nits in a small follow-up if you want the landing pages to age better through 0.28.0. diff --git a/eval/dataset/arms/cursor_composer25/pr5840.cost.json b/eval/dataset/arms/cursor_composer25/pr5840.cost.json new file mode 100644 index 000000000..d12df16ca --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5840.cost.json @@ -0,0 +1,15 @@ +{ + "arm": "cursor_composer2.5", + "model_requested": "composer-2.5", + "head": "75cd907089322c648cf1494a2423d706e20fec2a", + "base": "fc8946fcf542cf108fa8b2fdc69de737bbfd52d8", + "snapshot_sha256": "0250972d67ae968a6f260a906b426a0695bc5e6677d3f796a84a259d86925480", + "wall_s": 211.0, + "shell_commands": 5, + "file_reads": 24, + "cost_usd": null, + "audit_ok": true, + "audit_violations": [], + "artifact_chars": 6815, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_composer25/pr5840.md b/eval/dataset/arms/cursor_composer25/pr5840.md new file mode 100644 index 000000000..55aacfd6a --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5840.md @@ -0,0 +1,114 @@ +Reviewing PR #5840: I'll read the review skill, inspect the full diff, and trace how TeaCache integrates with MiniMax-H3. +## Review: PR #5840 — [MiniMax-H3] TeaCache support and Cache-DiT validation + +**Head:** `75cd90708932` · **Base:** `fc8946fcf542` · **State:** merged +**Verdict:** **Approve with minor follow-ups.** The TeaCache integration is technically sound: the extractor mirrors `MiniMaxH3DiTModel.forward`, partition gating is explicit, calibration data is documented, and CPU unit tests pass locally (`TestMiniMaxH3Extractor`, new backend tests). Nothing here is a merge blocker in hindsight, but two wiring/doc gaps mean the advertised `0.17` default does not always apply, and parallelism coverage is thin. + +**CI:** `buildkite/vllm-omni-amd-ci` failed at merge time; this PR is CUDA/MiniMax-H3–focused and does not touch AMD pipeline files — treat as an existing AMD gate failure, not evidence about this change. + +--- + +### What works well + +1. **Extractor fidelity** — `extract_minimax_h3_context` tracks the native forward path (`_embed` → `sp_prepare` / blocks / `sp_gather` → `final_layer` + masks). `test_extractor_matches_native_forward` and `test_teacache_reuses_h3_block_residual` give good confidence the hook path is equivalent when uncached and actually skips blocks on cache hits. + +2. **Partition policy** — `enable_minimax_h3_teacache` correctly hooks only `pipeline.transformer` (FL2VA), rejects `ref2va`-only serving, and warns on `combined`. That matches how `_transformer_for_task` routes Ref2VA to `transformers_ref`. + +3. **Model-specific defaults** — Coefficients and `_MODEL_DEFAULT_REL_L1_THRESH` for `MiniMaxH3DiTModel` are registered in `TeaCacheConfig`, and deferring `rel_l1_thresh=None` through `DiffusionCacheConfig` is the right abstraction. + +4. **Tests/docs** — Focused extractor tests, backend partition tests, an E2E smoke, recipe section, and the feature matrix update are appropriate for this scope. + +--- + +### Findings + +#### 1. Medium — Serve/Omni path still pins `rel_l1_thresh=0.2`, bypassing H3’s `0.17` default + +The PR defers threshold defaulting to `TeaCacheConfig` when `rel_l1_thresh` is `None`: + +```420:422:vllm_omni/diffusion/data.py + # None defers to the model-specific TeaCache default (0.2 fallback). + rel_l1_thresh: float | None = None +``` + +```96:99:vllm_omni/diffusion/cache/teacache/config.py +_DEFAULT_REL_L1_THRESH = 0.2 +_MODEL_DEFAULT_REL_L1_THRESH = { + "MiniMaxH3DiTModel": 0.17, +} +``` + +But `AsyncOmniEngine` still injects an explicit `0.2` whenever the user enables TeaCache without `--cache-config`: + +```853:856:vllm_omni/engine/async_omni_engine.py + if cache_backend == "tea_cache": + return { + "rel_l1_thresh": 0.2, + } +``` + +**Impact:** `vllm serve ... --cache-backend tea_cache` (no `--cache-config`) uses `0.2` for MiniMax-H3, not the calibrated `0.17` documented in `recipes/MiniMaxAI/MiniMax-H3.md`. The PR’s own online smoke also used `0.2`, while the recipe/examples say `0.17`. + +**Fix:** Stop injecting a global TeaCache default in `_get_default_cache_config` (return `{}` or `None` and let `TeaCacheConfig.__post_init__` resolve per model), or make the injection model-aware. Add a regression test for “empty cache_config → MiniMax gets 0.17”. + +--- + +#### 2. Low — No TeaCache + Ulysses/SP coverage + +TeaCache E2E is single-GPU only (`test_minimax_h3_t2va_teacache_single_gpu_smoke`). Ulysses smoke (`test_minimax_h3_t2va_ulysses8_smoke`) does not enable TeaCache. The extractor computes `modulated_input` on the full packed sequence while `run_transformer_blocks` goes through `sp_prepare` / `sp_gather` (`extractors.py:1341–1369`), which only become non-identity under sequence parallelism. + +**Impact:** TeaCache + `--usp N` is untested; cache decisions and residuals may diverge under SP sharding. + +**Fix:** Either document “TeaCache supported for FL2VA single-GPU / non-SP only” in the recipe/limitations, or add a multi-GPU smoke and verify extractor behavior when `ulysses_degree > 1`. + +--- + +#### 3. Low — Weak polynomial fit (documented, but worth monitoring) + +Calibration reports **R² ≈ 0.095** and RMSE ≈ 0.04 on the relative-L1 mapping. The fourth-order polynomial is a weak predictor of output error from embedding drift. + +**Impact:** Not a correctness bug — A/B metrics in the PR description look acceptable at the validated workload — but threshold tuning may be less predictable outside the 107-frame / 50-step / 256×448 envelope. + +**Fix:** No code change required; consider noting the calibrated workload bounds more prominently in `recipes/MiniMaxAI/MiniMax-H3.md` (duration, resolution, steps) so users know when re-calibration is needed. + +--- + +#### 4. Nit — User-facing TeaCache docs still imply a universal `0.2` default + +`docs/user_guide/diffusion_features.md` was updated, but `docs/user_guide/diffusion/cache_acceleration/teacache.md` still documents a single global default of `0.2` and does not mention model-specific defaults (e.g. MiniMax-H3 `0.17`). + +**Fix:** Add a short “model-specific defaults” subsection to `teacache.md`, or link to per-model recipes. + +--- + +#### 5. Nit — Combined serving behavior is correct but easy to misread + +```76:101:vllm_omni/diffusion/cache/teacache/backend.py +def enable_minimax_h3_teacache(pipeline: Any, config: DiffusionCacheConfig) -> None: + ... + apply_teacache_hook(pipeline.transformer, teacache_config) + if partition == "combined": + logger.warning( + "TeaCache is enabled only for MiniMax-H3 FL2VA; " + "Ref2VA requests on this combined pipeline will run without TeaCache" + ) +``` + +This is the right design (Ref2VA uses `transformers_ref`, which is intentionally unhooked). The recipe documents it. No change strictly required. + +--- + +### Validation performed + +| Check | Result | +|--------|--------| +| `tests/diffusion/cache/test_teacache_extractors.py -k TestMiniMaxH3Extractor` | 8 passed | +| `tests/diffusion/cache/test_cache_backends.py` (new MiniMax / generic default tests) | 6 passed | +| Extractor vs `MiniMaxH3DiTModel.forward` trace | Aligned on preprocessing, blocks, postprocess | +| GPU E2E TeaCache smoke | Not run (requires `VLLM_TEST_MINIMAX_H3_FL2VA_MODEL`) | + +--- + +### Summary + +This is a well-scoped TeaCache addition: registry entry, faithful extractor, partition-aware enabler, recalibrated coefficients, and solid CPU tests. The main gap is **default-threshold plumbing** — model-specific `0.17` is implemented in `TeaCacheConfig` but overridden by `AsyncOmniEngine`’s hardcoded `0.2` on the common serve path. Secondary gaps are **missing TeaCache+SP validation** and **stale global TeaCache docs**. None of these rise to a correctness blocker for the validated single-GPU FL2VA workload, which matches an approve-with-follow-ups verdict for an already-merged PR. diff --git a/eval/dataset/arms/cursor_composer25/pr5863.cost.json b/eval/dataset/arms/cursor_composer25/pr5863.cost.json new file mode 100644 index 000000000..d389a5db4 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5863.cost.json @@ -0,0 +1,15 @@ +{ + "arm": "cursor_composer2.5", + "model_requested": "composer-2.5", + "head": "0a4c96151148ccf3e4ed29ec53287fe36cbf2409", + "base": "48298030d4b3c320b858dd98934c01babdd320ce", + "snapshot_sha256": "47a66ad1e1d54d29f2c36c5017a36f5772be207b0f2b908d2f94e64a343275a5", + "wall_s": 161.3, + "shell_commands": 1, + "file_reads": 20, + "cost_usd": null, + "audit_ok": true, + "audit_violations": [], + "artifact_chars": 7630, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_composer25/pr5863.md b/eval/dataset/arms/cursor_composer25/pr5863.md new file mode 100644 index 000000000..cffecb225 --- /dev/null +++ b/eval/dataset/arms/cursor_composer25/pr5863.md @@ -0,0 +1,118 @@ +Reviewing PR #5863: I'll inspect the diff and read the changed documentation and any related code it references. +Verifying CLI flags, API parameters, and response headers against the implementation. +## Verdict: **Approve with minor follow-ups** + +This is a documentation-only PR that adds a single, well-structured hardware recipe (`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md`). The serving flags, parallelism layout, attention-backend choice, and HTTP examples align with the current MiniMax-H3 implementation. The benchmark section is unusually thorough: warmup discipline, stage breakdown, memory sampling, scaling analysis, and explicit “not yet measured” caveats. + +Two copy-paste issues would mislead operators on dual-socket / dual-server setups; everything else is polish and discoverability. Worth merging (already merged); address the P1 items in a small follow-up. + +--- + +## What works well + +**Parallelism configs are internally consistent and match the engine model.** For all profiles, `num_gpus = tensor_parallel_size × usp × ring` (with `cfg_parallel_size = 1`), and `text_encoder_tp_size` values (2/4/8) satisfy the Qwen3-VL divisibility checks in `pipeline_minimax_h3.py` (64 heads / 8 KV heads). + +**Attention backend guidance is correct for SM120.** `platform.py` defaults datacenter Blackwell (SM100/SM103) to `TRTLLM_ATTN` and consumer Blackwell (SM120, RTX PRO 6000) to `CUDNN_ATTN`. Explicit `--diffusion-attention-backend CUDNN_ATTN` is the right call and matches the 5090 recipe pattern. + +**API examples match the server contract.** T2VA’s required `aspect_ratio` is enforced in `pipeline_minimax_h3.py`. `/v1/videos/sync`, `extra_params.task`, and profiler headers `X-Stage-Durations` / `X-Peak-Memory-MB` match `api_server.py`. + +**Benchmark methodology is credible.** Warmup before measurement, profiler-enabled stage times, `nvidia-smi` peak sampling, and the honesty about unmeasured Ref2VA / device-order tuning are all good practice. + +--- + +## Issues to fix + +### P1 — `numactl` flags presented as `vllm serve` arguments + +```201:203:recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md +Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and +`--port 8091` for the FL2VA server. Use `CUDA_VISIBLE_DEVICES=4,6,5,7`, +`--cpunodebind=1 --membind=1`, and `--port 8092` for the Ref2VA server. +``` + +`--cpunodebind` and `--membind` are **not** `vllm serve` flags (confirmed by searching the codebase; they appear only in this file). They are `numactl` options. A user copying this literally will get an argparse error. + +**Change:** Wrap the command in `numactl`, e.g.: + +```bash +numactl --cpunodebind=0 --membind=0 \ + env CUDA_VISIBLE_DEVICES=0,2,1,3 VLLM_OMNI_VIDEO_SYNC_TIMEOUT=1800 \ + vllm serve "${MODEL_ROOT}/FL2VA" ... --port 8091 +``` + +### P1 — Dual-server layout omits `Ref2VA` model path + +In the same section (lines 193–205), the Ref2VA server is described as using “the four-GPU configuration” but only `CUDA_VISIBLE_DEVICES`, NUMA binding, and `--port` are changed. The four-GPU commands all set `MODEL="${MODEL_ROOT}/FL2VA"` (lines 112–113). Without an explicit `MODEL="${MODEL_ROOT}/Ref2VA"` (or `--task-type ref2va` on the hub ID), the second server would still load FL2VA. + +**Change:** State explicitly that the Ref2VA server uses `MODEL="${MODEL_ROOT}/Ref2VA"` (or equivalent `--task-type ref2va`), mirroring the single-server note at lines 215–216. + +### P2 — Ref2VA curl example uses non-portable `/root/` paths + +```336:345:recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md +curl -X POST "http://127.0.0.1:${PORT}/v1/videos/sync" \ + ... + -F "input_reference=@/root/hand.jpg;type=image/jpeg" \ + -F "audio_reference= 0`, but only when spawn goes through `_launch_omni_core_engines` / colocated `launch_stage_replica` (`1046–1065`, `1147`). + +Headless paths spawn subprocesses **without** that wrapper: + +- `launch_headless_llm_replica` → direct `StageEngineCoreProcManager(...)` then implicit start (`1216–1228`) +- `launch_headless_replica_group` only wraps `replica_device_env`, not compile cache (`1447–1449`) +- `launch_headless_diffusion_replica` → `StageDiffusionProcManager.launch_headless` (`1329–1342`) +- Distributed diffusion: `launch_diffusion_stage_replica` (`1587–1599`) + +On a headless host with `omni_dp_size_local > 1`, replicas auto-assigned as 0, 1, … can still compile concurrently into the **shared** default cache and reproduce the TritonBundler race the PR fixed for colocated startup (build 2954). + +**Change:** After registration returns `response.replica_id`, wrap headless LLM/diffusion proc `start()` in `scoped_spawn_device_env(None, None, stage_id=..., replica_id=response.replica_id)` (or set `VLLM_CACHE_ROOT` in the child from `omni_replica_id` in `stage_engine_core_proc.py:123`). Extend `test_stage_engine_startup_cache_env.py` with a headless-launcher integration test. + +--- + +### 2. **P3 — `async_tokens_to_discard` is written but never consumed** + +After the 0.27 migration to `num_stale_output_tokens`, assignments at: + +- `omni_ar_scheduler.py:479` (segment stop) +- `omni_ar_scheduler.py:634` (streaming session replacement) + +have **no readers** anywhere in `vllm_omni/`. Stale suppression is entirely via `num_stale_output_tokens` in `update_from_output` (`330–353`). + +**Change:** Remove the dead assignments (and narrow tests that only assert the field is set) to avoid implying a second discard path still exists. If something external still reads it, document that contract explicitly. + +--- + +### 3. **P3 — `stage_runtime._scoped_spawn_device_env` omits `stage_id` / `replica_id`** + +`stage_runtime.py:310–313` calls `scoped_spawn_device_env(physical_devices, self._spawn_device_lock)` without replica identity. Today it is only used for device-lock acquisition before `launch_stage_replica` (which does pass `replica_id` into the actual spawn). That is easy to misuse if someone later adds spawn logic inside `_stage_device_scope`. + +**Change:** Pass `stage_id` and `replica_id` through `_stage_device_scope` / `_scoped_spawn_device_env`, or rename the helper to `_scoped_device_visibility_env` so it is not confused with compile-cache isolation. + +--- + +### 4. **P3 — Open product issue: seed-TTS WER (acknowledged in PR)** + +The stale-drain fix is necessary and tested, but nightly 2957 WER (~0.39) stayed in the pre-fix band. The PR correctly tracks this as open. Do not treat the scheduler fix as closing the WER regression; continue bisecting torch 2.13 / codec-path changes in parallel. + +--- + +### 5. **Nit — Inductor patch probe failure installs unconditionally** + +`patch.py:491–494`: if the upstream-capability probe throws, the wrapper installs anyway. That is defensible (wrapper only adds `True`), but on a broken probe + future sympy API drift you could get silent no-op or unexpected behavior. + +**Change:** Log at warning when the probe fails, not only on success/skip paths. + +--- + +## Test coverage assessment + +| Area | Coverage | +|------|----------| +| Stale drain units | Strong (`test_omni_ar_scheduler_stale_drain.py`) | +| Thinker forward contract | Strong (CPU stubs) | +| Inductor divisibility | Strong | +| Compile cache env | Colocated spawn only; **headless gap** | +| seed-TTS WER | Metric still red; scheduler tests do not prove WER recovery | + +--- + +## Summary for maintainers + +Merge-quality rebase work: API drift, toolchain fallout, and omni-specific scheduler semantics are handled with evidence and tests. The one gap I would chase post-merge is **headless multi-replica compile-cache isolation** — the fix exists but is not wired on headless launch paths. Clean up dead `async_tokens_to_discard` writes. Track seed-TTS WER and non-CUDA CI on their respective gates. diff --git a/eval/dataset/arms/cursor_grok45/manifest.json b/eval/dataset/arms/cursor_grok45/manifest.json new file mode 100644 index 000000000..f30cac902 --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/manifest.json @@ -0,0 +1,13 @@ +{ + "arm": "cursor_grok45", + "model_requested": "cursor-grok-4.5-high", + "harness": "cursor-agent --print --trust --force stream-json", + "leakage_control": "DETECTIVE, not preventive: gh auth stripped (empty GH_CONFIG_DIR, tokens removed) and every shell command/read audited post-run; items with violations are failed. cursor-agent has no tool allowlist, so prevention equivalent to the claude baseline is not available.", + "pr_context": "frozen snapshot (no_discussion), byte-identical to the other arms", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "n_items": 1, + "concurrency": 3 +} diff --git a/eval/dataset/arms/cursor_grok45/pr5509.cost.json b/eval/dataset/arms/cursor_grok45/pr5509.cost.json new file mode 100644 index 000000000..845d52388 --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5509.cost.json @@ -0,0 +1,18 @@ +{ + "arm": "cursor_cursorgrok4.5high", + "model_requested": "cursor-grok-4.5-high", + "head": "58131b8dc1cb7273362574bee8472b08ea46c91f", + "base": "0c88f8a0b662ae497a6485db6f3b91b79620608c", + "snapshot_sha256": "7fbc3a6958a884db3fcc43bbeaa8163e7a1c6458ab51ac62c6a765b9a4c4ff27", + "wall_s": 216.5, + "shell_commands": 5, + "file_reads": 9, + "cost_usd": null, + "audit_ok": false, + "audit_violations": [ + "read outside worktree: /home/zhoutaichang/.claude/skills/imreview/SKILL.md", + "read outside worktree: /home/zhoutaichang/.cursor/projects/data-zhoutaichang-infermatrix-copilot-worktrees-vllm-omni-pr5509/terminals/694614.txt" + ], + "artifact_chars": 5795, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_grok45/pr5509.md b/eval/dataset/arms/cursor_grok45/pr5509.md new file mode 100644 index 000000000..fa03a1d9b --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5509.md @@ -0,0 +1,81 @@ +I'll review PR #5509 with InferMatrixCopilot Direct mode. First I'll read the review skill and freeze the PR snapshot before touching source or knowledge.Freezing the PR snapshot now: verifying HEAD and collecting the changed-file list before any source or knowledge reads.**Frozen snapshot** +- **Head SHA:** `58131b8dc1cb7273362574bee8472b08ea46c91f` (matches checkout) +- **Merge-base:** `0c88f8a0b662ae497a6485db6f3b91b79620608c` +- **CI / mergeability:** MERGED; gates reported clean +- **Changed files (5):** `trtllm_attn.py`, `data.py`, `attention_backends.md`, `test_attention_config.py`, `test_trtllm_attn.py` +- **Preliminary:** Typed SAGE quant on `TRTLLM_ATTN` (Q/K int8|fp8_e4m3, V per-channel fp8 via FlashInfer ≥0.6.16rc1). Focus: config validation, version/capability gates, kernel kwarg wiring, SM100/int8 constraints, test fidelity. + +Calling Direct `review` next, then source inspection within the execution budget.Direct routing selected configuration + diffusion. Reading the changed implementation and tests next (budget: 6 source files, 0 full knowledge pages).Findings are evidence-backed. Validating the Direct completion gate next (no discussion fetch — evaluation mode).## Verdict: **Request changes** + +Head `58131b8dc1cb`. The TRTLLM SAGE path itself is mostly sound (init-time FlashInfer gating, Dynamo-safe quantize resolve, kwargs only when enabled, unit tests pass). The blocker is the **shared `quant` public surface**: schema + docs advertise multi-backend behavior that the live code does not implement, so misconfigs silently stay dense. + +--- + +### 1. `FLASHINFER_ATTN` + `quant` is accepted and documented, but ignored + +```1447:1451:vllm_omni/diffusion/data.py + if self.quant is not None and self.backend.upper() not in ("TRTLLM_ATTN", "FLASHINFER_ATTN"): + raise ValueError( + f"quant is only supported by the TRTLLM_ATTN and FLASHINFER_ATTN backends, but " + f"backend={self.backend!r}. Remove quant or set a supported backend." + ) +``` + +```185:188:docs/user_guide/diffusion/attention_backends.md +The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the +fields that apply and rejects the rest. ... `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`), +`dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises. +``` + +`FlashInferAttentionImpl` never reads `backend_kwargs` / `quant` (`flashinfer_attn.py` ~54–66, 131–183); there is also no FLASHINFER quant section. Config like `{"backend":"FLASHINFER_ATTN","quant":{"dtype_qk":"bfloat16","dtype_vo":"fp8_e4m3"}}` round-trips (see `test_quant_superset_fields_passed_through`) and then silently runs dense. + +**Change:** Until FLASHINFER actually consumes `AttnQuantSpec`, allow `quant` only on `TRTLLM_ATTN`, drop unused `dtype_vo` / `flashinfer_backend` (or reject them on TRTLLM), and fix the docs (including the dead “see its section” pointer). Prefer reject-at-config over silent ignore. + +--- + +### 2. TRTLLM `dtype_vo`-only (and missing `dtype_qk`) silently disables SAGE + +```1472:1483:vllm_omni/diffusion/data.py + if self.quant is not None and self.quant.enabled: + q = self.quant + quant_kw: dict[str, Any] = { + "dtype_qk": q.dtype_qk, + ... + } + if q.dtype_vo is not None: + quant_kw["dtype_vo"] = q.dtype_vo +``` + +`AttnQuantSpec.enabled` is true if either dtype is set; `QuantConfig.enabled` requires `dtype_qk`. Reproduced at this head: `AttentionSpec(backend="TRTLLM_ATTN", quant={"dtype_vo":"fp8_e4m3"}).backend_kwargs()` → `dtype_qk: None`, so SAGE stays off with no error. + +**Change:** For `TRTLLM_ATTN`, require `dtype_qk in {"int8","fp8_e4m3"}` in `AttentionSpec` / `AttnQuantSpec` and reject `dtype_vo` / FLASHINFER-only fields. Do not serialize an “enabled” quant block that the TRTLLM path cannot apply. + +--- + +### 3. Docs claim wrong-set `dtype_qk` raises; config layer does not + +```1410:1416:vllm_omni/diffusion/data.py + _VALID_DTYPES = frozenset({"float16", "bfloat16", "int8", "fp8_e4m3"}) + ... + if v is not None and v not in self._VALID_DTYPES: +``` + +`AttentionSpec(backend="TRTLLM_ATTN", quant={"dtype_qk":"bfloat16"})` succeeds; failure is only later in `TrtllmAttentionImpl.__init__` (`trtllm_attn.py:199–203`). That contradicts the docs and delays feedback until model/layer build. + +**Change:** Backend-specific dtype validation at coerce time (same place as the `skip_softmax` backend gate). + +--- + +### 4. Unrelated doc deletions in this PR + +The SAGE section is fine, but the same diff removes **Known Limitations** (LTX-2 + `CUDNN_ATTN`/`torch.compile` workaround), validation guidance, and several usage examples. Those are still gone at EOF of `attention_backends.md`. + +**Change:** Restore that content (or move it explicitly); don’t bury it under a Kernel/SAGE change. + +--- + +### Non-blocking notes +- TRTLLM SAGE wiring (`_sage_kernel_available`, resolve quantize once, conditional `sage_kwargs`) looks correct; unit tests at this head: **13 passed** (`test_*quant*`, non-GPU). Env here is FlashInfer **0.6.14** without `trtllm_sage_attention_quantize` and SM90 — GPU SAGE numerical test not exercised locally. +- Docs say int8 is SM100-only; init does not arch-check, so SM103/int8 fails later with a kernel miss. A clearer init-time message would help; not a silent-correctness bug. + +**Smallest fix to unblock:** make `quant` TRTLLM-only with SAGE dtype/block validation at config time; trim unused FLASHINFER fields/claims; restore the deleted limitations/docs.Silent-misconfig cases reproduced at head (`dtype_vo`-only TRTLLM and FLASHINFER `quant` both serialize without enabling a real consumer). Targeted quant unit tests: **13 passed**. diff --git a/eval/dataset/arms/cursor_grok45/pr5550.cost.json b/eval/dataset/arms/cursor_grok45/pr5550.cost.json new file mode 100644 index 000000000..c99449abd --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5550.cost.json @@ -0,0 +1,15 @@ +{ + "arm": "cursor_cursorgrok4.5high", + "model_requested": "cursor-grok-4.5-high", + "head": "20ca4b93de6e61b42544f2b4ba1101711e467986", + "base": "c4a14091c9b8fd61f4da46022f683adc1dc91550", + "snapshot_sha256": "164875457cfa48b5dbdf9910ab1823eb23590cb709fe0dcf988c6e837f562a27", + "wall_s": 354.3, + "shell_commands": 7, + "file_reads": 32, + "cost_usd": null, + "audit_ok": true, + "audit_violations": [], + "artifact_chars": 5535, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_grok45/pr5550.md b/eval/dataset/arms/cursor_grok45/pr5550.md new file mode 100644 index 000000000..155d75954 --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5550.md @@ -0,0 +1,86 @@ +I'll review PR #5550 as an expert reviewer using the repo's review skill and the frozen checkout at the PR head.Freezing the snapshot first, then I'll route the Direct review with title, body, and changed files.**Frozen snapshot** +- Head: `20ca4b93de6e61b42544f2b4ba1101711e467986` +- Merge-base: `c4a14091c9b8fd61f4da46022f683adc1dc91550` +- PR state: MERGED; gates clean +- Diff census: 10 files (6 production, 4 tests) — Worker KV metadata DTOs + RPC plumbing for Scheduler-managed paged KV + +**Preliminary findings** +- Scope matches W0 control-plane slice claim (DTO + transport; no allocator/BlockTable install) +- Primary risk surface: optional metadata acceptance in paged mode vs dense rejection, request-ID agreement at Worker boundary, and RPC arity compatibility +- Discussion: none available (per frozen context); will mark `existing_feedback_status=disabled` + +Calling Direct review with title/body/changed files next.Routes selected (scope fallback): **diffusion** primary (`DIFF-4a/b/c`), plus configuration/serving maps. Serving map is truncated — I'll use the embedded excerpt and worktree refs only (no out-of-tree reads). Investigating the six production files next.Checking cache-mode typing and whether stepwise/batch paths drop metadata; also reviewing tests for contract gaps.Validating the Direct completion gate, then writing the review. The main contract gap: `execute_batch` / `execute_step` skip the pre-RPC identity checks that `execute_request` added.## Verdict + +**Approve with comments** (no merge blockers for this W0 slice). Head `20ca4b93de6e`; claimed scope matches the diff; dense RPC shape and optional metadata behavior look correct; new contract tests pass locally (17). One real contract hole on the primary engine dispatch paths. + +--- + +### P2 — Pre-RPC identity checks only cover `execute_request` + +`execute_request` correctly validates every envelope **before** RPC so identity failures never cross the process boundary or get wrapped as worker errors: + +```454:460:vllm_omni/diffusion/executor/multiproc_executor.py + # Validate every envelope before selecting a dispatch path. In + # particular, keep identity errors outside the RPC error wrapper so an + # invalid request cannot be forwarded or converted into a worker output. + from vllm_omni.diffusion.sched.interface import validate_new_request_data_identity + + for new_req in new_reqs: + validate_new_request_data_identity(new_req) +``` + +But the engine’s live `execute_fn` is `execute_batch` / `execute_step`, and those paths still forward first: + +```615:620:vllm_omni/diffusion/executor/multiproc_executor.py + result = self.collective_rpc( + "execute_model_batch", + args=(scheduler_output, self.od_config), + unique_reply_rank=0, + exec_all_ranks=True, + ) +``` + +```646:656:vllm_omni/diffusion/executor/multiproc_executor.py + def execute_step(self, scheduler_output: DiffusionSchedulerOutput) -> BaseRunnerOutput: + """Forward step-mode scheduler output to worker execute_stepwise RPC.""" + ... + result = self.collective_rpc( + "execute_stepwise", + args=(scheduler_output,), + ... + ) +``` + +**Trigger:** `max_num_seqs > 1` fused request-batch, or step mode, with envelope/`req`/`metadata.request_id` mismatch. +**Impact:** Invalid allocation envelopes are pickled to workers; failure becomes an RPC/`RuntimeError` instead of a local `ValueError`, contrary to this PR’s stated invariant. Single-request waves are fine (`execute_batch` → `execute_request`). +**Fix:** Run the same `validate_new_request_data_identity` loop at the top of `execute_batch` and `execute_step` (before any `collective_rpc`). Add a regression test on the multi-req `execute_batch` path mirroring `test_request_rpc_rejects_envelope_request_identity_mismatch`. + +--- + +### P3 — DLO DP waves silently serialize when any metadata is present + +```462:474:vllm_omni/diffusion/executor/multiproc_executor.py + has_diffusion_kv_metadata = any(new_req.diffusion_kv_metadata is not None for new_req in new_reqs) + ... + if ( + len(new_reqs) > 1 + and not has_diffusion_kv_metadata + and getattr(self.od_config, "enable_distributed_layerwise_offload", False) + ... + ): +``` + +**Trigger:** W1 starts attaching `DiffusionKVMetadata` under DLO+AllGather + `max_num_seqs > 1`. +**Impact:** Correctness is preserved via per-request RPC, but DP concurrency silently disappears with no log/metric. +**Fix:** Log at warning/info when the guard fires, or fail closed until the list RPC can carry per-request metadata. Cover with a unit test that asserts the serial 4-arg shape (not the list RPC) when metadata is attached. + +--- + +### What looks solid + +- Ownership split is clean: wire `DiffusionKVMetadata` vs Scheduler-owned `DiffusionKVRequest`; `prepared_layout` stays separate. +- Dense path keeps the 3-arg positional RPC; metadata is only appended when present (`multiproc_executor.py:544-546`, `diffusion_worker.py:462-465`). +- Mode gate is right for W0: dense rejects metadata; paged accepts optional metadata (`diffusion_model_runner.py:163-178`). +- Engine init failure now shuts down executor/scheduler if bring-up fails after executor start (`diffusion_engine.py:191-208`) — sensible adjacent hygiene. + +**Validation:** `pytest` on the new metadata/worker-contract tests + engine cleanup case — 17 passed at `20ca4b93de6e`. PR discussion intentionally not consulted. diff --git a/eval/dataset/arms/cursor_grok45/pr5610.cost.json b/eval/dataset/arms/cursor_grok45/pr5610.cost.json new file mode 100644 index 000000000..22f60ec69 --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5610.cost.json @@ -0,0 +1,17 @@ +{ + "arm": "cursor_cursorgrok4.5high", + "model_requested": "cursor-grok-4.5-high", + "head": "50126edf2f50a85e7e41b32b70da7d8e70b21aec", + "base": "900a7f0813d0482811b0e4dfd3cf7deabbe2429f", + "snapshot_sha256": "155a8a48d7ac9a99f8aa125d3ccb82c7bacd8524d84d04518b200e037950d936", + "wall_s": 262.8, + "shell_commands": 8, + "file_reads": 23, + "cost_usd": null, + "audit_ok": false, + "audit_violations": [ + "read outside worktree: /home/zhoutaichang/.claude/skills/imreview/SKILL.md" + ], + "artifact_chars": 4300, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_grok45/pr5610.md b/eval/dataset/arms/cursor_grok45/pr5610.md new file mode 100644 index 000000000..251eef129 --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5610.md @@ -0,0 +1,67 @@ +I'll review PR #5610 with InferMatrixCopilot Direct mode. Reading the imreview skill and freezing the PR snapshot first.Collecting the frozen snapshot (changed files, HEAD) so I can publish the progress update before calling Direct review.**Frozen snapshot** +- **Head SHA:** `50126edf2f50a85e7e41b32b70da7d8e70b21aec` +- **CI / gates:** clean (per frozen context) +- **Mergeability:** MERGED +- **Changed files (4, docs-only):** + - `docs/.nav.yml` + - `docs/design/feature/omni_async_output_materialization.md` + - `docs/design/index.md` + - `docs/source/architecture/qwen3-omni-async-output-step-gap.svg` +- **Preliminary:** documentation of async Omni output materialization (no runtime code in the diff). Docs-only path: skip dependency preflight/pytest; validate against live contracts in-tree. + +Calling Direct review next.Investigating the docs against the live contracts they claim — reading the diff and locating the referenced runtime APIs.Verifying the documented guards and opt-ins against the live runner and model contracts.Checking the blog numbers and a few remaining contract claims without reading files outside the worktree.Checking the SVG relative path — it may resolve outside `docs/`.Contracts check out. Validating the Direct completion gate, then writing the review.## Verdict: Approve (docs-accurate; two minor follow-ups) + +Head `50126edf2f50`. Docs-only PR; live contracts in `GPUARModelRunner._should_use_async_omni_output`, Qwen3-Omni/TTS opt-ins, deploy YAMLs, and NPU sync path match the page. Blog numbers (9.3→11.3 req/s, 655→631 ms TTFP, 0.63→0.47 RTF; ~2.8 ms→41 µs gap) check out. Diff hygiene clean. `mkdocs build --strict` remains blocked by the existing transformers pin conflict (noted in the PR). + +No blocking contract errors. Two small polish items below. + +--- + +### Comments + +**1. `docs/design/feature/omni_async_output_materialization.md:70-76` — figure URL convention** + +```70:76:docs/design/feature/omni_async_output_materialization.md +

+ Decode step gap before and after async output materialization +

+``` + +`../../../source/...` is correct for MkDocs (`docs/` site root → `/source/architecture/...`). Sibling design pages (e.g. `async_chunk.md`) use absolute `raw.githubusercontent.com/.../docs/source/architecture/...` URLs, so GitHub blob/preview of this page 404s the figure. + +**Change:** Prefer the same absolute raw URL pattern as `async_chunk.md`, or document that the relative path is MkDocs-only. + +--- + +**2. `docs/design/feature/omni_async_output_materialization.md:320-324` — warning incomplete vs guard table** + +```320:324:docs/design/feature/omni_async_output_materialization.md +!!! warning + Do not pass `--no-async-chunk` or enable prefix caching when you want this + optimization. Either change causes the runner to fall back to synchronous + output construction. Enabling async chunk alone cannot activate the feature + for a model that has not opted into async Omni output. +``` + +`_should_use_async_omni_output()` also returns `False` when `use_async_scheduling` is off (lines 1596–1597 in `gpu_ar_model_runner.py`), and the compatibility table already lists that. The warning callout does not. + +**Change:** Extend the warning to include disabling AR async scheduling (CLI/stage override), so the “how to keep it on” guidance matches the full guard set. + +--- + +### No-issue conclusions (checked) + +| Claim | Evidence | +|---|---| +| Auto-enable via deploy profile; no user flag | No `async_omni_output` config; opt-in via `use_async_omni_output` on Thinker/Talker / Qwen3-TTS Talker | +| YAML excerpts | `qwen3_omni_moe.yaml` / `qwen3_tts.yaml` match (`async_chunk: true`, prefix caching off, Code2Wav `async_scheduling: false` on Omni) | +| NPU stays sync | `NPUARModelRunner.sample_tokens` builds `OmniModelRunnerOutput` then wraps `AsyncGPUModelRunnerOutput` | +| XPU/MUSA may hit GPU path | `XPUARModelRunner(GPUARModelRunner)`; MUSA returns `GPUARWorker` | +| Full-payload branch unreachable under feature | `should_accumulate_full_payload_output` returns `False` when `async_chunk` | +| Nav / design index / SVG / `async_chunk.md` link | Present and parseable | + +**Subtraction:** none (documentation only). diff --git a/eval/dataset/arms/cursor_grok45/pr5703.cost.json b/eval/dataset/arms/cursor_grok45/pr5703.cost.json new file mode 100644 index 000000000..9071264dd --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5703.cost.json @@ -0,0 +1,19 @@ +{ + "arm": "cursor_cursorgrok4.5high", + "model_requested": "cursor-grok-4.5-high", + "head": "1c6ee28f627cef5e2659b3671fb3451e718562f6", + "base": "ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783", + "snapshot_sha256": "6959d8d9cb9a0f35dcd94367f9717276b8fe358f3147b030589ab34459a7564e", + "wall_s": 224.6, + "shell_commands": 9, + "file_reads": 9, + "cost_usd": null, + "audit_ok": false, + "audit_violations": [ + "read outside worktree: /home/zhoutaichang/.claude/skills/imreview/SKILL.md", + "read outside worktree: /data/zhoutaichang/.cache/uv/archive-v0/b1YBuFePm-5r6QpFFv7bF/lib/python3.12/site-packages/infermatrix_copilot/knowledge/repos/vllm-omni/components/serving/rule", + "read outside worktree: /home/zhoutaichang/.cursor/projects/data-zhoutaichang-infermatrix-copilot-worktrees-vllm-omni-pr5703/terminals/191251.txt" + ], + "artifact_chars": 5399, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_grok45/pr5703.md b/eval/dataset/arms/cursor_grok45/pr5703.md new file mode 100644 index 000000000..047cd940b --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5703.md @@ -0,0 +1,82 @@ +I'll review PR #5703 with InferMatrixCopilot Direct mode: freeze the snapshot first, then run the structured review without posting to GitHub.Collecting the frozen diff file list, then posting the snapshot update before any knowledge or source reads.**Frozen snapshot** +- Head: `1c6ee28f627cef5e2659b3671fb3451e718562f6` +- Merge-base: `ae7bee0c43c780a8e21d0b5e4b92a7eaeaea8783` +- State: MERGED +- CI: failing `buildkite/vllm-omni-amd-ci` (treat as gate status; not re-argued) +- Scope: `vae.py` RNG context + MiniMax-H3 MUSA recipe/README (3 files, +156/−7) + +**Preliminary (pre-knowledge):** Likely a small Hardware/MUSA fix for device-aware conditioned VAE RNG; AMD CI failure looks unrelated to this MUSA-focused diff. Next: Direct `review`, then source/validation.Serving route map is truncated (must open once). Parallel tracks: read that knowledge page, inspect the VAE RNG diff/callers, and start validation preflight.The code widens RNG handling beyond CUDA; next I'll verify `get_device_module`/`fork_rng` semantics and callers against DIFF-1b.`fork_rng` defaults to `device_type="cuda"` — checking whether the MUSA path actually restores device RNG, and how NPU is affected.Confirming a likely `fork_rng`/`device_type` bug and a recipe FL2VA/Ref2VA mismatch; running a targeted repro and tests next.Fixing the validation gate payload: correct subtraction action verbs and drop dispositions under `disabled` feedback.## Verdict: **Changes requested** (merged, but the VAE RNG fix is still incorrect for non-CUDA devices) + +Head `1c6ee28f627c`. Scope is small and the MUSA recipe is useful, but the encode RNG path does not actually make `fork_rng` device-aware. CI gate still red: `buildkite/vllm-omni-amd-ci` (status only; not re-litigated). Focused CPU tests here: **63 passed** (`contract`/`packing`/`parallel`); they do not cover accelerator RNG restore. + +--- + +### P1 — `fork_rng` still uses CUDA RNG state for non-CUDA devices + +`vllm_omni/diffusion/models/minimax_h3/vae.py:170-176` and `:214-220` + +```170:176:vllm_omni/diffusion/models/minimax_h3/vae.py + devices = [parameter.device] if parameter.device.type != "cpu" else [] + try: + with torch.random.fork_rng(devices=devices): + torch.default_generator.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED) + for device in devices: + with self.device_module.device(device): + self.device_module.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED) +``` + +`torch.random.fork_rng` defaults to `device_type="cuda"`. It saves/restores via `torch.cuda.get_rng_state` / `set_rng_state`, and that API uses **only `device.index`**, ignoring `device.type`. + +So for `parameter.device == musa:0` (or `npu:0`): +- seeding goes through `self.device_module` (hopefully musa/npu), +- but the context saves/restores **CUDA index 0** (or fails on Ascend CPU+NPU builds with no CUDA generators). + +That breaks the PR’s own claim that CPU and active-device RNG are restored, and it is a regression vs the old `device.type == "cuda"` gate for Ascend MiniMax H3 (`MiniMax-H3-NPU.md`: no CUDA runtime). + +**Fix:** allowlist explicitly, and pass the matching type: + +```python +device = parameter.device +device_type = device.type +devices = [device] if device_type in ("cuda", "musa") else [] +device_mod = torch.get_device_module(device) if devices else None +with torch.random.fork_rng(devices=devices, device_type=device_type if devices else "cuda"): + ... + if device_mod is not None: + with device_mod.device(device): + device_mod.manual_seed(MINIMAX_H3_KEYFRAME_ENCODE_SEED) +``` + +Same change in both `encode_image` and `encode_video`. + +--- + +### P2 — Init-time `get_device_module()` does not bind the VAE device + +`vllm_omni/diffusion/models/minimax_h3/vae.py:120` + +`self.device_module = torch.get_device_module()` takes no device, so it caches the process default accelerator, not `device` from `__init__` / `parameter.device`. Delete the field and resolve `torch.get_device_module(parameter.device)` at encode time (with the allowlist above). + +--- + +### P2 — Claimed CUDA/MUSA allowlist is not what the code does + +`vllm_omni/diffusion/models/minimax_h3/vae.py:170` / `:214` + +PR text says only CUDA and MUSA; code uses `!= "cpu"`. That pulls NPU/XPU/etc. into the broken `fork_rng` path. Keep an explicit `{"cuda", "musa"}` set; leave other backends on the previous CPU-only fork path until validated. + +--- + +### P3 — Recipe advertises Ref2VA TP4 but exports FL2VA + +`recipes/MiniMaxAI/MiniMax-H3-MUSA.md:72-76` + +The “validated Ref2VA” block sets `MODEL="${MODEL_ROOT}/FL2VA"`. Change to `Ref2VA` (or relabel the block as FL2VA). + +--- + +### Notes (non-blocking) + +- PR body mentions audio-loading coverage and `test_minimax_h3_vae.py`; this diff only touches `vae.py` + recipes. No accelerator RNG restore test landed, so the MUSA claim is unenforced in CI. +- Serving/config knowledge routes did not apply to this diff; no findings there. +- Existing PR discussion was intentionally unavailable (`disabled`); no duplicate suppression against GitHub threads.Focused MiniMax H3 tests finished successfully: **63 passed** (`test_minimax_h3_contract.py`, `test_minimax_h3_packing.py`, `test_minimax_h3_parallel.py`) under head `1c6ee28f627c`. Those runs stay on the CPU contract path and do not exercise the MUSA/`fork_rng(device_type=...)` restore issue called out in the review. diff --git a/eval/dataset/arms/cursor_grok45/pr5715.cost.json b/eval/dataset/arms/cursor_grok45/pr5715.cost.json new file mode 100644 index 000000000..f7a3e1465 --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5715.cost.json @@ -0,0 +1,17 @@ +{ + "arm": "cursor_cursorgrok4.5high", + "model_requested": "cursor-grok-4.5-high", + "head": "765caa2a121aec024201936c4decb1151225991c", + "base": "7a2007cc17d63ce9070e1feefd29fcca8a93cf34", + "snapshot_sha256": "3723b9e637bb5d67daa30874435afe2848c3e1ee689c5b0501634fb0eba751d1", + "wall_s": 143.0, + "shell_commands": 4, + "file_reads": 9, + "cost_usd": null, + "audit_ok": false, + "audit_violations": [ + "read outside worktree: /home/zhoutaichang/.claude/skills/imreview/SKILL.md" + ], + "artifact_chars": 3151, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_grok45/pr5715.md b/eval/dataset/arms/cursor_grok45/pr5715.md new file mode 100644 index 000000000..6999367ec --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5715.md @@ -0,0 +1,41 @@ +I'll review PR #5715 with InferMatrixCopilot Direct mode—reading the skill and freezing the PR snapshot first.Pinning the PR snapshot from the local checkout, then I'll post the progress update before any knowledge or source reads.**Frozen snapshot** +- **Head:** `765caa2a121aec024201936c4decb1151225991c` +- **Merge-base:** `7a2007cc17d63ce9070e1feefd29fcca8a93cf34` +- **CI / mergeability:** MERGED; gates clean (no failing checks) +- **Changed files (6, docs-only):** `README.md`, `docs/README.md`, `docs/configuration/README.md`, `docs/getting_started/installation/README.md`, `docs/getting_started/installation/gpu/cuda.inc.md`, `docs/getting_started/installation/npu/npu.inc.md` +- **Preliminary:** Documentation-only v0.26.0 release alignment; no code/runtime paths in the diff. Next: Direct `review`, then source checks against the frozen head. + +Calling InferMatrixCopilot Direct review now.Serving quick_map is truncated but this is docs-only; staying inside the worktree and checking the six changed files plus live version/model contracts.Evidence checks out at the frozen head. Validating the Direct completion gate, then writing the review.## Verdict: Approve (docs-only, merge-ready) + +Frozen head `765caa2a121aec024201936c4decb1151225991c` is a clean documentation alignment for the v0.26.0 line. Version pins, recipe links, Dockerfile base image, and the vLLM config URL match the live tree; MUSA/quickstart are untouched; full-duplex is correctly marked experimental in the news blurb. One optional accuracy nit below. + +--- + +### Optional: scope the full-duplex feature bullet + +**`README.md:54`** (and mirrored **`docs/README.md:55`**) + +```54:54:README.md +- Full-duplex realtime serving with streaming audio input and output (experimental) +``` + +The 0.26 news entry correctly scopes this to MiniCPM-o 4.5 (`README.md:19`). In the tree, `duplex_runtime_extension` is only set on the MiniCPM-o 4.5 pipeline (`vllm_omni/model_executor/models/minicpmo_4_5/pipeline.py`); no other pipeline registers it. + +**Suggested change:** mention MiniCPM-o 4.5 in the bullet (or “for supported omni models”), e.g. +`Full-duplex realtime serving for MiniCPM-o 4.5 with streaming audio input and output (experimental)` +so the feature list does not read as a general, model-agnostic capability. + +--- + +### Checked and clean (no further action) + +| Claim | Evidence at head | +|---|---| +| CUDA `BASE_IMAGE=vllm/vllm-openai:v0.26.0` | Matches `docker/Dockerfile.cuda` default | +| NPU pins `v0.26.0` / `releases/v0.26.0rc` + `cd vllm` / `cd vllm-omni` | Consistent with the from-scratch clone flow; drops broken `/vllm-workspace/vllm-omni` | +| Install note: Omni 0.26.x ↔ vLLM 0.26.x | Aligns with `cuda.inc.md` / `quickstart.md` pins | +| Config link → vLLM 0.26 docs | `https://docs.vllm.ai/en/v0.26.0/configuration/index.html` resolves | +| Recipe links MiniMax H3 / MiniCPM-o 4.5 / Cosmos3 | Files exist under `recipes/` | +| Model categorization (MiniMax H3 as diffusion; MiniCPM-o 4.5 as omni) | Matches registry/recipe roles | + +**Not investigated further:** PR discussion (disabled for this review); MkDocs rebuild (authors already reported strict build pass). diff --git a/eval/dataset/arms/cursor_grok45/pr5840.cost.json b/eval/dataset/arms/cursor_grok45/pr5840.cost.json new file mode 100644 index 000000000..44bf2295f --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5840.cost.json @@ -0,0 +1,18 @@ +{ + "arm": "cursor_cursorgrok4.5high", + "model_requested": "cursor-grok-4.5-high", + "head": "75cd907089322c648cf1494a2423d706e20fec2a", + "base": "fc8946fcf542cf108fa8b2fdc69de737bbfd52d8", + "snapshot_sha256": "0250972d67ae968a6f260a906b426a0695bc5e6677d3f796a84a259d86925480", + "wall_s": 351.4, + "shell_commands": 8, + "file_reads": 24, + "cost_usd": null, + "audit_ok": false, + "audit_violations": [ + "read outside worktree: /home/zhoutaichang/.claude/skills/imreview/SKILL.md", + "read outside worktree: /home/zhoutaichang/.cursor/projects/data-zhoutaichang-infermatrix-copilot-worktrees-vllm-omni-pr5840/terminals/219037.txt" + ], + "artifact_chars": 3646, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_grok45/pr5840.md b/eval/dataset/arms/cursor_grok45/pr5840.md new file mode 100644 index 000000000..275302fac --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5840.md @@ -0,0 +1,49 @@ +I'll review PR #5840 using the project's review skill and the frozen PR context you provided.Following the review workflow: freezing the snapshot, then loading process references and invoking Direct review.Pinned head: `75cd907089322c648cf1494a2423d706e20fec2a` +Base/comparison: `fc8946fcf542cf108fa8b2fdc69de737bbfd52d8` +CI: failing `buildkite/vllm-omni-amd-ci` (PR MERGED) +Mergeability: MERGED +Preliminary findings: TeaCache wiring for MiniMax-H3 across extractors/config/backend/hook + tests/recipe; investigating coefficient defaults, partition handling, and cache correctness next.Investigating the TeaCache production path next: extractors, config, backend, hook, and data wiring.Comparing the extractor to the live MiniMax-H3 forward and tracing `rel_l1_thresh=None` through callers.Checking whether the calibrated threshold can ever produce cache hits given the polynomial's constant term.Confirming the engine-default hole, then validating and writing the review.## Verdict + +**Looks good for the core TeaCache path; one actionable P2 default-contract hole.** Extractor ↔ `MiniMaxH3DiTModel.forward` parity, FL2VA/combined/ref2va gating, and residual reuse look correct. CI gate: `buildkite/vllm-omni-amd-ci` failing (do not re-litigate). CPU unit tests for the new MiniMax TeaCache surfaces passed at `75cd9070`. + +--- + +### P2 — Model default `0.17` never reaches Omni/serve + +`TeaCacheConfig` / recipe / e2e advertise MiniMax-specific `rel_l1_thresh=0.17` when unset: + +```96:99:vllm_omni/diffusion/cache/teacache/config.py +_DEFAULT_REL_L1_THRESH = 0.2 +_MODEL_DEFAULT_REL_L1_THRESH = { + "MiniMaxH3DiTModel": 0.17, +} +``` + +```420:422:vllm_omni/diffusion/data.py + # TeaCache parameters [tea_cache only] + # None defers to the model-specific TeaCache default (0.2 fallback). + rel_l1_thresh: float | None = None +``` + +But every Omni/serve path that omits `cache_config` goes through `AsyncOmniEngine._normalize_cache_config` → `_get_default_cache_config("tea_cache")`, which **forces** `0.2`: + +```853:856:vllm_omni/engine/async_omni_engine.py + if cache_backend == "tea_cache": + return { + "rel_l1_thresh": 0.2, + } +``` + +**Trigger:** `Omni(..., cache_backend="tea_cache")` or `vllm serve ... --cache-backend tea_cache` without `--cache-config`. +**Effect:** `_MODEL_DEFAULT_REL_L1_THRESH` is dead on the public entry path; only direct `TeaCacheBackend(DiffusionCacheConfig())` (as in unit tests) gets `0.17`. Recipe/e2e paper over this by hard-coding `0.17`. +**Fix:** For `tea_cache`, return `{}` / omit `rel_l1_thresh` so `None` can defer to the model map; or stop changing the shared default and set MiniMax’s `0.17` only inside `enable_minimax_h3_teacache` after the engine empty-config path stops pinning `0.2`. + +--- + +### What looked solid (no further comments) + +- `extract_minimax_h3_context` matches native forward (embed → blocks via `sp_prepare`/`sp_gather` → `final_layer` + masks); `test_extractor_matches_native_forward` and residual-hit coverage are the right checks. +- `enable_minimax_h3_teacache` correctly hooks only `pipeline.transformer` (FL2VA), allows `combined` with a warning, and rejects `ref2va`. +- Coefficients / registry wiring are consistent with other TeaCache models. + +**Validation gap (not a finding):** cache-hit path is not asserted numerically against full compute; production A/B quality claims were not re-run here. AMD CI remains red as reported.The preflight/collection run finished successfully: 8 `TestMiniMaxH3Extractor` cases collected (18 deselected). Those tests were later executed and all passed, which is already reflected in the review. diff --git a/eval/dataset/arms/cursor_grok45/pr5863.cost.json b/eval/dataset/arms/cursor_grok45/pr5863.cost.json new file mode 100644 index 000000000..312ee6f98 --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5863.cost.json @@ -0,0 +1,17 @@ +{ + "arm": "cursor_cursorgrok4.5high", + "model_requested": "cursor-grok-4.5-high", + "head": "0a4c96151148ccf3e4ed29ec53287fe36cbf2409", + "base": "48298030d4b3c320b858dd98934c01babdd320ce", + "snapshot_sha256": "47a66ad1e1d54d29f2c36c5017a36f5772be207b0f2b908d2f94e64a343275a5", + "wall_s": 182.5, + "shell_commands": 2, + "file_reads": 16, + "cost_usd": null, + "audit_ok": false, + "audit_violations": [ + "read outside worktree: /home/zhoutaichang/.claude/skills/imreview/SKILL.md" + ], + "artifact_chars": 3637, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_grok45/pr5863.md b/eval/dataset/arms/cursor_grok45/pr5863.md new file mode 100644 index 000000000..562ef508d --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5863.md @@ -0,0 +1,49 @@ +I'll review PR #5863 using InferMatrixCopilot Direct mode: first freezing the PR snapshot, then investigating the diff and related code.Freezing the PR snapshot from the local checkout, then reporting progress before the Direct review call.**Frozen snapshot** +- Head: `0a4c96151148ccf3e4ed29ec53287fe36cbf2409` +- Changed files: `recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md` (docs-only) +- PR state: MERGED; gates clean / mergeable +- Preliminary: documentation recipe for MiniMax-H3 on RTX PRO 6000 (2×/4×/8× configs); no runtime code in the diff + +Calling Direct `review` next, then validating against the live recipe and referenced contracts.Docs-only budget: checking the recipe against live CLI/config contracts, skipping pytest.Checking the Ref2VA form contract, aspect_ratio rules, attention defaults, and the dual-server numactl flags.Findings verified against live contracts. Validating the Direct completion gate, then writing the review.## Verdict: LGTM with follow-ups + +Docs-only recipe for MiniMax-H3 on RTX PRO 6000. The 2×/4×/8× serve commands, TP×USP×world-size math, CFG-distilled `cfg_parallel_size=1`, T2VA `aspect_ratio` requirement, cuDNN attention on SM120, and the measured table are consistent with the live CLI and H3 pipeline. Two copy-paste issues in the dual-server / Ref2VA sections should be fixed in a follow-up (PR is already merged). + +--- + +### Findings + +**1. Dual-server NUMA flags are not `vllm serve` arguments** +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:201-203` + +```201:203:recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md +Use `CUDA_VISIBLE_DEVICES=0,2,1,3`, `--cpunodebind=0 --membind=0`, and +`--port 8091` for the FL2VA server. Use `CUDA_VISIBLE_DEVICES=4,6,5,7`, +`--cpunodebind=1 --membind=1`, and `--port 8092` for the Ref2VA server. +``` + +`--cpunodebind` / `--membind` appear nowhere in `vllm_omni/entrypoints/cli/serve.py` (only in this recipe). They are `numactl` options. Pasting them into `vllm serve` will fail argparse. Show a full wrapper, e.g. `numactl --cpunodebind=0 --membind=0 vllm serve ... --port 8091`, or drop the flags and keep the CUDA device order only. + +**2. Ref2VA `audio_reference` example does not match the live form contract** +`recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md:335-345` + +```335:345:recipes/MiniMaxAI/MiniMax-H3-RTX-PRO-6000.md +```bash +curl -X POST "http://127.0.0.1:${PORT}/v1/videos/sync" \ + ... + -F "audio_reference= str: + if os.path.isdir(vocoder_name): + return vocoder_name + local_dir = resolve_model_directory( + model_path, + "bigvgan", + os.path.join("hf_cache", "bigvgan"), + ) + if local_dir is not None: + return local_dir + if os.path.isdir(model_path): + raise FileNotFoundError(f"IndexTTS BigVGAN assets are missing from local bundle {model_path!r}") + return vocoder_name +``` + +**Trigger:** any local model directory (IndexTTS-2 or 2.5) without vendored `bigvgan/` / `hf_cache/bigvgan/`. +**Before:** Stage 1 called `BigVGAN.from_pretrained("nvidia/bigvgan_v2_...")` and could fetch the public vocoder. +**Now:** hard `FileNotFoundError` on every local directory. Confirmed in this environment. +**Impact:** intentional for native 2.5 bundles, but this is shared `IndexTTS2S2MelDecoder` code — IndexTTS-2 local checkouts that previously worked without vendoring BigVGAN now fail at load/warmup. +**Fix:** scope the fail-fast to IndexTTS 2.5 / native bundles (or an explicit flag), and keep HF ID / IndexTTS-2 local fallback unless docs require vendoring for both. + +--- + +### 2. Bare scalar `duration_factor` is silently ignored (native speed no-ops) + +`vllm_omni/model_executor/models/indextts2/indextts2_talker.py:561-569` + +```561:569:vllm_omni/model_executor/models/indextts2/indextts2_talker.py + def _first(key, default=None): + v = info_dict.get(key) + return v[0] if isinstance(v, list) and v else default + + duration_factor = float(_first("duration_factor", 1.0)) + # Direct Omni callers bypass the serving adapter, so retain the + # model-side guard as the final fail-fast validation layer. + if not (INDEXTTS25_MIN_DURATION_FACTOR <= duration_factor <= INDEXTTS25_MAX_DURATION_FACTOR): + raise ValueError("IndexTTS 2.5 duration_factor must be between 0.5 and 2.0") +``` + +**Trigger:** offline / direct Omni `additional_information` with `"duration_factor": 0.5` (scalar) instead of `[0.5]`. +**Behavior:** `_first` treats any non-list as missing and returns `1.0`, so Stage 1 length scaling never runs; no error. Serving and the offline example wrap lists, so the happy path is fine. +**Impact:** contradicts the PR’s fail-fast claim for the public native-speed contract. +**Fix:** accept scalar or list (`v[0] if isinstance(v, list) else v`), and reject wrong types with an explicit `ValueError`. + +--- + +### Notes (non-blocking) + +- Native speed double-application looks correct: `IndexTTS25Adapter.native_speed_control` + `_audio_encode_speed()` → `1.0`, while `duration_factor = 1/speed` drives S2Mel `_target_lengths`. +- Request-end meta that emits conditioning only on `mel_len == 1` is safe with flattened `meta.*` keys (absent keys are not wiped). The “emit every step” comment in `make_omni_output` is misleading but not a shipping bug. +- Targeted CPU tests for duration metadata / full-payload / indextts adapter detection passed here on vLLM `0.26.0` at head `19a6e513`.Targeted validation at head `19a6e513` finished successfully: IndexTTS duration-metadata, full-payload bridge, and adapter tests passed (10 cases). The BigVGAN local-bundle fail-fast also reproduced as expected. diff --git a/eval/dataset/arms/cursor_grok45/pr5976.cost.json b/eval/dataset/arms/cursor_grok45/pr5976.cost.json new file mode 100644 index 000000000..189165e9a --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5976.cost.json @@ -0,0 +1,19 @@ +{ + "arm": "cursor_cursorgrok4.5high", + "model_requested": "cursor-grok-4.5-high", + "head": "25b80bb995bdae8bbff8f332e4ebb53232415436", + "base": "06e733d3cd4400bb0029f8418e2207f1441a39c0", + "snapshot_sha256": "3ab56af68c00d7c83d8a00d2ed140e1e80497b21e484dfaed8a81d3d9cb15912", + "wall_s": 281.7, + "shell_commands": 13, + "file_reads": 21, + "cost_usd": null, + "audit_ok": false, + "audit_violations": [ + "read outside worktree: /home/zhoutaichang/.claude/skills/imreview/SKILL.md", + "read outside worktree: /usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py", + "read outside worktree: /usr/local/lib/python3.12/dist-packages/vllm/v1/engine/utils.py" + ], + "artifact_chars": 5211, + "split": "holdout" +} diff --git a/eval/dataset/arms/cursor_grok45/pr5976.md b/eval/dataset/arms/cursor_grok45/pr5976.md new file mode 100644 index 000000000..da65cf99e --- /dev/null +++ b/eval/dataset/arms/cursor_grok45/pr5976.md @@ -0,0 +1,63 @@ +I'll review PR #5976 with the repo's review skill and InferMatrix Direct flow: freeze the snapshot first, then investigate the diff against the stated merge-base.Frozen snapshot: head `25b80bb995bd`, merge-base `06e733d3cd44`, MERGED, 4 failing Buildkite gates (amd/intel/npu/rebase). Collecting the file list and kicking off Direct review.**Snapshot frozen:** head `25b80bb995bdae8bbff8f332e4ebb53232415436`, MERGED, 55 files; failing gates: amd/intel/npu/rebase CI. Preliminary: rebase + runner/scheduler/diffusion/quantization fixes; open WER regression noted. Calling Direct review next.Serving route map is truncated; I'll stay inside the worktree and inspect the highest-risk paths (runners, scheduler drain, EP, compile cache, thinker capture).Checking a few likely defect sites: generation `maybe_randomize_inputs` default, stale-token double-seeding, NPU parity, and thinker capture consumers.Suspecting double-seeding of `num_stale_output_tokens` on the AR `replace_streaming_prompt` path — verifying that call chain and related high-risk spots.Confirmed a P1 double-seed on the MiniCPM-o `replace_streaming_prompt` path. Validating the Direct gate, then writing the review.## Verdict: **Request changes** (merge-blocking defect in the stale-drain fix) + +Head `25b80bb995bd` · base `06e733d3cd44` · state **MERGED** · failing gates: `buildkite/vllm-omni-amd-ci`, `intel-ci`, `npu-ci`, `rebase` (status only). Most of the 0.27 alignment work (dummy-run kwarg, thinker bare-tensor contract, EP `use_all2all`, compile-cache isolation, inductor CantSplit patch) looks sound and well-tested. One scheduler path undoes the WER-class fix the PR claims to land. + +--- + +### P1 — Double-seed of `num_stale_output_tokens` on `replace_streaming_prompt` + +**`vllm_omni/core/sched/omni_ar_scheduler.py:627-670`** + **`vllm_omni/core/sched/omni_scheduler_mixin.py:105`** + +**Trigger:** MiniCPM-o (and any stage) streaming update with `meta.replace_streaming_prompt=True` while `num_in_flight_tokens > 0`. + +**Current behavior:** `_update_request_as_session` always does: + +```627:629:vllm_omni/core/sched/omni_ar_scheduler.py + in_flight_tokens = int(getattr(session, "num_in_flight_tokens", 0) or 0) + if in_flight_tokens > 0: + session.num_stale_output_tokens += in_flight_tokens +``` + +then, on the replace path, calls `_replace_streaming_session`, which seeds again: + +```105:105:vllm_omni/core/sched/omni_scheduler_mixin.py + session.num_stale_output_tokens += int(getattr(session, "num_in_flight_tokens", 0) or 0) +``` + +With `num_in_flight_tokens=2`, the counter becomes **4**. After the two late frames drain, **2** leftovers remain and the drain swallows the new segment’s first valid frames — the same hole this PR’s `c2b65233` / stale-drain tests were written to close. + +**Impact:** Corrupted codec/audio on MiniCPM-o (and any `replace_streaming_prompt` consumer). Stage-0 seed-TTS path is not covered by this bug (tests hit `super()` without replace), which matches “drain fix landed but WER still open” only partially — this is a separate, reachable regression. + +**Fix:** Seed once. On the AR replace branch, skip the pre-seed (mixin already seeds); keep the pre-seed only for the early-return / `super()` paths. Add a CPU test that sets `num_in_flight_tokens`, calls `_update_request_as_session` with `replace_streaming_prompt=True`, and asserts `num_stale_output_tokens == num_in_flight_tokens` (not `2×`). + +Generation is fine: it only goes through the mixin (`omni_generation_scheduler.py:604`). + +--- + +### P2 — NPU `_dummy_run` missing `randomize_inputs` (GPU fixed, NPU not) + +**`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:641-656`** and **`npu_model_runner.py:112-128`** + +GPU runners accept/forward (or intentionally ignore) `randomize_inputs` for vLLM 0.27 warmup. NPU overrides still omit the kwarg. If the Ascend/warmup path mirrors upstream and passes it, warmup dies with the same `TypeError` this PR fixed on CUDA — consistent with the failing `npu-ci` gate. + +**Fix:** Mirror the GPU signatures: accept the kwarg; on the generation runner do not forward randomization into codec dummy ids. + +--- + +### P3 — Generation runner comment vs call + +**`vllm_omni/worker/gpu_generation_model_runner.py:840-847`** + +Comment says never randomize, but the call is still `maybe_randomize_inputs(input_ids, inputs_embeds)` with no `randomize_inputs=False`. On 0.27 the AR runner already passes the flag; pass `randomize_inputs=False` explicitly so env/DP paths cannot re-enable vocab-uniform dummy ids for code2wav. + +--- + +### What looks good (no change requested) + +- Thinker bare-tensor / staged-capture contract + `test_qwen3_omni_forward_contract.py` +- EP `use_all2all=True` (version-guarded) for HunyuanImage3 MoE +- Per-replica `VLLM_CACHE_ROOT` isolation at spawn (`test_stage_engine_startup_cache_env.py`) +- Inductor factorable-divisibility patch + self-extinguish probe +- Stale-drain unit tests for the **non-replace** path; base image pin to `v0.27.0` + +**Open (already acknowledged):** seed-TTS WER still not restored after the drain fix — do not treat that as newly introduced here, but fix the double-seed before trusting another nightly on MiniCPM-o / TTS quality. diff --git a/eval/dataset/arms/goal_debug_promo/manifest.json b/eval/dataset/arms/goal_debug_promo/manifest.json new file mode 100644 index 000000000..aaa6f73fd --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "goal_debug_promo", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "1" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/goal_debug_promo/pr4923.cost.json b/eval/dataset/arms/goal_debug_promo/pr4923.cost.json new file mode 100644 index 000000000..df1d46204 --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo/pr4923.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 402.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/goal_debug_promo/runs/pr4923/run-20260812-201415-002815", + "recorded_at": "2026-08-12T20:20:55", + "attempts": [ + { + "invocation_id": "inv-7b7e88ba", + "outcome": "run:run-20260812-201415-002815", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-201415-002815", + "llm_calls": 43, + "input_tokens": 695092, + "output_tokens": 60771, + "usd": 0.308437, + "source": "spans" + } + ], + "attempt_usd_total": 0.308437, + "llm_calls": 43, + "input_tokens": 695092, + "output_tokens": 60771, + "usd": 0.308437, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-201415-002815", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3084, + "minutes": 6.67, + "ci_minutes": 0, + "input_tokens": 695092, + "output_tokens": 60771, + "tool_calls": 84, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3647, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 43, + "by_role": { + "agent": { + "usd": 0.297002, + "input_tokens": 684223, + "output_tokens": 53072, + "calls": 41 + }, + "reducer": { + "usd": 0.011435, + "input_tokens": 10869, + "output_tokens": 7699, + "calls": 2 + } + }, + "cache_read_tokens": 1996800, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.26, + "gate": 2.26, + "review": 393.84, + "report": 0.06 + }, + "time_to_first_result_s": 400.4 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 21516 +} \ No newline at end of file diff --git a/eval/dataset/arms/goal_debug_promo/pr4923.md b/eval/dataset/arms/goal_debug_promo/pr4923.md new file mode 100644 index 000000000..196df3df5 --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo/pr4923.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4923, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] qwen3_tts_code_predictor_vllm.py:36: `use_cuda_graphs=current_omni_platform.is_npu()` correctly gates inner CUDA graphs to NPU only — verified in CodePredictorWrapper._setup_compile that GPU path skips _capture_cuda_graphs when use_cuda_graphs=False. +- [validated] qwen3_tts_talker.py:327: `talker_mtp_graph_safe = True` correctly enables outer CUDAGraphWrapper — runner reads this flag in gpu_model_runner.py:209 and wraps talker_mtp in CUDAGraphWrapper when has_full_cudagraphs(). +- [validated] qwen3_tts_talker.py:346-350: cudagraph_mode gating with getattr(..., None) default is defensive and consistent — when mode is PIECEWISE or None, has_full_cudagraphs() returns False or is not reached, preserving eager per-row generators. The runner's _init_talker_mtp asserts cudagraph_mode is non-None, so both sides agree. +- [validated] qwen3_tts_talker.py:351-352: use_async_omni_output and eager_omni_postprocess_before_async_output match Qwen3-Omni's pattern (qwen3_omni.py:134,163) and the runner reads them via _should_use_async_omni_output / _maybe_run_eager_omni_postprocess_before_async_output in gpu_ar_model_runner.py. +- [sweep] qwen3_tts.yaml: max_num_seqs=64 x max_num_batched_tokens=32768 is consistent (64*512=32768). The 512→32768 jump for stage 0 is explained in PR description as enabling larger decode batches. +- [sweep] qwen3_tts.yaml: NPU platform cudagraph_mode: PIECEWISE for both stages is consistent with NPU keeping inner code predictor graphs (is_npu()→True) and not using FULL outer wrap. +- [sweep] Diff hunk at qwen3_tts_code_predictor_vllm.py:39 — `use_cuda_graphs=current_omni_platform.is_npu()`: grep confirms Qwen3-Omni (`qwen3_omni_moe_code_predictor_mtp.py:21`) already uses same pattern; no other CodePredictorWrapper consumers exist. +- [sweep] Diff hunk at qwen3_tts_talker.py:327 — `talker_mtp_graph_safe = True`: grep confirms runner reads this at `gpu_model_runner.py:208` and `gpu_ar_model_runner.py`/`npu_ar_model_runner.py`; FishSpeech also sets it. All consumers handled. + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The comment at line 38 claims prefix caching is on by default and calls it 'a net throughput win', but the diff flips `enable_prefix_caching` from `true` to `false`. The comment now contradicts the config. Update it to explain that prefix caching is disabled by default to avoid bookkeeping overhead with full-graph MTP, and document when users may re-enable it. The body text about HS opt-out and deferred writes is still correct. (evidence: read_file vllm_omni/deploy/qwen3_tts.yaml lines 38-44; comment at line 38 contradicts `enable_prefix_caching: false` at line 46.) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:353` [minor] — The new `self.omni_pooler_payload_include_hidden = False` at line 353 duplicates the same assignment already present at line 317. The duplication is confusing — readers may wonder if one of them was meant to be `True` under some condition. Since the new placement groups it with the async output flags, remove the earlier assignment at line 317 (or add a comment noting the intentional grouping). (evidence: Read qwen3_tts_talker.py lines 310-360; line 317 already sets `self.omni_pooler_payload_include_hidden = False` before the PR-added block at 351-353.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — `max_num_seqs` was raised from 10 to 64 for both stages. The PR benchmark was run at a max concurrency of 8 (peak 17), so batches as large as 64 are untested. Either validate performance/stability at (or near) 64 concurrent requests with a sweep, or constrain the default to a value actually exercised (e.g., 32) to avoid latent issues in graph capture and per-row generator batching logic at high concurrency. (evidence: Read qwen3_tts.yaml: max_num_seqs changed 10→64; PR body benchmark shows `Maximum request concurrency: 8`, `Peak concurrent requests: 17` — not reaching 64.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The `buildkite/vllm-omni-npu-ci` check is failing (CI gate). This PR adds NPU-specific `cudagraph_mode: PIECEWISE` config and gates code predictor graphs on `is_npu()`. Please verify whether the failure is pre-existing on main. If introduced, the outer PIECEWISE wrapping combined with inner NPU device graphs may need a fix in the code predictor's graph setup. (evidence: gh_ci_read PR#4923 shows buildkite/vllm-omni-npu-ci FAILURE; diff adds NPU platform cudagraph_mode and is_npu() gating.) + +`vllm_omni/deploy/qwen3_tts.yaml:?` [minor] — The header comment at line ~6 says `max_num_batched_tokens=512`, but the diff now sets it to 32768. Update the comment to reference the new value and its purpose (full cudagraph token coverage, not latency tuning). (evidence: read_file qwen3_tts.yaml L1-12 (header comment) vs L49 (max_num_batched_tokens: 32768 in diff)) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_code_predictor_vllm.py:?` [nit] — The docstring change removes '(CUDA graphs' as part of the description, but on NPU the code predictor still enables inner device graphs via `use_cuda_graphs=current_omni_platform.is_npu()`. Consider noting the platform-dependent behavior: e.g. '(per-call sampling, projection; inner graphs on NPU only)'. (evidence: Read qwen3_tts_code_predictor_vllm.py: `use_cuda_graphs=current_omni_platform.is_npu()` means NPU keeps inner graphs; CodePredictorWrapper._setup_compile confirms NPU path uses eager + NPU graphs when use_cuda_graphs=True.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/goal_debug_promo/pr4923.trace.json.gz b/eval/dataset/arms/goal_debug_promo/pr4923.trace.json.gz new file mode 100644 index 000000000..bf33580bb Binary files /dev/null and b/eval/dataset/arms/goal_debug_promo/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/goal_debug_promo2/manifest.json b/eval/dataset/arms/goal_debug_promo2/manifest.json new file mode 100644 index 000000000..61fe853e4 --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo2/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "goal_debug_promo2", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/goal_debug_promo2/pr5009.cost.json b/eval/dataset/arms/goal_debug_promo2/pr5009.cost.json new file mode 100644 index 000000000..651139edd --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo2/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 391.4, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/goal_debug_promo2/runs/pr5009/run-20260812-202055-1985d3", + "recorded_at": "2026-08-12T20:27:24", + "attempts": [ + { + "invocation_id": "inv-1e36441f", + "outcome": "run:run-20260812-202055-1985d3", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-202055-1985d3", + "llm_calls": 45, + "input_tokens": 455854, + "output_tokens": 59674, + "usd": 0.241367, + "source": "spans" + } + ], + "attempt_usd_total": 0.241367, + "llm_calls": 45, + "input_tokens": 455854, + "output_tokens": 59674, + "usd": 0.241367, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-202055-1985d3", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2414, + "minutes": 6.49, + "ci_minutes": 0, + "input_tokens": 455854, + "output_tokens": 59674, + "tool_calls": 79, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3314, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 45, + "by_role": { + "agent": { + "usd": 0.227726, + "input_tokens": 442739, + "output_tokens": 50520, + "calls": 43 + }, + "reducer": { + "usd": 0.013642, + "input_tokens": 13115, + "output_tokens": 9154, + "calls": 2 + } + }, + "cache_read_tokens": 1949824, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 11.6, + "gate": 1.98, + "review": 375.44, + "report": 0.06 + }, + "time_to_first_result_s": 389.07 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 25587 +} \ No newline at end of file diff --git a/eval/dataset/arms/goal_debug_promo2/pr5009.md b/eval/dataset/arms/goal_debug_promo2/pr5009.md new file mode 100644 index 000000000..221bb74c9 --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo2/pr5009.md @@ -0,0 +1,44 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] platform.py:249-260 — removed `using_inductor` branch; new code unconditionally returns `["vllm_c", "native"]`, matching pre-rebase c45ac74d behavior. The `vllm_config` parameter is now accepted but unused (API compat). +- [validated] platform.py:249 — docstring updated from multi-line inductor-aware explanation to single-line "Prefer vllm_c CUDA kernels over native for diffusion IR ops", accurate for new behavior. +- [validated] qwen_image_transformer.py:17 — import `from vllm.model_executor.layers.layernorm import RMSNorm` removed; all 5 usages replaced with `nn.RMSNorm`. No stale references remain in the file. +- [validated] qwen_image_transformer.py:554-555,592-593,1006 — all 5 `nn.RMSNorm` calls pass `eps` kwarg, supported by both torch.nn.RMSNorm and the removed vLLM RMSNorm. Normalized shapes: `head_dim` (128) for norm_q/k/norm_added_q/k, `joint_attention_dim` (3584) for txt_norm — all compatible with standard default_weight_loader. +- [sweep] Verified Cosmos3 per-model override at `vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:612`: `get_cosmos3_ir_op_priority_func` forces `rms_norm=["native"]` and `fused_add_rms_norm=["native"]`, so the platform-level change to `["vllm_c", "native"]` is correctly overridden for Cosmos3. +- [sweep] Grepped for `get_default_ir_op_priority` — RocmOmniPlatform (rocm/platform.py:173) and XPUOmniPlatform (xpu/platform.py:107) still retain inductor-aware `["native"]`-when-compiling logic. This PR's scope is CUDA-only, so this is expected. +- [sweep] Grepped for `ir_op_priority_func` — confirmed `_DIFFUSION_IR_OP_PRIORITY_FUNCS` registry only contains Cosmos3 entry; no other model has a per-model override. +- [validated] tests/platforms/test_cuda_ir_op_priority.py — has `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`, ensuring CI selection. Tests parametrize over NONE/VLLM_COMPILE/STOCK_TORCH_COMPILE modes, all asserting same `["vllm_c", "native"]` output, plus oink override test. + +`vllm_omni/platforms/cuda/platform.py:?` [minor] — The method `get_default_ir_op_priority` now unconditionally returns `["vllm_c", "native"]` regardless of the `vllm_config` argument (the old inductor‑aware branch was removed). The parameter is retained only for call‑site compatibility. Consider adding a note in the docstring that `vllm_config` is no longer consulted, or plan deprecation in a follow‑up. (evidence: read_file vllm_omni/platforms/cuda/platform.py:249-262 — method body ignores vllm_config entirely) + +`vllm_omni/platforms/cuda/platform.py:252` [minor] — This change restores the global `["vllm_c", "native"]` default for **all** CUDA diffusion models (Flux, SD3, Wan, HunyuanVideo, etc.), not just Qwen‑Image. The PR benchmarks only Qwen‑Image and FLUX.1-dev. Although this reverts pre‑#4709 behavior and Cosmos3 has its own `["native"]` override, models added during the #4709 window were not tested with `vllm_c`. Please either: (a) smoke‑test at least FLUX, Wan, and SD3 families, or (b) add a comment linking a tracking issue confirming confidence from the pre‑rebase commit and stating no model changed RMSNorm expectations during the window. (evidence: Read vllm_omni/platforms/cuda/platform.py:249-268 (the hunk); grepped `get_default_ir_op_priority` callers → diffusion_worker.py:175 feeds ALL diffusion models; grepped `_DIFFUSION_IR_OP_PRIORITY_FUNCS` → only Cosmos3 has a per-model override (registry.py:549); all other models inherit this platform default.) + +`vllm_omni/platforms/cuda/platform.py:?` [minor] — The docstring says “Prefer ``vllm_c`` CUDA kernels over ``native`` for diffusion IR ops.” but `get_default_ir_op_priority` is the CUDA platform default inherited by all models that use vLLM IR ops (both AR and diffusion). Understating the scope may mislead maintainers. Suggestion: “Prefer ``vllm_c`` CUDA kernels over ``native`` for IR ops. Models needing ``native``-only must provide a per‑model ``ir_op_priority_func`` override (see Cosmos3).” (evidence: Read platform.py:252; grep for ir_op_priority_func shows Cosmos3 override at pipeline_cosmos3.py:612-624; call site at diffusion_worker.py:175 applies to all diffusion models, and AR models also inherit this default via the platform class.) + +`vllm_omni/platforms/cuda/platform.py:?` [minor] — The invariant that models needing `native`‑only must register a per‑model `ir_op_priority_func` override (Cosmos3 is the reference) is documented in the PR description but not in the code. Add a brief comment here (e.g., “# Models requiring native‑only (e.g., Cosmos3) provide a per‑model ir_op_priority_func override”) to help future maintainers know how to opt out. (evidence: Verified Cosmos3 has per-model override at pipeline_cosmos3.py:612-624 that hardcodes ['native'] for rms_norm. The override mechanism exists at diffusion_worker.py:177-178. The invariant is implicit but not stated in code.) + +`tests/platforms/test_cuda_ir_op_priority.py:23` [minor] — This CPU unit test validates `IrOpPriorityConfig` values, not actual kernel dispatch (dispatch is covered by the Qwen‑Image e2e serving test on H100 in CI). Consider adding a brief docstring note that the test is fast‑path config checking and the real dispatch path is exercised by `test_qwen_image_single_device` etc. (evidence: read_file tests/platforms/test_cuda_ir_op_priority.py — only asserts on list equality, no kernel-dispatch verification) + +`tests/platforms/test_cuda_ir_op_priority.py:30` [minor] — The OINK test only checks `CompilationMode.VLLM_COMPILE`. The non‑OINK test parametrizes all three compilation modes to guarantee correctness. Consider parametrizing the OINK test as well (or at least adding `CompilationMode.NONE`) to prevent a future regression where the `["oink", "vllm_c", "native"]` order is accidentally dropped for non‑compiling runs. (evidence: Read tests/platforms/test_cuda_ir_op_priority.py:30-38; compared with the parametrized test at lines 18-28 which covers all three CompilationMode values.) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:?` [minor] — Switching from `vllm.model_executor.layers.layernorm.RMSNorm` to `torch.nn.RMSNorm` is a deliberate performance choice (#4074 → #3588 revert chain) but the code carries no rationale. Add an inline comment—e.g., “# torch.nn.RMSNorm fuses better with inductor for small per‑head norms; avoid reverting to vLLM RMSNorm”—to prevent accidental reversal during future rebases. (evidence: PR description explains #4074 → #3588 revert → this fix chain. Read qwen_image_transformer.py showing all 5 RMSNorm sites changed. No inline comment exists at any of the 5 sites explaining the choice.) + +`.buildkite/pipeline.yml:?` [minor] — The gate report shows `buildkite/vllm-omni-amd-ci`, `intel-ci`, and `npu-ci` failing. These lanes should be unaffected by CUDA‑only platform and Qwen‑Image changes, but please confirm the failures are pre‑existing on main (e.g., from a broken baseline) before merging, to rule out an unexpected interaction in shared code. (evidence: Gate report lists 3 failing checks. PR changes are scoped to cuda/platform.py and qwen_image_transformer.py — neither AMD/ROCm, Intel/XPU, nor NPU platform code is touched. rocm/platform.py and xpu/platform.py retain their existing inductor-aware logic.) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/goal_debug_promo2/pr5009.trace.json.gz b/eval/dataset/arms/goal_debug_promo2/pr5009.trace.json.gz new file mode 100644 index 000000000..fcec79c50 Binary files /dev/null and b/eval/dataset/arms/goal_debug_promo2/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/goal_debug_promo3/manifest.json b/eval/dataset/arms/goal_debug_promo3/manifest.json new file mode 100644 index 000000000..618529ebf --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo3/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "goal_debug_promo3", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/goal_debug_promo3/pr5009.cost.json b/eval/dataset/arms/goal_debug_promo3/pr5009.cost.json new file mode 100644 index 000000000..e612f2e48 --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo3/pr5009.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 735.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/goal_debug_promo3/runs/pr5009/run-20260812-202823-174f5c", + "recorded_at": "2026-08-12T20:40:35", + "attempts": [ + { + "invocation_id": "inv-ec79b53c", + "outcome": "run:run-20260812-202823-174f5c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260812-202823-174f5c", + "llm_calls": 56, + "input_tokens": 702924, + "output_tokens": 89260, + "usd": 0.362777, + "source": "spans" + } + ], + "attempt_usd_total": 0.362777, + "llm_calls": 56, + "input_tokens": 702924, + "output_tokens": 89260, + "usd": 0.362777, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260812-202823-174f5c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3628, + "minutes": 12.21, + "ci_minutes": 0, + "input_tokens": 702924, + "output_tokens": 89260, + "tool_calls": 108, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5276, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 56, + "by_role": { + "agent": { + "usd": 0.34898, + "input_tokens": 690205, + "output_tokens": 79867, + "calls": 54 + }, + "reducer": { + "usd": 0.013798, + "input_tokens": 12719, + "output_tokens": 9393, + "calls": 2 + } + }, + "cache_read_tokens": 2770432, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.98, + "gate": 1.76, + "review": 726.73, + "report": 0.08 + }, + "time_to_first_result_s": 732.51 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 26044 +} \ No newline at end of file diff --git a/eval/dataset/arms/goal_debug_promo3/pr5009.md b/eval/dataset/arms/goal_debug_promo3/pr5009.md new file mode 100644 index 000000000..acd21e88a --- /dev/null +++ b/eval/dataset/arms/goal_debug_promo3/pr5009.md @@ -0,0 +1,46 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5009, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [validated] platform.py:249 — removed `using_inductor` branch restores pre-#4709 behavior (`["vllm_c", "native"]` always). +- [validated] pipeline_cosmos3.py:612-624 — Cosmos3 has per-model `ir_op_priority_func` override forcing `["native"]`, confirming the PR's claim that models needing native-only are protected. +- [validated] qwen_image_transformer.py:17,554-558,592-596,1006-1009 — all 5 vLLM RMSNorm instances replaced with `nn.RMSNorm` consistently; no partial conversion. +- [validated] test_cuda_ir_op_priority.py:23-28 — parametrized over all three CompilationMode values; correctly asserts `["vllm_c", "native"]` for each. +- [validated] test_cuda_ir_op_priority.py:30-37 — oink test correctly monkeypatches VLLM_USE_OINK_OPS and validates oink prepended. +- [validated] test_cuda_ir_op_priority.py:9 — pytestmark `[core_model, cpu]` matches CI "Simple · Other Test" lane `-m 'core_model and cpu'`; test will be collected. +- [sweep] Grepped all `nn.RMSNorm` uses under vllm_omni/diffusion — OvisImage, FLUX.2, SD3, HunyuanVideo, ErnieImage, Wan22 still import vLLM's RMSNorm and are affected by Part 1's priority change; consistent with pre-rebase state. +- [sweep] Grepped all `get_default_ir_op_priority` references — rocm/platform.py:173 and xpu/platform.py:107 still have the inductor-aware `["native"]` when inductor active branch, unchanged by this PR. + +`tests/platforms/test_cuda_ir_op_priority.py:9` [major] — The new regression test lives in `tests/platforms/`, a directory that CI never runs (`docs/contributing/ci/tests_style.md:123` explicitly states 'no tests/platforms/'). This means the test is silently deselected — a future breakage will not be caught. Move the test to a CI‑covered directory (e.g., `tests/diffusion/`) or add an explicit Buildkite step for `tests/platforms/`. (evidence: grep of `.buildkite/` for `tests/platforms` returned zero matches; docs at `docs/contributing/ci/tests_style.md:123` confirm `platforms/ → (no tests/platforms/`)) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:?` [minor] — The replacement of vLLM's `RMSNorm` with `torch.nn.RMSNorm` (available from PyTorch 2.1) at five call sites may break on environments with an older PyTorch. Please confirm that the project's minimum PyTorch version is ≥ 2.1; otherwise users will see `AttributeError` at import time. (evidence: Read qwen_image_transformer.py lines 554-558, 592-596, 1006-1009 showing all 5 instances of `nn.RMSNorm`; could not verify torch lower bound in pyproject.toml due to budget.) + +`tests/platforms/test_cuda_ir_op_priority.py:23` [minor] — Part 2 replaces vLLM's `RMSNorm` with `torch.nn.RMSNorm` at five call sites in Qwen-Image, but no test validates the class identity. Add a simple assertion in the existing Qwen‑Image model test (e.g., `isinstance(cross_attn.norm_q, nn.RMSNorm)`) to prevent future reverts that silently reintroduce the fused `RMSNorm` (as in #3588 → #4074 → #4709). (evidence: Read the full diff; Part 2 changes 5 vLLM RMSNorm instances to nn.RMSNorm. No test in this PR validates the class identity. The linked issue #4964 shows this exact pattern (revert→regression) happened before.) + +`vllm_omni/platforms/rocm/platform.py:?` [minor] — The ROCm platform (`vllm_omni/platforms/rocm/platform.py:182`) and XPU platform (`xpu/platform.py:112`) still gate `rms_norm` dispatch on inductor activity: `default = ["native"] if using_inductor else ["vllm_c", "native"]`. The CUDA fix removed this gating because it forced a slower native decomposition for diffusion denoise loops. Consider removing the `using_inductor` branch on ROCm/XPU as well, or add a comment explaining why the divergence is intentional (the PR targets CUDA-only and AMD/Intel CI failures are pre‑existing). (evidence: Read vllm_omni/platforms/rocm/platform.py:173 — `default = ["native"] if using_inductor else ["vllm_c", "native"]`; same pattern as vllm_omni/platforms/xpu/platform.py:107.) + +`vllm_omni/platforms/cuda/platform.py:252` [minor] — The `get_default_ir_op_priority` change now applies to all CUDA diffusion models except Cosmos3 (which has a per‑model override in `registry.py:545‑549`). The PR validated only Qwen-Image. Please confirm that at least one other model (e.g., SD3, FLUX, Wan) was smoke‑tested against this branch, or note that the risk is considered low because the change restores pre‑#4709 behaviour. (evidence: Read registry.py:545-549 — only Cosmos3 has a per-model ir_op_priority override; all other ~50+ models inherit the platform default. PR only reports Qwen-Image evidence.) + +`vllm_omni/platforms/cuda/platform.py:252` [minor] — The old docstring for `get_default_ir_op_priority` explained the upstream inductor‑aware logic; the new one‑liner drops that rationale. Add a sentence (e.g., 'Diffusion denoise loops are RMSNorm‑heavy and `vllm_c` CUDA kernels outperform native decomposition, even when inductor is compiling.') to prevent a future rebase from re‑inserting the `using_inductor` branch and silently regressing again. (evidence: Compared old (removed) docstring "Copied from upstream CudaPlatformBase with inductor-aware logic…" with new one-liner; checked consumer chain via `diffusion_worker.py:174-179` confirming platform-wide blast radius.) + +`tests/platforms/test_cuda_ir_op_priority.py:23` [minor] — The test name `test_cuda_default_ir_op_priority_prefers_vllm_c_when_inductor_backend` is misleading because it parametrizes `CompilationMode.NONE` (no inductor) and asserts `['vllm_c', 'native']` in every case. Rename to `test_cuda_default_ir_op_priority_always_prefers_vllm_c` to accurately describe the post‑fix behaviour. (evidence: Read test file at lines 18-27 — parametrize includes NONE, VLLM_COMPILE, STOCK_TORCH_COMPILE; assertion is identical for all.) + +`tests/platforms/test_cuda_ir_op_priority.py:9` [minor] — The test is placed in `tests/platforms/`, which documentation warns is never run by CI; however, CI 'Simple' lanes use marker‑based selection (`-m 'core_model and cpu'`) and the config contains no path‑based exclusion for `tests/platforms/`. This means the test may already be executing in CI, contrary to the documentation and the kept comment. Please confirm whether this test actually runs in the current CI pipeline and update the comment/docs accordingly. (evidence: [sweep] Read .buildkite/test-merge.yml and test-ready.yml — CI "Simple" lanes use `-m 'core_model and cpu'` which covers `tests/platforms/`; no path-based exclusion.) + +`vllm_omni/platforms/cuda/platform.py:251` [nit] — The `vllm_config` parameter is now unused by the method body (only `envs.VLLM_USE_OINK_OPS` is checked, which doesn't need it). It's kept for API compatibility (callers at `diffusion_worker.py:175` and test mocks pass it). Consider prefixing with `_` (`_vllm_config`) or adding a brief comment that it's retained for signature compatibility with per-model overrides. Evidence: read the full method body at lines 251-263 — no access to `vllm_config`. (evidence: Read `platform.py:249-263` — method body only uses `cls` and `envs.VLLM_USE_OINK_OPS`; `vllm_config` is never read.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/goal_debug_promo3/pr5009.trace.json.gz b/eval/dataset/arms/goal_debug_promo3/pr5009.trace.json.gz new file mode 100644 index 000000000..c8ce4074f Binary files /dev/null and b/eval/dataset/arms/goal_debug_promo3/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/goal_smoke_verify/manifest.json b/eval/dataset/arms/goal_smoke_verify/manifest.json new file mode 100644 index 000000000..7316da212 --- /dev/null +++ b/eval/dataset/arms/goal_smoke_verify/manifest.json @@ -0,0 +1,20 @@ +{ + "arm": "goal_smoke_verify", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "(default)" +} \ No newline at end of file diff --git a/eval/dataset/arms/goal_smoke_verify/pr4870.cost.json b/eval/dataset/arms/goal_smoke_verify/pr4870.cost.json new file mode 100644 index 000000000..e480eca56 --- /dev/null +++ b/eval/dataset/arms/goal_smoke_verify/pr4870.cost.json @@ -0,0 +1,107 @@ +{ + "wall_s": 1310.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/goal_smoke_verify/runs/pr4870/run-20260813-113121-53d40d", + "recorded_at": "2026-08-13T11:53:09", + "attempts": [ + { + "invocation_id": "inv-372a89a1", + "outcome": "run:run-20260813-113121-53d40d", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260813-113121-53d40d", + "llm_calls": 97, + "input_tokens": 1549655, + "output_tokens": 179910, + "usd": 0.776957, + "source": "spans" + } + ], + "attempt_usd_total": 0.776957, + "llm_calls": 97, + "input_tokens": 1549655, + "output_tokens": 179910, + "usd": 0.776957, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260813-113121-53d40d", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.777, + "minutes": 21.8, + "ci_minutes": 0, + "input_tokens": 1549655, + "output_tokens": 179910, + "tool_calls": 199, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.8776, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 97, + "by_role": { + "agent": { + "usd": 0.756032, + "input_tokens": 1533155, + "output_tokens": 164969, + "calls": 95 + }, + "reducer": { + "usd": 0.020925, + "input_tokens": 16500, + "output_tokens": 14941, + "calls": 2 + } + }, + "cache_read_tokens": 5949952, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 4.02, + "gate": 1.91, + "review": 1302.1, + "report": 0.07 + }, + "time_to_first_result_s": 1308.08 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 32718 +} \ No newline at end of file diff --git a/eval/dataset/arms/goal_smoke_verify/pr4870.md b/eval/dataset/arms/goal_smoke_verify/pr4870.md new file mode 100644 index 000000000..a3679772e --- /dev/null +++ b/eval/dataset/arms/goal_smoke_verify/pr4870.md @@ -0,0 +1,40 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 4870, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [sweep] serving_speech.py:3495-3510 read in situ — gating `qwen3_full_payload = _tts_model_type == 'qwen3_tts' and not bool(async_chunk)` and `is_streaming_request = request.is_streaming() and not qwen3_full_payload` matches the stated intent; residual default mismatch captured in review comment. +- [validated] protocol/audio.py is_streaming read — `stream_format='audio'` + `response_format='pcm'` yields streaming True, so both new serving tests drive the real branch. +- [validated] entrypoints/utils.py coerce_param_message_types read — `is_streaming=False` maps to FINAL_ONLY; qwen3 full-payload path gets FINAL_ONLY as claimed. +- [sweep] gpu_ar_model_runner.py:1676/1731/1787/1873/2039 read in situ — `seq_len` param removed everywhere; `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)` substituted at 1734 and passed at 1790; no stale `seq_len` references remain in this function. +- [validated] utils/mm_outputs.py to_payload_element read — tensor is sliced only when `element.shape[0] == seq_len`; using total scheduled tokens makes batched `codes.audio` slice per request for tail-only hidden states. +- [sweep] gpu_ar_model_runner.py callers grep — `_build_omni_model_runner_output_from_snapshot` referenced only from `sample_tokens` and the test file; signature removal leaves no stale caller. +- [sweep] tests/worker/test_gpu_ar_model_runner.py:438-480 read — new test calls the real `_build_omni_model_runner_output_from_snapshot` with tail-only `hidden_states` shape [1] vs total 3 and asserts per-request `codes.audio` slices; it exercises the changed gate. +- [sweep] indexed access tests/worker line 478 — `output.inter_stage_outputs[0]` is non-null because `needs_pooler_payload` is true and pooler_inter is built from pooler_output; no off-by-one in the asserted slices. + +`vllm_omni/entrypoints/openai/serving_speech.py:3508` [major] — Confirmed at serving_speech.py:3508-3509: stream=True qwen3_tts with async_chunk=False now sets is_streaming_request=False (FINAL_ONLY), but the streaming response still consumes the generator through _generate_audio_chunks. That function's list branch (serving_speech.py:2751, prev_count=0) emits every tensor in the FINAL_ONLY full-payload list with no empty-tensor filter, while the non-streaming path explicitly handles non-async Qwen3-TTS cumulative history by keeping only the latest non-empty tensor. This can send leading empty snapshots and repeated/overlapping waveform data to streaming clients. Mirror the latest-non-empty handling in _generate_audio_chunks (or pass an explicit no-async flag) and add a stream=True + async_chunk=False chunk-emission test; the new unit tests only assert the coercion argument, not the emitted chunks. (evidence: read vllm_omni/entrypoints/openai/serving_speech.py 2700-2860 and 3818-3835; the diff hunk at 3495-3510.) + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:?` [major] — GPU runner now derives the multimodal split length from `scheduler_output.total_num_scheduled_tokens`, but NPUARModelRunner still computes `seq_len = hidden_states.shape[0]` at vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905 and passes it to `to_payload_element` via `seq_len=seq_len` at line 1144 (call at line 1138). This leaves the pre-fix full-payload `codes.audio` split behavior on Ascend for concurrent Qwen3-TTS `async_chunk=False` requests. Verified: qwen3_tts_talker.py:311 sets `requires_full_prefix_cached_hidden_states = False`, and NPU patches qwen3_tts at vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py. Port the same `total_num_scheduled_tokens` fix with an NPU regression test, or explicitly scope the fix as GPU-only with a tracking issue. (evidence: read_file vllm_omni/platforms/npu/worker/npu_ar_model_runner.py; grep requires_full_prefix_cached_hidden_states) + +`vllm_omni/entrypoints/openai/serving_speech.py:?` [minor] — vllm_omni/entrypoints/openai/serving_speech.py:3507 introduces `async_chunk = getattr(self.model_config, "async_chunk", True)`, but the same file's streaming path uses `getattr(self.engine_client.model_config, "async_chunk", False)` at line 3823 and the worker defaults missing async_chunk to False. If the attribute is ever absent, `qwen3_full_payload` becomes False and the qwen3_tts full-payload fix silently reverts to DELTA coercion — the opposite of the fallback used at line 3823. Also unify the source: the new tests set `engine_client.model_config.async_chunk`, so if `self.model_config` is not the same object the test won't exercise the production read. Use `getattr(self.engine_client.model_config, "async_chunk", False)` (or guarantee `self.model_config` aliases it). Note: `self.model_config` is not assigned here but is read at 2466, 3283, and 3600. (evidence: read_file vllm_omni/entrypoints/openai/serving_speech.py; grep self.model_config) + +`tests/worker/test_gpu_ar_model_runner.py:478` [minor] — The added test `test_build_omni_output_splits_mm_by_scheduled_tokens_when_hidden_is_tail_only` (tests/worker/test_gpu_ar_model_runner.py:438-480) uses total_num_scheduled_tokens=3 with uniform num_scheduled_tokens {r1:1,r2:1,r3:1} and query_start_loc_cpu=[0,1,2], so all per-request slice offsets are identical (0:1, 1:2, 2:3). It proves the total_num_scheduled_tokens gate and one-token-per-request slicing but cannot catch an off-by-one/cumulative-offset error in the start/end loop. Add a non-uniform case, e.g. codes shaped (4,16), num_scheduled_tokens={r1:1,r2:2,r3:1}, query_start_loc_cpu=[0,1,3], asserting slices codes[0:1], codes[1:3], codes[3:4]. (evidence: Read tests/worker/test_gpu_ar_model_runner.py:438-480; the added test drives the real `_build_omni_model_runner_output_from_snapshot` but only with equal per-request scheduled tokens.) + +`tests/worker/test_gpu_ar_model_runner.py:?` [minor] — The new test exercises the changed tail-only split with `_async_chunk = False`, where `pooler_inter = pooler_client = pooler_output` (vllm_omni/worker/gpu_ar_model_runner.py:1803) and both `inter_stage_outputs` and `multimodal_outputs` are built from the same list (1813-1814). It currently asserts only `output.inter_stage_outputs[0..2]["codes.audio"]`. Since the client-facing `output.multimodal_outputs` is also populated on this non-async-chunk path (the consumer-visible wire path), add matching `output.multimodal_outputs[0..2]["codes.audio"] == codes[0:1]` (and r2/r3) assertions so the regression covers that path too. (evidence: read_file vllm_omni/worker/gpu_ar_model_runner.py; read_file tests/worker/test_gpu_ar_model_runner.py) + +`vllm_omni/worker/gpu_ar_model_runner.py:1734` [nit] — gh_ci_read(4870) shows only buildkite/vllm-omni-amd-ci FAILING while buildkite/vllm-omni, intel, npu, release, docs, build, pre-commit, and DCO pass. This diff removes the seq_len parameter from _build_omni_model_runner_output_from_snapshot and replaces it with mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) at vllm_omni/worker/gpu_ar_model_runner.py:1734, a shared AR-runner payload-slicing path an AMD lane plausibly exercises. Confirm against main whether that lane was already red before this PR; if pre-existing, record it on the PR so it does not mask a new AMD regression from this signature/slicing change. (evidence: gh_ci_read(4870): buildkite/vllm-omni-amd-ci = FAILURE; diff touches vllm_omni/worker/gpu_ar_model_runner.py.) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/goal_smoke_verify/pr4870.trace.json.gz b/eval/dataset/arms/goal_smoke_verify/pr4870.trace.json.gz new file mode 100644 index 000000000..1bf3ad40d Binary files /dev/null and b/eval/dataset/arms/goal_smoke_verify/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/smoke_cursor_backend/manifest.json b/eval/dataset/arms/smoke_cursor_backend/manifest.json new file mode 100644 index 000000000..3327d7cdc --- /dev/null +++ b/eval/dataset/arms/smoke_cursor_backend/manifest.json @@ -0,0 +1,21 @@ +{ + "arm": "smoke_cursor_backend", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "holdout" + ], + "kinds": [ + "pr_review", + "issue_answer" + ], + "n_items": 1, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "(default)", + "pr_context_mode": "no_discussion" +} \ No newline at end of file diff --git a/eval/dataset/arms/smoke_cursor_backend/pr5884.cost.json b/eval/dataset/arms/smoke_cursor_backend/pr5884.cost.json new file mode 100644 index 000000000..0648e61db --- /dev/null +++ b/eval/dataset/arms/smoke_cursor_backend/pr5884.cost.json @@ -0,0 +1,101 @@ +{ + "wall_s": 593.9, + "split": "holdout", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/smoke_cursor_backend/runs/pr5884/run-20260814-180645-698688", + "recorded_at": "2026-08-14T18:16:38", + "attempts": [ + { + "invocation_id": "inv-51ae9b51", + "outcome": "run:run-20260814-180645-698688", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260814-180645-698688", + "llm_calls": 20, + "input_tokens": 94292, + "output_tokens": 8443, + "usd": 0.064035, + "source": "spans" + } + ], + "attempt_usd_total": 0.064035, + "llm_calls": 20, + "input_tokens": 94292, + "output_tokens": 8443, + "usd": 0.064035, + "source": "spans", + "metrics": { + "schema": 1, + "run": "run-20260814-180645-698688", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.064, + "minutes": 9.89, + "ci_minutes": 0, + "input_tokens": 94292, + "output_tokens": 8443, + "tool_calls": 208, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3336, + "cost_partial": false, + "unpriced_calls": 0, + "source": "spans", + "llm_calls": 20, + "by_role": { + "moa_member": { + "usd": 0.064035, + "input_tokens": 94292, + "output_tokens": 8443, + "calls": 20 + } + }, + "cache_read_tokens": 548928, + "cache_creation_tokens": 0 + }, + "timings": { + "steps": { + "fetch": 3.64, + "gate": 2.35, + "review": 587.07, + "report": 0.06 + }, + "time_to_first_result_s": 593.11 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5884.trace.json.gz", + "trace_bytes": 19254 +} \ No newline at end of file diff --git a/eval/dataset/arms/smoke_cursor_backend/pr5884.md b/eval/dataset/arms/smoke_cursor_backend/pr5884.md new file mode 100644 index 000000000..328b0660d --- /dev/null +++ b/eval/dataset/arms/smoke_cursor_backend/pr5884.md @@ -0,0 +1,50 @@ +# Run report + +- task: {'kind': 'pr_review', 'mode': 'eco', 'repo': 'vllm-omni', 'pr': 5884, 'issue': None, 'report_only': False, 'post': False, 'params': {}} + +- checkout: PR-TIME TREE (head 5dc24e7f6a0c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [validated] backend.py:37-52 _dit_module_names uses attrgetter in try/except for dotted paths; catches AttributeError and skips None — correct +- [validated] backend.py:56-67 cache_summary uses attrgetter to resolve and BlockAdapter.is_cached to filter — matches model_specific.py:591 pattern +- [validated] backend.py:75-80 _make_pipeline_transformer_getter uses attrgetter — consistent with the rest of the file +- [validated] test_cache_backends.py:120-135 test_summary_skips_uncached_nested_dit: sets up two _dit_modules, only one passes is_cached, asserts summary called once — correctly exercises the skip path +- [validated] test_cache_backends.py:139-167 test_enable_and_refresh_nested_declared_dit: uses dotted path 'language_model.model', verifies enable_cache, refresh_context, and summary all resolve to the correct transformer — end-to-end coverage of the fix +- [validated] module_collector.py already uses attrgetter for dotted _dit_modules resolution (offloader path) +- [validated] All existing cache-backend tests (30) still pass conceptually: existing non-dotted tests are unaffected by attrgetter (attrgetter works on simple names too) +- [sweep] lora/manager.py:378 declared_components from _dit_modules are used with hasattr/pipeline.component_name — dotted paths like 'language_model.model' would silently be skipped (pre-existing, not introduced by this PR) + +`vllm_omni/diffusion/cache/cachedit/backend.py:57` [minor] — Update the `cache_summary` docstring at backend.py:57: behavior now skips uncached modules via `BlockAdapter.is_cached` (lines 61-62), so it should say only cached transformers are summarized—not every transformer on the pipeline. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:57 `"""Log Cache-DiT statistics for every transformer on the pipeline."""`; vllm_omni/diffusion/cache/cachedit/backend.py:60-63 `for transformer in transformers: + if not BlockAdapter.is_cached(transformer): + continue + cache_dit.summary(transformer, details=details)`) + +`tests/diffusion/cache/test_cache_backends.py:32` [minor] — gate_report shows buildkite/vllm-omni-amd-ci red at PR head; this PR adds two new mocked CPU tests in tests/diffusion/cache/test_cache_backends.py (lines 120–161). That file is selected by the AMD ready lane via module-level `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` and `.buildkite/amd/test-amd-ready.yml` runs `tests/diffusion -m 'core_model and cpu'`. Can you confirm whether the AMD CI failure predates this branch or is attributable to these changes before merge? (evidence: tests/diffusion/cache/test_cache_backends.py:32 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; .buildkite/amd/test-amd-ready.yml:19 `timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py`) + +`vllm_omni/diffusion/cache/cachedit/backend.py:48` [minor] — This PR switches Cache-DiT backend helpers to `attrgetter` for dotted `_dit_modules` (backend.py:47-48), but parallel consumers still use top-level `getattr`/`hasattr` and silently skip nested paths: `diffusion_model_runner._compile_transformer` (diffusion_model_runner.py:153), `lora/manager` declared_components (manager.py:386-388), and `registry._apply_sequence_parallel_if_enabled` (registry.py:442-451, with `find_module_with_attr` only when `hasattr` fails). Is aligning those call sites in scope for this PR, or tracked as follow-up so Bagel/SoulX nested paths get consistent behavior beyond cache enable/summary? (evidence: backend.py:47-48 `module = attrgetter(name)(pipeline)` — dotted `_dit_modules` resolved in Cache-DiT backend; diffusion_model_runner.py:153 `model = getattr(self.pipeline, attr_name, None)` — top-level getattr when compiling `_dit_modules` entries (called from lines 286-287); manager.py:386-388 `if not hasattr(self.pipeline, component_name): continue` / `component = getattr(self.pipeline, component_name)` — top-level only for declared components; registry.py:442-451 `if not hasattr(model, attr):` / `module = find_module_with_attr(model, attr)` / `transformer = getattr(model, attr)` — still top-level getattr after optional fallback) + +`tests/diffusion/cache/test_cache_backends.py:139` [minor] — This diff replaces `getattr` with `operator.attrgetter` in `_dit_module_names`, `cache_summary`, and `_make_pipeline_transformer_getter` (backend.py:43-53, 59, 81) so dotted `_dit_modules` paths resolve; the new unit tests cover only two-segment paths (`language_model.model`, `transformer.language_model`). SoulX Singer's production path `cfm_decoder.model.diff_estimator` is a three-segment consumer of the same fix—add a parametrized nested-path case (or SoulX fixture) so triple-segment resolution is CI-guarded. (evidence: vllm_omni/diffusion/models/soulx_singer/pipeline_soulx_singer_base.py:63 `_dit_modules: ClassVar[list[str]] = ["cfm_decoder.model.diff_estimator"]`; tests/diffusion/cache/test_cache_backends.py:146 `_dit_modules=["language_model.model"],` — only two-segment nested path tested, no `cfm_decoder` case in tests/diffusion/cache/) + +`tests/diffusion/cache/test_cache_backends.py:154` [minor] — This diff appends `cache_summary(pipeline)` to `test_enable_and_refresh_nested_declared_dit` (test_cache_backends.py:154) alongside the new `BlockAdapter.is_cached` gate at backend.py:61, but the test patches `cachedit.backend.BlockAdapter` (test_cache_backends.py:137), so `is_cached` is a default Mock (always truthy) and `mock_cache_dit.summary.assert_called_once_with(...)` cannot fail if the skip regresses. Set `mock_block_adapter.is_cached.side_effect` explicitly before calling `cache_summary`, or drop the summary assert here and rely on `test_summary_skips_uncached_nested_dit` (lines 120–135) for gating coverage. (evidence: ['tests/diffusion/cache/test_cache_backends.py:137 `@patch("vllm_omni.diffusion.cache.cachedit.backend.BlockAdapter")`', 'tests/diffusion/cache/test_cache_backends.py:154 `cache_summary(pipeline)`', 'tests/diffusion/cache/test_cache_backends.py:161 `mock_cache_dit.summary.assert_called_once_with(transformer, details=True)`', 'vllm_omni/diffusion/cache/cachedit/backend.py:61 `if not BlockAdapter.is_cached(transformer):`', 'tests/diffusion/cache/test_cache_backends.py:120 `@patch("vllm_omni.diffusion.cache.cachedit.backend.cache_dit")` — no `BlockAdapter` patch on `test_summary_skips_uncached_nested_dit`']) + +`vllm_omni/diffusion/cache/cachedit/backend.py:65` [nit] — After resolving dotted DiT paths into a non-empty `transformers` list, the loop skips uncached modules (`continue`) without logging; when every resolved module is uncached, `cache_summary` returns silently because `if not transformers` only covers zero resolved modules, not zero post-filter summaries. An optional debug log for that all-skipped case could help ops. (evidence: vllm_omni/diffusion/cache/cachedit/backend.py:59 `transformers = [attrgetter(name)(pipeline) for name in _dit_module_names(pipeline)]`; vllm_omni/diffusion/cache/cachedit/backend.py:61-62 `if not BlockAdapter.is_cached(transformer):` / ` continue`; vllm_omni/diffusion/cache/cachedit/backend.py:65-66 `if not transformers:` / ` logger.warning("CacheDiT summary failed; this pipeline has no defined transformer attribute")`) + +**Verdict:** COMMENT + +## skill candidates awaiting curation + +- **behavior-lens-device-rng-review**: Behavior lens for PRs that flip a device guard from ==cuda to !=cpu: enumerate platform inheritance and validate the claimed allowlist +- **verification-lens-pytestmark-deselection**: PR review verification lens: new test files without pytestmark are silently deselected by marker-filtered CI; check CI selection command vs each added test file's markers. + +(promote with SkillStore.promote(name); candidates are never auto-activated) diff --git a/eval/dataset/arms/smoke_cursor_backend/pr5884.trace.json.gz b/eval/dataset/arms/smoke_cursor_backend/pr5884.trace.json.gz new file mode 100644 index 000000000..05e1d3505 Binary files /dev/null and b/eval/dataset/arms/smoke_cursor_backend/pr5884.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/manifest.json b/eval/dataset/baselines/claudecode_opus5/manifest.json index c4a423031..502830746 100644 --- a/eval/dataset/baselines/claudecode_opus5/manifest.json +++ b/eval/dataset/baselines/claudecode_opus5/manifest.json @@ -37,10 +37,8 @@ "pr_context": "frozen snapshot (no_discussion); gh pr view and gh pr checks removed from the allowlist", "dataset": "vllm_omni_dataset.yaml", "splits": [ - "test", - "train", - "val" + "holdout5" ], - "n_items": 12, - "concurrency": 3 + "n_items": 1, + "concurrency": 1 } diff --git a/eval/dataset/baselines/claudecode_opus5/pr5509.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5509.cost.json new file mode 100644 index 000000000..7321911fe --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5509.cost.json @@ -0,0 +1,53 @@ +{ + "calls": 53, + "input_tokens": 3929843, + "output_tokens": 29005, + "cost_usd": 3.6522110000000003, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout", + "head": "58131b8dc1cb7273362574bee8472b08ea46c91f", + "base": "0c88f8a0b662ae497a6485db6f3b91b79620608c", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "7fbc3a6958a884db3fcc43bbeaa8163e7a1c6458ab51ac62c6a765b9a4c4ff27", + "validation_commands": [ + "git diff --stat 0c88f8a0b662ae497a6485db6f3b91b79620608c HEAD", + "git diff 0c88f8a0b662ae497a6485db6f3b91b79620608c HEAD -- vllm_omni/diffusion/data.py vllm_omni/diffusion/attention/backends/trtllm_attn.py", + "python -c \"import flashinfer, inspect; print(flashinfer.__version__); print(flashinfer.__file__)\" 2>&1 | tail -5", + "cd /usr/local/lib/python3.12/dist-packages/flashinfer && python -c \"\nfrom flashinfer.prefill import trtllm_ragged_attention_deepseek as f\nimport inspect\nprint(list(inspect.signature(f).parameters))\n\" 2>&1 | tail -5; grep -rn \"sage\" --include=*.py /usr/local/lib/python3.12/dist-packages/flashinfer/ |", + "python -c \"\nfrom flashinfer.prefill import trtllm_ragged_attention_deepseek as f\nimport inspect\nprint(list(inspect.signature(f).parameters))\n\"", + "python -c \"\nimport flashinfer\nprint(hasattr(flashinfer,'trtllm_sage_attention_quantize'))\n\"; grep -rn \"sage_attention_quantize\" /usr/local/lib/python3.12/dist-packages/flashinfer/*.py /usr/local/lib/python3.12/dist-packages/flashinfer/**/*.py 2>/dev/null | head -20", + "python -c \"\nimport flashinfer\nprint('has quantize fn:', hasattr(flashinfer,'trtllm_sage_attention_quantize'))\n\"", + "ls /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/vllm_omni/diffusion/attention/backends/", + "grep -rn \"backend_kwargs\" /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/vllm_omni --include=*.py | head -40", + "sed -n 1,140p /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/vllm_omni/diffusion/attention/backends/flashinfer_attn.py", + "git diff 0c88f8a0b662ae497a6485db6f3b91b79620608c HEAD -- tests/ docs/", + "grep -n \"^#\" /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/docs/user_guide/diffusion/attention_backends.md", + "grep -rn \"LTX-2\\|bench_attention_backends\\|Validation Guidance\" /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/docs/ | head -20", + "ls /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/benchmarks/diffusion/ 2>/dev/null | head -20", + "timeout 900 python -m pytest tests/diffusion/attention/test_attention_config.py tests/diffusion/attention/test_trtllm_attn.py -q 2>&1 | tail -25", + "grep -rln \"sage\" /usr/local/lib/python3.12/dist-packages/flashinfer/*.py /usr/local/lib/python3.12/dist-packages/flashinfer/jit/*.py 2>/dev/null | head", + "python - <<'EOF'\nimport subprocess, vllm, os\np = os.path.dirname(vllm.__file__)\nprint(p)\nEOF", + "python -c \"\nfrom vllm_omni.diffusion.data import AttentionSpec\nfrom vllm_omni.diffusion.attention.backends.trtllm_attn import QuantConfig\n\ns = AttentionSpec(backend='TRTLLM_ATTN', quant={'dtype_vo':'fp8_e4m3'})\nbk = s.backend_kwargs()\nprint('backend_kwargs:', bk)\nqc = QuantConfig.from_backend_kwargs", + "sed -n 140,260p /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/vllm_omni/diffusion/attention/layer.py", + "git log --oneline 0c88f8a0b662ae497a6485db6f3b91b79620608c..HEAD -- docs/user_guide/diffusion/attention_backends.md", + "git show --stat e54759c2 0cb1d75e -- docs/user_guide/diffusion/attention_backends.md | head -40", + "git log --format=\"%h %an %s\" 0c88f8a0b662ae497a6485db6f3b91b79620608c..HEAD | head", + "sed -n 60,300p /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/tests/diffusion/attention/test_trtllm_attn.py", + "grep -n \"_run_ring_attention\" -A 30 /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/vllm_omni/diffusion/attention/layer.py | head -50", + "grep -rn \"vllm-omni serve\" /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/docs/ | head -10", + "sed -n 168,212p /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/docs/user_guide/diffusion/attention_backends.md | cat -n", + "grep -n \"test_quant_superset_fields_passed_through\" -A 10 /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5509/tests/diffusion/attention/test_attention_config.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "4ff5b2a69ed0abc2" + }, + "recorded_at": "2026-08-13T18:02:07", + "arm": "baseline_opus5", + "wall_s": 507.7, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [], + "artifact_chars": 9372 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5509.md b/eval/dataset/baselines/claudecode_opus5/pr5509.md new file mode 100644 index 000000000..53c212dbb --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5509.md @@ -0,0 +1,117 @@ +## Verdict + +**Approve with follow-ups needed.** The core mechanism is right: the sfs tuple ordering `(q, k, p, v)` with `num_elts = (q_blk, k_blk, 0, 1)` matches how FlashInfer actually unpacks these args (`flashinfer/prefill.py:4044-4049` — note its *docstring* at 3916-3920 claims `(q_sf, k_sf, v_block_sum, p_block_sum)`, which contradicts its own code; the PR follows the code, which is correct), `bmm1_scale`/`bmm2_scale` are consistent with per-block sfs carrying the dequant, gating the kwargs behind `self.quant.enabled` keeps older FlashInfer builds working, and resolving the quantize fn at `__init__` is the right fix for the Dynamo break. Tests pass locally (`68 passed, 7 skipped`). + +What holds it back is that the config surface is wider than the implementation, in ways that fail **silently** — three distinct configurations are accepted, documented as working, and quietly produce dense BF16. I verified each by running the code. + +--- + +## Blocking + +### 1. Ring SP silently drops SAGE — the guard that exists for skip-softmax wasn't extended + +`vllm_omni/diffusion/attention/layer.py:319-326` + +```python +skip = getattr(self.attention, "skip", None) +if skip is not None and getattr(skip, "configured", False): + raise NotImplementedError( + "Skip-Softmax (TRTLLM_ATTN) is not supported with ring sequence parallelism: " + "the ring path bypasses the backend, so the skip config would be silently ignored. ...") +``` + +The ring path delegates to `self.ring_runner.run_attention(...)` and never touches `self.attention`, so with `parallel_config.ring_degree > 1` a `quant` config is ignored exactly the way skip-softmax would be — the failure mode this guard was written to prevent. The user sees no error, no warning, full-speed dense attention, and concludes SAGE gave them 0% speedup. + +Add the parallel check: + +```python +quant = getattr(self.attention, "quant", None) +if quant is not None and getattr(quant, "enabled", False): + raise NotImplementedError( + "SAGE quant (TRTLLM_ATTN) is not supported with ring sequence parallelism: " + "the ring path bypasses the backend, so the quant config would be silently ignored. " + "Use Ulysses SP instead, or remove the quant config.") +``` + +### 2. `FLASHINFER_ATTN` is allow-listed for `quant`, but nothing consumes it + +`vllm_omni/diffusion/data.py:1447-1451` permits `quant` on `FLASHINFER_ATTN`. But `FlashInferAttentionImpl.__init__` (`vllm_omni/diffusion/attention/backends/flashinfer_attn.py:54-65`) swallows `backend_kwargs` into `**extra_impl_args` and reads only `causal`/`softmax_scale` — no warning, unlike `SDPAImpl`/`FlashAttentionImpl`, which at least log `"ignoring backend_kwargs"`. + +Verified: + +```python +AttentionSpec(backend='FLASHINFER_ATTN', quant={'dtype_qk':'fp8_e4m3'}).backend_kwargs() +# {'quant': {'dtype_qk': 'fp8_e4m3', 'q_block_size': 1, 'k_block_size': 16}} → dropped on the floor +``` + +This is compounded by the docs (`docs/user_guide/diffusion/attention_backends.md:185-188`) stating `FLASHINFER_ATTN` "reads `dtype_qk` (`float16`/`bfloat16`), `dtype_vo`, and `flashinfer_backend`", and by `tests/diffusion/attention/test_attention_config.py:65-73`, which asserts the pass-through as if it were functional. + +Pick one: drop `FLASHINFER_ATTN` from the allow-list at `data.py:1447` until the consumer lands (my preference — this PR is TRTLLM-only), or have `FlashInferAttentionImpl` raise/`warning_once` on a `quant` key. Whichever you pick, the doc paragraph and that test should follow. + +### 3. `dtype_vo`-only config is a silent no-op on TRTLLM too + +`AttnQuantSpec.enabled` (`data.py:1423-1425`) is `dtype_qk is not None or dtype_vo is not None`, but `QuantConfig.enabled` (`trtllm_attn.py:129-131`) is `dtype_qk is not None`. Verified: + +```python +AttentionSpec(backend='TRTLLM_ATTN', quant={'dtype_vo':'fp8_e4m3'}).backend_kwargs() +# {'quant': {'dtype_qk': None, 'q_block_size': 1, 'k_block_size': 16, 'dtype_vo': 'fp8_e4m3'}} +QuantConfig.from_backend_kwargs(...) # enabled: False → dense, no error +``` + +Same for `flashinfer_backend`, and for `dtype_vo: "int8"` combined with a valid `dtype_qk` (V is unconditionally fp8; the request is dropped without comment). `QuantConfig.from_backend_kwargs` at `trtllm_attn.py:121-127` should raise on quant keys it does not honor, rather than ignoring them: + +```python +unsupported = set(q) - {"dtype_qk", "q_block_size", "k_block_size"} +if unsupported: + raise ValueError(f"TRTLLM_ATTN quant (SAGE) does not support {sorted(unsupported)}; " + "V is always per-channel fp8_e4m3.") +``` + +Better still, do it in `AttentionSpec.__post_init__`, where the backend name is already known — including the `dtype_qk in {float16, bfloat16}` case, which currently passes config parsing and only blows up later at layer construction (`trtllm_attn.py:199-203`). Failing at config-parse time beats failing after the model has started loading. + +--- + +## Should fix + +### 4. Unrelated doc deletion — 86 lines removed + +The two web-UI commits `0cb1d75e` and `e54759c2` delete content unrelated to SAGE from `docs/user_guide/diffusion/attention_backends.md`: **Known Limitations** (incl. the LTX-2 `CUDNN_ATTN`-under-`torch.compile` crash and its `FLASHINFER_ATTN` workaround, tracked as #3121), **Mixed backends across roles**, **Compare against FlashAttention**, **Validation Guidance**, and **Reproducing the Benchmark Table** (which points at `benchmarks/diffusion/bench_attention_backends.py` — still present in the tree). I grepped `docs/`: none of it was moved elsewhere, it's just gone. Unless this was deliberate, restore those sections. + +### 5. Doc claims that don't match behavior + +`docs/user_guide/diffusion/attention_backends.md:185-188`: "each reads the fields that apply and **rejects the rest**" — neither backend rejects anything today (finding 3). And "`flashinfer_backend` (see its section)" is a dangling reference — there is no `FLASHINFER_ATTN` quant section in the file. + +### 6. Output dtype silently changes under SAGE + +`trtllm_attn.py:301-322`. FlashInfer allocates `out` as `bfloat16` whenever `query.dtype` is fp8/int8 (`prefill.py:3950-3963`), while the dense path returns `query.dtype`. `float16` is a supported model dtype here (`data.py:957-959`), so an fp16 DiT gets bf16 back from attention only when SAGE is on — a dtype mismatch injected into the middle of the transformer block. One line closes it: + +```python +return out.reshape(batch, q_len, num_q_heads, head_dim).to(query.dtype) +``` + +(no-op for the bf16 path). + +### 7. No arch preflight for int8 + +The docs correctly say int8 QK kernels are SM100-only, but `__init__` (`trtllm_attn.py:198-216`) checks only FlashInfer capability. On SM103/SM120 the failure arrives as an opaque `"Missing TRTLLM-GEN kernel"` from inside the kernel launcher on the first forward — after the model has loaded. The new test at `tests/diffusion/attention/test_trtllm_attn.py:308-312` has to string-match that message to skip, which is a good signal the check belongs at init. Add a `torch.cuda.get_device_capability()` check alongside the FlashInfer checks. + +--- + +## Questions / test gaps + +### 8. Sequence boundaries in the quantize call + +`trtllm_attn.py:297` passes `q`/`k`/`v` already flattened to `(batch*len, heads, dim)` and gives `trtllm_sage_attention_quantize` no `cu_seq_lens`, while the kernel itself receives `cum_seq_lens_q`/`cum_seq_lens_kv`. If the quantizer blocks over the flat token axis while the kernel indexes sfs per sequence, the two disagree whenever `batch > 1` **and** `kv_len % k_block_size != 0` — and a K-smoothing mean computed across the flattened batch would mix CFG branches regardless of divisibility. I couldn't verify this locally (this box has FlashInfer 0.6.14, which has the `sage_attn_sfs` params but not `trtllm_sage_attention_quantize`), so it may well be fine. But the only e2e test, `test_sage_quant_matches_sdpa` (`test_trtllm_attn.py:305-321`), uses `B, S = 2, 256` — `256 % 16 == 0`, precisely the case that can't expose it. Please add a shape like `B=2, S=250` (or 258) to that parametrization. + +### 9. The "kwargs only when enabled" fix is untested + +Commit `4a6ab1a0`'s behavior — dense path passes *no* sage kwargs, so old FlashInfer builds keep working — has no test. A cheap one: monkeypatch `tg.trtllm_ragged_attention_deepseek` with a recorder and assert `"sage_attn_sfs" not in kwargs` for `_impl()`. This runs on CPU, so it protects the guarantee on every CI machine, not just Blackwell. + +--- + +## Nits + +- `trtllm_attn.py:121-127`: `SkipSoftmaxConfig` re-validates its values at the backend boundary via `_validate_control`; `QuantConfig` does not — it accepts any `q_block_size`/`k_block_size` and `int(x or 1)` silently turns `0` into `1`. Mirror the sibling's stricter posture. +- `trtllm_attn.py:200/205/212`: `RuntimeError` for what are user config errors; the surrounding config layer uses `ValueError`. +- `data.py:1553-1560`: `spec_keys` gained `"quant"` but the error text still reads "cannot mix backend/skip_softmax with nested role keys" — add `quant`. +- `trtllm_attn.py:88-95`: `_sage_kernel_available` probes via `inspect.signature`, which silently returns `False` if a future FlashInfer wraps the function without preserving its signature — i.e. it fails closed into a confusing "your build lacks the kernel" error on a build that has it. Worth a comment noting the assumption. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5509.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5509.trace.json.gz new file mode 100644 index 000000000..5cb15d304 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5509.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5550.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5550.cost.json new file mode 100644 index 000000000..b78c5ea1c --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5550.cost.json @@ -0,0 +1,42 @@ +{ + "calls": 38, + "input_tokens": 2307568, + "output_tokens": 22494, + "cost_usd": 2.4085164999999997, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout", + "head": "20ca4b93de6e61b42544f2b4ba1101711e467986", + "base": "c4a14091c9b8fd61f4da46022f683adc1dc91550", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "164875457cfa48b5dbdf9910ab1823eb23590cb709fe0dcf988c6e837f562a27", + "validation_commands": [ + "git diff --stat c4a14091c9b8fd61f4da46022f683adc1dc91550 HEAD", + "git diff c4a14091c9b8fd61f4da46022f683adc1dc91550 HEAD -- vllm_omni/", + "ls vllm_omni/diffusion/executor/ vllm_omni/diffusion/diffusion_kv/ vllm_omni/diffusion/sched/", + "grep -rn \"diffusion_kv_mode\" --include=*.py vllm_omni/ | head -50", + "grep -n \"def close\\|def shutdown\" vllm_omni/diffusion/sched/base_scheduler.py vllm_omni/diffusion/sched/request_scheduler.py vllm_omni/diffusion/sched/step_scheduler.py vllm_omni/diffusion/executor/abstract.py vllm_omni/diffusion/executor/multiproc_executor.py", + "git diff c4a14091c9b8fd61f4da46022f683adc1dc91550 HEAD -- tests/diffusion/test_diffusion_engine_cleanup.py tests/diffusion/test_diffusion_model_runner.py", + "grep -rn \"def execute_model\" vllm_omni/ --include=\"*.py\" | head -30; echo ---; ls vllm_omni/diffusion/worker/", + "grep -n \"class .*ModelRunner\" vllm_omni/experimental/ar_diffusion/runner.py; grep -rn \"diffusion_model_runner_cls\\|default_diffusion_model_runner_cls\" vllm_omni/ --include=\"*.py\" | head -20", + "grep -rn \"get_diffusion_model_runner_cls\" vllm_omni/platforms/*.py vllm_omni/platforms/**/*.py 2>/dev/null | head; echo ---; grep -rn \"class .*(DiffusionModelRunner)\" vllm_omni/ --include=\"*.py\"", + "grep -n \"execute_fn\\|execute_request\\|execute_batch\\|def step\" vllm_omni/diffusion/diffusion_engine.py | head -40", + "sed -n 1,60p vllm_omni/diffusion/executor/multiproc_executor.py", + "grep -n \"^from\\|^import\" vllm_omni/diffusion/sched/request_scheduler.py | head -20", + "timeout 900 python -m pytest tests/diffusion/diffusion_kv/test_metadata.py tests/diffusion/diffusion_kv/test_worker_contract.py -q 2>&1 | tail -20", + "grep -n \"SchedulerRequestState(\" -A6 vllm_omni/diffusion/sched/*.py | head -40", + "grep -n \"from typing import\" vllm_omni/diffusion/worker/diffusion_worker.py; timeout 300 python -c \"\nimport vllm_omni.diffusion.worker.diffusion_worker as w\nimport inspect\nprint(inspect.signature(w.DiffusionWorker.execute_model))\nprint(inspect.signature(w.WorkerWrapperBase.execute_model))\n\" 2>&1 | t", + "grep -n \"diffusion_kv_metadata\" vllm_omni/diffusion/worker/diffusion_worker.py vllm_omni/diffusion/executor/multiproc_executor.py vllm_omni/diffusion/worker/diffusion_model_runner.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "4ff5b2a69ed0abc2" + }, + "recorded_at": "2026-08-13T18:00:07", + "arm": "baseline_opus5", + "wall_s": 387.7, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [], + "artifact_chars": 11331 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5550.md b/eval/dataset/baselines/claudecode_opus5/pr5550.md new file mode 100644 index 000000000..cd618e0ae --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5550.md @@ -0,0 +1,95 @@ +# Review: PR #5550 — Worker contracts and RPC plumbing for Scheduler-managed paged KV cache + +## Verdict + +**Approve with required follow-ups.** The scoping is genuinely well done: the transport shape (`NewRequestData.diffusion_kv_metadata` → optional 4th RPC arg → runner) is the right seam, the dense path is left byte-identical, and the "what W0 does not validate and who owns it later" section of the description is the kind of thing most staged PRs skip. The 16 new tests pass locally (`pytest tests/diffusion/diffusion_kv/ -q` → 16 passed). + +Two things should be fixed before the W1 producer lands, because both are latent-until-then and will surface as confusing runtime failures rather than contract violations: the un-updated `ARDiffusionModelRunner.execute_model` override, and the total absence of DTO-local invariants on the new metadata classes (which the sibling DTOs from #5541 do enforce). Everything else is polish. + +--- + +## Required + +### 1. `ARDiffusionModelRunner.execute_model` was not updated — `vllm_omni/experimental/ar_diffusion/runner.py:237` + +```python +def execute_model(self, req: OmniDiffusionRequest, kv_prefetch_job: KVPrefetchJob | None = None) -> DiffusionOutput: +``` + +This is the only `DiffusionModelRunner` subclass in-tree, and it overrides `execute_model` without the new parameter. `DiffusionWorker.execute_model` (`diffusion_worker.py:463-465`) injects `diffusion_kv_metadata=` as a keyword when metadata is present, so a `paged_scheduler` request routed to the AR-Diffusion runner raises `TypeError: execute_model() got an unexpected keyword argument 'diffusion_kv_metadata'`. That TypeError is then swallowed by the executor's blanket `except Exception` at `multiproc_executor.py:574` and returned as `DiffusionOutput(error="execute_model() got an unexpected keyword argument...")` — a stringified signature error surfacing as a user-facing generation failure. + +It's unreachable today (no producer emits metadata), which is precisely why it will be missed until W1. Fix now: add the parameter to the override and forward it to `super().execute_model(...)`, or raise an explicit `NotImplementedError("ARDiffusionModelRunner does not support Scheduler-managed paged KV")` mirroring the style already used at `runner.py:281` and `runner.py:287`. + +### 2. The new DTOs drop every invariant their Scheduler-side twins enforce — `vllm_omni/diffusion/diffusion_kv/metadata.py:24-57` + +Compare with `diffusion_kv/request.py:13-89` from the merged #5541. `DiffusionKVContext` is `frozen=True` with a `__post_init__` that rejects empty `context_id`/`cache_role` and non-positive `num_tokens`; `DiffusionKVRequest.__init__` rejects negative `prefix_len`, non-positive `target_len`/`seq_len`, `prefix_len + target_len > seq_len`, and duplicate `context_id`s. The new `DiffusionKVSequenceMetadata`/`DiffusionKVContextMetadata` carry literally the same fields and check none of it, and are not frozen despite being described as "one allocation snapshot." + +The description's validation-ownership argument is correct for *cross-object* checks — comparing against `SchedulerRequestState.diffusion_kv_requests` or rank-local `KVCacheConfig` really does need state W0 lacks. It does not cover *DTO-local* checks. The clearest example is referential integrity: `DiffusionKVSequenceMetadata.context_ids` is documented at `metadata.py:36-37` as referencing "request-scoped context allocations from the enclosing `DiffusionKVMetadata`," and nothing enforces it. That check needs zero external state — it is a set membership test inside one object. A typo'd or stale `context_id` today produces a silently wrong (or empty) context block table in W1, at attention time, far from the cause. + +Suggested minimum on `DiffusionKVMetadata`: + +```python +@dataclass(frozen=True) +class DiffusionKVMetadata: + ... + def __post_init__(self) -> None: + context_ids = [c.context_id for c in self.contexts] + if len(context_ids) != len(set(context_ids)): + raise ValueError(f"contexts must have unique context_id values, got {context_ids}") + sequence_ids = [s.sequence_id for s in self.sequences] + if len(sequence_ids) != len(set(sequence_ids)): + raise ValueError(f"sequences must have unique sequence_id values, got {sequence_ids}") + known = set(context_ids) + for seq in self.sequences: + unknown = [cid for cid in seq.context_ids if cid not in known] + if unknown: + raise ValueError(f"sequence {seq.sequence_id} references unknown context_ids {unknown}") +``` + +plus the same local length/emptiness checks the twins already apply. `frozen=True` on all three also makes the "snapshot" claim real (note `block_ids: tuple[list[int], ...]` still has mutable inner lists — worth `tuple[tuple[int, ...], ...]` if immutability matters). + +### 3. Field semantics are undocumented, and the tests model them inconsistently — `metadata.py:41-45` + +`prefix_len`, `target_len`, `seq_len` have no field docs. `DiffusionKVRequest` documents them (`request.py:41-44`) and enforces `prefix_len + target_len <= seq_len`; the new tests use `prefix_len=4, target_len=2, seq_len=8` (`test_metadata.py:22-29`, `test_worker_contract.py:32-39`), leaving a W1 implementer unable to tell whether `seq_len` is meant to equal the sum, bound it, or be independent. Similarly, nothing in the class says the outer dimension of `block_ids` is native KV **cache-group** order — that's only in the PR description, which won't survive into the codebase. Add field docstrings. + +Two smaller consistency points in the same block: `cache_role: str` is free-form where the set of roles is closed enough to be an enum, and `context_id: str` vs `sequence_id: int` is an unexplained asymmetry between two sibling identifiers. + +--- + +## Should fix + +### 4. The DLO fallback is silent and untested — `multiproc_executor.py:462,471` + +`has_diffusion_kv_metadata` disables the DLO+AllGather DP broadcast for the entire wave whenever any request carries metadata. The fallback is functionally correct (the serial loop uses `exec_all_ranks=True`, so every rank still participates in each weight AllGather) but costs `dp_size`× redundant compute, with no log line and no test — `make_executor()` at `test_worker_contract.py:62-65` sets `enable_distributed_layerwise_offload=False`, so this branch is never exercised. Add a `logger.warning` on the fallback and a test. The inline comment should also state *why* (the broadcast RPC has one args tuple for N requests and no per-request metadata slot). + +### 5. `cache_mode.value` assumes a normalized enum — `diffusion_model_runner.py:169-173` + +`DiffusionKVCacheMode` is a `str, Enum`, and `OmniDiffusionConfig.__post_init__` (`data.py:933`) normalizes the field — so this is correct on the production path. But if a config ever reaches the runner unnormalized (a duck-typed config, as the tests themselves use at `test_worker_contract.py:45`, or a config path that skips `__post_init__`), the `is not` identity check is a false negative for the string `"paged_scheduler"` → it wrongly raises, and then `cache_mode.value` raises `AttributeError` instead of the intended `ValueError`. Normalize once (`DiffusionKVCacheMode(cache_mode)`) or use `==` and `getattr(cache_mode, "value", cache_mode)` in the message. + +### 6. Batch path validates against a different config than it executes with — `diffusion_model_runner.py:638` vs `645` + +`_validate_diffusion_kv_metadata` reads `self.od_config`, while `_execute_request_list` is called with the `od_config` argument. If those ever diverge, the mode gate is checked against the wrong config. Thread `od_config` into the validator. + +### 7. Two overlapping identity checks with two different messages + +`validate_new_request_data_identity` (`sched/interface.py:191-197`) and `_validate_diffusion_kv_metadata` (`diffusion_model_runner.py:175-178`) both assert `metadata.request_id == req.request_id`, and both run in the batch (`:637-641`) and step (`:751-754`) paths. Two error strings for one condition. Collapse into one helper. + +### 8. The comment at `multiproc_executor.py:454-456` overstates what raising buys + +Raising out of `execute_request` doesn't avoid conversion into worker output — the engine busy loop catches it at `diffusion_engine.py:461` and fabricates an error `DiffusionOutput` for **every** `scheduled_request_id` in the wave. So one malformed envelope fails all co-scheduled requests. Either fail just the offending request (append its `RunnerOutput` and continue) or reword the comment to say the goal is a distinguishable engine-level error rather than a per-request RPC error. + +Worth noting: in-tree, envelope `request_id` always equals `req.request_id` (`base_scheduler.py:76`, `:114`, `step_scheduler.py:54`), so the first branch is purely defensive against injected schedulers. That's fine; it's cheap. + +--- + +## Minor + +9. **`multiproc_executor.py:457`** — the function-local import is unnecessary. `sched.request_scheduler` is already imported at module level (`:26`) and itself imports `sched.interface` at module level, so there is no cycle to break. Move it to the top-level block; it currently re-runs on every dispatch. + +10. **Conditional-arg pattern** (`multiproc_executor.py:544-546`, `diffusion_worker.py:462-464`, `:1371-1373`) — three branches to append a positional/keyword arg only when non-`None`, in an RPC where both ends are always the same build. `kv_prefetch_job` is already passed unconditionally even when `None`. Always passing `diffusion_kv_metadata=metadata` would delete all three branches — and would turn finding #1 into an immediate loud failure instead of a latent one. If the conditional exists specifically to tolerate third-party runners with narrower signatures (which `test_dense_worker_call_does_not_extend_model_runner_signature` hints at), say so in a comment, because it isn't self-evident. + +11. **`tests/diffusion/diffusion_kv/test_metadata.py:15-110`** — all three tests construct a dataclass and assert its attributes equal the constructor arguments; they exercise `@dataclass`, not this code. If #2 lands, rewrite them as invariant tests (unknown `context_id` rejected, duplicate `sequence_id` rejected, bad lengths rejected) — the CFG shared-context and branch-context scenarios are the right *scenarios*, they just need an assertion with teeth. + +12. **Design asymmetry worth recording in a comment** — request mode transports metadata as an explicit 4th RPC arg (`multiproc_executor.py:546`), batch mode transports it implicitly inside `scheduler_output` where the runner validates and then discards it (`diffusion_model_runner.py:636-642`). Two transports for one payload. The description acknowledges this; the code should too, so W1 doesn't have to rediscover it. + +13. **`diffusion_engine.py:191-208`** — a real fix (init failure previously leaked worker processes), well tested, but unrelated to "Worker contracts and RPC plumbing" and would have been cleaner as its own PR. Two notes on it: when the caller injects `scheduler=`, the engine now calls `close()` on an object it does not own (`:201`) — surprising ownership, worth documenting or restricting to self-created schedulers. And `_init_executor` (`:190`) sits outside the `try`, which is correct but reads as an oversight; a one-line comment would preempt the question. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5550.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5550.trace.json.gz new file mode 100644 index 000000000..da41c11b6 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5550.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5608.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5608.cost.json new file mode 100644 index 000000000..eaad45f1c --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5608.cost.json @@ -0,0 +1,46 @@ +{ + "calls": 28, + "input_tokens": 910765, + "output_tokens": 21823, + "cost_usd": 1.4993044999999998, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout4", + "head": "de8517f6f2bc0ae2f8bfeaad0832c1863884c6af", + "base": "b3f4fbf9e217ab6174fc0c475e33a478006ac34c", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "43b9382d8441192b80578d1f686953bd7d4e1cde60428c7e0a12a46cdbc5424d", + "validation_commands": [ + "git diff b3f4fbf9e217ab6174fc0c475e33a478006ac34c HEAD --stat", + "git diff b3f4fbf9e217ab6174fc0c475e33a478006ac34c HEAD", + "ls vllm_omni/platforms/npu/ && ls vllm_omni/platforms/npu/layers/ 2>&1 && ls vllm_omni/platforms/npu/models/ | head -50", + "sed -n '1,80p' tests/platforms/npu/test_310p_patches.py", + "cat vllm_omni/platforms/npu/layers/__init__.py; echo \"---\"; python -c \"import torch; print(torch.__version__)\" 2>&1 | tail -2", + "grep -n \"apply_rotary_pos_emb\\|unsqueeze_dim\\|transpose(1, 2)\\|head_dim\\|num_heads\\|def forward\\|scaled_dot_product\\|npu_fusion\" vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py | head -60", + "sed -n '120,175p;340,400p' vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py", + "sed -n '1,120p' vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/configuration_qwen3_tts_tokenizer_v2.py | grep -n \"num_attention_heads\\|hidden_size\\|head_dim\\|num_key_value\" ", + "grep -rn \"codec_chunk_ramp\" --include=*.py | head -20", + "grep -rn \"num_code_groups\" --include=\"*.py\" . | head -20; echo \"=== code predictor forward callers ===\"; grep -rn \"CodePredictor\\|code_predictor\" --include=\"*.py\" vllm_omni/model_executor/models/qwen3_tts/ | head -30", + "sed -n '160,215p' vllm_omni/model_executor/models/qwen3_tts/configuration_qwen3_tts.py", + "grep -n \"def forward\" -A 40 vllm_omni/model_executor/models/common/qwen3_code_predictor.py | sed -n '1,40p'; echo \"=== wrapper ===\"; grep -n \"class CodePredictorWrapper\" -A 80 vllm_omni/model_executor/models/common/qwen3_code_predictor.py | grep -n \"seq\\|forward\\|cat\\|step\" | head -40", + "sed -n '430,560p' vllm_omni/model_executor/models/common/qwen3_code_predictor.py", + "head -20 vllm_omni/platforms/npu/layers/fused_moe.py; echo \"=== ruff cfg ===\"; grep -n \"line-length\\|\\[tool.ruff\" -A3 pyproject.toml | head -20", + "timeout 600 python -m pytest tests/platforms/npu/test_310p_patches.py -q -k tokenizer 2>&1 | tail -20", + "cd /tmp && cat > t.py <<'EOF'\nimport sys, types, torch, importlib.util\nm = types.ModuleType(\"torch_npu\")\ndef rot_half(x):\n x1, x2 = x[..., :x.shape[-1]//2], x[..., x.shape[-1]//2:]\n return torch.cat((-x2, x1), dim=-1)\ndef npu_rotary_mul(x, cos, sin):\n return x*cos + rot_half(x)*sin\nm.npu_ro", + "python3 -c '\nimport sys, types, torch, importlib.util\nm = types.ModuleType(\"torch_npu\")\ndef rot_half(x):\n x1, x2 = x[..., :x.shape[-1]//2], x[..., x.shape[-1]//2:]\n return torch.cat((-x2, x1), dim=-1)\ndef npu_rotary_mul(x, cos, sin):\n return x*cos + rot_half(x)*sin\nm.npu_rotary_mul = npu_ro", + "ls tests/platforms/npu/; echo \"=== docs mention ===\"; grep -rn \"BNSD\\|BSND\" --include=\"*.py\" vllm_omni | head -20" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-15T06:19:37", + "arm": "baseline_opus5", + "wall_s": 314.0, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [ + "Write: path outside the worktree: /tmp/t.py in {\"file_path\": \"/tmp/t.py\", \"content\": \"import sys, types, torch, importlib.util\\nm = types.ModuleType(\\\"torch_npu\\\")\\ndef rot_half(x):\\n x1, x2 = x[..., :x.s" + ], + "artifact_chars": 6950 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5608.md b/eval/dataset/baselines/claudecode_opus5/pr5608.md new file mode 100644 index 000000000..db5ae1f5f --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5608.md @@ -0,0 +1,96 @@ +## Verdict + +The approach is sound and the transform is provably correct — I stubbed `torch_npu.npu_rotary_mul` with the rotate-half reference and confirmed the BSND round-trip is **bit-identical** to the BNSD path, and that the inbound `.contiguous()` is a no-op for the real caller (q/k arrive as `.view(B,S,N,D).transpose(1,2)` views, so transposing back recovers the original contiguous buffer). So the fix is cheap and safe for the path it targets. + +But it ships a hand-transcribed CANN tiling rule as the gate for a fast path, and that rule doesn't hold up against a sibling call site in the same model. Combined with a `_load_source_module` shim in the test that leaves the actual new logic untested, this is **approve-with-required-follow-ups** rather than clean. It's already merged, so the items below are follow-up work. + +--- + +## Blocking-severity follow-ups + +### 1. The "shared" helper has exactly one caller; two identical BNSD sites are untouched — and one of them contradicts the predicate + +`vllm_omni/model_executor/models/common/qwen3_code_predictor.py:260-261` + +```python +cos = cos.unsqueeze(1) # [batch, 1, seq_len, head_dim] +sin = sin.unsqueeze(1) +if current_omni_platform.is_npu(): + q = torch_npu.npu_rotary_mul(q, cos, sin) + k = torch_npu.npu_rotary_mul(k, cos, sin) +``` + +This is the same fused op, same BNSD layout, same rotate-half mode. The code predictor re-prefills the full growing sequence each AR step (`qwen3_code_predictor.py:444` — "len 2 -> num_code_groups+1"), with `head_dim=128`, `num_attention_heads=16` (`configuration_qwen3_tts.py:198-200`), and batch = concurrency. At the **first** AR step, `seq_len=2`: + +- `batch*heads = 4*16 = 64` vs `seq_len*8 = 16` at concurrency 4 +- `batch*heads = 16*16 = 256` vs `16` at concurrency 16 + +`head_dim//2 = 64` is 32-byte aligned in bf16, so `_bnsd_rotary_shape_is_supported` returns `False` here — a *far worse* violation than the tokenizer decoder's (head_dim 64, 16 heads at `hidden_size=1024`). Yet the PR's own benchmark table shows c=4/8/16 completing without a `DoRotateHalfTiling` abort. + +Exactly one of these is true, and the PR should state which: + +- **The predicate is not CANN's real condition.** Then the fast path is being disabled for shapes that are fine, and the tokenizer crash's actual trigger is something else (or narrower) than `B*N <= S*8`. +- **The code predictor carries the same latent crash** and just wasn't exercised in this configuration. Then it needs the helper wired in — which is what the commit "Extract shared rotary fallback helper" implies but does not do. + +`vllm_omni/platforms/npu/_310p/patch/qwen3_tts.py:184-185` is the same pattern on the 310P path and should be dispositioned alongside. + +### 2. The tiling rule is uncited and silently version-coupled + +`vllm_omni/platforms/npu/layers/rotary_embedding.py:15-17` + +``` +For the aligned-head-dimension branch, CANN requires +``batch * num_heads <= seq_len * 8``. +``` + +Both magic numbers — the `* 8` and the `32 // element_size` half-dim alignment — are reverse-engineered from `rope_rotate_half_tiling.cpp` with no reference to a CANN version, doc, or source line. The PR body says CANN 9.0.0; the code doesn't. Failure modes if CANN changes the tiler: the crash silently returns, or every shape permanently takes the transpose path, with no signal either way. + +Please add the CANN version + source/doc reference to the docstring, and a `logger.warning_once` on the fallback branch so a perf regression is diagnosable from logs. + +--- + +## Correctness + +### 3. `ndim != 4` is conflated with "use BSND" and produces a garbage failure + +`rotary_embedding.py:19-20` returns `False` for non-4D input, and `:44` treats `False` as "take the transpose path". For a 3-D tensor with `unsqueeze_dim=1`, `:51` transposes dims 1/2 and `:54` unsqueezes `cos` at dim 2 — I confirmed this raises `RuntimeError: The size of tensor a (4) must match the size of tensor b (8) at non-singleton dimension 3`, and for shapes where the broadcast happens to succeed it would return silently wrong numbers. + +The current caller always passes 4-D, but this is now a shared module. Fix in `npu_rotary_mul_with_bsnd_fallback`: + +```python +if unsqueeze_dim != 1 or hidden_states.ndim != 4 or _bnsd_rotary_shape_is_supported(hidden_states): + return torch_npu.npu_rotary_mul(...) +``` + +so "shape I can't reason about" degrades to today's behavior rather than into the transpose path. + +### 4. `rotary_embedding.py:57` returns a non-contiguous tensor + +`output_bsnd.transpose(1, 2)` — verified non-contiguous. The current consumers tolerate it (`modeling_qwen3_tts_tokenizer_v2.py:365` HF `Cache.update`, and the eager/SDPA matmuls), but any NPU fused-attention consumer needs contiguous BNSD and will either copy implicitly or fail. Since the inbound `.contiguous()` at `:51` is already a no-op for the real caller, the symmetric and actually-useful placement is on the output. At minimum, document that the returned tensor is a view. + +--- + +## Test coverage + +### 5. The new logic has no direct test; the one assertion changed doesn't cover the bug being fixed + +`tests/platforms/npu/test_310p_patches.py:538-547` + +The test's `q = torch.zeros(1, 2, 3, 2)` is fp32 with `head_dim=2`, so `head_dim//2 = 1` against a `half_dim_alignment` of 8 — it takes the fallback because of the **alignment** clause, never reaching `batch*heads <= seq_len*8`. The `B*N` rule that is the entire point of this PR has zero coverage, and the fast BNSD path is no longer covered either. + +The value assertions can't catch a broken transpose round-trip: `q` is all zeros and `k` all ones, so `assert_close(q_out, q + 1)` holds under any permutation of dims 1/2. + +Add a dedicated test module (e.g. `tests/platforms/npu/test_npu_rotary_embedding.py`) with: +- `_bnsd_rotary_shape_is_supported` boundary cases: `B*N == S*8` (supported) vs `B*N == S*8 + 1` (not), across fp16/bf16/fp32 and aligned/unaligned `head_dim` (64 and 80 in bf16 are a good pair — I verified they split True/False). +- A fallback round-trip with **asymmetric** random data, asserting the result equals the reference `x*cos + rotate_half(x)*sin` computed in BNSD. +- One case that stays on the BNSD fast path, asserting `cos` reaches the op as `[B,1,S,D]`. + +### 6. Minor: the module-loading shim + +`test_310p_patches.py:492-498` loads `rotary_embedding.py` by path and injects it into `sys.modules`. It works (the tokenizer test passes locally: `1 passed, 6 deselected`), but it means the test is validating a hand-loaded copy of the module rather than the import graph. Acceptable given the file's `import torch_npu` at `:9`, but a real unit test that fakes `torch_npu` once and imports normally would be more durable. + +--- + +## Gate + +`buildkite/vllm-omni-amd-ci` is red on the merged head. The tokenizer test passes locally on CPU, so this doesn't look attributable to the diff, but it's a failing required check on merged code and needs to be dispositioned rather than left. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5608.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5608.trace.json.gz new file mode 100644 index 000000000..884ed1eb1 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5608.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5610.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5610.cost.json new file mode 100644 index 000000000..9117c2853 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5610.cost.json @@ -0,0 +1,75 @@ +{ + "calls": 55, + "input_tokens": 2672664, + "output_tokens": 26838, + "cost_usd": 2.7515824999999996, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout", + "head": "50126edf2f50a85e7e41b32b70da7d8e70b21aec", + "base": "900a7f0813d0482811b0e4dfd3cf7deabbe2429f", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "155a8a48d7ac9a99f8aa125d3ccb82c7bacd8524d84d04518b200e037950d936", + "validation_commands": [ + "git diff --stat 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD", + "git diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- docs/.nav.yml docs/design/index.md", + "git diff --stat 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- '*.svg' && ls docs/source/architecture/ 2>/dev/null | head -30 && echo \"---\" && find . -name 'qwen3-omni-async-output-step-gap.svg'", + "cat vllm_omni/deploy/qwen3_omni_moe.yaml && echo \"=====TTS====\" && cat vllm_omni/deploy/qwen3_tts.yaml", + "grep -n \"_should_use_async_omni_output\" -A 80 vllm_omni/worker/gpu_ar_model_runner.py | head -140", + "grep -rn \"use_async_omni_output\\|eager_omni_postprocess_before_async_output\\|omni_pooler_payload_include_hidden\" --include=*.py --include=*.yaml -l | head -30", + "grep -rn \"use_async_omni_output\\|eager_omni_postprocess_before_async_output\\|omni_pooler_payload_include_hidden\" --include='*.py' --include='*.yaml' . | grep -v \"^./docs\" | head -40", + "grep -n \"should_accumulate_full_payload_output\" -r --include='*.py' . | head -20", + "sed -n 40,70p vllm_omni/worker/omni_connector_model_runner_mixin.py; echo ---; sed -n 740,760p vllm_omni/worker/omni_connector_model_runner_mixin.py", + "sed -n 95,175p vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py", + "grep -rn \"omni_prefix_cache\" --include='*.py' vllm_omni/worker/gpu_ar_model_runner.py | head -20; echo ---; grep -rn \"omni_prefix_cache\\s*=\" --include='*.py' vllm_omni/ | head -20", + "grep -rn \"async_scheduling\" vllm_omni/config/stage_config.py | head -20", + "sed -n 160,180p vllm_omni/worker/gpu_model_runner.py; echo \"=== stage_config 925-950 ===\"; sed -n 920,955p vllm_omni/config/stage_config.py", + "grep -rn \"source/architecture\" docs/design/feature/*.md docs/design/*.md 2>/dev/null | head -20", + "sed -n 1,60p mkdocs.yaml 2>/dev/null || ls | head -30", + "grep -n \"class OmniAsyncGPUModelRunnerOutput\" -A 90 vllm_omni/worker/gpu_ar_model_runner.py", + "sed -n 2120,2190p vllm_omni/worker/gpu_ar_model_runner.py", + "grep -n \"get_omni_connector_output\\|AsyncGPUModelRunnerOutput\\|OmniModelRunnerOutput(\" vllm_omni/platforms/npu/worker/npu_ar_model_runner.py | tail -20", + "grep -rn \"GPUARModelRunner\" vllm_omni/platforms/ vllm_omni/worker/ --include='*.py' | grep -i \"class \\|import\" | head -20", + "grep -n \"_get_or_create_omni_payload_copy_stream\" -A 25 vllm_omni/worker/gpu_ar_model_runner.py | head -40; echo \"=== snapshot helper ===\"; grep -n \"_snapshot_tensor_payload_to_cpu_async\" -A 60 vllm_omni/worker/gpu_ar_model_runner.py | head -90", + "grep -rn \"GPUARWorker\" vllm_omni/platforms/ --include='*.py' | head; echo ---; grep -rn \"musa\" -il vllm_omni/platforms/ | head", + "python -c \"import vllm,sys;print(vllm.__version__, vllm.__file__)\" 2>&1 | tail -2", + "ls docs/; echo ---; sed -n 1,120p docs/DOCS_GUIDE.md 2>/dev/null", + "find docs -iname '*DOCS_GUIDE*' -o -iname '*style*guide*' | head; echo ---; ls docs/contributing/ | head -20", + "grep -rln \"^## Table of Contents\" docs/ | head -20", + "grep -n -i \"image\\|figure\\|svg\\|png\\|relative\\|link\" docs/contributing/DOCS_GUIDE.md | head -40", + "grep -rn \"src=\\\"\\.\\./\\|](\\.\\./.*\\.\\(png\\|svg\\|jpg\\)\" docs/ --include='*.md' | head -20", + "grep -rn \"\\.svg\" docs/ --include='*.md' | grep -v logos | head -20", + "head -40 docs/source/architecture/qwen3-omni-async-output-step-gap.svg; echo \"....\"; wc -l docs/source/architecture/qwen3-omni-async-output-step-gap.svg", + "grep -o '>[^<]*<' docs/source/architecture/qwen3-omni-async-output-step-gap.svg | sed 's/^>//; s/<$//' | grep -v '^\\s*$' | head -80", + "grep -oE ']*>[^<]+' docs/source/architecture/qwen3-omni-async-output-step-gap.svg | head -90", + "sed -n 1725,1760p vllm_omni/worker/gpu_ar_model_runner.py; echo \"=== builder end ===\"; sed -n 1840,1960p vllm_omni/worker/gpu_ar_model_runner.py", + "sed -n 749,800p vllm_omni/worker/omni_connector_model_runner_mixin.py", + "grep -n \"_lock\" vllm_omni/worker/omni_connector_model_runner_mixin.py | head -20", + "grep -n \"def get_omni_connector_output\" -A 40 vllm_omni/worker/omni_connector_model_runner_mixin.py", + "grep -n \"_broadcast_tp_payload_packet\" -A 30 vllm_omni/worker/omni_connector_model_runner_mixin.py | head -50", + "grep -n \"def attach_omni_connector_output\" -A 15 vllm_omni/worker/omni_connector_model_runner_mixin.py", + "grep -rn \"attach_omni_connector_output\\|get_omni_connector_output\" --include='*.py' vllm_omni/ | head -20", + "sed -n 1085,1135p vllm_omni/worker/gpu_ar_model_runner.py", + "grep -rn \"no-async-chunk\\|async_chunk\" --include='*.py' vllm_omni/config/*.py vllm_omni/entrypoints/ 2>/dev/null | grep -i \"arg\\|flag\\|add_argument\\|no-async\" | head -10; echo \"=== TTS model names ===\"; grep -rn \"Qwen3-TTS-12Hz-1.7B\" docs/ examples/ --include='*.md' --include='*.py' -l | head", + "grep -n -i \"async output\\|materializ\\|step gap\\|41\" docs/design/qwen3_omni_tts_performance_optimization.md | head -20; echo \"=== head ===\"; sed -n 1,40p docs/design/qwen3_omni_tts_performance_optimization.md", + "ls docs/design/figures/omni | head -5; echo ---; grep -rn \"9.3\\|11.3\\|0.47\\|655\\|631\" docs/design/qwen3_omni_tts_performance_optimization.md | head -20", + "grep -rn \"vllm.ai/blog\" docs/ README.md 2>/dev/null | head -10", + "curl -sS -o /dev/null -w \"%{http_code} %{redirect_url}\\n\" --max-time 8 https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization 2>&1 | tail -2", + "grep -rn \"ASYNC_OMNI\\|OMNI_ASYNC\" --include='*.py' vllm_omni/ | head", + "grep -rn \"def _model_omni_flag\\|def _runner_model_omni_flag\" -A 20 vllm_omni/worker/gpu_ar_model_runner.py | head -45", + "awk 'NR>=155 && NR<=169 {printf \"%d|%s|\\n\", NR, $0}' docs/design/feature/omni_async_output_materialization.md | cat -A | head -20", + "python3 - <<'EOF'\np='docs/design/feature/omni_async_output_materialization.md'\nlines=open(p).read().split('\\n')\nfor i in range(154,169):\n l=lines[i]\n idx=l.find('+--',10)\n print(i+1, repr(l), 'rightcol=',idx)\nEOF", + "grep -n \"attach_omni_connector_output\" vllm_omni/worker/gpu_ar_model_runner.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "4ff5b2a69ed0abc2" + }, + "recorded_at": "2026-08-13T18:00:10", + "arm": "baseline_opus5", + "wall_s": 391.2, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [], + "artifact_chars": 8807 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5610.md b/eval/dataset/baselines/claudecode_opus5/pr5610.md new file mode 100644 index 000000000..fa96adc54 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5610.md @@ -0,0 +1,78 @@ +## Verdict + +**Approve with minor changes.** I checked every substantive technical claim in this doc against the code at this commit, and they hold up — the guard list, the output lifecycle, the NPU/XPU/MUSA platform statements, the YAML excerpts, and the "full-payload branch is unreachable" argument are all accurate. This is unusually well-verified documentation. + +The changes I'd ask for are: one image reference that will render broken on GitHub, and two safety claims in the *Live Runner State and Ownership* section that are stated more strongly than the code supports. Neither blocks merge; both are worth a follow-up. + +--- + +## Findings + +### 1. Image `src` renders in the MkDocs site but breaks on GitHub — `docs/design/feature/omni_async_output_materialization.md:73` + +```html +src="../../../source/architecture/qwen3-omni-async-output-step-gap.svg" +``` + +The three `../` are correct for the built site (MkDocs defaults to `use_directory_urls: true`, so the page URL is `/design/feature/omni_async_output_materialization/` and this resolves to `/source/architecture/...`). But when the file is viewed on GitHub, the base is `docs/design/feature/`, so three levels up is the repo root and `source/architecture/...` does not exist — the figure will show as broken on the PR page and in the repo browser. + +Every other doc in the repo sidesteps this by using an absolute raw URL, e.g. `docs/design/feature/async_chunk.md:85-86`. This is the only relative `src` and the only `.svg` reference in `docs/`. Suggest matching the convention: + +```html +... +``` + +Note that `mkdocs build --strict` does not validate raw-HTML `src` attributes, so the PR's "image references: passed" check would not have caught this either way. + +While you're there: the SVG hardcodes light-mode colors (`.canvas { fill: #ffffff }`, dark text). It stays legible in the Material dark theme because it paints its own white canvas, but the `` + `prefers-color-scheme` pattern used in `async_chunk.md` is available if you want it to blend. + +### 2. The stated ownership invariant is stronger than the code guarantees — `omni_async_output_materialization.md:193-196` + +> Correctness therefore depends on that builder being the only consumer that drains connector signals for its output cycle. + +`get_omni_connector_output()` is drained from three other places on the same runner, all via `attach_omni_connector_output()`: + +- `gpu_ar_model_runner.py:1104` — EC-transfer early return in `execute_model()` +- `gpu_ar_model_runner.py:1129` — the `num_scheduled_tokens <= 0` early return +- `gpu_ar_model_runner.py:1920` — the `execute_model_state is None` path in `sample_tokens()` + +Under async scheduling, step `N+1`'s `execute_model()` runs before step `N`'s `get_output()` joins the builder — that overlap is the entire point of the feature — so an empty-token or PP step can in principle drain while a builder is still in flight. Either state why that can't happen (if e.g. a zero-token step can't be scheduled while an output is outstanding, say so — that's the load-bearing fact), or soften to something like "the builder is the only consumer that drains connector signals for a step that produced model output; the early-return paths in `execute_model()`/`sample_tokens()` drain for steps that produced none." + +### 3. No TP > 1 caveat, though the background builder issues a collective — `omni_async_output_materialization.md:189-198` and the guard table at `336-354` + +`get_omni_connector_output()` is not purely CPU-local: with `async_chunk` and TP world size > 1 it calls `tp_group.broadcast_object(...)` (`omni_connector_model_runner_mixin.py:1521-1527`, via `_broadcast_tp_payload_packet` at `:622-628`). Under this feature that collective now executes on the `omni-async-output-builder` thread while the main thread is running the next decode step. + +`_should_use_async_omni_output()` has no TP guard, and the doc's compatibility table is presented as exhaustive ("only when **all** of the following runtime conditions hold"). Meanwhile the validated configuration is TP=1 throughout (bundled CUDA profiles, one replica per stage), while `qwen3_omni_moe.yaml` uses `tensor_parallel_size: 4` for stage 0 on XPU and `2` on NPU. Worth one sentence in *Live Runner State and Ownership* noting that the builder participates in a TP collective when TP > 1, plus stating that the validation was TP=1 — that scopes the "validated on CUDA and ROCm" claim honestly. + +### 4. Blog URL host looks wrong — `omni_async_output_materialization.md:83`, `119`, `383` + +`https://vllm.ai/blog/2026-07-01-qwen3-omni-optimization` appears nowhere else in the repo, and the vLLM project blog is normally `blog.vllm.ai`. The entire Performance section's provenance rests on this link, and `mkdocs --strict` does not check external links, so a bad URL survives CI silently. Please confirm the host/slug. + +Related: the repo already has `docs/design/qwen3_omni_tts_performance_optimization.md`, which is where Qwen3-Omni/TTS performance results live. It's worth cross-linking from *Performance* / *Related Files* (and a back-link from `async_chunk.md`, since the overview at line 16 positions this doc as its complement). + +### 5. ASCII diagram is off by one column — `omni_async_output_materialization.md:163` + +`+-- slice per request` starts one column right of the branch column shared by line 162's `|` and lines 164-168's `+--`. Shift it left by one space. + +### 6. TTS YAML excerpt drops a key it shows for the sibling stage — `omni_async_output_materialization.md:300-310` + +The excerpt shows `async_scheduling: true` on stage 0 but omits it on stage 1, whereas `vllm_omni/deploy/qwen3_tts.yaml` sets it on both. Since the surrounding prose is about which stage gets the async path, the omission reads as a meaningful difference when it isn't. Either include it on stage 1 or drop it from stage 0 and let the prose carry it. (The Qwen3-Omni excerpt at `260-275` is exact — `async_scheduling: false` on stage 2 matches, and the "defaults to true for AR stages" comment is confirmed by `vllm_omni/config/stage_config.py:941-944`.) + +### 7. Nits + +- `:349` "These checks are evaluated per stage" — `_should_use_async_omni_output()` is actually called on every `sample_tokens()` invocation (`gpu_ar_model_runner.py:2077`). The inputs are effectively static, so the outcome is per-stage, but "evaluated per step, from per-stage-static inputs" is more precise. +- `:108-112` The **Change** row bolds `-4%` and `-25%` as wins alongside `+22%`. A "lower is better" marker on the TTFP/RTF columns would prevent a skim from reading them as regressions. + +--- + +## What I verified as correct (no action needed) + +- Guard table at `:339-347` matches `_should_use_async_omni_output()` exactly (`gpu_ar_model_runner.py:1595-1619`): async scheduling, `async_chunk`, `use_async_omni_output`, `omni_prefix_cache is None`, `speculative_config is None`, `enable_return_routed_experts`, and the `has_postprocess` → `eager_omni_postprocess_before_async_output` condition. +- Lifecycle steps 1-7 (`:139-153`) match `OmniAsyncGPUModelRunnerOutput` (`:151-239`) and the tail of `sample_tokens()` (`:2120-2153`), including the ordering claim that the background thread starts before `set_async_sampled_token_ids()`, and that `get_output()` joins, re-raises, then delegates to `super().get_output()`. +- The full-payload-branch unreachability argument (`:200-206`) is right: `should_accumulate_full_payload_output()` returns `False` when `async_chunk` is set (`omni_connector_model_runner_mixin.py:60-61`). +- Connector drain ordering (`:214-215`) matches the builder — `get_omni_connector_output()` runs after `_build_multimodal_outputs()` (`gpu_ar_model_runner.py:1874-1900`). +- NPU claim (`:356-359`) matches `npu_ar_model_runner.py:1273-1316`. XPU inheritance (`xpu_ar_model_runner.py:10`) and MUSA → `GPUARWorker` (`platforms/musa/platform.py:27`) both check out. +- Prefix-cache note (`:326-332`) is accurate: `omni_prefix_cache` is created only under `cache_config.enable_prefix_caching` (`gpu_model_runner.py:170-176`). +- Model opt-ins match `qwen3_omni.py:134` (Thinker), `:162-164` (Talker), and `qwen3_tts_talker.py:351-353`, including the `omni_pooler_payload_include_hidden = False` basis for the "skips hidden-state D2H" claim. +- Figure and caption are self-consistent (2.8 ms → 41 µs, ~68×), and the performance table percentages compute correctly (+21.5%, -3.7%, -25.4%). +- All six TOC anchors resolve, `async_chunk.md` exists at the relative path used, `--no-async-chunk` is a real flag, `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` matches `docs/models/supported_models.md`, and both nav entries are well-formed. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5610.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5610.trace.json.gz new file mode 100644 index 000000000..4766037e1 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5610.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5676.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5676.cost.json new file mode 100644 index 000000000..cc6c568ef --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5676.cost.json @@ -0,0 +1,70 @@ +{ + "calls": 76, + "input_tokens": 10164234, + "output_tokens": 46300, + "cost_usd": 8.166000499999999, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout5", + "head": "44e60c00090ef2848983a1bc249e4ffdb80a7cfd", + "base": "09b6a1604a3f7f32c8592cd6ce115c4b89610e92", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "152c8d784ba3c7be3c6efcf1e946ea47a777112e62d936ab21a445e49ae2a9f9", + "validation_commands": [ + "git diff --stat 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- vllm_omni/diffusion/request.py vllm_omni/diffusion/sched/ vllm_omni/engine/", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py", + "wc -l vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py vllm_omni/diffusion/worker/request_batch.py vllm_omni/diffusion/model", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py", + "grep -n \"batch_size\" vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py | head -50", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py", + "sed -n 278,340p vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py; echo ---; grep -n \"def check_inputs\" -A 60 vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py", + "grep -n \"def encode_audio\" -A 60 vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py", + "grep -n \"def check_inputs\" -A 40 vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py; echo \"=== prepare_latents ===\"; grep -n \"def prepare_latents\" -A 40 vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py", + "grep -n \"def forward\" -A 80 vllm_omni/diffusion/models/wan2_2/wan2_2_s2v_transformer.py | sed -n '1,200p'", + "grep -n \"hidden_states\\|def forward\\|cond_states\\|motion_latents\\|ref_latents\\|def encode_audio\\|patchify\\|unpatchify\" vllm_omni/diffusion/models/wan2_2/wan2_2_s2v_transformer.py | sed -n '60,200p'", + "sed -n 333,420p vllm_omni/diffusion/models/wan2_2/wan2_2_s2v_transformer.py; echo \"=== zero_timestep ===\"; grep -n \"zero_timestep\" vllm_omni/diffusion/models/wan2_2/wan2_2_s2v_transformer.py", + "sed -n 1400,1450p vllm_omni/engine/stage_init_utils.py; echo \"=== startup ===\"; sed -n 1520,1600p vllm_omni/engine/stage_engine_startup.py", + "grep -rn \"max_num_seqs\" vllm_omni/engine/stage_init_utils.py vllm_omni/engine/stage_engine_startup.py vllm_omni/diffusion/config.py 2>/dev/null | head -40; echo ===; grep -rn \"max_num_seqs\" vllm_omni/ --include=*.py | grep -v test | head -40", + "grep -rn \"max_num_seqs\" vllm_omni/ --include=*.py | head -40", + "grep -rn \"max_num_seqs\\|def _max_num_seqs\" vllm_omni/diffusion/ | head -30; echo \"=== build_diffusion_config ===\"; grep -n \"def build_diffusion_config\" -A 60 vllm_omni/engine/stage_init_utils.py | head -80", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- tests/engine/test_async_omni_engine_stage_init.py tests/engine/test_single_stage_mode.py", + "sed -n 100,130p vllm_omni/engine/async_omni_engine.py; grep -n \"diffusion_batch_size\" vllm_omni/engine/async_omni_engine.py | head; echo \"=== stage_pool 680-700 ===\"; sed -n 675,700p vllm_omni/engine/stage_pool.py", + "grep -rn \"max_num_seqs\" vllm_omni/deploy/ examples/ 2>/dev/null | head -20; echo \"=== from_kwargs ===\"; grep -n \"def from_kwargs\" -A 30 vllm_omni/diffusion/data.py", + "grep -n \"_normalize_pipeline_outputs\" -A 40 vllm_omni/diffusion/worker/diffusion_model_runner.py | head -60", + "grep -rn \"def resolve_wan_sample_solver\" -A 30 vllm_omni/diffusion/ ; echo ===; grep -rn \"def resolve_wan_flow_shift\" -A 30 vllm_omni/diffusion/", + "grep -rn \"def predict_noise_maybe_with_cfg\" -A 60 vllm_omni/diffusion/ | head -80", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- tests/diffusion/test_diffusion_scheduler.py tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py tests/diffusion/test_diffusion_engine_rpc_routing.py", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py | head -200", + "sed -n 980,1035p vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py", + "sed -n 690,720p vllm_omni/diffusion/diffusion_engine.py; echo \"=== build_wan_scheduler ===\"; grep -rn \"def build_wan_scheduler\" -A 30 vllm_omni/diffusion/", + "grep -n \"self.scheduler\\s*=\\|scheduler\" vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py | head -20", + "sed -n 50,90p vllm_omni/diffusion/sched/base_scheduler.py; grep -n \"sampling_params_key\" vllm_omni/diffusion/sched/base_scheduler.py | head -20", + "sed -n 495,545p vllm_omni/diffusion/worker/diffusion_worker.py", + "timeout 900 python -m pytest tests/diffusion/test_diffusion_scheduler.py tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py -q 2>&1 | tail -20", + "timeout 1800 python -m pytest tests/diffusion/models/wan2_2/ -q 2>&1 | tail -15", + "sed -n 1,70p vllm_omni/deploy/audex_tta.yaml", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py tests/diffusion/models/wan2_2/test_wan22_vace_pipeline.py | grep -n \"^+def test\\|^+ def test\\|last_image\\|latents=\" | head -6", + "git diff 09b6a1604a3f7f32c8592cd6ce115c4b89610e92 HEAD -- tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py | sed -n '540,660p'", + "grep -n \"test_s2v_forward_batches_request_local_inputs_and_splits_outputs\" -A 90 tests/diffusion/models/wan2_2/test_wan22_s2v_pipeline.py", + "grep -rn \"class .*Wan22Pipeline\\|class .*Wan22I2VPipeline\\|class .*Wan22S2VPipeline\\|class .*Wan22VACEPipeline\" vllm_omni/ ; echo \"=== DMD2 ===\"; grep -rn \"class DMD2PipelineMixin\" -A 40 vllm_omni/diffusion/ | head -60", + "grep -rn \"def scheduler_step_maybe_with_cfg\" -A 30 vllm_omni/diffusion/distributed/cfg_parallel.py", + "sed -n 1055,1075p vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py; echo \"=== stochastic step ===\"; grep -rn \"stochastic_sampling\" -A 15 vllm_omni/diffusion/models/dmd2/*.py | head -50", + "sed -n 60,120p vllm_omni/diffusion/models/dmd2/mixin.py", + "sed -n 41,125p vllm_omni/diffusion/output_formatter.py", + "grep -n 'prompt=\\[\"\"\\] \\* req.num_reqs' vllm_omni/diffusion/models/wan2_2/*.py; echo ---; grep -n \"isinstance(clip_generator, torch.Generator)\" vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py; echo ---; grep -n \"astype(np.float32\" vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py; ec", + "sed -n 44,58p vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-16T00:51:42", + "arm": "baseline_opus5", + "wall_s": 736.4, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [], + "artifact_chars": 13126 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5676.md b/eval/dataset/baselines/claudecode_opus5/pr5676.md new file mode 100644 index 000000000..28daf8bf6 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5676.md @@ -0,0 +1,173 @@ +I've finished the investigation. Here's the review. + +--- + +## Overall verdict: **Approve with changes** — the batching core is sound; a few config/inheritance edges need fixing before this is safe in production + +I traced the whole path: `pre_process_func` → `batch_compatibility_key` → `RequestBatchSamplingParamsKey` → `RequestScheduler` grouping → `execute_model_batch` → `_execute_request_list` → `pipeline.forward(batch)` → `split_diffusion_output_by_request`, plus the S2V/VACE transformers' batch handling. The new tests pass locally (86 scheduler/helper + 76 wan2_2 pipeline tests). + +**What's right, and non-trivially so:** +- The `repeat_interleave(num_outputs_per_prompt, dim=0)` ordering on user-supplied embeds/images matches `encode_prompt`'s `repeat(1, n, 1) + view(B*n, ...)` interleave semantics, matches `collate_sampling_param_generators`' flat ordering, and matches the `[idx*n : (idx+1)*n]` slicing in `split_diffusion_output_by_request`. All four axes line up. +- Dropping `latent_condition.repeat(batch_size, …)` in favour of `repeat_interleave` on `image_tensor` before the VAE is the correct fix, and the `first_frame_mask` shape change from `1` to `batch_size` is consistent with it. +- VACE's per-request `prepare_video_latents(request_generator)` inside the loop, followed by `prepare_latents([gen0, gen1])`, preserves the *per-generator consumption order* of the single-request path — so single-vs-batch determinism is genuinely preserved there. That's easy to get wrong and wasn't. +- Removing the `squeeze(0)`/`unsqueeze(0)` pairs in S2V `predict_noise`/`diffuse` is correct; the S2V transformer's `forward` was already batch-shaped. +- The known-limitation section in the description is honest and I agree with the diagnosis: the batch-shape BF16 divergence is upstream of anything this PR controls. + +The findings below are ordered by severity. + +--- + +### 1. `od_config.max_num_seqs = batch_size` clobbers configured values — breaks in both directions + +`vllm_omni/engine/stage_init_utils.py:1444` and `vllm_omni/engine/stage_engine_startup.py:1561` + +```python +od_config = build_diffusion_config(model, stage_cfg, metadata) +od_config.max_num_seqs = batch_size +``` + +`max_num_seqs` is a real `OmniDiffusionConfig` field (`vllm_omni/diffusion/data.py:852`) settable from `engine_args` via `from_kwargs`, and stage configs already expose it (`vllm_omni/config/stage_config.py:310`). `batch_size` here traces back to `AsyncOmni(diffusion_batch_size=...)`, which **defaults to 1** (`vllm_omni/engine/async_omni_engine.py:113`). Two consequences: + +- A diffusion stage that sets `engine_args.max_num_seqs: 4` is now silently forced back to `1` unless the caller also passes `diffusion_batch_size` — i.e. the exact knob this PR exists to enable is disabled by this PR for YAML-configured deployments. +- Conversely, `diffusion_batch_size=4` with a non-batching pipeline now hard-fails engine startup at `vllm_omni/diffusion/diffusion_engine.py:229-234` (`raise ValueError(... does not support request-level batching)`), where previously that argument only sized the client. + +**Change to:** reconcile rather than overwrite, e.g. + +```python +od_config.max_num_seqs = max(int(getattr(od_config, "max_num_seqs", 1) or 1), int(batch_size or 1)) +``` + +and state the new precedence in the docstring of `initialize_diffusion_stage`. The two new tests (`test_initialize_diffusion_stage_applies_client_batch_size_to_engine`, `test_launch_diffusion_stage_replica_applies_batch_size_to_config`) pin the clobbering behaviour, so they'd need updating too. + +--- + +### 2. S2V: `scheduler.step(generator=…)` is now unconditionally `None` + +`vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_s2v.py:1081` + +```python +generator=clip_generator if isinstance(clip_generator, torch.Generator) else None, +``` + +`clip_generator` is always the **list** returned by `_make_clip_generators` (`:1317` → `:1390`), so `isinstance(..., torch.Generator)` is always `False` and the generator is never forwarded — including for a batch of one, where it used to be. This is inert today because `FlowUniPCMultistepScheduler` (`:548`) is deterministic and ignores `generator`, but it's a silent trap: the moment a stochastic solver is wired in, S2V loses reproducibility with no signal. + +**Change to** either drop the parameter entirely with a comment explaining that the S2V scheduler is deterministic, or thread per-sample generators properly (`clip_generator[0]` when `len(clip_generator) == 1`, otherwise a per-sample step). + +--- + +### 3. DMD2 Wan variants silently inherit `supports_request_batch = True` + +`vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:1055` (`WanT2VDMD2Pipeline`) and `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py:1027` (`WanI2VDMD2Pipeline`) + +Adding `supports_request_batch = True` to the base classes propagates to both DMD2 subclasses, which are not exercised by any test in this PR. That matters because `DMD2PipelineMixin.__init_dmd2__` installs `DMD2EulerScheduler(stochastic_sampling=(solver == "sde"))` (`vllm_omni/diffusion/models/dmd2/mixin.py:31-36`), and `Wan22Pipeline.diffuse` calls `scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)` (`pipeline_wan2_2.py:564`) with **no** generator. For the SDE solver the step noise is therefore drawn from the global RNG as one draw over the whole batch tensor, making each request's output depend on batch composition — the exact isolation property this PR claims to establish. + +**Change to:** set `supports_request_batch = False` explicitly on `WanT2VDMD2Pipeline` and `WanI2VDMD2Pipeline` until validated, or plumb per-request generators into `scheduler_step_maybe_with_cfg`. + +(Separately and pre-existing: `DMD2PipelineMixin.forward` at `mixin.py:85-93` passes `guidance_scale=`/`num_inference_steps=` to `super().forward()`, which `Wan22Pipeline.forward(self, req)` does not accept. Those classes look untested end-to-end — worth a follow-up issue, not this PR.) + +--- + +### 4. Admission keys are coarser than forward-time validation, so one bad pairing kills both requests + +The condition keys admit requests that `forward` then rejects, and the resulting `ValueError` propagates out of `_execute_request_list` for the *whole* batch — an individually-valid request gets failed because of its batch-mate. + +Most concrete instance, `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py:142`: + +```python +request.batch_compatibility_key = ("wan22_i2v_last_image", last_image is not None) +``` + +`_normalize_i2v_last_image` (`:88-110`) explicitly accepts a raw `torch.Tensor`, and the pre-processor never resizes `last_image`. Two requests with differently-shaped tensor last-images share this key, pass admission, and then blow up in `DiffusionRequestBatch.collate_tensors` at `:669`. + +**Change to:** mirror the VACE approach — reuse `_vace_condition_structure` (or an equivalent) so the key carries shape/size/dtype, not just presence. The same reasoning applies to S2V's forward-time `raw_audio_waveforms`/sample-rate check (`pipeline_wan2_2_s2v.py:1263`) and VACE's reference-count check (`pipeline_wan2_2_vace.py:657-660`): prefer keeping incompatible requests out of the same batch over aborting the batch. + +Worth deciding explicitly (and documenting) whether `execute_model_batch` should isolate per-request failures at all, or whether whole-batch abort is the accepted contract. + +--- + +### 5. `output_type="latent"` payload key changed — an output-contract change bundled into a perf PR + +`vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:207-208` and `pipeline_wan2_2_i2v.py:68-69` + +```python +if sampling_params is not None and sampling_params.output_type is not None: + output_type = sampling_params.output_type +``` + +This is a genuine bug fix — previously `sampling_params.output_type` was ignored and `postprocess_video` ran on raw latents. But the fixed path now returns a bare tensor, which `normalize_diffusion_postprocess_output` (`vllm_omni/diffusion/output_formatter.py:76-79`) wraps as `{"output": tensor}` with `primary_key="output"` instead of `"video"`. Latent-mode clients will see a different payload key. + +**Change to:** nothing in the code — but call this out explicitly in the PR description / release notes. It's unrelated to batching and will surprise someone. + +--- + +### 6. `sampling.extra_args` accessed without `getattr` guard + +`vllm_omni/diffusion/sched/request_scheduler.py:48` + +```python +extra_args = sampling.extra_args or {} +``` + +The surrounding lines are defensive (`getattr(sampling, "lora_request", None)`, `getattr(request, "batch_compatibility_key", None)`), and `resolve_wan_sample_solver`/`resolve_wan_flow_shift` both use `getattr(req.sampling_params, "extra_args", {}) or {}`. This inconsistency already forced a fix to an existing stub-based test (`tests/diffusion/test_diffusion_engine_rpc_routing.py:126` adds `sampling_params["extra_args"] = {}`), which is the signal. + +**Change to:** `extra_args = getattr(sampling, "extra_args", None) or {}`. + +--- + +### 7. `None` in the solver key means "current pipeline default", but that default is mutable + +`vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py:689-694` + +```python +sample_solver = resolve_wan_sample_solver(first_request, default=self._sample_solver) +... + self._sample_solver = sample_solver + self._flow_shift = flow_shift +``` + +The key deliberately preserves `None` for "unspecified" so that it resolves against the pipeline default — but that default *drifts*: once any request overrides `sample_solver`, `self._sample_solver` is permanently rebound, and all later `None`-key batches silently run the previously-overridden solver. Pre-existing, but the new key design now depends on this being stable. + +**Change to:** resolve `None` against an immutable `self._default_sample_solver` / `od_config.flow_shift` captured at init, rather than the mutable current value. + +--- + +### 8. No shape validation on collated explicit `latents` + +`req.collate_request_tensors("latents", None)` concatenates on dim 0 and the result flows straight into `prepare_latents`, which only does `.to(device, dtype)` (`pipeline_wan2_2_i2v.py:904-907`; S2V bypasses `prepare_latents` entirely at `pipeline_wan2_2_s2v.py:1319-1320`). The implicit convention — visible only in the new tests (`latents_a = torch.zeros(2, 4, 1, 2, 2)` for `num_outputs_per_prompt=2`) — is that each request supplies a leading `num_outputs_per_prompt` dim. Anything else silently yields a wrong-shaped batch or a confusing failure deep in the transformer. + +**Change to:** assert `request_latents.shape[0] == batch_size` right after collation in each of the four `forward`s, and document the convention on `collate_request_tensors`. Given the test plan leans on `--explicit-latents`, this path deserves a guard. + +--- + +### 9. `_vace_condition_structure` fallthrough loses shape for unknown types + +`vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py:55` + +```python +return (type(value).__name__,) +``` + +An `np.ndarray` condition returns `("ndarray",)` regardless of shape, so two differently-shaped numpy conditions get the same key and collide at collation time (see finding 4). **Change to:** add an `np.ndarray` branch carrying `(shape, dtype)`, and for genuinely unknown types be conservative rather than optimistic. + +--- + +### 10. Wasted full text-encoder pass in the negative-embed fallback + +`pipeline_wan2_2.py:674-682`, `pipeline_wan2_2_i2v.py:582-590`, `pipeline_wan2_2_vace.py:619-627` all do: + +```python +_, negative_prompt_embeds = self.encode_prompt( + prompt=[""] * req.num_reqs, ..., do_classifier_free_guidance=True, ... +) +``` + +That's a full UMT5 forward over `num_reqs` empty prompts whose positive result is discarded, on every batch that supplies `prompt_embeds` without `negative_prompt_embeds`. In a perf PR this is worth a small negative-only encode helper. + +--- + +### Minor / nits + +- **`pipeline_wan2_2_s2v.py:1257`** — `astype(np.float32, copy=False)` returns the caller's array unchanged when it's already float32, and the new test asserts `np.shares_memory(outputs[0].output[1], audio_a)`. Handing a view of request-owned input back out in the output payload is surprising; confirm it's intentional. +- **`_make_clip_generators` production path is untested.** `OmniDiffusionRequest.__post_init__` (`vllm_omni/diffusion/request.py:58-59`) always assigns a seed when both seed and generator are absent, and the runner materializes a generator from it (`diffusion_model_runner.py:445-453`). So in production `seed is not None` and the *request* generator is discarded in favour of `Generator().manual_seed(seed + clip_index)`. The new test sets `seed=None`, exercising only the other branch. Add a seeded case. +- **`request_batch.py:48`** — `stage_durations` is shared by reference across every split output, so per-request timing metrics for a batch are actually batch aggregates. A comment would stop that being misread downstream. +- **`pipeline_wan2_2.py:238`** — for the base T2V pipeline the image key splits batches even when `self.expand_timesteps` is `False`, in which case the image is ignored entirely. Harmless, slightly reduces batching opportunity. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5676.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5676.trace.json.gz new file mode 100644 index 000000000..f54f76b47 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5676.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5678.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5678.cost.json new file mode 100644 index 000000000..ab5268dc4 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5678.cost.json @@ -0,0 +1,81 @@ +{ + "calls": 61, + "input_tokens": 5669032, + "output_tokens": 48258, + "cost_usd": 5.295251500000002, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout3", + "head": "eeaa5614481860505459fcafca0e44e1357ab015", + "base": "a874b8e09bb5251bb16e3de8bc22edf3880eec09", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "b161cebc4cc81c01eeea44294997eac5cb59b37d5b5a9b905bc3bcfbc5893de1", + "validation_commands": [ + "git diff --stat a874b8e09bb5251bb16e3de8bc22edf3880eec09 HEAD", + "git diff a874b8e09bb5251bb16e3de8bc22edf3880eec09 HEAD -- vllm_omni/config/omni_config.py", + "git diff a874b8e09bb5251bb16e3de8bc22edf3880eec09 HEAD -- vllm_omni/engine/stage_init_utils.py vllm_omni/engine/stage_runtime.py vllm_omni/entrypoints/cli/serve.py", + "sed -n '1,80p' vllm_omni/config/stage_config.py", + "grep -n \"class StageType\" -A 12 vllm_omni/config/stage_config.py; grep -n \"class StageExecutionType\" -A 12 vllm_omni/config/stage_config.py", + "wc -l vllm_omni/config/omni_config.py vllm_omni/engine/stage_init_utils.py; sed -n '1,60p' vllm_omni/config/omni_config.py", + "sed -n '760,1000p' vllm_omni/engine/stage_init_utils.py", + "grep -n \"_to_dict\\|def _get_attr_or_item\" -A 12 vllm_omni/engine/stage_init_utils.py | head -60", + "grep -n \"def _to_dict\" -A 20 vllm_omni/entrypoints/stage_utils.py", + "grep -rn \"tensor_parallel_size\" vllm_omni/engine/stage_init_utils.py | head -30; echo ---; grep -n \"def get_stage_tp_size\" -A 25 vllm_omni/entrypoints/stage_utils.py", + "sed -n '181,215p' vllm_omni/engine/stage_init_utils.py; echo ---; sed -n '1170,1250p' vllm_omni/engine/stage_init_utils.py", + "sed -n '1000,1200p' vllm_omni/config/omni_config.py", + "sed -n '1029,1170p' vllm_omni/engine/stage_init_utils.py", + "sed -n '1,180p' tests/engine/test_stage_engine_args.py", + "sed -n '180,400p' tests/engine/test_stage_engine_args.py", + "sed -n '400,520p' tests/engine/test_stage_engine_args.py", + "grep -rn \"from_pipeline_config\\|VllmOmniConfig\" --include=*.py vllm_omni/ | head -30; echo \"=== global_stage_cli_fields\"; grep -rn \"_global_stage_cli_fields\\|_stage_cli_overrides\" --include=*.py vllm_omni/ tests/ | head -20", + "grep -n \"VllmOmniConfig\\|from_pipeline_config\" vllm_omni/entrypoints/utils.py vllm_omni/config/config_factory.py | head -30", + "sed -n '280,400p' vllm_omni/config/config_factory.py", + "ls /data/zhoutaichang/.infermatrix-copilot/worktrees/ 2>/dev/null; ls -d ../.venv* 2>/dev/null; which python3; python3 -c \"import vllm; print(vllm.__version__)\" 2>&1 | tail -2", + "ls -d /data/zhoutaichang/.venv-vllm-omni-main /data/zhoutaichang/.infermatrix-copilot/.venv-vllm-omni-main 2>&1", + "python3 -c \"import vllm, sys; print(sys.executable, vllm.__version__)\" 2>&1 | tail -3", + "timeout 300 python3 -c \"\nfrom vllm_omni.config.omni_config import _STAGE_ENGINE_FIELDS, _LLM_STAGE_ENGINE_FIELDS\nprint(sorted(_LLM_STAGE_ENGINE_FIELDS))\n\" 2>&1 | tail -20", + "sed -n '385,420p' vllm_omni/entrypoints/utils.py; echo \"=== _stage_cli_overrides\"; grep -n \"_stage_cli_overrides\" -A 25 vllm_omni/config/omni_config.py | head -40", + "timeout 600 python3 -c \"\nfrom vllm_omni.config.pipeline_registry import resolve_pipeline_config, OMNI_PIPELINES\nfrom vllm_omni.config.omni_config import VllmOmniConfig\np = None\nfor name in sorted(OMNI_PIPELINES):\n c = resolve_pipeline_config(name)\n if c is not None and len(c.stages) >= 2 and c", + "timeout 600 python3 -c \"\nfrom dataclasses import fields\nfrom vllm_omni.engine.arg_utils import OmniEngineArgs\nnames = {f.name for f in fields(OmniEngineArgs)}\nfor n in ['devices','num_replicas','env','num_gpus','duplex_max_sessions','async_chunk','swap_space','max_logprobs','enable_lora','scheduler_", + "grep -n \"scheduler_cls\" vllm_omni/config/stage_config.py | head -20; echo \"=== StagePipelineConfig fields\"; grep -n \"class StagePipelineConfig\" -A 70 vllm_omni/config/stage_config.py | head -80", + "sed -n '930,1040p' vllm_omni/config/stage_config.py", + "timeout 900 python3 - <<'EOF' 2>&1 | tail -30\nimport copy\nfrom vllm_omni.config.pipeline_registry import resolve_pipeline_config\nfrom vllm_omni.config.omni_config import VllmOmniConfig\nfrom vllm_omni.config.stage_config import merge_pipeline_deploy, build_stage_runtime_overrides, load_deploy_config,", + "timeout 900 python3 -c '\nimport copy\nfrom pathlib import Path\nfrom vllm_omni.config.omni_config import VllmOmniConfig\nfrom vllm_omni.config.pipeline_registry import resolve_pipeline_config\nfrom vllm_omni.config.stage_config import DeployConfig, build_stage_runtime_overrides, load_deploy_config, merg", + "timeout 900 python3 -c '\nfrom vllm_omni.config.pipeline_registry import OMNI_PIPELINES, resolve_pipeline_config\nok=[]\nfor n in sorted(OMNI_PIPELINES):\n try:\n c = resolve_pipeline_config(n)\n except Exception as e:\n c = None\n if c is not None:\n ok.append((n, c.default_dep", + "timeout 900 python3 -c '\nimport copy\nfrom pathlib import Path\nfrom vllm_omni.config.omni_config import VllmOmniConfig\nfrom vllm_omni.config.pipeline_registry import OMNI_PIPELINES, resolve_pipeline_config\nfrom vllm_omni.config.stage_config import DeployConfig, build_stage_runtime_overrides, load_dep", + "timeout 900 python3 -c '\nimport copy\nfrom pathlib import Path\nfrom dataclasses import fields\nfrom vllm_omni.diffusion.data import OmniDiffusionConfig, DiffusionParallelConfig\nfrom vllm_omni.config.omni_config import OmniStageDiffusionParallelConfig\nd = DiffusionParallelConfig()\nprint(\"DiffusionParal", + "grep -n \"class DiffusionParallelConfig\" -A 80 vllm_omni/diffusion/data.py | head -110", + "sed -n '226,330p' vllm_omni/diffusion/data.py", + "timeout 900 python3 -c '\nimport copy\nfrom pathlib import Path\nfrom vllm_omni.config.omni_config import VllmOmniConfig\nfrom vllm_omni.config.pipeline_registry import resolve_pipeline_config\nfrom vllm_omni.config.stage_config import DeployConfig, build_stage_runtime_overrides, load_deploy_config, merg", + "grep -n \"class OmniStageParallelConfig\" -A 60 vllm_omni/config/omni_config.py", + "git diff a874b8e09bb5251bb16e3de8bc22edf3880eec09 HEAD -- tests/config/test_omni_config.py", + "sed -n '330,430p' vllm_omni/engine/stage_runtime.py", + "sed -n '376,470p' vllm_omni/engine/stage_init_utils.py", + "grep -n \"def _build_engine_args\" -A 60 vllm_omni/config/stage_config.py", + "ls vllm_omni/deploy/ | head -80; ls vllm_omni/deploy | wc -l", + "timeout 900 python3 -c '\nimport copy, traceback\nfrom pathlib import Path\nfrom vllm_omni.config.omni_config import VllmOmniConfig\nfrom vllm_omni.config.pipeline_registry import resolve_pipeline_config, OMNI_PIPELINES\nfrom vllm_omni.config.stage_config import load_deploy_config\nDEPLOY = Path(\"vllm_omn", + "timeout 900 python3 -c '\nimport yaml\nfrom pathlib import Path\nfrom vllm_omni.config.omni_config import _LLM_STAGE_ENGINE_FIELDS, _DIFFUSION_OWNED_STAGE_ENGINE_FIELDS, _LEGACY_STAGE_METADATA_EXTRA_FIELDS\nfrom vllm_omni.config.stage_config import StageDeployConfig\nfrom dataclasses import fields\nsd = s", + "timeout 900 python3 -c '\nfrom dataclasses import fields\nfrom vllm_omni.engine.arg_utils import OmniEngineArgs, orchestrator_field_names\nfrom vllm_omni.config.omni_config import _LLM_STAGE_ENGINE_FIELDS, _global_stage_cli_fields\ngsf = _global_stage_cli_fields()\ncommon = [\"kv_cache_dtype\",\"block_size\"", + "timeout 900 python3 -c '\nimport copy\nfrom pathlib import Path\nfrom vllm_omni.config.pipeline_registry import resolve_pipeline_config\nfrom vllm_omni.config.stage_config import build_stage_runtime_overrides, load_deploy_config, merge_pipeline_deploy\nfrom vllm_omni.engine.stage_init_utils import build_", + "timeout 1200 python3 -m pytest tests/engine/test_stage_engine_args.py -q -x --no-header 2>&1 | tail -20", + "sed -n '20,60p' vllm_omni/deploy/hunyuan_image_3_moe.yaml", + "timeout 900 python3 -c '\nfrom vllm_omni.config.pipeline_registry import resolve_pipeline_config\np = resolve_pipeline_config(\"hunyuan_image_3_moe\")\nfor s in p.stages:\n print(s.stage_id, s.execution_type, \"owns_tokenizer=\", s.owns_tokenizer, \"final_output=\", s.final_output, s.final_output_type)\n' 2", + "sed -n '1040,1130p' vllm_omni/config/stage_config.py", + "sed -n '3040,3070p' vllm_omni/entrypoints/openai/serving_chat.py", + "grep -n \"_apply_rocm_attention_backend\\|TRITON_ATTN\\|ROCM_AITER_FA\" vllm_omni/engine/stage_init_utils.py; echo ---; grep -n \"def _validate_stage_engine_override_ownership\\|no structured config owner\\|_LEGACY_STAGE_METADATA_EXTRA_FIELDS\\|def _build_cache_config\\|def _build_scheduler_config\\|def sched", + "sed -n '1375,1400p' vllm_omni/config/omni_config.py; echo \"---940\"; sed -n '938,950p' vllm_omni/engine/stage_init_utils.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-15T01:52:09", + "arm": "baseline_opus5", + "wall_s": 1002.4, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [ + "Write: path outside the worktree: /tmp/probe1.py in {\"file_path\": \"/tmp/probe1.py\", \"content\": \"import copy\\nfrom pathlib import Path\\n\\nfrom vllm_omni.config.omni_config import VllmOmniConfig\\nfrom vllm_omni.con" + ], + "artifact_chars": 8709 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5678.md b/eval/dataset/baselines/claudecode_opus5/pr5678.md new file mode 100644 index 000000000..bf304293f --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5678.md @@ -0,0 +1,87 @@ +## Verdict + +**Approve for the live path; the typed projection needs changes before PR8 flips the consumer.** + +The production-reachable part of this PR is small and sound: `stage_runtime.py` / `serve.py` are comment-only, `_apply_rocm_attention_backend` is a faithful extraction, and `build_legacy_engine_args_dict` fixes a real wart (it no longer mutates `stage_config.engine_args` when dropping a `None` TP size) with a test for it. I traced the three legacy readers of `tensor_parallel_size` (`_tp_size_for_stage`, `get_stage_tp_size`, `acquire_device_locks`) and all of them tolerate the key now being present-and-`None`, so that change is safe. + +The typed projection is dormant today, which is the right call — but two things in it will bite at the cutover, and one of them is a user-visible regression I was able to reproduce. I ran the new suite locally (vLLM 0.26.0): `63 passed, 1 skipped`. The one red gate is `buildkite/vllm-omni-amd-ci`; see finding 4, which is adjacent enough to be worth checking against it. + +--- + +## Must fix before the PR8 cutover + +### 1. Ownership validation rejects ~226 engine args that work today +`vllm_omni/config/omni_config.py:872` (`_validate_stage_engine_override_ownership`) + +`_COMMON_STAGE_ENGINE_FIELDS` is a hand-curated ~62-field set, but `OmniEngineArgs` has 249 fields. Any explicitly-passed arg outside the curated set is now a hard `ValueError`. Reproduced against `qwen3_tts` at this commit: + +``` +{'max_logprobs': 5} -> ValueError: Stage 0 (llm_ar) has explicit engine argument(s) + with no structured config owner: max_logprobs +{'stage_0_swap_space': 8}-> ValueError: ... no structured config owner: swap_space +``` + +And these all reach `OmniEngineArgs` intact through today's legacy path (verified end to end through `filter_dataclass_kwargs`), but are unowned for LLM stages: + +`kv_cache_dtype`, `block_size`, `cpu_offload_gb`, `served_model_name`, `disable_log_stats`, `num_gpu_blocks_override`, `disable_custom_all_reduce`, `structured_outputs_config`, `kv_transfer_config`, `speculative_config`, `enable_lora`, `max_num_partial_prefills`, `long_prefill_token_threshold`, `mm_encoder_tp_mode`. + +`--kv-cache-dtype fp8` is the one that worries me most — it's a routine flag, it works today, and `tests/config/test_omni_config.py:236` locks in rejecting it for LLM stages (it's currently only owned as the diffusion `diffusion_kv_cache_dtype` alias). + +The strictness is defensible as a design goal, and I confirmed every shipped `vllm_omni/deploy/*.yaml` passes (0 unowned keys across all 81), so nothing in-repo breaks. But user CLIs and user deploy YAMLs are unconstrained. Before the consumer switches, either give the structured stage a typed passthrough bucket for backend-owned-but-unmodeled args, or extend ownership to the full `OmniEngineArgs` surface. A note in the PR8 description that "unowned args now hard-fail" is not enough — this is a CLI compatibility break. + +### 2. `scheduler_cls` diverges from legacy when `async_scheduling` arrives via CLI +`vllm_omni/config/omni_config.py:1067` + +Legacy `merge_pipeline_deploy` (`stage_config.py:948-952`) resolves `sched_cls` from *deploy-only* engine args and CLI overrides are layered on afterwards in `to_omegaconf`, so legacy ends up with `async_scheduling=False` **and** `OmniARAsyncScheduler`. The typed path merges CLI into `engine` before building `scheduler_config`, so the property resolves the sync scheduler. Swept all 51 resolvable pipelines × 9 CLI override sets; with `--async-scheduling=false` or `--stage-0-async-scheduling=false`: + +``` +personaplex stage 0 scheduler_cls: OmniARAsyncScheduler -> OmniARScheduler +qwen2_5_omni stage 0,1 (same) +qwen3_tts stage 0 (same) +step_audio_2 / step_audio_2_asr / voxtral_tts / qwen2_5_omni_thinker_only (same) +``` + +The typed behavior is almost certainly the *correct* one — legacy is self-contradictory. But this PR's thesis is bit-for-bit parity, and this is an unannounced, untested runtime scheduler swap. Either state it as an intentional fix with a test asserting the new behavior, or reproduce the legacy resolution order. + +### 3. Parity tests are one-directional and never exercise CLI overrides +`tests/engine/test_stage_engine_args.py:221`, `:470` + +`assert {name: typed_args[name] for name in legacy_args} == legacy_args` only proves legacy ⊆ typed; extra typed keys are unchecked. The registry test (`:470`) is the strong one — it compares `_effective_backend_values` in both directions — but it always builds with `cli_overrides={"model": "/tmp"}`. That's exactly the blind spot that hides finding 2. Parametrize the registry test over a handful of representative CLI overrides (`async_scheduling`, `tensor_parallel_size`, `enable_chunked_prefill`, `gpu_memory_utilization`) and it will catch this class of drift automatically. + +--- + +## Should fix + +### 4. ROCm AITER backend is unreachable — and AMD CI is the failing gate +`vllm_omni/engine/stage_init_utils.py:368-376` + +```python +if rocm_aiter_ops.is_enabled(): + engine_args["attention_backend"] = "ROCM_AITER_FA" +# ... +engine_args["attention_backend"] = "TRITON_ATTN" # unconditional — clobbers the line above +``` + +Pre-existing (carried over verbatim from `extract_legacy_stage_metadata`), so it isn't a regression, but this PR is what turns it into a named, reused helper called from two places, which makes it the natural moment to fix: add `return` after the `ROCM_AITER_FA` assignment. Given `buildkite/vllm-omni-amd-ci` is the one red check, worth confirming against that job's log before assuming they're unrelated. + +### 5. Generation-stage cache defaults are split across two layers +`vllm_omni/config/omni_config.py:1383` vs `vllm_omni/engine/stage_init_utils.py:947-948` + +`_build_cache_config` hoists `disable_hybrid_kv_cache_manager=True` into the structured config for `LLM_GENERATION`, while its twin `enable_prefix_caching=False` stays behind as a `setdefault` in `_finalize_engine_args_dict`. Engine args come out identical, but a consumer reading `cache_config` directly — the entire point of the structured config — sees `disable_hybrid_kv_cache_manager=True` next to `enable_prefix_caching=None`. Put both defaults in the same place; the structured config is the better home now that it owns one of them. + +### 6. Three engine_extras keys are silently dropped while everything else hard-errors +`vllm_omni/config/omni_config.py:64` and `:934` + +`hunyuan_image_3_moe.yaml:39-41` sets `is_comprehension: true` / `final_output: true` / `final_output_type: text`, and the YAML's own comment says stage 0 "must be marked True for the serving path to set AR seed/stop_token_ids." The topology says `owns_tokenizer=False`, so the typed path resolves `is_comprehension=False`. + +I checked and this is **not** a regression — `StageConfig.to_omegaconf()` (`stage_config.py:1063`) already overwrites the top-level key from topology, and `serving_chat.py:3060` even documents that the engine_args-nested copy is invisible. So legacy is effectively `False` too. But the inconsistency is worth closing: everything else unowned raises, these three vanish. At minimum log a warning when they appear in `engine_extras`; ideally fix the shipped YAML, since it's currently asking for behavior it doesn't get. + +--- + +## Nits + +**7.** `_project_omni_config_fields` (`stage_init_utils.py:768`) skips every `None`, so the typed representation can't express an explicit `None` for `Any`-typed fields (`attention_backend`, `hf_overrides`, `dtype`). Fine for now given the defaults, but worth a docstring line so a future field with meaningful-`None` semantics doesn't get silently swallowed. + +**8.** Diffusion `parallel_config` (`stage_init_utils.py:836`) materializes all 17 defaults where legacy emitted only what the user set — e.g. `--tensor-parallel-size 2` on `wan2_2_ti2v` gives legacy `{'tensor_parallel_size': 2}` vs the full typed dict. I diffed `OmniStageDiffusionParallelConfig` against `DiffusionParallelConfig` field by field and they agree today, and both derive `sequence_parallel_size` identically (`init=False` post-init on the Omni side), so this is currently value-neutral. But it pins one default set to the other with nothing enforcing it — add a test asserting the two default sets match, or the next `DiffusionParallelConfig` default change becomes a silent behavior fork. + +**9.** `build_engine_args_dict` is now a pure pass-through whose docstring says callers should "not bind directly to a representation-specific implementation" — while the two comments added at `stage_runtime.py:373` and `serve.py:938` spell out the exact legacy binding it's meant to hide. Harmless, just redundant; one of the three can go. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5678.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5678.trace.json.gz new file mode 100644 index 000000000..ebecf8dda Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5678.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5691.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5691.cost.json new file mode 100644 index 000000000..0d9b63c8c --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5691.cost.json @@ -0,0 +1,71 @@ +{ + "calls": 57, + "input_tokens": 8082357, + "output_tokens": 37508, + "cost_usd": 6.760932999999998, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout3", + "head": "0eee3b18645af4de79b6fbc2b1999e0294f0ad24", + "base": "67c54777bb22e9e7e08fdf7c47a64f06b566fc47", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "ce4a996a5a6efbe53f21032e1c89bb4997826bdb1363f0c79ebb703c46f85c86", + "validation_commands": [ + "git diff --stat 67c54777bb22e9e7e08fdf7c47a64f06b566fc47 HEAD", + "git diff 67c54777bb22e9e7e08fdf7c47a64f06b566fc47 HEAD -- vllm_omni/entrypoints/ vllm_omni/engine/ vllm_omni/config/ vllm_omni/diffusion/registry.py vllm_omni/diffusion/data.py vllm_omni/diffusion/model_metadata.py pyproject.toml", + "sed -n 2960,3200p vllm_omni/entrypoints/openai/api_server.py", + "sed -n 100,200p vllm_omni/entrypoints/openai/serving_video.py", + "git diff 67c54777bb22e9e7e08fdf7c47a64f06b566fc47 HEAD -- vllm_omni/diffusion/attention/", + "sed -n 1,140p vllm_omni/diffusion/attention/backends/abstract.py", + "sed -n 1,320p vllm_omni/diffusion/attention/backends/flash_attn.py", + "sed -n 1,200p vllm_omni/diffusion/attention/layer.py", + "sed -n 200,340p vllm_omni/diffusion/attention/layer.py", + "sed -n 1,200p vllm_omni/diffusion/attention/parallel/ring.py", + "sed -n 1,150p vllm_omni/diffusion/models/minimax_h3/encoder.py", + "grep -n \"class \\|def \\|load_model\\|rank_in_group\\|world_size\" vllm_omni/diffusion/models/minimax_h3/encoder.py | sed -n 1,200p", + "sed -n 898,1214p vllm_omni/diffusion/models/minimax_h3/encoder.py", + "git log --format='%H%n%s%n%b---' 67c54777bb22e9e7e08fdf7c47a64f06b566fc47..HEAD | head -40; echo \"=== accelerator usage ===\"; grep -rn \"torch.accelerator\" --include=*.py vllm_omni/ | head -20", + "sed -n 1,200p vllm_omni/diffusion/models/minimax_h3/vae.py", + "grep -rn \"trust_remote_code\" vllm_omni/diffusion/ | head -30", + "git diff 67c54777bb22e9e7e08fdf7c47a64f06b566fc47 HEAD -- tests/model_tests/diffusion/test_alignment.py tests/config/test_omni_config.py tests/diffusion/attention/test_flash_attn.py", + "(ruff --version && ruff format --check $(git diff --name-only 67c54777bb22e9e7e08fdf7c47a64f06b566fc47 HEAD | grep '\\.py$') 2>&1 | tail -20; echo \"--- lint ---\"; ruff check $(git diff --name-only 67c54777bb22e9e7e08fdf7c47a64f06b566fc47 HEAD | grep '\\.py$') 2>&1 | tail -40) || echo \"ruff not availab", + "git diff --name-only 67c54777bb22e9e7e08fdf7c47a64f06b566fc47 HEAD | grep '\\.py$' > /tmp/changed.txt; wc -l /tmp/changed.txt; ruff --version", + "ruff --version", + "python -c \"import torch, vllm; print(torch.__version__, vllm.__version__)\" 2>&1 | tail -3", + "timeout 900 python -m pytest tests/diffusion/models/minimax_h3/ -x -q 2>&1 | tail -30", + "timeout 900 python -m pytest tests/diffusion/attention/test_flash_attn.py tests/diffusion/attention/test_ring_fa4.py tests/entrypoints/openai_api/test_video_server.py -q 2>&1 | tail -25", + "sed -n 1,80p tests/model_tests/diffusion/test_alignment.py", + "cat vllm_omni/diffusion/models/minimax_h3/time_request.py vllm_omni/diffusion/models/minimax_h3/packed_tokens.py", + "git diff 67c54777bb22e9e7e08fdf7c47a64f06b566fc47 HEAD -- docs/ tests/entrypoints/openai_api/test_video_server.py", + "grep -n -i \"sage\" docs/user_guide/diffusion/attention_backends.md | head -20", + "grep -n \"fn(\\|attn_fn\\|select_flash_attn_impl\\|deterministic\\|def ring_flash_attn_func\" vllm_omni/diffusion/attention/backends/ring_flash_attn.py | head -40", + "sed -n 60,130p vllm_omni/diffusion/attention/backends/ring_flash_attn.py", + "sed -n 150,235p vllm_omni/diffusion/attention/backends/ring/ring_kernels.py", + "grep -rn \"def update_out_and_lse\" -A 30 vllm_omni/diffusion/attention/backends/ | head -50", + "grep -rn \"vae_patch_parallel_size\" --include=\"*.py\" vllm_omni/ | grep -v test | wc -l; echo \"---\"; grep -rln \"vae_patch_parallel_size\" --include=\"*.py\" vllm_omni/; echo \"=== text_encoder_tp_size ===\"; grep -rln \"text_encoder_tp_size\" --include=\"*.py\" vllm_omni/", + "grep -rn \"vae_patch_parallel_size\" vllm_omni/engine/arg_utils.py vllm_omni/config/stage_config.py vllm_omni/config/omni_config.py", + "grep -rn \"DiffusionParallelConfig(\" --include=\"*.py\" vllm_omni/ | head -10", + "grep -rn \"def init_world_group\" -A 25 vllm_omni/diffusion/distributed/parallel_state.py", + "grep -rn \"class GroupCoordinator\" -A 60 vllm_omni/diffusion/distributed/group_coordinator.py | head -80", + "grep -n \"text-encoder-tp-size\\|text_encoder_tp_size\" recipes/MiniMaxAI/MiniMax-H3.md", + "sed -n 110,175p recipes/MiniMaxAI/MiniMax-H3.md", + "grep -rn \"def get_dit_group\" -A 12 vllm_omni/diffusion/distributed/parallel_state.py", + "grep -rn \"_DIT = \\|_DIT: \\|_DIT =\" vllm_omni/diffusion/distributed/parallel_state.py | head", + "sed -n 1,90p vllm_omni/diffusion/models/minimax_h3/presentation.py", + "grep -n \"def minimax_h3_multi_image_presentation_ids\" -A 25 vllm_omni/diffusion/models/minimax_h3/presentation.py; grep -n \"def minimax_h3_multi_image_presentation_token_tags\" -A 25 vllm_omni/diffusion/models/minimax_h3/presentation.py", + "grep -n \"valid values on a 4-GPU\" recipes/MiniMaxAI/MiniMax-H3.md; grep -n \"enable_cudnn_sdp\" vllm_omni/diffusion/models/minimax_h3/encoder.py; grep -n \"missing = sorted\" vllm_omni/diffusion/models/minimax_h3/encoder.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-15T02:10:22", + "arm": "baseline_opus5", + "wall_s": 598.2, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [ + "Write: path outside the worktree: /tmp/gc_check.py in {\"file_path\": \"/tmp/gc_check.py\", \"content\": \"import os\\nimport torch.distributed as dist\\nimport torch.multiprocessing as mp\\n\\n\\ndef worker(rank, world):\\n " + ], + "artifact_chars": 10705 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5691.md b/eval/dataset/baselines/claudecode_opus5/pr5691.md new file mode 100644 index 000000000..120293734 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5691.md @@ -0,0 +1,115 @@ +Investigation complete. Here is my review. + +--- + +# Review: PR #5691 — [Model] Add MiniMax H3 diffusion support + +## Verdict: **Request changes** + +This is a large, generally well-structured model addition (~7.4k lines). The packed-sequence layout, the RF scheduler, and the DiT itself read carefully and the CPU unit tests are good (I ran `tests/diffusion/models/minimax_h3`, `tests/diffusion/attention/test_flash_attn.py`, `test_ring_fa4.py`, and `tests/entrypoints/openai_api/test_video_server.py` locally — 106 passed, 1 skipped). + +Three things should block merge: a **documented `--text-encoder-tp-size` value crashes at startup**, **ring SP silently produces wrong attention** for this model, and the **text-encoder loader downgrades a missing-weight checkpoint to a warning**. The rest are follow-ups. + +--- + +## Blocking + +### 1. `--text-encoder-tp-size N` crashes every DiT rank outside the encoder group when `1 < N < dit_world` + +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:561-579` + +`_build_text_encoder_group` calls `init_world_group(ranks=list(range(N)))` on *every* DiT rank. But `GroupCoordinator.__init__` (`vllm_omni/diffusion/distributed/group_coordinator.py:109-122`) only sets `self.ranks`, `self.world_size`, `self.cpu_group`, `self.device_group` when `self.rank in ranks`, and then does: + +```python +assert self.cpu_group is not None +assert self.device_group is not None +``` + +So on a 4-GPU server with `--text-encoder-tp-size 2`, ranks 2 and 3 raise `AssertionError` during pipeline construction. The docstring's premise ("ranks outside the group never run encoder collectives") is true, but they never survive construction to get there. This is exactly the case `_SingleRankEncoderGroup` was written to dodge for `N == 1` — it just wasn't generalized. + +The recipe advertises the broken values: `recipes/MiniMaxAI/MiniMax-H3.md:151` — *"valid values on a 4-GPU server are 1, 2, and 4"*. + +**Fix:** still call `torch.distributed.new_group` on every rank (it is a collective over the default group), but return a non-member stub — mirroring `_SingleRankEncoderGroup` — on ranks outside the encoder rank set, instead of a `GroupCoordinator`. + +Also at `pipeline_minimax_h3.py:235` — `ranks: list[int] = [0]` is a mutable **class** attribute on `_SingleRankEncoderGroup`; make it an instance attribute. + +### 2. Ring sequence parallelism silently discards the packed-document boundaries + +`vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:405-419` + +The DiT publishes `cu_seqlens_q/k` + `max_seqlen_q/k` via `AttentionMetadata.extra`, plus a fallback `attn_mask`. Only `FlashAttentionImpl.forward_cuda` reads them (`vllm_omni/diffusion/attention/backends/flash_attn.py:277-291`). `RingParallelAttention.run_attention` (`vllm_omni/diffusion/attention/parallel/ring.py:93-165`) ignores **both** `extra` and `attn_mask` and calls `ring_flash_attn_func` on the full tensor. + +With `--ring N>1`, the up-to-63 alignment padding rows become attendable keys for every query. They are not benign zeros: their `token_tags` is `-1`, clamped to `0` at `minimax_h3_transformer.py:1057`, so `_modulate_scale_shift` gives them the video-modality `shift` (`:674`) — non-zero content that dilutes the softmax on every real row, every layer, every step. + +`MiniMaxH3DiTModel.__init__` validates `ulysses_degree` (`:851-859`) but never `ring_degree`, so this is accepted silently. The recipe only uses `--ring 1`, so this is untested-and-wrong rather than tested-and-wrong. + +**Fix:** raise in `_validate_tp_config` when `parallel_config.ring_degree > 1` (matching the existing `cfg_parallel_size` guard at `pipeline_minimax_h3.py:276-277`), or thread `cu_seqlens` into the ring path. + +### 3. Text-encoder weight loading treats a broken checkpoint as a warning + +`vllm_omni/diffusion/models/minimax_h3/encoder.py:1021-1027` + +```python +missing = sorted(set(params) - loaded) +if missing: + logger.warning("MiniMax H3 text encoder weights not loaded (retained-layer subset): ...") +``` + +Every parameter is allocated with `torch.empty` (`encoder.py:81`, `:129`, `:170`, …), so any missing shard yields uninitialized memory and a plausible-but-garbage video. The "retained-layer subset" framing doesn't apply: `_map_weight_name` (`:955-993`) already filters `lm_head`, `model.language_model.norm`, and layers ≥ 50, so `missing` should be empty on a healthy checkpoint. + +**Fix:** raise. Compare with the DiT side, which does enforce this (`minimax_h3_transformer.py:894-896`, `_mark_missing_params_required`). + +--- + +## Should fix before/soon after merge + +### 4. Reference-video sampling is one ffmpeg process per frame, with a linear seek each time + +`vllm_omni/diffusion/models/minimax_h3/reference_video.py:225-244` + +Each sampled frame spawns a fresh `ffmpeg -i /FL2VA/model_index.json`. Verified: + +``` +partition: combined +root: /tmp/.../h3-fl2va +model_path: /tmp/.../h3-fl2va/FL2VA exists: False +``` + +Pre-PR this exact directory worked (`Path(model)/"model_index.json"` was read directly). Fix: in `_resolve_minimax_h3_model_root` / `__init__`, if `/model_index.json` exists, treat `` as the partition root and take the partition from its `_minimax_h3.partition` field instead of inferring it from the folder name. At minimum, raise a message that names the missing file and the expected layout. + +### 4. `omni_snapshot_download` gates on a filename, not on the pipeline +`vllm_omni/entrypoints/omni_base.py:80-85` + +Any repo containing `modular_model_index.json` now skips download entirely, but only `MiniMaxH3Pipeline` implements self-download. This is an implicit contract between an entrypoint file-existence probe and one model's `__init__`. Prefer keying on the resolved pipeline class (e.g. a `_downloads_own_weights: ClassVar[bool]`), so adding an unrelated modular repo doesn't silently produce a server with no weights. Two smaller points on the same block: the probe passes `revision=None` regardless of `--revision`, and it adds an HF round trip to every non-modular startup. + +### 5. Two `get_diffusion_model_index` callers lost their exception guard +`vllm_omni/diffusion/data.py:298-301`, `vllm_omni/diffusion/utils/hf_utils.py:101` + +Both call sites previously wrapped `get_hf_file_to_dict` in `try/except`. `get_hf_file_to_dict` swallows hub/network errors and returns `None`, but a malformed index still raises `json.JSONDecodeError`. In `is_diffusion_model` that now aborts detection before Strategy 3, making the Bagel/DreamZero `config.json` fallback (`hf_utils.py:121`) unreachable. Wrap the loop body in `get_diffusion_model_index` and keep returning `None`. + +### 6. `supported_tasks` fallback is asymmetric between the two partitions +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:582` + +`ref2va_release.get("tasks", ["ref2va"])` yields `[]` when the key is present but empty, so combined mode would construct and load `transformers_ref` and then reject every `ref2va` request. Line 570 already handles the FL2VA side with `if not supported_tasks:` — use the same guard here. + +--- + +## Worth considering + +### 7. Default flip to combined mode doubles weights and download +`pipeline_minimax_h3.py:133-134` maps `auto → combined`. Per the recipe that's ~270 GiB on disk and both DiTs resident (`recipes/MiniMaxAI/MiniMax-H3.md:66-69`). The MUSA recipe compensates with prose only — *"do not use the modular default (combined) mode on MUSA until its two-DiT memory and performance profile is validated"* (`MiniMax-H3-MUSA.md:24-26`) — with no code-side guard. I'd either make combined explicit (`--task-type combined`, default single-partition) or emit a loud startup log stating the extra HBM/disk before the download starts. Related: `_minimax_h3_partition_for_task` accepts `"combined"` but the error message at line 139 doesn't list it. + +### 8. `--task-type` typos for diffusion surface very late +`vllm_omni/config/model.py:213-218` validates only `Qwen3TTSTalkerForConditionalGenerationARVLLM`, and `serve.py:213` dropped `choices=`. `--task-type ref2v` is only rejected inside worker `__init__`, potentially after a multi-GB download. A model-arch-keyed validator table alongside `_QWEN3_TTS_TASK_TYPES` would catch it at parse time. + +### 9. SP/compile dotted-path blind spot (latent, not yet firing) +`registry.py:437-439` and `diffusion_model_runner.py:283-287` have the same flat-lookup issue as #1 — `find_module_with_attr` uses `hasattr`, so Cosmos3's `"transformer.language_model"` and SoulX's `"cfm_decoder.model.diff_estimator"` are skipped. I checked: today Cosmos3 has a non-dotted second entry, and Bagel/SenseNova define neither `_sp_plan` nor `_repeated_blocks`, so behavior is unchanged — but this becomes a silent no-op the moment one of those models grows an SP plan. Apply the same `attrgetter` fix. + +### 10. `cache_summary` fallback narrowed +`backend.py:40` defaults to `("transformer",)` for pipelines with no `_dit_modules`; the old code covered `transformer_2` too. A diffusers-adapter Wan MoE now loses `transformer_2` from the summary. Keep `("transformer", "transformer_2")` as the fallback. + +### 11. `load_weights` contiguity coupling +`pipeline_minimax_h3.py:694-701`. The `groupby` assumption holds for `DiffusersPipelineLoader.get_all_weights` (`diffusers_loader.py:293-299` yields sources in order), so this is correct today — but it also requires `MiniMaxH3DiTModel.load_weights` to fully drain each sub-iterator, which is an invariant nothing enforces. Routing into a `{prefix: list}` dict would drop both that coupling and the "is not contiguous" error branch. + +### 12. Naming +`transformers_ref` (plural noun) reads like a collection and breaks the `transformer` / `transformer_2` convention; `transformer_ref` would be clearer. It's baked into weight prefixes and `_dit_modules`, so renaming later is a migration — cheap to fix now, not later. + +--- + +## Docs + +- `recipes/MiniMaxAI/MiniMax-H3.md:194` still passes `MODEL_ROOT=/path/to/MiniMax-H3` to `run_h3_2gpu_all_tasks.sh`, but the Prerequisites block that defined `MODEL_ROOT` and the `hf download --local-dir` command was deleted in this PR. That invocation now references an undefined variable. +- `recipes/MiniMaxAI/MiniMax-H3-NPU.md` drops `HF_HUB_OFFLINE=1`/`TRANSFORMERS_OFFLINE=1` and rewrites *"Validated on eight NPUs of an Atlas 800I A3 server"* → *"Use ..."*. That's a self-acknowledged downgrade from a validated recipe to a suggested one, on the same PR where `buildkite/vllm-omni-npu-ci` is failing. +- Both `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are red — those gates need to go green or be explicitly waived. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5720.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5720.trace.json.gz new file mode 100644 index 000000000..760d17f94 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5720.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5723.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5723.cost.json new file mode 100644 index 000000000..e6f89ba3a --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5723.cost.json @@ -0,0 +1,68 @@ +{ + "calls": 46, + "input_tokens": 2286681, + "output_tokens": 32087, + "cost_usd": 2.8094650000000008, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout4", + "head": "93c0112c114ec8a1e0bf9913e883f8f3580a541c", + "base": "a874b8e09bb5251bb16e3de8bc22edf3880eec09", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "71fc8f2a4be1868ee95aba893e56e3f3aa3b33f1d13085db9424cce3ecc4598d", + "validation_commands": [ + "git diff --stat a874b8e09bb5251bb16e3de8bc22edf3880eec09 HEAD", + "git diff a874b8e09bb5251bb16e3de8bc22edf3880eec09 HEAD", + "ls && ls docs/ && cat mkdocs.yaml 2>/dev/null | head -60", + "grep -n \"sup>\" docs/models/supported_models.md; echo ---; grep -n \"slugify\\|toc\\|markdown_extensions\" -A12 mkdocs.yml | head -60", + "grep -rn \"recipes.vllm.ai\" --include=*.md --include=*.yml . | head -30", + "sed -n 140,200p mkdocs.yml", + "grep -rn \"recipes.vllm.ai\" . --include=\"*.md\" --include=\"*.yml\" 2>/dev/null | head -20", + "python -c \"\nfrom markdown.extensions.toc import slugify\nprint(repr(slugify('AMD ROCm (gfx942 / gfx950)','-')))\n\" 2>&1 | tail -3; echo \"---nav---\"; grep -n \"recipes\" mkdocs.yml | head; echo \"---readme---\"; sed -n 1,40p recipes/README.md", + "ls .github/workflows/ && grep -rln \"markdown\\|lint\" .github/workflows/ | head", + "grep -n \"MiniMax\" recipes/README.md; echo \"=== docs links to recipes ===\"; grep -rn \"recipes/\" docs/*.md docs/**/*.md 2>/dev/null | grep -v \"recipes.vllm.ai\" | head -20; echo \"=== show prev commit ===\"; git show 93c0112c --stat && git show 93c0112c", + "cat .pre-commit-config.yaml | head -80", + "ls docker/; echo ===; grep -rn \"VLLM_OMNI_TARGET_DEVICE\" --include=\"*.py\" --include=\"*.md\" --include=\"*.sh\" . | head -20", + "grep -rn \"aiter\" vllm_omni/ --include=\"*.py\" -l | head -30", + "grep -rn \"VLLM_OMNI_TARGET_DEVICE\" -r . --include=\"*.py\" --include=\"*.md\" --include=\"*.sh\" --include=\"Dockerfile*\" 2>/dev/null | head -20; echo \"=== rocm dockerfile head ===\"; head -40 docker/Dockerfile.rocm", + "sed -n 1,120p vllm_omni/diffusion/attention/backends/utils/fa.py", + "sed -n 1,80p vllm_omni/platforms/rocm/patch/worker/patch_groupnorm.py", + "grep -rn \"torchcodec\\|TorchCodec\" docker/Dockerfile.rocm docker/Dockerfile.cuda requirements/*.txt 2>/dev/null | head -20; echo \"=== full rocm dockerfile ===\"; sed -n 40,200p docker/Dockerfile.rocm", + "ls vllm_omni/diffusion/models/minimax_h3/ 2>/dev/null || grep -rln \"minimax\" vllm_omni/diffusion/ | head -20", + "grep -n \"self\\.vae\\|vae\" vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | head -40", + "timeout 12 curl -s \"https://hub.docker.com/v2/repositories/vllm/vllm-omni-rocm/tags?page_size=100\" | head -c 1500; echo; echo \"exit=$?\"", + "grep -rn \"GroupNorm\" vllm_omni/diffusion/models/minimax_h3/vae.py | head; echo \"=== platform rocm ===\"; sed -n 1,120p vllm_omni/platforms/rocm/platform.py", + "sed -n 120,190p vllm_omni/platforms/rocm/platform.py", + "grep -n \"class \\|norm\\|Norm\" vllm_omni/diffusion/models/minimax_h3/vae.py | head -50", + "sed -n 105,200p vllm_omni/diffusion/models/minimax_h3/vae.py", + "grep -rn \"short_edge\" vllm_omni/ --include=\"*.py\" | head -30", + "grep -rln \"minimax_h3\\|minimax-h3\\|MiniMaxH3\" vllm_omni/entrypoints vllm_omni/diffusion/utils 2>/dev/null | head -20", + "sed -n 730,860p vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py", + "grep -n \"_resolve_minimax_h3_aspect_ratio\" -A 45 vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | sed -n 1,70p", + "grep -rn \"audio_reference\\|MAX_IMAGES\\|images \u22649\\|len(images)\\|too many\\|at most\" vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | head -40", + "sed -n 305,330p vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py; echo \"=== 1560-1600 ===\"; sed -n 1560,1600p vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py", + "grep -rn \"FLASHINFER_DISABLE_VERSION_CHECK\" . --include=\"*.md\" --include=\"*.py\" --include=\"*.sh\" 2>/dev/null | head", + "grep -rn \"cfg_parallel\\|cfg-parallel\" vllm_omni/diffusion/models/minimax_h3/*.py | head; echo \"=== ring ===\"; grep -rn \"ring\" vllm_omni/diffusion/models/minimax_h3/*.py | head", + "sed -n 1,45p recipes/MiniMaxAI/MiniMax-H3-MUSA.md", + "git log --oneline -8 -- recipes/MiniMaxAI/MiniMax-H3-MUSA.md; echo ===; git log --oneline -5 -- docs/models/supported_models.md", + "grep -n \"partition\\|supported_tasks\" vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | head -40", + "sed -n 520,600p vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py", + "grep -n \"MODEL\\|vllm serve\" recipes/MiniMaxAI/MiniMax-H3-MUSA.md | head -30", + "grep -n \"_resolve_minimax_h3_model_root\" -A 60 vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | sed -n 1,80p", + "sed -n 120,145p vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py", + "ls requirements/; grep -rn \"aiter\" requirements/ pyproject.toml setup.py 2>/dev/null | head", + "cat requirements/rocm.txt; python -c \"import vllm, inspect; from vllm import _aiter_ops; print(inspect.getsource(_aiter_ops.is_aiter_found_and_supported))\" 2>&1 | tail -20", + "git diff a874b8e09bb5251bb16e3de8bc22edf3880eec09 HEAD | grep -n \" $\" | head; echo \"no-trailing-ws-above\"; awk 'NR>=394 && NR<=490 {print length($0)\"\\t\"NR}' recipes/MiniMaxAI/MiniMax-H3.md | sort -rn | head -5" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-15T06:21:49", + "arm": "baseline_opus5", + "wall_s": 446.1, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [], + "artifact_chars": 10223 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5723.md b/eval/dataset/baselines/claudecode_opus5/pr5723.md new file mode 100644 index 000000000..020b13dd3 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5723.md @@ -0,0 +1,93 @@ +I've finished investigating. Here's the review. + +--- + +# Review: PR #5723 — [Recipe] Document MiniMax H3 ROCm (gfx942 / gfx950) BF16 serving + +## Verdict + +**Request changes** (already merged, so treat these as follow-ups). + +The ROCm serving content is welcome and the parts I could check against code hold up well: the gfx942/gfx950 scoping matches the actual gate in `vllm_omni/platforms/rocm/platform.py:95-99` (`is_aiter_found_and_supported() and 90 < capability < 100`), `FLASH_ATTN` really does resolve to AITER packed varlen (`vllm_omni/diffusion/attention/backends/utils/fa.py:38-45`), and the CFG / VAE-tile-mode caveats match `pipeline_minimax_h3.py:551` and `vae.py:156-166`. + +But three things are wrong rather than merely imprecise: + +1. `supported_models.md` does **not** actually mark AMD support — the footnote is orphaned, so the PR's headline change didn't land. +2. A rewritten "Known limitations" bullet asserts a reference-count restriction the code does not have, contradicting the same file's input matrix and its own curl examples. +3. Both ROCm `vllm serve` commands point `MODEL` at the `FL2VA` partition, which makes `task=ref2va` a hard error — directly under a paragraph claiming "All tasks work out of the box". + +--- + +## Blocking + +### 1. The AMD checkmark and the footnote marker are missing — `docs/models/supported_models.md:47`, `:101` + +Line 47 is unchanged by this PR: + +``` +| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, ... | `MiniMaxAI/MiniMax-H3` | ✅︎ | | | | +``` + +The AMD GPU cell is still empty, and `H3` appears nowhere in the table (`grep 'sup>'` returns only line 24's `1` and the new footnote itself). So the file now carries a dangling footnote that explains a support claim the table never makes. The PR description says "mark MiniMax H3 AMD GPU supported" — that edit is absent from the diff. + +Fix (matching the existing marker convention on line 24): + +``` +| `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VAH3 | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | +``` + +### 2. The reference-limit bullet is factually wrong — `recipes/MiniMaxAI/MiniMax-H3.md:727-731` + +> "the current vLLM-Omni path takes exactly one image plus one audio reference, or one or more videos with no separate `audio_reference`" + +`_validate_ref2va_reference_counts` (`pipeline_minimax_h3.py:309-326`, called at `:1587`) accepts up to 9 images, 3 videos, 3 standalone audios, 12 total, with the only structural rule being "at least one image or video". There is no one-image/one-audio restriction and no bar on video + standalone `audio_reference`. + +This also contradicts two other places in the same file: the input matrix at line 663 (`images ≤9, videos ≤3, audios ≤3, total ≤12`) and the examples at lines 562-645 ("Image-only Ref2VA omits `audio_reference`…", "A video may use its embedded soundtrack, a separate `audio_reference`, or both"). + +The stale claim predates this PR, but the PR rewrote and expanded these exact lines into a more authoritative-sounding paragraph, so it should have been deleted instead. Drop the bullet; the input matrix already documents the real contract. + +### 3. `MODEL` points at the FL2VA partition, so Ref2VA is rejected — `recipes/MiniMaxAI/MiniMax-H3.md:426`, `:446` (vs. `:416-418`) + +`export MODEL="${MODEL_ROOT}/FL2VA"` resolves to `partition="fl2va"` (`pipeline_minimax_h3.py:125-141`, plus the parent-dir handling at `:150-152`), so `supported_tasks` is `{t2va, fl2va}` and any `task=ref2va` request fails at `:734-736` with *"checkpoint partition 'fl2va' supports ['fl2va', 't2va']"*. Meanwhile lines 416-418 tell the reader "All tasks work out of the box: the image bundles TorchCodec (for image+audio Ref2VA) and `ffmpeg` (for video-reference Ref2VA)", and the HTTP examples the reader goes to next include two Ref2VA requests. The evidence table also has no Ref2VA row, so nothing here supports the "all tasks" claim. + +Two secondary problems in the same lines: `MODEL_ROOT` is never defined anywhere in this recipe (it appears once at line 205 as a script env var), and every other serve command in the file uses `export MODEL=MiniMaxAI/MiniMax-H3`. + +I suspect the partition path was deliberate: under pure Ulysses each rank holds a full DiT copy, so combined serving is 2 × ~134 GiB per GPU, which does not fit MI300X's 192 GiB. If so, say that explicitly, e.g.: + +> These commands serve the `FL2VA` partition only (T2VA and FL2VA). Combined serving replicates both DiTs on every Ulysses rank (~270 GiB), which does not fit gfx942/gfx950 HBM without offload; run a second server on `${MODEL_ROOT}/Ref2VA` for `ref2va`. Set `MODEL_ROOT` to a local MiniMax-H3 checkout containing the sibling `FL2VA/` and `Ref2VA/` directories. + +and soften "All tasks work out of the box" to a statement about the image's decoder dependencies rather than about these servers. + +--- + +## Should fix + +### 4. "when the model shards across the GPUs" contradicts the bullet this PR added — `recipes/MiniMaxAI/MiniMax-H3.md:440-443` + +The four-GPU command uses `--usp 4` with TP left at 1, so the DiT is replicated, not sharded — exactly what the new limitation at lines 740-742 says ("Pure Ulysses still replicates the full DiT on every rank"). Suggest: "…with no CPU offload; each rank holds a full copy of the FL2VA DiT, and only the text encoder is sharded (`--text-encoder-tp-size 4`)." + +### 5. `aiter` is a hard requirement for the documented path, and its absence is nearly silent — `recipes/MiniMaxAI/MiniMax-H3.md:400-402`, `:410-414` + +`aiter` is not in `requirements/rocm.txt` — it comes from the ROCm base image. Users who follow the wheel install (lines 410-414) may not have it, in which case `platform.py:110-118` logs a single `warning` and silently returns `TORCH_SDPA`, i.e. roughly half the throughput per this PR's own comparison. The CUDA sections model the right pattern ("Confirm the server log contains `Defaulting to diffusion attention backend TRTLLM_ATTN`"); the ROCm equivalents are `logger.debug`, so grepping for them won't work by default. Add something like: + +> `FLASH_ATTN` requires the `aiter` library (present in the ROCm image; install it separately on the wheel path). It is already the ROCm default when aiter is usable (`platform.py:126-129`); passing the flag is explicit belt-and-braces. If the server log shows *"Flash Attention requires `aiter` library which is only supported on gfx942 and gfx950. Falling back to TORCH_SDPA backend."*, the measurements below do not apply. + +### 6. "The VAE uses AITER GroupNorm on ROCm" is broader than what happens — `recipes/MiniMaxAI/MiniMax-H3.md:406` + +`patch_groupnorm.py:41-52` only rewrites `model.vae`, which for H3 is `self.vae = self.video_vae` (`pipeline_minimax_h3.py:677`); the audio VAE is untouched. The swap is also skipped silently (`except Exception: logger.warning("Failed to apply AITER GroupNorm to VAE.")`) when aiter is missing. Say "video VAE", and point at the `AITER GroupNorm is enabled for VAE.` info log as the confirmation — or drop the sentence, since it doesn't change anything the reader types. + +### 7. The gfx950 evidence row isn't reproducible from this recipe's own API — `recipes/MiniMaxAI/MiniMax-H3.md:481` + +"T2VA, 832x480, ~4 s, 40 steps": T2VA requires an explicit named aspect ratio even when `width`/`height` are given (`pipeline_minimax_h3.py:346-348`, `:354-355`), and 832x480 is 26:15 — not one of the six supported names. So the row can only have been produced by passing some named ratio that disagrees with the canvas. Since this is the *sole* evidence behind the "gfx950 validated" claim in the supported-models footnote, either state the exact request used, or re-report at 1344x768/50 steps so it lines up with the gfx942 rows and the documented T2VA curl. As written, a reader copying the §1 curl gets a very different shape than the one behind "~0.73 s/denoise-step". + +--- + +## Nits + +- **`docs/models/supported_models.md:105`** — every other docs→in-repo-recipe link uses `https://github.com/vllm-project/vllm-omni/blob/main/recipes/...` (6 occurrences, e.g. `docs/user_guide/diffusion/cpu_offload.md:388`), and `recipes/README.md:17-20` says only the TTS recipes are published to recipes.vllm.ai. So `https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3` most likely resolves to the *upstream* `vllm-project/recipes` page — a different document that does not contain the ROCm commands the footnote promises — or 404s. The blob URL fixes that and still avoids the mkdocs `--strict` warning that motivated 93c0112c. +- **`recipes/MiniMaxAI/MiniMax-H3.md:486`** — "MI325X (gfx942) and other MI355X SKUs" reads as if MI325X were an MI355X SKU. Suggest "MI325X (gfx942) and MI355X (gfx950) are not listed until their own evidence is added." Line 481 and the footnote should say MI350X, not MI350. +- **`recipes/MiniMaxAI/MiniMax-H3.md:63` / `:394`** — `#amd-rocm-gfx942--gfx950` is right for GitHub, but python-markdown's slugify collapses the double hyphen, so it breaks anywhere this file is rendered by MkDocs. Renaming the heading to `## AMD ROCm (gfx942, gfx950)` yields `amd-rocm-gfx942-gfx950` under both slugifiers. +- **`recipes/MiniMaxAI/MiniMax-H3.md:404-406`** — "drop the CUDA-only `FLASHINFER_DISABLE_VERSION_CHECK`" refers to a variable that appears in none of this recipe's CUDA commands; it's set automatically in `stage_engine_core_proc.py:122`. Remove it from the delta list. +- **`recipes/README.md:48`** — the index row still reads "1x GPU with CPU offload / 4x B300"; add the AMD hardware so ROCm support is discoverable from the recipe index. (Note the repo's other non-CUDA H3 target got its own file, `MiniMax-H3-MUSA.md`; an inline section is fine, but the index should reflect it either way.) +- **`recipes/MiniMaxAI/MiniMax-H3.md:416`** — `vllm/vllm-omni-rocm:minimax-h3` doesn't match the documented tagging convention for that Docker Hub repo (release versions, `installation/gpu/rocm.inc.md:117-133`). `docker/Dockerfile.rocm` on main already installs ffmpeg (line 24) and TorchCodec (lines 41-59), so consider pointing at the standard image or the Dockerfile, or note who publishes and maintains this one-off tag. +- New prose wraps at ~90 columns while the rest of the file wraps at ~78. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5723.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5723.trace.json.gz new file mode 100644 index 000000000..b8a2ba998 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5723.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5732.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5732.cost.json new file mode 100644 index 000000000..247d1f5a0 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5732.cost.json @@ -0,0 +1,55 @@ +{ + "calls": 35, + "input_tokens": 1450933, + "output_tokens": 34613, + "cost_usd": 2.1203485000000004, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout5", + "head": "82a142ad19c2b17e0a0a02109d69ef05b8a0d2ee", + "base": "e0c2b9d73fa3840e0b6083901a69f13f62a99d9d", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "1ad7dd62888049857fdc09ac87be6332f293efe18fd9166ecbca59f2d70bc14e", + "validation_commands": [ + "git diff e0c2b9d73fa3840e0b6083901a69f13f62a99d9d HEAD --stat", + "git diff e0c2b9d73fa3840e0b6083901a69f13f62a99d9d HEAD", + "python -c \"\nimport numpy as np\nprint(np.__version__)\nfor n in [2, 31, 32, 33, 64, 65, 100, 362]:\n try:\n r = np.result_type(*([np.dtype('float32')]*n))\n print(n, 'ok', r)\n except Exception as e:\n print(n, 'FAIL', type(e).__name__, e)\n\"", + "grep -rn \"numpy\" requirements/*.txt setup.py pyproject.toml 2>/dev/null | head -40", + "timeout 600 python -m pytest -q tests/entrypoints/openai_api/test_video_api_utils.py 2>&1 | tail -20", + "ls requirements* 2>/dev/null; grep -rn \"numpy\" --include=*.txt --include=*.toml --include=*.cfg . 2>/dev/null | grep -v node_modules | head -30", + "grep -rn \"numpy\" requirements/ pyproject.toml 2>/dev/null | head -30", + "ls; echo ---; ls requirements 2>/dev/null || ls -d */requirements* 2>/dev/null; echo ---; find . -maxdepth 2 -name \"requirements*\" -o -maxdepth 2 -name \"pyproject.toml\" | head", + "cat requirements/common.txt; echo \"=== npu ===\"; cat requirements/npu.txt", + "grep -rn \"numpy\" docker/ docs/ setup.py pyproject.toml 2>/dev/null | head -20; echo \"=== python -c limit test ===\"; python -c \"\nimport numpy as np\ntry:\n print(np.result_type(*([np.dtype('float32')]*5000)))\nexcept Exception as e: print('FAIL', e)\nprint(np.__version__)\n\"", + "pip download \"numpy==1.26.4\" -d /tmp/npdl --no-deps -q 2>&1 | tail -3; ls /tmp/npdl 2>/dev/null", + "grep -rn \"_coerce_video_to_uint8_frames\\|_encode_video_bytes\\|_coerce_video_to_frames\" --include=*.py . | grep -v \"video_api_utils.py\"", + "grep -rn \"_coerce_video_to_uint8_frames\\|_encode_video_bytes\" vllm_omni/entrypoints/openai/video_api_utils.py", + "cat > /tmp/diff_test.py <<'EOF'\nimport numpy as np\n\ndef old(frames):\n frames_np = np.stack(frames, axis=0)\n if frames_np.ndim == 4 and frames_np.shape[-1] == 4:\n frames_np = frames_np[..., :3]\n if frames_np.dtype == np.uint8:\n frames_u8 = frames_np\n else:\n frames_np ", + "python - <<'EOF'\nimport numpy as np\n\ndef old(frames):\n frames_np = np.stack(frames, axis=0)\n if frames_np.ndim == 4 and frames_np.shape[-1] == 4:\n frames_np = frames_np[..., :3]\n if frames_np.dtype == np.uint8:\n frames_u8 = frames_np\n else:\n frames_np = np.clip(frame", + "python -c '\nimport numpy as np\ndef old(frames):\n a = np.stack(frames, axis=0)\n if a.ndim == 4 and a.shape[-1] == 4:\n a = a[..., :3]\n if a.dtype == np.uint8:\n u = a\n else:\n a = np.clip(a, 0.0, 1.0) * 255.0\n u = np.round(a).astype(np.uint8)\n return np.asconti", + "ls .buildkite 2>/dev/null; find . -path ./node_modules -prune -o -name \"*.yaml\" -path \"*buildkite*\" -print 2>/dev/null | head; ls -a | head -30", + "grep -rn \"video_api_utils\\|entrypoints/openai_api\" .buildkite/npu .buildkite/amd .buildkite/common 2>/dev/null | head -20", + "ls -R .buildkite | head -60", + "grep -rn \"test_video_api_utils\\|openai_api\\|entrypoints\" .buildkite/npu/test-npu-ready.yml .buildkite/amd/test-amd-ready.yml 2>/dev/null | head -20", + "head -60 .buildkite/npu/test-npu-ready.yml; echo \"=== AMD ===\"; head -60 .buildkite/amd/test-amd-ready.yml", + "grep -rn \"tests/entrypoints\" .buildkite/ | head -20", + "python -c '\nimport resource, torch, numpy as np, time\nfrom vllm_omni.entrypoints.openai import video_api_utils as v\nF,H,W = 120, 768, 768\nvid = torch.rand(F,H,W,3, dtype=torch.float32)*2-1\nbase = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024\nt=time.time()\nout = v._coerce_video_to_uint8_fra", + "python -c '\nimport resource, torch, time\nfrom vllm_omni.entrypoints.openai import video_api_utils as v\nF,H,W = 120, 768, 768\nvid = torch.empty(F,H,W,3, dtype=torch.float32).uniform_(-1,1)\nbase = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024\nt=time.time()\nout = v._coerce_video_to_uint8_fram", + "python -c '\nimport resource, torch, numpy as np, time\nimport vllm_omni.entrypoints.openai.video_api_utils as v\n\ndef patched(video_tensor):\n video_tensor = video_tensor.detach().cpu()\n if video_tensor.dim() == 5: raise ValueError\n if video_tensor.is_floating_point():\n video_tensor = v", + "python -c '\nimport resource, torch, time\nimport vllm_omni.entrypoints.openai.video_api_utils as v\n\ndef patched(video_tensor):\n video_tensor = video_tensor.detach().cpu()\n if video_tensor.is_floating_point():\n video_tensor = video_tensor.float().clamp(-1, 1)\n video_tensor.mul_(0.5", + "python -c '\nimport numpy as np, time\ndef old(frames):\n a = np.stack(frames, axis=0)\n if a.ndim == 4 and a.shape[-1] == 4: a = a[..., :3]\n if a.dtype == np.uint8: return np.ascontiguousarray(a)\n a = np.clip(a, 0.0, 1.0); a *= 255.0\n return np.ascontiguousarray(np.round(a).astype(np.uin" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-16T00:48:42", + "arm": "baseline_opus5", + "wall_s": 556.4, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [ + "Write: path outside the worktree: /tmp/diff_test.py in {\"file_path\": \"/tmp/diff_test.py\", \"content\": \"import numpy as np\\n\\ndef old(frames):\\n frames_np = np.stack(frames, axis=0)\\n if frames_np.ndim == 4 and " + ], + "artifact_chars": 6317 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5732.md b/eval/dataset/baselines/claudecode_opus5/pr5732.md new file mode 100644 index 000000000..857d24c83 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5732.md @@ -0,0 +1,74 @@ +## Verdict + +**Approve — the change is correct and the win is real.** I verified this rather than taking the description's word for it: + +- **Differential fuzz vs. the old implementation** across 20 input classes (float16/32/64 × 1/3/4 channels, 2-D frames, non-contiguous per-frame views, out-of-range values, NaN, bool, ragged) — byte-identical output in every case reachable through `_coerce_video_to_frames`, inputs unmutated, output C-contiguous. The only divergence is an unreachable one (finding 2). +- **Speed**: 0.86 s → 0.24 s (~3.6×) on an in-RAM 810 MiB float32 video, output identical. So the headline 79 s → 0.97 s is swap-dominated and environment-specific, but directionally right. +- `pytest -q tests/entrypoints/openai_api/test_video_api_utils.py` → **13 passed** locally (the test plan says 11; stale count). + +The main thing I'd push back on is *scope framing*: the PR is titled "bound memory during video frame conversion," but after this change the dominant allocation on the video path is still unbounded, upstream of the function that was touched. Details below. + +--- + +## Comments + +### 1. `vllm_omni/entrypoints/openai/video_api_utils.py:315` — the now-dominant peak is untouched (medium, follow-up) + +`_coerce_video_to_uint8_frames` streams frames, but `_coerce_video_to_frames` still materializes the whole normalized float video first. `video_tensor.float().clamp(-1, 1) * 0.5 + 0.5` keeps **two live full-size float32 temporaries** (`clamp` result alive while `*0.5` allocates, then that alive while `+0.5` allocates). + +Measured on this checkout, 120×768×768×3 float32 (810 MiB video), peak RSS delta over the whole `_coerce_video_to_uint8_frames` call: + +| variant | peak delta | ×video | +|---|---|---| +| this PR as merged | 1623 MiB | 2.00× | +| + in-place normalize (below) | 1029 MiB | 1.27× | + +```python +video_tensor = video_tensor.float().clamp(-1, 1) +video_tensor.mul_(0.5).add_(0.5) +``` + +Output is bit-identical (`np.array_equal` → True). This is safe: `.clamp()` always returns a fresh tensor, so the in-place ops can never reach the caller's tensor even when `.float()` returned `self`. Same pattern applies to the numpy path at `video_api_utils.py:338` (`np.clip(...) * 0.5 + 0.5` → `arr = np.clip(...); arr *= 0.5; arr += 0.5`) and to `:317` (`.to(torch.float32).div_(255.0)`). + +At the PR's own 362-frame 1344×768 scale that's roughly **3.3 GiB more off the peak — larger than what this PR already saved**. Not blocking, but it should be the immediate follow-up, and the PR description shouldn't read as if the conversion path is now bounded: the remaining 1.27× is one full float copy plus the uint8 output. + +Minor, same area: for the tensor path, `_normalize_single_video_array:337` re-scans the array with `.min()`/`.max()` even though `_normalize_video_tensor` just guaranteed `[0, 1]` — two wasted full passes. + +### 2. `video_api_utils.py:444` — uint8 fast path contradicts the promotion the next lines emulate (low) + +```python +if frame.dtype == np.uint8: + frames_u8[index] = frame + continue +``` + +The gate is per-frame, but the old `np.stack` behavior it claims to preserve was per-video. For a mixed `[uint8, float32]` list the old path promoted to float32, so uint8 values 0–255 clipped to `[0, 1]` → 0 or 255; the new path copies them through verbatim. My fuzz shows `maxdiff 250` on exactly that input — the single divergence found. + +It's **not a live bug**: `_normalize_frames:373-374` and `_normalize_single_video_array:339-340` convert every integer dtype to float32, so no uint8 frame ever reaches here and the branch is dead in both old and new code. But it's odd to have a branch that silently picks per-frame semantics ten lines above a comment explaining why per-frame promotion had to be avoided. Either gate on `common_dtype == np.uint8`, or drop the branch and leave a comment that integer frames are normalized upstream. + +### 3. `video_api_utils.py:441-442` — new error path is untested, and validation runs after allocation (low) + +`ValueError("All video frames must have the same shape.")` replaces `np.stack`'s message. It's a nicer message, but no test covers it, and it fires *after* `np.empty` (`:434`) and `np.result_type` (`:435`) — a ragged input now allocates the full output buffer before failing. Cheap fix: validate shapes in the `common_dtype` pass before allocating, and add a test asserting the `ValueError`. A uint8-passthrough test would also be worth having if finding 2 keeps that branch. + +### 4. `video_api_utils.py:451` — one heap allocation per frame remains (nit) + +`np.clip(...)` allocates a fresh temporary every iteration (F allocations). A single reusable scratch buffer keeps the semantics exactly and allocates once: + +```python +scratch = np.empty(output_shape, dtype=common_dtype) +... +np.clip(frame, 0.0, 1.0, out=scratch) # widening cast into out is safe +scratch *= 255.0 +np.rint(scratch, out=scratch) +frames_u8[index] = scratch +``` + +Small, but it's free and it's the same idea the PR is already applying. + +### 5. `tests/entrypoints/openai_api/test_video_api_utils.py:63` — this patches the global numpy module (nit) + +`video_api_utils.np` **is** the `numpy` module object, not a module-local alias, so `monkeypatch.setattr(video_api_utils.np, "stack", fail_stack)` breaks `numpy.stack` process-wide for the duration of the test. It's restored by `monkeypatch` and pytest-xdist isolates by process, so it passes today — but it's a global mutation used to assert an implementation detail, and it will fail spuriously if a future refactor legitimately uses `np.stack` on a bounded slice. Prefer asserting the observable property (e.g. `tracemalloc` peak stays well under the full-video float size), or at minimum add a comment that the global patch is deliberate. + +### 6. CI gates + +`buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci` are red and this merged anyway. For what it's worth, `tests/entrypoints` only runs in the CUDA pipelines (`.buildkite/cuda/test-ready.yml:23`, `.buildkite/cuda/test-merge.yml:23`) — neither `.buildkite/amd/test-amd-ready.yml` nor `.buildkite/npu/test-npu-ready.yml` runs it, so those failures are very unlikely to be caused by this diff. That's an argument for triaging them, not for assuming them pre-existing; they're the merge gate and should be confirmed green or explicitly waived. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5732.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5732.trace.json.gz new file mode 100644 index 000000000..8a165b8e8 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5732.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5752.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5752.cost.json new file mode 100644 index 000000000..314a72e7d --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5752.cost.json @@ -0,0 +1,56 @@ +{ + "calls": 53, + "input_tokens": 6486356, + "output_tokens": 38697, + "cost_usd": 5.80282, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout5", + "head": "57bcbfe1bcc150b1ea132c1eb16b4977539efa2b", + "base": "e788ef6e76a93e8fe73c3072064f985ea0b50889", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "411788ca57f4e9c32069ff40797e543c85af4e341ffe593082bab37d89d4249d", + "validation_commands": [ + "git diff --stat e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD", + "git diff e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD -- vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py", + "wc -l vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py vllm_omni/diffusion/models/minimax_h3/reference_video.py vllm_omni/entrypoints/openai/api_server.py vllm_omni/entrypoints/openai/video_api_utils.py vllm_omni/entrypoints/openai/serving_video.py", + "grep -n \"def get_minimax_h3_post_process_func\" -A 80 vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | head -100", + "grep -n \"_minimax_h3_post_process\" -r vllm_omni | head; grep -n \"def _minimax_h3_post_process\" -A 60 vllm_omni/diffusion/models/minimax_h3/*.py", + "grep -rn \"audio_sample_rate\" vllm_omni | grep -v minimax_h3 | head -30", + "git diff e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD -- vllm_omni/entrypoints/openai/serving_video.py vllm_omni/entrypoints/openai/protocol/videos.py vllm_omni/entrypoints/openai/video_api_utils.py", + "git diff e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD -- vllm_omni/entrypoints/openai/api_server.py > /tmp/api.diff; wc -l /tmp/api.diff", + "git diff e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD -- vllm_omni/entrypoints/openai/api_server.py", + "git diff e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD -- vllm_omni/diffusion/models/minimax_h3/reference_video.py vllm_omni/diffusion/models/minimax_h3/condition_noise.py vllm_omni/diffusion/data.py vllm_omni/diffusion/model_metadata.py vllm_omni/engine/async_omni_engine.py", + "grep -rn \"keyframe_frame_indices\" vllm_omni/diffusion/models/minimax_h3/ | head -20", + "grep -rn \"num_outputs_per_prompt\" vllm_omni/ | head -20", + "grep -rn \"def encode_video_with_audio\\|audio_array\\|\\\"audio\\\"\" vllm_omni/entrypoints/openai/video_api_utils.py | head -30", + "grep -rn \"audio\" vllm_omni/entrypoints/openai/serving_video.py | head -40", + "grep -rn \"def decode_latent\" -A 25 vllm_omni/diffusion/models/minimax_h3/*.py | head -60", + "git diff e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD -- tests/diffusion/models/minimax_h3/ tests/engine/test_orchestrator_error_handling.py", + "git diff e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD -- tests/entrypoints/openai_api/test_video_server.py", + "grep -n \"seconds\" vllm_omni/entrypoints/openai/protocol/videos.py | head -20", + "git diff e788ef6e76a93e8fe73c3072064f985ea0b50889 HEAD -- recipes/ docs/", + "grep -n \"async def decode_image_url\" -A 30 vllm_omni/entrypoints/openai/video_api_utils.py", + "timeout 600 python -c \"\nfrom types import SimpleNamespace\nfrom vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline\np = object.__new__(MiniMaxH3Pipeline)\ns = SimpleNamespace(fps=24, num_frames=209, height=768, width=1344, extra_args={'task':'t2va'})\ntry:\n print(p._re", + "timeout 600 python -c \"\nfrom types import SimpleNamespace\nfrom vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline\np = object.__new__(MiniMaxH3Pipeline)\nfor kw in [dict(num_frames=29, extra={'aspect_ratio':'16:9'}), dict(num_frames=1, extra={'duration':15.0,'aspect_rat", + "sed -n 1,60p vllm_omni/errors.py", + "grep -rin \"heif\\|heic\" requirements/ pyproject.toml setup.py 2>/dev/null | head", + "grep -n \"video_audio\\|kind\" vllm_omni/diffusion/models/minimax_h3/packed_sequence.py | head -40", + "grep -rn \"multi_modal_data\\[\\\"video\\\"\\]\\|multi_modal_data.get(\\\"video\\\")\\|\\\"video\\\"\" vllm_omni/diffusion/models/ --include=*.py | grep -v minimax_h3 | head -20", + "grep -n \"_video_payload_to_frames\" -A 30 vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py | head -45", + "grep -n \"def _pil_to_rgb\" -A 20 vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-16T00:49:36", + "arm": "baseline_opus5", + "wall_s": 610.2, + "audit_ok": false, + "audit_violations": [ + "Read: path outside the worktree: /tmp/cc-arm-home-cvwq0d17/.claude/projects/-data-zhoutaichang--infermatrix-copilot-worktrees-vllm-omni-pr5752/d8b74667-71f9-43e5-a63c-8d407f936107/tool-results/bdxfbp2r3.txt in {\"file_path\": \"/tmp/cc-arm-home-cvwq0d17/.claude/projects/-data-zhoutaichang--infermatrix-copilot-worktrees-vllm-omni-pr5752/d8b74667-71f9-43e5-a63c-8d407f93610" + ], + "audit_blocked_attempts": [], + "artifact_chars": 10981 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5752.md b/eval/dataset/baselines/claudecode_opus5/pr5752.md new file mode 100644 index 000000000..bd8e185ad --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5752.md @@ -0,0 +1,116 @@ +I've finished investigating. Here's the review. + +--- + +# Review: PR #5752 — [Feature] Align MiniMax H3 official input matrix + +## Verdict: **Request changes** (merged — these should be follow-ups) + +The core H3 work is good. The packing layer already supported `[0]/[-1]/[0,-1]` keyframes and `image`/`audio`/`video`/`video_audio` ref blocks (`packed_sequence.py:62`, `:338`), so most of this PR is plumbing that was genuinely missing, and the plumbing is coherent: `condition_labels` ordering matches `ref_blocks` ordering, the multi-output fan-out concatenates on the batch dim in a way that `_extract_audio_outputs`/`_minimax_h3_post_process` actually split correctly, the `ValueError → OmniClientError` sweep is the right call (and safe, since `OmniClientError` subclasses `ValueError`), and the rank-0/broadcast discipline in `_encode_audio_conditions` is correct with the `video_count`-vs-`prepared_videos` comment explaining the subtle part. + +But the PR reaches outside MiniMax H3 into shared entrypoint code, and one of those changes breaks other models. Two required gates are also red (`buildkite/vllm-omni-amd-ci`, `buildkite/vllm-omni-npu-ci`) — those need to be green or explained regardless of everything below. + +--- + +## Blocking + +### 1. Typed `video_reference` now hands every model a file path instead of frames — breaks Wan VACE and Cosmos3 V2V +`vllm_omni/entrypoints/openai/api_server.py:3325-3345`, `vllm_omni/entrypoints/openai/video_api_utils.py:219-268` + +`decode_video_url` now always writes the downloaded/data-URL bytes to a temp `.mp4` and sets `source_path`. `_parse_video_form` then unconditionally prefers that path and **throws the decoded frames away**: + +```python +if media_data.source_path is not None: + video_paths.append(media_data.source_path) +else: + ... + video_frames = list(media_data) +``` + +So for *all* models, `multi_modal_data["video"]` changes from `[PIL.Image, ...]` to `["/tmp/xxx.mp4"]`. Downstream: + +- `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py:143` — `PIL.Image.open(f)` on each string entry → `UnidentifiedImageError` on an MP4. +- `vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:479` — `_pil_to_rgb(raw_video_frames[0])`, which for a `str` does `PIL.Image.open(value)` (`:301`) → same failure. + +The rewritten test bakes this in: `tests/entrypoints/openai_api/test_video_server.py:513` changed from `assert input_video[0].size == (32, 24)` to `assert all(isinstance(item, str) ...)`. A fake engine passing doesn't mean Wan/Cosmos pass. + +Secondary damage: `decode_spec` (`max_video_frames`, `keep="first"|"last"`, derived from stage configs) becomes a no-op for URL video references, since the frames it selected are discarded. + +**Fix:** gate the path-vs-frames choice on capability. `VideoFrames` already carries `source_path`, so the information is available at the call site — pass paths only when `supports_mixed_reference_inputs` (or a dedicated `requires_source_container` flag) is set, and keep `ReferenceVideo(data=frames)` otherwise. Restore the frame-based assertions in `test_r9_typed_video_reference_form` for a non-H3 model and add the path assertions under an H3 model class. + +### 2. `t2va` now rejects requests that fully specify the canvas +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:652-661`, `:210-212` + +`_resolve_minimax_h3_aspect_ratio` is called *before* the `if height is None or width is None` branch, so the "t2va requires an explicit aspect_ratio" error fires even when width and height are both given. Verified against this checkout: + +``` +_resolve_shape('t2va', fps=24, num_frames=209, height=768, width=1344, extra={'task':'t2va'}) +→ OmniClientError: t2va requires an explicit aspect_ratio +``` + +Over HTTP this means `-F 'width=1344' -F 'height=768'` returns 400 demanding a parameter that cannot affect the result. `tests/diffusion/models/minimax_h3/test_minimax_h3_e2e.py:65` had to add `"aspect_ratio": "16:9"` to a test that already sets `height=256, width=448` — that diff line is the regression. + +**Fix:** move the ratio resolution inside the `height is None or width is None` branch (and keep the `short_edge` validation there too — right now `short_edge=720` is silently ignored when width/height are supplied but rejected when they aren't). + +--- + +## Should fix + +### 3. Hard 4 s output floor, and the bound is checked on the wrong quantity +`vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py:636-640` + +Measured on this checkout: + +| input | result | +|---|---| +| `num_frames=29` | `OmniClientError: duration must be in [4, 15] seconds, got 1.208` | +| `duration=15.0` | 362 frames = **15.083 s** — accepted | +| `duration=4.0` | 107 frames = 4.458 s | + +Two problems. First, the check is applied to `requested_frames` *before* `minimax_h3_align_frame_count`, so the function emits clips outside the very window it enforces. Second, an explicit small `num_frames` is now unusable — every smoke test or debug run must render ≥4 s. The e2e test's `num_frames=29` had to be replaced with `duration=4.0` for exactly this reason. + +**Fix:** validate `num_frames / fps` after alignment (or clamp alignment downward at the top of the range), and consider treating an explicit `sampling.num_frames` as an opt-out from the API-level duration policy. + +### 4. Image format/size contract is enforced on only one of three input paths +`pipeline_minimax_h3.py:161-171`, `api_server.py:3038-3060` + +`_load_images` checks the 30 MiB cap and the JPEG/PNG/WEBP/HEIC/HEIF allowlist **only for path/PathLike inputs**. The multipart path enforces it in `_validate_minimax_h3_image_payload`. But typed `image_reference` (data URL or HTTP) is decoded to a `PIL.Image` in `decode_image_url` (`video_api_utils.py:170`) before it ever reaches the pipeline, so it hits the `isinstance(value, Image.Image)` branch and skips both checks. A BMP is a 400 via `input_references` (`test_h3_multipart_rejects_bmp_image_reference`) and a silent success via `image_reference` — and the 30 MiB limit documented in `recipes/MiniMaxAI/MiniMax-H3.md` is unenforced on that path entirely. + +Also: HEIC/HEIF are in the allowlist and the docs, but there's no `pillow-heif` in `requirements/` or `pyproject.toml`, so Pillow can't decode them at all; and when it can, `Image.format` is `"HEIF"`, never `"HEIC"` — that allowlist entry is dead either way. + +### 5. List-form references are silently dropped by the streaming endpoint +`vllm_omni/entrypoints/openai/serving_video_output_stream.py:563-577` + +`image_reference` was widened to `ImageReference | list[ImageReference] | None` (`protocol/videos.py:119`), but this path passes it straight to `decode_input_reference`, which only matches `UrlImageReference`/`FileImageReference`. A list falls through every branch and returns `None` (`video_api_utils.py:356`) — the reference is dropped with no error and generation proceeds unconditioned. Either normalize with `_reference_list` here too, or reject lists explicitly. + +### 6. H3-specific constraints hardcoded into the shared request schema +`vllm_omni/entrypoints/openai/protocol/videos.py:146-165` + +`num_outputs_per_prompt` gets `le=10` — MiniMax H3's limit — applied to every video model. `aspect_ratio`, `short_edge`, and `start_time_seconds` are added as top-level fields whose descriptions all begin "MiniMax H3 …". This is the kind of thing `extra_params`/`video_params` exists for; the per-model bound belongs in the pipeline (where `_resolve_minimax_h3_num_outputs` already re-checks it), not in the generic schema. + +--- + +## Minor + +**7. `ref_audio_t` is clobbered by a loop variable.** `pipeline_minimax_h3.py:1510` — `for ref_audio_t in external_audio_lengths:` reuses the name of the value passed to `diffuse()` at `:1549`. After the loop it holds the last external audio length instead of `None`. Harmless today only because every path that reaches it also sets `ref_blocks`, which short-circuits the `ref_audio_t` fallback in `diffuse` (`:1191-1198`). Rename the loop variable. + +**8. Audio-length bounds re-derived from VAE latents.** `pipeline_minimax_h3.py:1517-1523` rejects any reference whose encoded length is `<80 or >600` latent frames. Durations were already validated to 2–15 s upstream, so this re-derives the same constraint through resampling and VAE striding — a reference at exactly 2.0 s can land on 79 or 80 depending on rounding and 400 out. It also runs *after* every video/audio VAE encode, so bad input pays full encode cost. Validate the known durations in `prepare_reference_videos` / `validate_reference_audio_*` instead. + +**9. Per-rank duplicated ffprobe and audio decode.** `pipeline_minimax_h3.py:1441-1444` — `validate_reference_audio_files` (an ffprobe subprocess per file) and `_load_audios` run on every DiT rank, but only rank 0 encodes (`_encode_audio_conditions:1024`). Video prep is already rank-0 gated via `_prepare_reference_videos:921`; audio should match. On 8 ranks that's 8× ffprobe + 8× decode per audio file. + +**10. `_probe_video` lost `-select_streams v:0` but kept `-count_frames`.** `reference_video.py:46-60` — ffprobe now full-decodes and counts frames for *every* stream, including audio, on every reference. Use a targeted `-select_streams v:0 -count_frames` probe plus a separate cheap probe for the stream/format list. + +**11. Dead code / inconsistencies.** +- `reference_video.py:317` — `target_frame_count` is a required keyword that is immediately `del`'d. Remove it and update the three call sites, or keep it and document why. +- `pipeline_minimax_h3.py:635` — `duration = requested_frames / fps` is assigned and never read. +- `pipeline_minimax_h3.py:1371` — `audio_is_waveform_pair` explicitly anticipates a list-form `[waveform, sample_rate]`, but `_load_audio:197` only accepts the *tuple* form, so that input is counted as one audio and then rejected as unparseable. + +**12. FL2VA silently ignores `aspect_ratio`.** Documented and asserted (`test_minimax_h3_contract.py:659`), and defensible — but silently discarding an explicit client parameter is surprising. Rejecting a conflicting explicit ratio would be friendlier than ignoring it. + +--- + +## Testing + +The 403 new contract-test lines are almost entirely pure-helper tests: `_resolve_fl2va_keyframe_indices`, `_validate_ref2va_reference_counts`, `_resolve_output_canvas`, `_validate_reference_video_metadata` with hand-built metadata dicts. Those are useful and well-parameterized against the R1–R10/F1–F2/G1–G4 matrix. + +What isn't covered is where the risk actually is: nothing exercises `forward()`'s new `condition_labels` + `ref_blocks` assembly (image blocks → video/video_audio blocks → standalone audio blocks), the two-keyframe FL2VA wiring into `minimax_h3_packed_sequence`, or the `num_outputs>1` concat/split round-trip. Those are all mockable at the same level as `test_distributed_video_vae_encodes_references_sequentially` already does — worth adding a `forward()`-level test with fake VAEs that asserts the emitted `ref_blocks` sequence for the R5 mixed case. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5752.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5752.trace.json.gz new file mode 100644 index 000000000..1067b30f3 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5752.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5756.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5756.cost.json new file mode 100644 index 000000000..edc5a8e80 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5756.cost.json @@ -0,0 +1,77 @@ +{ + "calls": 52, + "input_tokens": 2842203, + "output_tokens": 34422, + "cost_usd": 3.0533745000000003, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout4", + "head": "43592108ac79fb20e1254ef02eff6db9755dc6b1", + "base": "900a7f0813d0482811b0e4dfd3cf7deabbe2429f", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "c9a9f03d9dd8e72c03725e050fe30ab736d2fb1adc4c1707c7d2d8d7d6db1516", + "validation_commands": [ + "git diff --stat 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD", + "git diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- apps/ComfyUI-vLLM-Omni/__init__.py apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/", + "sed -n 1,60p apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py && echo ---- && sed -n 100,240p apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py", + "sed -n 1,60p apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py && echo ==== && sed -n 240,420p apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py", + "sed -n 250,400p apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py", + "grep -n \"def video_to_bytes\\|def video_to_base64\\|def audio_to_base64\" -A 25 apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py", + "grep -rn \"input_references\\|audio_reference\\|extra_params\\|input_reference\" --include=*.py -l | head -30", + "grep -rn \"input_references\" --include=\"*.py\" . | head -20; echo ---; grep -rn \"audio_reference\" --include=\"*.py\" . | head -20; echo ---; grep -rn \"extra_params\" --include=\"*.py\" . | head -30", + "sed -n 2950,3070p vllm_omni/entrypoints/openai/api_server.py", + "sed -n 1,200p vllm_omni/entrypoints/openai/protocol/videos.py", + "sed -n 3070,3160p vllm_omni/entrypoints/openai/api_server.py; echo ---PROTO---; sed -n 200,280p vllm_omni/entrypoints/openai/protocol/videos.py", + "grep -rn \"\\\"task\\\"\\|'task'\" --include=\"*.py\" vllm_omni/ | grep -i \"minimax\\|video\\|extra\" | head -30; echo ---; grep -rln \"minimax_h3\\|minimax-h3\\|MiniMaxH3\" --include=\"*.py\" vllm_omni/ | head -30", + "grep -n \"_resolve_task\" -A 40 vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | head -80; echo ---; grep -n \"audio_flow_shift\\|flow_shift\" vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | head -30", + "sed -n 130,180p tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py; echo ---; ls tests/e2e/accuracy/minimax_h3/", + "grep -rn \"flow_shift\" vllm_omni/entrypoints/openai/serving_video.py vllm_omni/entrypoints/openai/*.py | head -20", + "cat apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/validators.py; echo ---MODELS---; sed -n 1,140p apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py", + "cd apps/ComfyUI-vLLM-Omni && python3 -c \"\nimport re\npats = {r'BAGEL-7B-MoT':1, r'Qwen2.5-Omni*':2, r'Qwen3-Omni*':3, r'MiniMax-H3(/FL2VA|/Ref2VA)?':4}\nC={re.compile(k):v for k,v in pats.items()}\ndef lookup(model):\n normalized = model.rstrip('/').lstrip('/')\n key = normalized.split('/',1)[-1]\n ", + "git diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/e2e/features/comfyui/test_comfyui_integration.py apps/ComfyUI-vLLM-Omni/README.md", + "git diff --stat 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- \"*test_comfyui_integration.py\" \"*README.md\"; git diff --name-only 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD", + "git diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/e2e/features/comfyui/test_comfyui_integration.py apps/ComfyUI-vLLM-Omni/README.md | cat", + "git --no-pager diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/e2e/features/comfyui/test_comfyui_integration.py > /tmp/t.diff; wc -l /tmp/t.diff; head -200 /tmp/t.diff", + "git --no-pager diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/e2e/features/comfyui/test_comfyui_integration.py", + "git diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/e2e/features/comfyui/test_comfyui_integration.py", + "git diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/ | head -250", + "git diff --stat 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/; git log --oneline -1; git status --short | head", + "ls -la tests/e2e/features/comfyui/ 2>&1 | head; git config --get core.sparseCheckout; git ls-files tests/e2e/features/comfyui/ | head", + "ls tests/e2e/ | head -30; echo ---; ls tests/e2e/features 2>&1 | head", + "pwd; ls; ls tests 2>&1 | head", + "cd /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5756 && git diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/ README.md apps/ComfyUI-vLLM-Omni/README.md", + "git -C /data/zhoutaichang/.infermatrix-copilot/worktrees/vllm-omni-pr5756 diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- tests/ apps/ComfyUI-vLLM-Omni/README.md", + "git diff 900a7f0813d0482811b0e4dfd3cf7deabbe2429f HEAD -- ../../tests/ ./README.md", + "python3 -c \"import comfy_api, comfy, execution; print(comfy_api.__file__)\" 2>&1 | head -3; pip show comfyui 2>/dev/null | head -3; ls ../../tests/e2e/features/comfyui/", + "python3 -c \"import comfy_api; print(comfy_api.__file__)\" 2>&1 | tail -2", + "ls ../../tests/e2e/features/comfyui/", + "sed -n 1,80p ../../tests/e2e/features/comfyui/conftest.py", + "python3 -c \"\nimport json\nd=json.load(open('example_workflows/vLLM-Omni Video Generation.json'))\nprint(type(d), list(d)[:10])\nnodes=d.get('nodes',[])\nprint('num nodes', len(nodes))\nfor n in nodes:\n print(n.get('id'), n.get('type'), [i.get('name') for i in n.get('inputs',[])], n.get('widgets_values", + "git show 900a7f0813d0482811b0e4dfd3cf7deabbe2429f:\"apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json\" > /tmp/old_wf.json; python3 -c \"\nimport json\nd=json.load(open('/tmp/old_wf.json'))\nfor n in d['nodes']:\n print(n.get('id'), n.get('type'), [i.get('name') for i in n.get('in", + "git show 900a7f0813d0482811b0e4dfd3cf7deabbe2429f:\"apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json\" | python3 -c \"\nimport json,sys\nd=json.load(sys.stdin)\nfor n in d['nodes']:\n print(n.get('id'), n.get('type'), [i.get('name') for i in n.get('inputs',[])], str(n.get('widget", + "python3 -c \"\nimport json\nd=json.load(open('example_workflows/vLLM-Omni Video Generation.json'))\nbyid={n['id']:n for n in d['nodes']}\nfor n in d['nodes']:\n for i in n.get('inputs',[]):\n if i.get('link') is not None or i.get('type','').isupper():\n print(n['id'], n['type'], i['name", + "grep -rn \"minimax_h3\\|\\\"type\\\"\\|'type'\" comfyui_vllm_omni/ web/ --include=\"*.py\" --include=\"*.js\" | grep -v \"input_type\\|content\\[.type.\\]\\|\\\"type\\\": \\\"text\\\"\\|image_url\" | head -30", + "grep -rn \"max_poll_duration\" comfyui_vllm_omni/ | head; echo ---RUFF---; python3 -m ruff check comfyui_vllm_omni/ ../../tests/e2e/features/comfyui/ 2>&1 | tail -20", + "grep -n \"line-length\\|\\[tool.ruff\" -A3 ../../pyproject.toml | head -20; awk 'length > 120 {print FILENAME\": \"FNR\": \"length}' comfyui_vllm_omni/nodes.py comfyui_vllm_omni/utils/api_client.py comfyui_vllm_omni/utils/models.py", + "grep -n \"line-length\" ../../pyproject.toml", + "awk 'length > 120 {print FILENAME\": \"FNR\": \"length}' comfyui_vllm_omni/nodes.py comfyui_vllm_omni/utils/api_client.py comfyui_vllm_omni/utils/models.py", + "ls web/; grep -rn \"VIDEO_PARAMS\\|SAMPLING_PARAMS\\|REMOTE_LORA\" web/ | head -10", + "grep -n \"image\\|GenerateVideo\\|frame\" web/main.js | head -20", + "ls ../../recipes/MiniMaxAI/ 2>/dev/null; grep -n \"ref2va\\|audio_reference\\|input_references\\|flow_shift\\|num_frames\" ../../recipes/MiniMaxAI/MiniMax-H3.md | head -40", + "sed -n 180,300p ../../recipes/MiniMaxAI/MiniMax-H3.md", + "grep -rn \"lookup_model_spec\" ../../tests/ | head", + "grep -n \"VALIDATE_INPUTS\\|\\\"frame\\\": (\\|\\\"references\\\": (\\|model_params\\[.type.\\]\\|params\\[\\\"type\\\"\\]\\|class VLLMOmniVideoReferences\\|image_2\\|audio_2\\|def get_references\" comfyui_vllm_omni/nodes.py", + "grep -n \"max_poll_duration\\|Provide only one of frame\\|references is empty\\|Invalid references combination\\|audio_reference\\\"\\|input_references\\\"\\|params_builder\\|lookup_model_spec\\|video_task = \\|lookup_key\\|MiniMax-H3(\" comfyui_vllm_omni/utils/api_client.py comfyui_vllm_omni/utils/models.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-15T06:22:16", + "arm": "baseline_opus5", + "wall_s": 472.9, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [], + "artifact_chars": 10282 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5756.md b/eval/dataset/baselines/claudecode_opus5/pr5756.md new file mode 100644 index 000000000..3a0e3fc2b --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5756.md @@ -0,0 +1,117 @@ +I've read the full diff, the client/server contract, and the H3 recipe. Here's my review. + +--- + +# Review: [Frontend] Add ComfyUI support for r2v (MiniMax H3 as example) — PR #5756 + +## Verdict + +**Approve with follow-ups.** The core design is sound and the wire format matches the server exactly: `input_reference` (image file), `audio_reference` (JSON `{"audio_url": data-url}`), repeated `input_references` (video files), `flow_shift` as a top-level form field, and `task`/`audio_flow_shift` inside `extra_params` — all line up with `_parse_video_form` (`vllm_omni/entrypoints/openai/api_server.py:2978`), `serving_video.py:189`, and `recipes/MiniMaxAI/MiniMax-H3.md`. The `extra_body` → `extra_params` rename is a real fix: the server never had an `extra_body` form field, so that payload was being silently dropped by FastAPI. + +What I'd want addressed: one silent-failure risk from the `image` → `frame` rename, a validation hook that can never fire, a path-matching regression for a plausible H3 model string, two node inputs that always error, and an example workflow that ships a machine-specific path. + +--- + +## Significant + +### 1. `image` → `frame` rename silently degrades existing workflows to t2v +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:168` + +ComfyUI stores input names in the saved workflow JSON. Every existing user workflow (and the pre-PR template, which used `Wan-AI/Wan2.2-TI2V-5B-Diffusers` with an `image` link) references an input that no longer exists. The link is dropped on load; because `frame` is `optional` and `generate()` has a `**kwargs` catch-all that only logs at INFO (`nodes.py:205-208`), nothing errors — the job just runs as text-to-video and returns a plausible-looking but wrong result. Silent wrong output is the worst failure mode here. + +Suggested change: keep `image` as a deprecated optional alias and fold it in: +```python +"frame": ("IMAGE",), +"image": ("IMAGE",), # deprecated alias for frame +... +if image is not None: + if frame is not None: + raise ValueError("Provide only one of frame or image (image is deprecated).") + logger.warning("'image' input is deprecated; reconnect to 'frame'.") + frame = image +``` + +### 2. `VALIDATE_INPUTS` override never fires, and it disables widget range checks +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:180-187` + +Two problems: + +- `frame` and `references` are link-only inputs. During validation ComfyUI calls `get_input_data(..., outputs=None)`, which marks linked inputs missing and passes `None`. So `frame is not None and references is not None` is unreachable — the exclusivity check never triggers. (The real enforcement is `api_client.py:277`, which does work, so behavior is correct — the override just adds nothing.) +- `**_kwargs` sets `varkw` on the argspec, and ComfyUI skips its own per-widget checks (`min`/`max`, combo membership) for *every* input whose name is covered by `VALIDATE_INPUTS`. With a catch-all that's all of them, so `width`/`height`/`fps`/`num_frames` lose their `min: 1` enforcement on this node. + +Simplest fix: delete the override entirely and rely on `api_client.py:277`. If you want to keep it, at minimum drop `**_kwargs` and keep the signature to `(cls, url, model)`. + +### 3. `lookup_model_spec` drops the first path component — `MiniMax-H3/Ref2VA` no longer matches +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py:126-131` + +`normalized.split("/", 1)[-1]` discards the first component to strip the HF org prefix. That breaks two-component relative paths where the first component *is* the model — exactly the shape of the H3 partition dirs. I ran the new logic: + +| model string | matched | +|---|---| +| `MiniMaxAI/MiniMax-H3` | ✅ | +| `/path/to/MiniMax-H3/Ref2VA` | ✅ | +| `./MiniMax-H3/Ref2VA` | ✅ | +| `MiniMax-H3/Ref2VA` | ❌ **no match** | + +On a miss the `params_builder` is skipped, so `audio_flow_shift` goes out as a bare form field that FastAPI discards, and `task` is never sent — silent parameter loss, no warning. + +Secondly, matching a multi-component remainder with `re.search` is now much looser than the old last-component match: `/data/MiniMax-H3/derived/Wan2.2-T2V-A14B` matches the H3 spec and would route a Wan model through `_minimaxh3_params_builder`, dropping `guidance_scale_2`/`boundary_ratio` with only a log warning. This function is also on the understanding path (`api_client.py:629`, `api_client.py:657`), so the blast radius covers BAGEL/Qwen too. + +I'd just search the full normalized string — it fixes the miss and keeps every case in the `__main__` demo working: +```python +normalized = model.strip("/") +for pattern, spec in MODEL_PIPELINE_SPECS.items(): + if pattern.search(normalized): + return spec, pattern.pattern +``` +Either way, this needs a real unit test — `grep` shows zero test coverage for `lookup_model_spec`; the `__main__` block at `models.py:143` isn't run by CI. + +### 4. `image_2` and `audio_2` are dead inputs that always raise +`apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py:788,790` + `api_client.py:303-329` + +The only accepted combinations are (videos only) and (exactly one image + exactly one audio). Any use of `image_2` or `audio_2` falls into the `else` at `api_client.py:325` and raises "Invalid references combination" — and only after the user hits Run. This matches the server (`pipeline_minimax_h3.py:1041-1050` takes a single image), so the restriction is right, but the UI shouldn't advertise sockets that can never be used. Drop `image_2`/`audio_2`, or rename to make the multi-video-only intent obvious (e.g. keep `image`/`audio` singular plus `video_1`/`video_2`). + +--- + +## Worth fixing + +### 5. Example template hardcodes a local snapshot path and drops the Wan example +`apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Video Generation.json` + +The `VLLMOmniGenerateVideo` model widget is `/workspace/models/hub/models--MiniMaxAI--MiniMax-H3/snapshots/428986c4aa5280855b9ac26eff8d1a46b5999cd0/FL2VA`. This ships to users via **sidebar → Template → vLLM-Omni Video Generation**; nobody else has that path or that snapshot hash. Use `MiniMaxAI/MiniMax-H3` (which the new tests use and which matches fine) or an obvious placeholder. + +Also, this replaced the Wan example rather than adding to it, while `README.md:108` still says "(e.g., Wan, MiniMax-H3)" and the node default is still `Wan-AI/Wan2.2-T2V-A14B-Diffusers`. Consider shipping both, or updating the README to say the template is H3-specific. + +One more on the MarkdownNote (node 22): "connect this Video Reference block ... then disconnect the `frame` input" is incomplete — you must *also* repoint `model` at the `Ref2VA` partition, since `_resolve_task` rejects `ref2va` against an FL2VA checkpoint (`pipeline_minimax_h3.py:387-390`). + +### 6. No way to set `duration`, and `num_frames` default is badly wrong for r2v +`nodes.py:163` (`num_frames` default 41) + `README.md:120` + +The recipe drives H3 length via `extra_params.duration` and explicitly warns: *"The requested duration should cover the complete audio. If duration is shorter, the reference soundtrack is truncated."* The node only exposes `num_frames`, defaulting to 41 — at H3's fixed 24 fps that's ~1.7 s. A user wiring a 15 s audio reference with defaults gets a ~2 s clip with the audio chopped, no warning. (`_resolve_shape` at `pipeline_minimax_h3.py:403-410` only falls back to `num_frames` when `duration` is absent.) + +Related: the node's `fps` default is 16, but H3 hard-errors on anything but 24 (`pipeline_minimax_h3.py:400`). That one at least fails loudly. + +Minimum: document both in the README r2v tip. Better: add `duration` to `VLLMOmniMiniMaxH3Params` and let the builder forward it into `extra_params` (the commit message says duration/num_frames were "conflicting" — worth revisiting, since duration is the parameter the recipe actually recommends). + +### 7. Dead code and an inaccurate comment +`nodes.py:223-225` and `nodes.py:777` + +The comment says `model_params["type"]` "is used in api_client" — it isn't. `api_client.py:359` and `models.py:44` both pop and discard it, and the builder is selected from the model name via `lookup_model_spec`, not from `type`. So `params["type"] = "minimax_h3"` at `nodes.py:777` writes state nothing reads, and the commented-out pop is confusing. Remove both; if you'd rather keep `type` for future dispatch, say so in the comment instead of claiming a use that doesn't exist. + +--- + +## Nits + +- `models.py:103` — `r"MiniMax-H3(/FL2VA|/Ref2VA)?"`: under `re.search` the optional group is a no-op (`MiniMax-H3` alone already matches every string the group would). Simplify to `r"MiniMax-H3"`. +- `api_client.py:71` — bumping `max_poll_duration` 5 → 30 min is reasonable for H3, but it's a global default change for all video jobs. Consider exposing it as a node widget rather than a constant. +- `api_client.py:341-345` — the audio reference goes out as a base64 data URL inside a multipart body (~33 % inflation) after a lossy 128 kbps MP3 re-encode (`format.audio_to_base64` default). For a lip-sync driving signal that's a lossy round-trip on the one input that matters most; consider a higher bitrate or a lossless container, since the server accepts any data URL. +- `models.py:36` — `for key in ("audio_flow_shift",):` over a one-element tuple; either add the other extras H3 accepts (`duration`) or write it as a plain `if`. +- Naming: PR text says "Video Reference", display name is "Video References" (`__init__.py:56`), template note says "Video Reference block". Pick one. + +## Tests + +The two new cases are good — they assert the *decoded server-side* shape (`multi_modal_data["image"]` is a PIL image, `["audio"]` is a path, `["video"]` is a 2-element path list) plus `task=ref2va` in the extras, which is exactly the contract that matters. Gaps I'd close: + +- No coverage of the three `ValueError` branches in `generate_video` (`api_client.py:277`, `:310`, `:325`) — cheap to add and they're the only thing enforcing exclusivity now that `VALIDATE_INPUTS` can't (item 2). +- Nothing asserts `task=fl2va` for the `frame` path. `test_video_generation_node` uses Wan, which has no `params_builder`, so H3 first-frame routing is untested. +- No test for `lookup_model_spec` (item 3) — this PR changed matching logic shared with BAGEL/Qwen and there's no regression net under it. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5756.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5756.trace.json.gz new file mode 100644 index 000000000..be0f2754e Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5756.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5779.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5779.cost.json new file mode 100644 index 000000000..68f204416 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5779.cost.json @@ -0,0 +1,66 @@ +{ + "calls": 47, + "input_tokens": 3813331, + "output_tokens": 35064, + "cost_usd": 3.7285454999999983, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout4", + "head": "f092a92eb57921b642f4651d96c355198df852b6", + "base": "a9ba47d8ce557caae567291b3386f287c0f26f81", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "931db1657047029d40563e3817ce4e0729775fc72a7c8a5a735b57bf9b84f2c5", + "validation_commands": [ + "git diff --stat a9ba47d8ce557caae567291b3386f287c0f26f81 HEAD", + "git diff a9ba47d8ce557caae567291b3386f287c0f26f81 HEAD -- vllm_omni/ | head -500", + "grep -rn \"cu_seqlens_q\\|attn_mask\\|max_seqlen\" vllm_omni/diffusion/models/minimax_h3/ | head -50", + "grep -rn \"supports_prefix_kv_slicing\\|valid_kv_length\" --include=*.py | head -30", + "grep -rn \"supports_prefix_kv_slicing\\|valid_kv_length\" vllm_omni/ tests/ | head -40", + "git diff a9ba47d8ce557caae567291b3386f287c0f26f81 HEAD -- tests/", + "grep -rn \"allgather_degree\" vllm_omni/ | head -30", + "sed -n 30,110p vllm_omni/diffusion/attention/parallel/factory.py; echo ---; grep -rn \"class AllGather\" -A 60 vllm_omni/diffusion/attention/parallel/*.py | head -80", + "sed -n 80,140p vllm_omni/diffusion/attention/parallel/allgather_kv.py", + "grep -rn \"query_ranges\" vllm_omni/diffusion/attention/backends/*.py | head -20", + "grep -rn \"attention_mask_free\" vllm_omni/ | head -20; echo \"=== SkipSoftmaxSpec.enabled usage ===\"; grep -rn \"\\.enabled\" vllm_omni/diffusion/data.py | head; grep -rn \"skip_softmax\" vllm_omni/diffusion/attention/selector.py | head -20", + "grep -rn \"allow_trtllm_default\" -B4 -A25 vllm_omni/diffusion/attention/selector.py | head -70", + "git diff a9ba47d8ce557caae567291b3386f287c0f26f81 HEAD -- docs/ recipes/", + "sed -n 1400,1600p vllm_omni/diffusion/data.py", + "grep -rn \"SkipSoftmaxSpec\" vllm_omni/ tests/ | grep -v \"data.py:\" | head; echo \"=== mixin ===\"; sed -n 195,260p vllm_omni/diffusion/forward_context.py", + "grep -rn \"sigmas_video\" vllm_omni/diffusion/models/minimax_h3/*.py | head -20", + "grep -rn \"video_sigmas\" vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py | head; echo ---; grep -rn \"def .*sigmas\\|_build_sigmas\\|linspace\" vllm_omni/diffusion/models/minimax_h3/*.py | head -20", + "sed -n 30,70p vllm_omni/diffusion/models/minimax_h3/time_request.py", + "timeout 600 python -m pytest tests/diffusion/attention/test_trtllm_attn.py -x -q -k \"packed or denoise_progress or mask_free or masked_layer\" 2>&1 | tail -30", + "timeout 600 python - <<'EOF' 2>&1 | tail -40\nimport torch\nfrom vllm_omni.diffusion.attention.backends import trtllm_attn as tg\nfrom vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata\nfrom vllm_omni.diffusion.attention.backends.trtllm_attn import TrtllmAttentionImpl\n\ndef impl(**", + "timeout 600 python - <<'PYEOF' 2>&1 | tail -20\nimport torch\nfrom vllm_omni.diffusion.attention.backends import trtllm_attn as tg\nfrom vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata\nfrom vllm_omni.diffusion.attention.backends.trtllm_attn import TrtllmAttentionImpl\n\ndef impl(", + "timeout 600 python -c \"\nimport torch\nfrom vllm_omni.diffusion.attention.backends import trtllm_attn as tg\nfrom vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata\nfrom vllm_omni.diffusion.attention.backends.trtllm_attn import TrtllmAttentionImpl\ndef impl(**kw):\n return Trtllm", + "grep -rn \"cu_seqlens\" vllm_omni/diffusion/models/minimax_h3/packed_sequence.py | head -20", + "sed -n 205,245p vllm_omni/diffusion/models/minimax_h3/packed_sequence.py; echo \"=== 530-565 ===\"; sed -n 530,565p vllm_omni/diffusion/models/minimax_h3/packed_sequence.py", + "grep -rn \"AllGather-KV sequence parallelism\\|allgather\" tests/diffusion/attention/ | head -20; echo \"=== recipes/yaml with allgather ===\"; grep -rln \"allgather\" recipes/ vllm_omni/deploy/ examples/ 2>/dev/null | head", + "grep -rn \"allgather\" recipes/MiniMaxAI/MiniMax-H3.md recipes/MiniMaxAI/MiniMax-H3-5090.md | head -20", + "grep -rn \"flashinfer\\|fa4\" pyproject.toml requirements*.txt requirements/*.txt 2>/dev/null | head -20", + "ls; echo ---; grep -rn \"flashinfer\\|fa4 =\\|\\[fa4\\]\\|optional-dependencies\" -A5 pyproject.toml | head -40", + "grep -rn \"flashinfer\" requirements/*.txt pyproject.toml 2>/dev/null | head -10; ls requirements/", + "for f in vllm_omni/diffusion/attention/backends/*.py; do echo \"== $f\"; grep -n \"def __init__\" -A 20 \"$f\" | grep -n \"extra_impl_args\\|\\*\\*kwargs\\|\\*\\*extra\" | head -2; done", + "grep -rn \"extra_impl_args\\|\\*\\*kwargs\" vllm_omni/diffusion/attention/backends/*.py | grep -i \"def\\|kwargs)\" | head -30", + "grep -rn \"class AttentionImpl\" -A 30 vllm_omni/diffusion/attention/backends/abstract.py | head -40", + "timeout 300 python -c \"\nimport inspect, importlib, pkgutil\nimport vllm_omni.diffusion.attention.backends as b\nfor m in pkgutil.iter_modules(b.__path__):\n try:\n mod = importlib.import_module('vllm_omni.diffusion.attention.backends.' + m.name)\n except Exception as e:\n print('skip',", + "grep -rn \"extra\\|attn_mask\\|cu_seqlens\" vllm_omni/diffusion/attention/parallel/ring.py | head -30", + "ls vllm_omni/diffusion/attention/parallel/; grep -rn \"class RingParallelAttention\" -A 40 vllm_omni/diffusion/attention/parallel/*.py | grep -n \"def \\|attn_metadata\\|extra\" | head -20", + "sed -n 61,140p vllm_omni/diffusion/attention/parallel/ring.py", + "grep -rn \"on_step\\b\\|on_step=\" vllm_omni/ tests/ | head -10; echo \"=== record_denoise_step callers ===\"; grep -rn \"record_denoise_step\" vllm_omni/ | head -10", + "grep -n \"within 2%\\|uv pip install -e\\|Stable measurements\\|defaults to dense\" recipes/MiniMaxAI/MiniMax-H3.md | head" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-15T06:28:55", + "arm": "baseline_opus5", + "wall_s": 558.1, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [ + "Write: path outside the worktree: /tmp/probe1.py in {\"file_path\": \"/tmp/probe1.py\", \"content\": \"import torch\\nfrom vllm_omni.diffusion.attention.backends import trtllm_attn as tg\\nfrom vllm_omni.diffusion.attenti" + ], + "artifact_chars": 8914 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5779.md b/eval/dataset/baselines/claudecode_opus5/pr5779.md new file mode 100644 index 000000000..55883b693 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5779.md @@ -0,0 +1,107 @@ +## Verdict + +**Approve with required follow-ups.** The core diagnosis is right and the fix is the correct shape: H3's alignment padding is structural, not a second document, and excluding it before quantization is the only way to keep SAGE from folding garbage into a block scale. The prefix-mask validation is genuinely strict, the trailing-empty-sequence normalization handles the zero-padding case, and the hardware tests cover the case that actually matters (`used=194`, a partial trailing K block). I ran the CPU-testable subset locally: `7 passed, 2 skipped`. + +Two things should not have shipped as-is: the packed path adds ~10 host↔device syncs per attention call in exactly the code path whose caller documents "do not do this," and `max_seqlen_q/k` went from a guaranteed-safe upper bound to unvalidated producer metadata. I verified both against the code rather than by inspection alone. + +--- + +## Required + +### 1. ~10 device syncs per attention call, and the metadata to avoid them is already in `extra` + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:300-341,364` + +Per call, on CUDA tensors: two `.item()` at :300, two more in the `while` condition at :305-308, `flat_mask.sum().item()` at :320, `torch.equal` at :322, two `.nonzero()` at :325-326 (nonzero is a synchronizing op), two `.item()` at :329-330, and `torch.all(...).item()` at :364. Each drains the launch queue. Plus a full-length `arange` allocation and elementwise compare over 58,816 elements at :321. + +The caller says exactly this, one frame up: + +``` +vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:394-396 +# max_seqlen is already the first (real) packed document length. Do +# not read the CUDA cu_seqlens scalars here: this function runs once +# per layer and .item() would serialize every attention launch. +``` + +The PR moves those reads one layer down, where they run once per layer per step per rank. And the value being reconstructed is already sitting in the metadata as a plain Python int: + +``` +vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py:417 +"valid_kv_length": used, +``` + +`valid_tokens` at :320 *is* `used`. Read `extra["valid_kv_length"]` instead of deriving it from the mask; the mask then only needs a shape check, not a reconstruction. Same for the SAGE gate — have the producer publish `min_seqlen` rather than reducing `seq_lens` on device. The `while` loop at :305-311 is producer-side normalization that belongs in the producer. + +This matters for the PR's own claim: the recipe asserts TRTLLM and FA4 are "within 2% of each other," while the PR body says the TRTLLM run was cold and "its end-to-end latency is not reported as steady-state backend performance." The sync cost was never measured. + +### 2. `max_seqlen_q/k` reach the kernel unvalidated + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:342-343,384` + +Before this PR, `max_q_len`/`max_kv_len` were `q_len`/`kv_len` — always an upper bound on any sequence in the batch. Now they are producer metadata, passed straight through. I confirmed the gap: + +``` +cu_seqlens = [0, 2, 8], max_seqlen_q = max_seqlen_k = 2 +→ no error; max_kv_len=2 reaches the kernel while seq_lens = [2, 6] +``` + +Not reachable from H3 today — `packed_sequence.py:224` and `:543` both emit exactly `[0, used, seq_len]`, so `cu[1]` happens to be the max. But `minimax_h3_transformer.py:394` documents `max_seqlen` as "the first (real) packed document length," which is not "the maximum," and this backend is generic. Add a check after :341: + +```python +if max_kv_len < int(seq_lens.max().item()): + raise ValueError(...) +``` + +or just derive both from `seq_lens` (you have already paid for the sync). + +### 3. An all-False mask silently launches a zero-token kernel + +`vllm_omni/diffusion/attention/backends/trtllm_attn.py:320-338` + +Confirmed: + +``` +attn_mask = all False, cu_seqlens = [0, 6, 8] +→ no error; batch_size=0, seq_lens=[], cum_seq_lens_q=[0], query has 0 tokens +``` + +`valid_tokens == 0` matches `cu_seq_lens_q[0]`, so the boundary check passes and everything downstream degenerates. Worse, `torch.all(empty >= k_block_size)` is `True`, so `use_sage` stays on and quantizes empty tensors. Every other malformed input in this block raises a clear `ValueError`; this one shouldn't be the exception. Add `if valid_tokens == 0: raise ValueError(...)` after :320. + +--- + +## Should fix + +### 4. The SAGE gate adds a sync to the dense (non-packed) path + +`trtllm_attn.py:364` — in the `else` branch, `seq_lens` is built at :346 from the Python int `kv_len`, then `torch.all(...).item()` syncs to answer `kv_len >= k_block_size`. That's a new per-call sync for every Wan/other TRTLLM+SAGE user, for a comparison that is host-side. Compute `use_sage` from `kv_len` directly in that branch. + +### 5. The SAGE gate ignores `q_block_size` + +`trtllm_attn.py:364` — `q_block_size` may be 4 or 16 (`data.py:1447`, `_VALID_BLOCK_SIZES`), but the guard only inspects KV sequence lengths. For a role where `q_len != kv_len` that's the wrong tensor. Gate on both. + +### 6. The AllGather-KV reject is unrelated scope, name-based, and untested + +`vllm_omni/diffusion/attention/layer.py:96-100` — a hard `ValueError` in the shared `Attention.__init__`, keyed on `get_name() == "TRTLLM_ATTN"`, now firing for models that never asked for TRTLLM (it is their platform default). The TODO above it already acknowledges the name-based dispatch. For non-causal attention with no joint tensors, local-Q × gathered-KV is what TRTLLM already computes correctly; the real gap is that TRTLLM ignores `attn_metadata.query_ranges`, which only `flash_attn.py:274` and `flash_attn_hub.py:216,383` honor. Either narrow the check to `query_ranges is not None` at forward time — which also removes the backend-name coupling — or keep it and add a test plus a line in `docs/user_guide/diffusion/attention_backends.md`, since this is a new user-visible hard error. + +### 7. Documentation claims outrun the evidence + +`recipes/MiniMaxAI/MiniMax-H3.md:247-248` — "Stable measurements … within 2% of each other" is contradicted by the PR body's own caveat that the TRTLLM run was cold and not steady-state. Cite a steady-state measurement or drop the number. + +Same section: this flips H3's *default* backend on the strength of a comparison that produced PSNR 27.10 dB / SSIM 0.888 versus FA4. That is a visible output change for existing users and the recipe doesn't mention it. One sentence noting outputs differ from the previous FA4 default would be honest. + +`recipes/MiniMaxAI/MiniMax-H3.md:44` — `uv pip install -e .` does not pull flashinfer; it's in neither `pyproject.toml` nor `requirements/*.txt`. So ":239 defaults to dense BF16 TRTLLM_ATTN; no attention backend flag is required" holds only on images that already ship it. The SAGE example additionally needs flashinfer ≥ 0.6.16rc1 or the server hard-fails at init (`trtllm_attn.py:216`). State both. + +--- + +## Minor + +- **`attention_mask_free=True` is now false as stated.** `model_metadata.py:46` — H3 does build a mask (`minimax_h3_transformer.py:409`). The field's only consumer is `allow_trtllm_default` (`layer.py:83`), so nothing breaks, but the name asserts something untrue and a future reader could use it to skip mask plumbing. Rename to `allows_trtllm_default`, matching the docs wording you already changed to "declares a compatible packed/mask-free path." +- **`SkipSoftmaxSpec.enabled` is dead code.** `data.py:1436-1439` — no reference in `vllm_omni/` or `tests/`. Drop it or wire it up. +- **Skip-Softmax has no short-sequence guard.** `trtllm_attn.py:356` — SAGE warns and falls back below one K block; skip-softmax silently applies `threshold * 14` to the 14-token refiner under a bare `default` spec. The recipe tells users to override `per_role`, but the asymmetry with the SAGE handling is surprising. A matching `warning_once` would be consistent. +- **Padding-row semantics diverge from CUDNN.** TRTLLM now zeroes padding rows (:398-405); CUDNN slices K/V only (`cudnn_attn.py:76-77`) and leaves padding queries with real outputs. Both are fine for H3, which discards those rows, but the shared `valid_kv_length` contract in `abstract.py:122` doesn't say which is intended. Worth pinning down in that comment. + +--- + +## Test gap + +The added coverage for the timestep fix (`test_trtllm_attn.py:146`) calls `record_denoise_step` directly. The actual fix for issue #5771's second symptom is the wiring — `on_step_start` at `denoise_loop.py:210` into `pipeline_minimax_h3.py:1376` — and nothing tests it. A test that drives `minimax_h3_denoise_loop` with a stub model and asserts `denoise_timestep == sigmas_video[step]` would lock in both the plumbing and the descending-sigma convention that commit `cd7e29ef` had to correct. I did verify the convention is right: `_time_shift_sigmas` (`time_request.py:47`) runs 1.0 → 0.0, matching the scheduler-normalized semantics `resolve_factor` expects at `trtllm_attn.py:71`. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5779.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5779.trace.json.gz new file mode 100644 index 000000000..50c9e2258 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5779.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5801.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5801.cost.json new file mode 100644 index 000000000..346ff16dd --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5801.cost.json @@ -0,0 +1,63 @@ +{ + "calls": 50, + "input_tokens": 3394803, + "output_tokens": 42371, + "cost_usd": 3.6079160000000003, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout4", + "head": "1ef0cb1562a67a0ecb1d773fb582865f386fd8c5", + "base": "9235b0ae651423bd1b53dac491b564b8afdf8c57", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "46572b4487f8b1d9cec42a4831bd9cd163f2207cb85478a53727d5f26122331b", + "validation_commands": [ + "git diff --stat 9235b0ae651423bd1b53dac491b564b8afdf8c57 HEAD", + "git diff 9235b0ae651423bd1b53dac491b564b8afdf8c57 HEAD", + "sed -n 150,260p vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", + "grep -n \"_norm(\\|_apply_rope\\|q_norm\\|k_norm\\|rope\" vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py | head -60", + "sed -n 400,480p vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", + "python -c \"import vllm, os; print(os.path.dirname(vllm.__file__))\" 2>/dev/null || echo \"no vllm\"", + "python -c \"import torch;print(torch.__version__)\" 2>&1 | tail -2", + "sed -n 1,120p /usr/local/lib/python3.12/dist-packages/vllm/vllm_flash_attn/layers/rotary.py", + "grep -n \"stride\\|contiguous\\|assert\" /usr/local/lib/python3.12/dist-packages/vllm/vllm_flash_attn/ops/triton/rotary.py | head -60", + "timeout 900 python -m pytest tests/diffusion/layers/test_norm.py tests/diffusion/layers/test_rope_broadcast.py tests/diffusion/models/minimax_h3/test_minimax_h3_parallel.py -q 2>&1 | tail -25", + "timeout 120 python -c \"\nimport torch\nprint('cuda', torch.cuda.is_available(), torch.cuda.device_count() if torch.cuda.is_available() else 0)\n\" 2>&1 | tail -3", + "timeout 300 python -c \"\nimport torch\nfrom vllm._custom_ops import rms_norm\nx = torch.randn(8,128, device='cuda', dtype=torch.bfloat16)\nout = torch.empty_like(x)\nw32 = torch.ones(128, device='cuda', dtype=torch.float32)\ntry:\n rms_norm(out, x, w32, 1e-5)\n print('fp32 weight OK')\nexcept Exception", + "timeout 600 python -c \"\nimport torch\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\n\ndef old_apply_rope(x, freqs):\n rot_dim = freqs.shape[-1]\n x_rot, x_pass = x[..., :rot_dim], x[..., rot_dim:]\n cos = torch.cos(freqs).to(x.dtype).unsqueeze(1)\n sin = torch.sin(freqs).to(x.dty", + "timeout 600 python -c \"\nimport torch\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\nrope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)\nT,H,D = 1000, 8, 128\nx = torch.randn(T,H,D, device='cuda', dtype=torch.bfloat16)\nhalf = torch.randn(T,48, device='cuda')\nfreqs = torch.cat(", + "timeout 600 python -c \"\nimport torch\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\nrope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)\nT,H,D = 1000, 8, 128\nx = torch.randn(T,H,D, device='cuda', dtype=torch.bfloat16)\nhalf = torch.randn(T,48, device='cuda')\nfreqs = torch.cat(", + "timeout 600 python -c \"\nimport torch, time\nimport torch.nn as nn\nfrom vllm_omni.diffusion.layers.norm import RMSNorm\nT,H,Dh = 8192, 8, 128\nqkv = torch.randn(T, H*Dh*3, device='cuda', dtype=torch.bfloat16)\nq = qkv.split([H*Dh, H*Dh, H*Dh], dim=-1)[0].view(T,H,Dh)\nprint('q contiguous', q.is_contiguous", + "timeout 600 python -c \"\nimport torch, time\nimport torch.nn as nn\nfrom vllm_omni.diffusion.layers.norm import RMSNorm\ndef bench(fn, it=200):\n for _ in range(20): fn()\n torch.cuda.synchronize(); t=time.perf_counter()\n for _ in range(it): fn()\n torch.cuda.synchronize(); return (time.perf_co", + "timeout 600 python -c \"\nimport torch, time\nfrom vllm_omni.diffusion.layers.norm import RMSNorm\ndef bench(fn, it=200):\n for _ in range(20): fn()\n torch.cuda.synchronize(); t=time.perf_counter()\n for _ in range(it): fn()\n torch.cuda.synchronize(); return (time.perf_counter()-t)/it*1e6\nT,Hs", + "sed -n 55,105p vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", + "sed -n 40,56p vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py", + "timeout 600 python -c \"\nimport torch, time\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\nrope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)\nrope_half = RotaryEmbedding(is_neox_style=True)\nT,H,D = 16384, 56, 128\nx = torch.randn(T,H,D, device='cuda', dtype=torch.bfloat16)\nha", + "sed -n 150,175p vllm_omni/diffusion/worker/diffusion_model_runner.py", + "sed -n 272,292p vllm_omni/diffusion/worker/diffusion_model_runner.py", + "timeout 900 python -c \"\nimport torch\nimport torch.nn as nn\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\nfrom vllm_omni.diffusion.layers.norm import RMSNorm\n\nclass M(nn.Module):\n def __init__(self):\n super().__init__()\n self.rope = RotaryEmbedding(is_neox_style=True, h", + "timeout 900 python -c \"\nimport torch, time\nimport torch.nn as nn\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\n\nclass Old(nn.Module):\n def forward(self, x, freqs):\n rot = freqs.shape[-1]\n xr, xp = x[..., :rot], x[..., rot:]\n co = torch.cos(freqs).to(x.dtype).uns", + "timeout 900 python -c \"\nimport torch, time\nimport torch.nn as nn\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\nclass Old(nn.Module):\n def forward(self, x, freqs):\n rot = freqs.shape[-1]\n xr, xp = x[..., :rot], x[..., rot:]\n co = torch.cos(freqs).to(x.dtype).unsq", + "grep -n \"def load_weights\" -A 60 vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py | head -80", + "timeout 900 python -c \"\nimport torch, time\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\nrope = RotaryEmbedding(is_neox_style=True, half_head_dim=False)\nT,H,D = 16384, 56, 128\nx = torch.randn(T,H,D, device='cuda', dtype=torch.bfloat16)\nh = torch.randn(T,48, device='cuda'); f = torch.ca", + "timeout 300 python -c \"\nimport torch\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\nr = RotaryEmbedding(is_neox_style=True, half_head_dim=False)\nr._forward_method = r.forward_native\nx = torch.randn(17,3,128, dtype=torch.bfloat16)\nh = torch.randn(17,48); f = torch.cat((h,h),dim=-1)\nco=to", + "grep -rn \"RMSNorm(\" vllm_omni/diffusion/models/z_image/z_image_transformer.py vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py vllm_omni/diffusion/models/cosmos3/transformer_cosmos3.py | head -20", + "grep -rn \"set_default_dtype\\|torch_dtype\\|params_dtype\" vllm_omni/diffusion/model_loader/diffusers_loader.py | head -20", + "sed -n 350,375p vllm_omni/diffusion/model_loader/diffusers_loader.py", + "timeout 300 python -c \"\nimport torch\nfrom vllm.utils.torch_utils import set_default_torch_dtype\nfrom vllm_omni.diffusion.layers.norm import RMSNorm\nwith set_default_torch_dtype(torch.bfloat16):\n n = RMSNorm(64)\n print('PR default under bf16 ctx ->', n.weight.dtype)\n print('torch.ones(4).dty", + "grep -n \"_repeated_blocks\" vllm_omni/diffusion/models/z_image/z_image_transformer.py; sed -n 758,770p vllm_omni/diffusion/models/z_image/z_image_transformer.py", + "grep -n \"def test_rmsnorm_dtype_matches\\|def test_rmsnorm_npu_receives\\|captured\\[.epsilon.\\]\" tests/diffusion/layers/test_norm.py; grep -n \"def test_mindie_full_dim\" tests/diffusion/layers/test_rope_broadcast.py" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-15T06:34:53", + "arm": "baseline_opus5", + "wall_s": 784.1, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [ + "Write: path outside the worktree: /tmp/chk_rope.py in {\"file_path\": \"/tmp/chk_rope.py\", \"content\": \"import torch\\n\\nfrom vllm_omni.diffusion.layers.rope import RotaryEmbedding\\n\\nrope = RotaryEmbedding(is_neox_styl" + ], + "artifact_chars": 11551 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5801.md b/eval/dataset/baselines/claudecode_opus5/pr5801.md new file mode 100644 index 000000000..f7a0dd5c1 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5801.md @@ -0,0 +1,133 @@ +## Verdict + +**Request changes (as a follow-up — this is merged with a red gate).** + +The H3-side change is correct and a real win. I verified the RoPE rewrite numerically against an fp64 reference on GPU: the new path is not just equivalent, it's marginally *more* accurate than the code it replaces (max err vs fp64: `0.02297` new vs `0.02468` old, bf16, T=1000/H=8/D=128), and it's 2.5× faster on the op (1420 µs → 567 µs eager at T=16384/H=56/D=128; 320 µs → 220 µs under `torch.compile`, which is the default since `_compile_transformer` runs unless `--enforce-eager`). No dynamo graph break is introduced — I checked with `torch._dynamo.explain`, `graph_count 1, break_count 0`. + +Two things need fixing, one of which is a cross-model regression that this PR introduces silently: + +1. `norm.py:68` changes the default weight dtype from *ambient* to *hard fp32*, which disables the fused RMSNorm kernel for every other diffusion model that uses this layer. +2. `minimax_h3_transformer.py:361-365` leaves ~2.7× of the RoPE win on the table with an unnecessary slice+cat. + +Everything else below is smaller. + +--- + +## Blocking + +### 1. `vllm_omni/diffusion/layers/norm.py:68` — new default `dtype=torch.float32` silently regresses every other model + +Before: `nn.Parameter(torch.ones(hidden_size))` — `torch.ones` follows `torch.get_default_dtype()`, and the diffusion loader builds models inside `set_default_torch_dtype(self.od_config.dtype)` (`vllm_omni/diffusion/model_loader/diffusers_loader.py:358`). So in production the weight was **bf16**. After this PR it is **always fp32**, regardless of that context: + +``` +with set_default_torch_dtype(torch.bfloat16): + RMSNorm(64).weight.dtype -> torch.float32 # was bfloat16 pre-PR + torch.ones(4).dtype -> torch.bfloat16 +``` + +That matters because `_forward_fused` → `vllm._custom_ops.rms_norm` requires weight dtype == input dtype. I confirmed on GPU: + +``` +rms_norm(out_bf16, x_bf16, weight_fp32, eps) + -> RuntimeError: const_data_ptr, expected scalar type BFloat16 but found Float +``` + +which is swallowed by the bare `except Exception` at `norm.py:96` and falls through to `forward_native`. Measured cost at `[8192, 3072]` bf16 on this box: **fused 32 µs vs native 347 µs (~10×)**; at `[32768, 3072]`, 111 µs vs 1303 µs. + +Affected call sites, all of which construct with the default: `z_image_transformer.py:291,292,438,439,441,442,764`, `wan2_2_transformer.py:395,396,528,529,555`, `hunyuan_image3_transformer.py:1757,1758`, `transformer_cosmos3.py`, `indextts2/.../gpt_fast/model.py`. Some of these are outside `_repeated_blocks` and therefore never compiled — e.g. `z_image_transformer.py:764` (`cap_embedder`, `_repeated_blocks = ["ZImageTransformerBlock"]`) — so they run eager and eat the full 10×. + +There is also a plausible harder break on NPU: `forward_npu` (`norm.py:116`) now hands `torch_npu.npu_rms_norm` an fp32 `gamma` with a bf16 `x` for those same models, where it previously got bf16. And an fp32 param sitting among bf16 params can upset FSDP2 mixed-dtype all-gather groups under `use_hsdp`. + +**Change to:** + +```python +def __init__(self, hidden_size: int, eps: float = 1e-6, dtype: torch.dtype | None = None) -> None: + super().__init__() + # ``dtype=None`` follows ``torch.get_default_dtype()``, i.e. the loader's + # ``set_default_torch_dtype(od_config.dtype)`` context. Pass an explicit + # dtype only to pin gamma independently of that context. + self.weight = nn.Parameter(torch.ones(hidden_size, dtype=dtype)) +``` + +`torch.ones(n, dtype=None)` uses the default dtype, so this restores the old behaviour exactly while keeping H3's explicit `dtype=_BF16_DTYPE` working. Please also add the `dtype` param to a docstring — it controls both checkpoint storage *and* kernel selection, which is not obvious. + +While you're in here: `norm.py:92-97` / `99-108` catching bare `Exception` per call is how this stayed invisible. A `logger.warning_once` in the `except` branch (or an explicit `self.weight.dtype == x.dtype` precondition instead of try/except) turns a silent 10× into a discoverable one, and stops real kernel bugs from being masked. + +--- + +## Perf (the point of the PR) + +### 2. `minimax_h3_transformer.py:361-365` — the slice + `cat` costs 2.7× of the RoPE win + +`apply_rotary` already handles partial rotary: `vllm/vllm_flash_attn/ops/triton/rotary.py:188-190` allocates `empty_like(x)` and copies `x[..., rotary_dim:]` itself. `apply_rotary_emb_torch` (`rope.py:31-37`) does the same for the native path. So the manual split and re-concat is pure overhead — it materialises a second full `[T, H, 128]` tensor per call, twice per layer, 50 layers. + +Measured, T=16384/H=56/D=128 bf16, bit-identical output (`maxdiff 0.0`, and `torch.equal` on the native path): + +| | µs | +|---|---| +| current (`x[..., :96]` → rope → `cat`) | 566.5 | +| pass full `x` (rotary_dim inferred from cos) | **212.6** | + +```python +def _apply_rope(self, x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: + """Rotate the first freqs.shape[-1] head dims; the kernel passes the rest through.""" + cos = torch.cos(freqs).to(x.dtype) + sin = torch.sin(freqs).to(x.dtype) + return self.rope(x, cos, sin) +``` + +Caveat: I could not verify that `mindiesd.rotary_position_embedding` supports `rotary_dim < head_dim`. If it doesn't, keep the split for the NPU branch only and say so in a comment. + +### 3. `rope.py:124-127` + `minimax_h3_transformer.py:335,362` — `half_head_dim=False` buys nothing on CUDA/HIP + +On CUDA the new flag only *undoes* itself: `_prepare_half_head_dim_cos_sin` slices the 96-wide cos back to 48 before handing it to a kernel that wants half-dim anyway. So the model computes `cos`/`sin` over 96 dims to throw half away, and the resulting `cos[..., :48]` is **non-contiguous** (`is_contiguous() == False`), which forces the `cos.contiguous()` copy at `vllm_flash_attn/ops/triton/rotary.py:180` on every call. + +Using the existing default path (`RotaryEmbedding(is_neox_style=True)` with `cos = torch.cos(freqs[..., :rot_dim // 2])`) is bit-identical (`maxdiff 0.0`) and marginally faster (557 µs vs 567 µs). The flag's only genuine payoff is the mindiesd branch, where full-dim cos skips the `repeat` at `rope.py:64-65`. That's a legitimate reason to keep it — but please say so in the `half_head_dim` docstring, because as written it reads like a CUDA optimisation and it isn't. + +--- + +## Correctness / robustness + +### 4. `rope.py:168-177` — `forward_npu` never normalises `x` to 4D, unlike `forward_cuda`/`forward_hip` + +`forward_cuda` and `forward_hip` both call `_ensure_batch_dim` (`rope.py:69-76`) precisely because omni call sites pass `[S, H, D]`. `forward_npu` → `apply_rotary_emb_mindiesd` does not, and H3 passes 3D (`x: [T, heads, head_dim]`). `rotary_position_embedding(..., head_first=False)` presumably wants `[B, S, H, D]`. + +The new test at `tests/diffusion/layers/test_rope_broadcast.py:255-260` passes a 3D `x` of shape `(11, 3, 96)` and asserts it is forwarded verbatim — so it *locks in* the 3D contract against a stub, without establishing that mindiesd accepts it. Either add `_ensure_batch_dim`/`_restore_batch_dim` to `forward_npu`, or state explicitly that H3's fused RoPE is CUDA/HIP-only. + +Related, pre-existing but now inherited by the base class: `apply_rotary_emb_mindiesd` hands the same op a 4D `[1, S, 1, D]` cos when `half_head_dim=True` and a 2D `[S, D]` cos when `False`. + +### 5. `rope.py:124-127` — the halving assumption is silent + +For `half_head_dim=False, interleaved=False` the code takes `cos[..., :D//2]`, which is only valid because `MiniMaxH3Rope.forward` returns `cat((half, half), -1)`. I confirmed the failure mode is silent: feed a non-duplicated 96-wide `freqs` and the output diverges by 10.97 with no error. Worth a one-line comment stating the layout contract (a debug assert is probably too hot for the inner loop). + +### 6. `rope.py:125-126` — the `interleaved` branch is unreachable in-tree + +`cos[..., ::2]` only runs for `interleaved=True and half_head_dim=False`. The only base-class user with `half_head_dim=False` is H3 (`is_neox_style=True`), and `RotaryEmbeddingWan` overrides every forward except `forward_xpu`/`forward_musa`, which route to its own `forward_native`. Untested and unused — cover it or drop it. + +### 7. `rope.py:220-222` — `RotaryEmbeddingWan.__init__` shadows the base field + +```python +super().__init__(is_neox_style=is_neox_style) # sets self.half_head_dim = True +self.half_head_dim = half_head_dim # then overwrites it +``` +Works, but now that the base owns the field, forward it: `super().__init__(is_neox_style=is_neox_style, half_head_dim=half_head_dim)` and delete the second line. + +--- + +## Comments and tests + +- **`minimax_h3_transformer.py:174-176`** — stale: "torch.nn.RMSNorm upcasts reduced-precision inputs for the variance reduction, matching that accumulation semantic." `_norm` no longer returns `nn.RMSNorm`. The fp32-accumulation claim still holds (the vllm kernel accumulates variance in float; I measured identical max error vs fp64 for `nn.RMSNorm` and the fused path, `0.01562` both), but the comment should name `vllm_omni...norm.RMSNorm` and note that native multiplies gamma in fp32 while the fused kernel does it in bf16. +- **`minimax_h3_transformer.py:357-358`** — "In the unfused path, cos/sin are cast to the activation dtype before the elementwise math" is now unconditional, and the `# [T, rot_dim]` annotation on line 362 is misleading since the rope halves it internally. +- **`test_minimax_h3_parallel.py:73`** — `attention.rope._forward_method = attention.rope.forward_native` means the test never exercises `forward_cuda`, which is the path this PR exists for. Reasonable for a `cpu`-marked test, but there's no GPU-marked counterpart. A tolerance-based equivalence test against the old formula would pass — I measured max abs diff `0.03125` in bf16, well inside `rtol=2e-2`. +- **`test_rope_broadcast.py:242`** — the test calls `apply_rotary_emb_mindiesd` directly, so the actual new plumbing (`self.half_head_dim` threaded through at `rope.py:175`) is not covered. Route it through `RotaryEmbedding(is_neox_style=True, half_head_dim=False).forward_npu(...)` instead. +- **No test asserts the fused CUDA path is actually reached with `dtype=bf16`.** The NPU test checks gamma dtype; the CUDA path — the one H3 runs — has no equivalent, which is exactly why finding #1 slipped through. A test that monkeypatches `vllm._custom_ops.rms_norm` and asserts it was called (and that a default-constructed `RMSNorm` under `set_default_torch_dtype(bf16)` also reaches it) would pin both. +- **`test_norm.py:330,340`** — `captured["epsilon"] = torch.tensor(epsilon)` then `.item()`; just store the float. +- **Nit, `rope.py:104,111`** — `RotaryEmbedding.__init__` runs two `find_spec` calls per instance; H3 now creates 50 of them. `norm.py:12` already does the module-level `_HAS_MINDIESD = find_spec(...) is not None` pattern — worth matching. + +--- + +## Gate + +`buildkite/vllm-omni-amd-ci` is red. `forward_hip` is squarely on the changed path (`_prepare_half_head_dim_cos_sin` now runs there, and `RMSNorm._forward_fused` is now actually reached on ROCm for H3 with a bf16 gamma), so that gate needs to be green or explained before the ROCm behaviour of this change can be trusted. + +**Repro for the measurements above:** all run on this checkout with 8 visible CUDA devices; RMSNorm timings `[8192,3072]`/`[32768,3072]` bf16, RoPE timings `T=16384, H=56, D=128` bf16, 100–200 iters after warmup with `torch.cuda.synchronize()`. \ No newline at end of file diff --git a/eval/dataset/baselines/claudecode_opus5/pr5801.trace.json.gz b/eval/dataset/baselines/claudecode_opus5/pr5801.trace.json.gz new file mode 100644 index 000000000..b75850324 Binary files /dev/null and b/eval/dataset/baselines/claudecode_opus5/pr5801.trace.json.gz differ diff --git a/eval/dataset/baselines/claudecode_opus5/pr5833.cost.json b/eval/dataset/baselines/claudecode_opus5/pr5833.cost.json new file mode 100644 index 000000000..e8baff269 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5833.cost.json @@ -0,0 +1,53 @@ +{ + "calls": 35, + "input_tokens": 805123, + "output_tokens": 18957, + "cost_usd": 1.2639175, + "is_error": false, + "terminal_reason": "completed", + "split": "holdout4", + "head": "ac67e8874d00cffa4e8315d4860bc5aeec8a9821", + "base": "9235b0ae651423bd1b53dac491b564b8afdf8c57", + "resolved_model": "claude-opus-5", + "snapshot_sha256": "8401efea92c6650fe5139597ce9c6d116a63e495ce267d045feb78b746b18723", + "validation_commands": [ + "git diff 9235b0ae651423bd1b53dac491b564b8afdf8c57 HEAD --stat", + "git diff 9235b0ae651423bd1b53dac491b564b8afdf8c57 HEAD", + "ls -R docs/design | head -60; echo \"---\"; ls docs/user_guide/quantization/", + "ls docs/design/module; echo \"---mkdocs---\"; ls | head -40; echo; ls mkdocs.yaml mkdocs.yml 2>/dev/null", + "cat mkdocs.yml | head -80; echo \"=== nested nav files ===\"; find docs -name \".nav.yml\" -o -name \".pages\" | head", + "for f in docs/design/feature/pipeline_parallel.md docs/design/feature/expert_parallel.md docs/design/feature/tensor_parallel.md docs/design/feature/sequence_parallel.md docs/design/feature/hsdp.md docs/design/feature/async_chunk.md docs/design/feature/async_diffusion_output.md docs/design/feature/pr", + "head -6 docs/design/feature/pipeline_parallel.md docs/design/feature/expert_parallel.md docs/design/feature/tensor_parallel.md docs/design/feature/sequence_parallel.md docs/design/feature/hsdp.md", + "head -8 docs/design/feature/async_chunk.md docs/design/feature/async_diffusion_output.md docs/design/feature/prefix_caching.md docs/design/ar_diffusion_pipeline_capability.md docs/design/feature/skip_softmax.md", + "python -c \"import mkdocs, mkdocs.__main__ as m; print(mkdocs.__version__)\" 2>&1 | tail -2; pip list 2>/dev/null | grep -i -E \"mkdocs|awesome|material\" ", + "sed -n 80,140p mkdocs.yml", + "python -c \"import mkdocs; print('mkdocs', mkdocs.__version__)\" 2>&1 | tail -3", + "python -c \"import mkdocs_awesome_nav, os; print(os.path.dirname(mkdocs_awesome_nav.__file__))\" 2>&1 | tail -3", + "ls requirements/; echo ===; find .github -name \"*doc*\"; echo ===; grep -rn \"mkdocs\" .github/workflows/ 2>/dev/null | head -20", + "grep -rn \"nav\\|sidebar\\|title\" docs/contributing/DOCS_GUIDE.md | head -40", + "ls .github/workflows/ 2>/dev/null; ls .readthedocs.yaml .readthedocs.yml 2>/dev/null", + "cat .readthedocs.yml; echo \"=== precommit ===\"; grep -n \"markdown\\|mdformat\\|lint\" .pre-commit-config.yaml | head -20", + "timeout 25 pip download mkdocs-awesome-nav -d /tmp/dl --no-deps -q 2>&1 | tail -3; ls /tmp/dl 2>/dev/null", + "timeout 30 pip download mkdocs-awesome-nav -d /tmp/dl5833 --no-deps -q", + "grep -n -A 25 \"docs = \\[\" pyproject.toml", + "grep -rn \"quantization/overview\" docs/ --include=*.md --include=*.yml | head -20", + "head -40 docs/user_guide/quantization/overview.md", + "grep -m1 -H \"^# \" docs/design/architecture_overview.md docs/design/metrics.md docs/design/qwen3_omni_tts_performance_optimization.md docs/design/module/*.md docs/design/feature/*.md docs/design/feature/omni_connectors/*.md", + "git diff --check 9235b0ae651423bd1b53dac491b564b8afdf8c57 HEAD; echo \"check-exit:$?\"", + "head -25 docs/design/module/async_omni_architecture.md", + "head -20 docs/design/feature/ray_based_execution.md", + "git log --oneline -3 -- docs/design/ar_diffusion_pipeline_capability.md", + "python -c \"\nimport markdown\nsrc=open('docs/design/index.md').read()\ni=src.index('#### Quantization'); j=src.index('## Infrastructure')\nprint(markdown.markdown(src[i:j]))\n\" 2>&1 | tail -20" + ], + "ran_validation": true, + "claude_md_seen": { + "/data/zhoutaichang/CLAUDE.md": "27ca8ea8056c231e" + }, + "recorded_at": "2026-08-15T06:26:29", + "arm": "baseline_opus5", + "wall_s": 253.0, + "audit_ok": true, + "audit_violations": [], + "audit_blocked_attempts": [], + "artifact_chars": 9334 +} diff --git a/eval/dataset/baselines/claudecode_opus5/pr5833.md b/eval/dataset/baselines/claudecode_opus5/pr5833.md new file mode 100644 index 000000000..e8decd5e2 --- /dev/null +++ b/eval/dataset/baselines/claudecode_opus5/pr5833.md @@ -0,0 +1,107 @@ +I have what I need. Here's the review. + +--- + +# Review: PR #5833 — docs: reorganize vLLM-Omni design navigation + +## Verdict + +**Approve with changes** — the reorganization is a net improvement and the scope discipline is good (only `.nav.yml` + `index.md`, no doc content touched, no URLs moved). It also quietly fixes two real orphans: `design/metrics.md` and `design/qwen3_omni_tts_performance_optimization.md` were previously not in `.nav.yml` at all, and `design/feature/vae_parallel.md` was listed twice. + +But there is **one concrete rendering bug** in `design/index.md` that puts three unrelated docs under a "Quantization" heading, and **one nav entry that duplicates a page already in the nav**, which is both a semantic and a MkDocs-mechanics problem. Both are small fixes. + +--- + +## Must fix + +### 1. `docs/design/index.md:49-55` — Cache-DiT / TeaCache / Diffusion Continuous Batching render *inside* the Quantization section + +```markdown +49 #### Quantization +50 +51 - [Quantization Overview](../user_guide/quantization/overview.md) +52 +53 - [Cache-DiT](feature/cache_dit.md) +54 - [TeaCache](feature/teacache.md) +55 - [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md) +``` + +The blank line at L52 does not close the list — in CommonMark and Python-Markdown a blank line between items just makes the list *loose*. So all four bullets render as a single (extra-spaced) list under the `#### Quantization` heading, and the page ToC will show Cache-DiT / TeaCache / Diffusion Continuous Batching as children of "Quantization". Even if you read it as two separate `