From b0c54c03f59cf11ccc28754c8f321abfc7db1c30 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Tue, 27 Oct 2026 19:11:16 +0800 Subject: [PATCH] rebase: port harness-tier support, wheel hardening and bridge host env to main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the operative rebase work from feat/rebase-main-v0320 and fixes the six engine defects found while reviewing the v0.33 → main campaign runs: 1. Harness tiers (cursor/zcode/claude-code) are first-class rebase backends: `_tier_client` / `_harness_completion_client` route a `kind="harness"` tier straight to the session transport instead of an OpenAI-compatible client with an empty api_key; reviewer routing and the CI debug attempt follow the same path and carry their own completion signal. 2. Module attempt budgets are consistent end to end: the adapter declares `module_timeout_sec == module_max_timeout_sec`, the live prompt tells the agent the real `TEST_TIMEOUT_SEC`, and bridge polls stay below the client-side deadline (test pins the invariant). 3. The MCP tool bridge rebuilds the agent shell env from explicit spec facts (`host_env`: CUDA_VISIBLE_DEVICES / HF_HOME) rather than from whatever survived the transport's sanitized environment — no `os.environ` mutation, so the env guardrail stays intact. 4. Session read audit distinguishes reads outside the declared roots from refusals: upstream/venv roots are declared via `read_roots`, and an audit note is traced (`harness_read_audit`) instead of vetoing a completed module. 5. cursor-agent sessions that exit without a result event are reported as failures (exit code + stderr tail) rather than as empty successes. 6. Wheel step: `packaging` dependency declared, baseline venv for main-tracking campaigns, scratch kept on non-done exits so a blocked or crashed run is resumable; the knowledge parent-store block that blocked prelude on a fresh host is gone. Also: fixture trees under test/fixtures are excluded from collection (their conftest shadowed the suite's), repo-vocabulary ceilings updated for the identifier-level parity names only (prose neutralized), SPEC pages re-verified against 2026-10-27. --- adapters/vllm_omni/manifest.yaml | 55 +- adapters/vllm_omni/rebase/api_drift_guard.py | 47 +- adapters/vllm_omni/rebase/prompt_data.yaml | 30 +- adapters/vllm_omni/rebase/shell_golden.json | 2 +- .../templates/module_rebase.live.prompt.tmpl | 147 +- .../vllm_omni/testing/watchdog_patterns.yaml | 4 +- doc/DRIFT_TRIAGE.md | 97 +- doc/RUNBOOK-rebase.md | 37 + doc/architecture/SPEC/app/core.md | 10 +- doc/architecture/SPEC/ci/buildkite.md | 9 +- doc/architecture/SPEC/cli.md | 7 +- .../SPEC/engine/steps/rebase_v3.md | 22 + doc/architecture/SPEC/providers/base.md | 8 +- .../SPEC/providers/claude_code.md | 11 +- doc/architecture/SPEC/providers/cursor.md | 14 +- doc/architecture/SPEC/rebase_engine.md | 19 + doc/architecture/SPEC/tool_bridge.md | 11 +- knowledge/repos/vllm-omni/rebase/workflow.md | 15 +- pyproject.toml | 4 + src/infermatrix_copilot/app/core.py | 55 +- src/infermatrix_copilot/ci/buildkite.py | 54 +- src/infermatrix_copilot/cli/entry.py | 21 +- .../engine/steps/rebase_v3.py | 407 +++- src/infermatrix_copilot/providers/base.py | 4 + .../providers/claude_code.py | 48 +- src/infermatrix_copilot/providers/cursor.py | 48 +- src/infermatrix_copilot/providers/zcode.py | 31 +- .../rebase_engine/ci_loop.py | 8 +- .../rebase_engine/module_pytest.py | 168 +- .../rebase_engine/module_rebase.py | 189 +- .../rebase_engine/prompt_builder.py | 38 +- .../rebase_engine/rebase_tools.py | 55 +- .../rebase_engine/remote_ci_owner.py | 260 +++ .../rebase_engine/test_loop.py | 73 +- .../rebase_engine/wheel.py | 393 +++- .../testing/cpu_ci_resources.py | 46 + src/infermatrix_copilot/testing/runner.py | 10 +- src/infermatrix_copilot/tool_bridge.py | 66 +- .../remote_ci_owner/adapter_manifest.yaml | 430 +++++ test/fixtures/remote_ci_owner/conftest.py | 1 + .../remote_ci_owner/expected106_routes.json | 1207 ++++++++++++ .../remote_ci_owner/native73_manifest.json | 1635 +++++++++++++++++ .../repo/.buildkite/cuda/test-merge.yml | 221 +++ .../repo/.buildkite/cuda/test-nightly.yml | 547 ++++++ .../repo/.buildkite/cuda/test-ready.yml | 287 +++ .../tests/dfx/perf/scripts/run_benchmark.py | 669 +++++++ .../perf/scripts/run_diffusion_benchmark.py | 1055 +++++++++++ .../perf/tests/test_cosmos3_vllm_omni.json | 288 +++ .../test_hunyuanvideo15_i2v_vllm_omni.json | 124 ++ .../test_hunyuanvideo15_t2v_vllm_omni.json | 104 ++ .../tests/test_lingbot_video_vllm_omni.json | 50 + .../dfx/perf/tests/test_minicpmo_4_5.json | 172 ++ .../test_minicpmo_4_5_duplex_seed_tts.json | 50 + .../perf/tests/test_minicpmo_4_5_ready.json | 81 + .../perf/tests/test_minimax_h3_vllm_omni.json | 300 +++ .../tests/test_qwen3_omni_async_chunk.json | 325 ++++ .../tests/test_qwen3_omni_no_async_chunk.json | 186 ++ .../perf/tests/test_qwen_image_vllm_omni.json | 408 ++++ .../repo/tests/dfx/perf/tests/test_tts.json | 551 ++++++ .../perf/tests/test_wan22_i2v_vllm_omni.json | 179 ++ ...est_hunyuanvideo15_i2v_video_similarity.py | 266 +++ ...est_hunyuanvideo15_t2v_video_similarity.py | 214 +++ .../minicpmo_4_5/test_minicpmo_4_5.py | 215 +++ .../test_minimax_h3_i2va_ref2va_similarity.py | 387 ++++ .../accuracy/qwen3_omni/test_qwen3_omni.py | 133 ++ .../test_diffusers_backend_similarity.py | 436 +++++ .../e2e/accuracy/test_hunyuan_image3_lora.py | 110 ++ .../test_hunyuan_image3_pixel_accuracy.py | 324 ++++ .../tests/e2e/accuracy/test_qwen_image.py | 276 +++ .../test_wan22_i2v_video_similarity.py | 295 +++ .../offline_inference/test_text_to_image.py | 77 + .../online_serving/test_text_to_image.py | 140 ++ .../remote_ci_owner/source_provenance.json | 137 ++ test/test_adapter_knowledge.py | 99 +- test/test_api_drift_guard.py | 90 + test/test_assembly.py | 388 +++- test/test_ci_wiring.py | 136 ++ test/test_cpu_ci_resources.py | 33 + test/test_engine_core.py | 54 + test/test_module_pytest_outcomes.py | 485 +++++ test/test_phase1_cluster.py | 81 + test/test_provider_claude_code.py | 82 +- test/test_provider_cursor.py | 44 + test/test_rebase_campaign_prompt.py | 205 +++ test/test_rebase_harness_backend.py | 115 ++ test/test_rebase_harness_completion.py | 185 ++ test/test_rebase_harness_env.py | 313 ++++ test/test_rebase_release_target.py | 439 +++++ test/test_rebase_resume_selection.py | 183 ++ test/test_rebase_shell_runtime.py | 51 + test/test_rebase_zcode_routing.py | 126 ++ test/test_remote_ci_owner.py | 348 ++++ test/test_repo_vocabulary.py | 12 +- test/test_shell_golden.py | 69 +- test/test_testing_substrate.py | 63 +- test/test_tool_bridge.py | 53 + test/test_v3_complete_e2e.py | 51 +- 97 files changed, 17767 insertions(+), 341 deletions(-) create mode 100644 src/infermatrix_copilot/rebase_engine/remote_ci_owner.py create mode 100644 src/infermatrix_copilot/testing/cpu_ci_resources.py create mode 100644 test/fixtures/remote_ci_owner/adapter_manifest.yaml create mode 100644 test/fixtures/remote_ci_owner/conftest.py create mode 100644 test/fixtures/remote_ci_owner/expected106_routes.json create mode 100644 test/fixtures/remote_ci_owner/native73_manifest.json create mode 100644 test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-merge.yml create mode 100644 test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-nightly.yml create mode 100644 test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-ready.yml create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/scripts/run_benchmark.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/scripts/run_diffusion_benchmark.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_tts.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/test_diffusers_backend_similarity.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/test_hunyuan_image3_lora.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/test_qwen_image.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/examples/offline_inference/test_text_to_image.py create mode 100644 test/fixtures/remote_ci_owner/repo/tests/examples/online_serving/test_text_to_image.py create mode 100644 test/fixtures/remote_ci_owner/source_provenance.json create mode 100644 test/test_api_drift_guard.py create mode 100644 test/test_cpu_ci_resources.py create mode 100644 test/test_module_pytest_outcomes.py create mode 100644 test/test_rebase_campaign_prompt.py create mode 100644 test/test_rebase_harness_completion.py create mode 100644 test/test_rebase_harness_env.py create mode 100644 test/test_rebase_release_target.py create mode 100644 test/test_rebase_resume_selection.py create mode 100644 test/test_rebase_shell_runtime.py create mode 100644 test/test_rebase_zcode_routing.py create mode 100644 test/test_remote_ci_owner.py diff --git a/adapters/vllm_omni/manifest.yaml b/adapters/vllm_omni/manifest.yaml index 94868f04f..9ac0bc3ad 100644 --- a/adapters/vllm_omni/manifest.yaml +++ b/adapters/vllm_omni/manifest.yaml @@ -25,11 +25,8 @@ upstream: kind: fork_tracking repo_path: ${VLLM_UPSTREAM_REPO} remote: origin - # v0.30.0 alignment campaign (2026-09-17): wheel pick and commit - # assignment target the release branch, not upstream main. The branch - # is cut but NOT yet tagged, so this tracks the branch head and is - # re-pinned to the tag commit once v0.30.0 is released. - target_branch: releases/v0.30.0 + # v0.32.0 alignment tracks upstream main until the release branch is cut. + target_branch: main # Repo-specific slice of the SHARED, human-curated knowledge base (the vendored # community docs at the repo-root `knowledge/` = settings.knowledge_dir; @@ -67,10 +64,10 @@ knowledge: # scoping) may only ever gain coverage here, never lose it. modules: worker_runner: - local_paths: [vllm_omni/worker/] + local_paths: [vllm_omni/worker/, vllm_omni/worker_v2/] upstream_paths: [vllm/v1/worker/] - test_paths: [tests/worker/, tests/diffusion/test_diffusion_worker.py, tests/e2e/offline_inference/custom_pipeline/, tests/e2e/online_serving/test_qwen3_tts_base.py, tests/entrypoints/test_omni_sleep_mode.py] - import_check: "from vllm_omni.worker.gpu_model_runner import *; from vllm_omni.worker.gpu_generation_model_runner import *; from vllm_omni.worker.gpu_ar_model_runner import *; from vllm_omni.worker.gpu_ar_worker import *; print('Worker/Runner imports OK')" + test_paths: [tests/worker/, tests/worker_v2/, tests/diffusion/test_diffusion_worker.py, tests/e2e/features/custom_pipeline/, tests/e2e/online_serving/test_qwen3_tts_base.py, tests/entrypoints/test_omni_sleep_mode.py] + import_check: "from vllm_omni.worker.gpu_model_runner import *; from vllm_omni.worker.gpu_generation_model_runner import *; from vllm_omni.worker.gpu_ar_model_runner import *; from vllm_omni.worker.gpu_ar_worker import *; from vllm_omni.worker_v2.omni_ar_model_runner import *; from vllm_omni.worker_v2.omni_generation_model_runner import *; print('Worker/Runner imports OK')" wave: 1 risk: high model_executor: @@ -82,14 +79,14 @@ modules: risk: high input_output: local_paths: [vllm_omni/inputs/, vllm_omni/outputs/, vllm_omni/utils/mm_outputs.py, vllm_omni/data_entry_keys.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/serialization.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_engine_core_client.py, vllm_omni/engine/output_processor.py, vllm_omni/request.py] - upstream_paths: [vllm/inputs/, vllm/v1/engine/input_processor.py, vllm/v1/engine/output_processor.py, vllm/v1/request.py] - test_paths: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/test_async_omni_engine_input.py, tests/test_outputs.py] + upstream_paths: [vllm/inputs/, vllm/renderers/, vllm/v1/engine/input_processor.py, vllm/v1/engine/output_processor.py, vllm/v1/request.py] + test_paths: [tests/inputs/, tests/entrypoints/test_omni_new_request_data.py, tests/engine/test_async_omni_engine_input.py, tests/engine/test_omni_request_output.py] import_check: "from vllm_omni.inputs.data import *; from vllm_omni.inputs.preprocess import *; from vllm_omni.engine.async_omni_engine import AsyncOmniEngine; from vllm_omni.engine.orchestrator import Orchestrator; from vllm_omni.engine.serialization import serialize_additional_information; print('Input/Output imports OK')" wave: 1 scheduler: local_paths: [vllm_omni/core/] upstream_paths: [vllm/v1/core/sched/] - test_paths: [tests/distributed/, tests/diffusion/distributed/test_ulysses_uaa_perf.py] + test_paths: [tests/core/sched/, tests/distributed/, tests/model_executor/test_cfg_pairing.py, tests/diffusion/distributed/test_ulysses_uaa_perf.py] import_check: "from vllm_omni.core.sched.omni_ar_scheduler import *; from vllm_omni.core.sched.omni_generation_scheduler import *; print('Scheduler imports OK')" wave: 1 risk: high @@ -213,18 +210,10 @@ rebase: # adapter is active; this whole `rebase:` section is HIGH-RISK (human-only # writes) so an agent can never self-install hooks hooks: rebase/hooks.py - # READ-ONLY parent-store read-compat (Rev 8 §9 PR4c, delivered by the - # completion design D2): v3 knowledge retrieval unions these layers LAST - # (runtime > adapter seed > parent) until the PR4d migration executes, - # after which the owner REMOVES these two keys (RUNBOOK step) and the - # compat layer dies. The v3 prelude fails CLOSED when a declared layer - # is unreadable and records open/close logical digests (§8 fairness). - knowledge: - parent_debug_db: ${REBASE_AGENT_ROOT}/agent/store/debug_memory.db - parent_skills_dir: ${REBASE_AGENT_ROOT}/agent/skills - # the parent store's own column carrying the upstream commit — repo - # data, so the neutral read-compat/migration code stays literal-free - parent_upstream_column: vllm_commit + # The READ-ONLY parent-store read-compat `knowledge:` block (Rev 8 §9 + # PR4c) is removed (owner decision 2026-10-01): the parent rebase-agent + # checkout no longer exists, so v3 runs on its own runtime + adapter seed + # knowledge only. An absent declaration attests nothing and blocks nothing. # the shared checkout flock name: v3, the v1 backend, EXT1's external # startup guard, and PR7 archival ALL take locks/.lock inside # the checkout, so exclusion is mutual (repo data — the core stays neutral) @@ -244,7 +233,10 @@ rebase: import_check_modules: [vllm._C_stable_libtorch, vllm._C] ops_namespace: _C stale_artifact_globs: ["_C*.so", "*.abi3.so", "_moe_C*.so"] - install_env: {VLLM_USE_PRECOMPILED: "1"} + install_env: + VLLM_USE_PRECOMPILED: "1" + VLLM_PRECOMPILED_WHEEL_COMMIT: "{commit}" + VLLM_PRECOMPILED_WHEEL_VARIANT: "{variant}" reinstall_retries: 3 pin: dockerfile: docker/Dockerfile.ci @@ -260,7 +252,7 @@ rebase: # trigger env (parent create_build env + trigger_and_monitor extra_env, # verbatim): opts the FULL suite in — a webhook build without these # runs fewer tests, which is why pushed rounds create their own build - build_env: {NIGHTLY: "1", RUN_HUNYUAN_IMAGE3_PERF: "1"} + build_env: {NIGHTLY: "1", RUN_HUNYUAN_IMAGE3_PERF: "1", EMAIL_DISTRIBUTION: "0"} # monitor classification data (parent monitor.py constants — EXCEPT # the pipeline upload/load lanes, moved to structural: their failure # suppresses every dynamic test job, so ignoring it was a false-green @@ -310,9 +302,9 @@ rebase: # behavioral routing replay). assignment_paths: model_config: [tests/entrypoints/test_stage_utils.py, tests/metrics/test_stats.py] - input_output: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/, tests/test_outputs.py] + input_output: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/, tests/engine/test_omni_request_output.py] scheduler: [tests/distributed/] - worker_runner: [tests/worker/, tests/diffusion/test_diffusion_worker.py, tests/e2e/offline_inference/custom_pipeline/] + worker_runner: [tests/worker/, tests/worker_v2/, tests/diffusion/test_diffusion_worker.py, tests/e2e/features/custom_pipeline/] model_executor: [tests/model_executor/, tests/diffusion/, tests/e2e/offline_inference/] online_serving: [tests/entrypoints/, tests/e2e/online_serving/] benchmarks: [tests/benchmarks/test_serve_cli.py] @@ -332,6 +324,13 @@ rebase: # full offline inference suite takes about four hours on the shared host, # so the previous 7200s default could never produce a terminal result. module_timeout_sec: 21600 + # The wrapper refuses a TEST_TIMEOUT_SEC above this cap; it must not be + # lower than the planned budget above or every required command is refused. + module_max_timeout_sec: 21600 + # Phase-2 module agents run on the CUDA wheel/runtime above. Broad pytest + # paths must retain CUDA-capable and platform-neutral items while leaving + # NPU/ROCm/XPU/MUSA-only items to their native CI lanes. + module_test_platform: cuda # root-level artifacts tests leak between runs (test_artifact_cleanup.sh # TEST_ARTIFACT_GLOBS, verbatim) — gitignored upstream but pile up on disk artifact_globs: ["audio_chatcmpl-*.wav", "audio_[0-9]*ch_*.wav", @@ -372,7 +371,7 @@ rebase: candidates: [tests/entrypoints/test_stage_utils.py, tests/metrics/test_stats.py, tests/entrypoints/test_omni_llm.py] fallback: tests/entrypoints/test_stage_utils.py input_output: - candidates: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/test_async_omni_engine_input.py, tests/entrypoints/test_omni_input_preprocessor.py, tests/test_outputs.py] + candidates: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/test_async_omni_engine_input.py, tests/entrypoints/test_omni_input_preprocessor.py, tests/engine/test_omni_request_output.py] fallback: tests/entrypoints/test_omni_new_request_data.py online_serving: candidates: [tests/entrypoints/test_async_omni.py, tests/entrypoints/test_async_omni_diffusion_config.py, tests/entrypoints/test_omni_diffusion.py, tests/entrypoints/test_omni_stage_diffusion_config.py, tests/entrypoints/openai_api/] diff --git a/adapters/vllm_omni/rebase/api_drift_guard.py b/adapters/vllm_omni/rebase/api_drift_guard.py index d4405ba1e..55cb11abd 100755 --- a/adapters/vllm_omni/rebase/api_drift_guard.py +++ b/adapters/vllm_omni/rebase/api_drift_guard.py @@ -2,10 +2,8 @@ # api_drift_guard.py — Inheritance/constructor/unpack drift guard for vllm-omni. # Ported from the rebase agent (agent/lib/api_drift_guard.py); runs # standalone with the TARGET repo python (cwd or OMNI_PATH = vllm-omni root). -# Verbatim except two documented divergences: the pooling constructor entry -# (parent's spelling could never import) and constructor-check import -# failures counting as mismatches instead of silent SKIPs (see inline -# comments at both sites). +# Documented divergences cover the pooling constructor/import checks and +# current renderer/EngineClient contracts (see doc/DRIFT_TRIAGE.md). # Repo-specific maps (INHERITANCE_MAP etc.) are the point of this file living # in the adapter tree; the copilot core never imports it. # Run with cwd = vllm-omni repo root (same as tasks/41_check_api_drift.sh). @@ -19,13 +17,10 @@ from pathlib import Path # --------------------------------------------------------------------------- -# Block flash attention C extensions from loading to prevent SIGBUS (signal 7) -# when the .so was compiled for a different CUDA / GPU architecture. -# -# The guard only inspects method signatures — it does not need the real -# flash attention kernels. By raising ImportError (instead of letting the -# process crash with SIGBUS), the existing try/except in the INHERITANCE_MAP -# loop can skip affected classes gracefully. +# Use real extensions by default: current vLLM refuses to import its flash +# attention package when both extensions are blocked, which skips worker and +# serving signatures. For a known incompatible binary, an operator may opt +# into the blocker with IMX_API_DRIFT_BLOCK_FLASH_ATTN=1. That run is partial. # --------------------------------------------------------------------------- class _BlockFlashAttnExtensions: @staticmethod @@ -40,14 +35,20 @@ def find_spec(fullname, path, target=None): return None -sys.meta_path.insert(0, _BlockFlashAttnExtensions) +_block_flash_attn = os.environ.get("IMX_API_DRIFT_BLOCK_FLASH_ATTN", "0") +if _block_flash_attn not in ("0", "1"): + raise ValueError("IMX_API_DRIFT_BLOCK_FLASH_ATTN must be 0 or 1") +if _block_flash_attn == "1": + sys.meta_path.insert(0, _BlockFlashAttnExtensions) + print("PARTIAL Flash attention extensions blocked by operator request; " + "affected import checks may be skipped.") # Ensure relative paths (vllm_omni/...) resolve when invoked from any cwd. if "OMNI_PATH" in os.environ: os.chdir(os.environ["OMNI_PATH"]) INHERITANCE_MAP = [ - ("vllm.inputs.preprocess", "InputPreprocessor", "vllm_omni.inputs.preprocess", "OmniInputPreprocessor"), + ("vllm.renderers", "BaseRenderer", "vllm_omni.inputs.preprocess", "omni_renderer_cls"), ("vllm.v1.engine.core_client", "AsyncMPClient", "vllm_omni.engine.stage_engine_core_client", "StageEngineCoreClient"), ("vllm.engine.protocol", "EngineClient", "vllm_omni.entrypoints.async_omni", "AsyncOmni"), ("vllm.v1.worker.gpu_model_runner", "GPUModelRunner", "vllm_omni.worker.gpu_model_runner", "OmniGPUModelRunner"), @@ -219,12 +220,17 @@ def _forbidden_test_imports() -> list[str]: return findings -def main() -> int: +def _check_inheritance() -> list[str]: mismatches: list[str] = [] for up_mod, up_cls_name, omni_mod, omni_cls_name in INHERITANCE_MAP: try: UpCls = import_class(up_mod, up_cls_name) OmniCls = import_class(omni_mod, omni_cls_name) + # OmniRenderer is a mixin; the actual upstream subclass is + # constructed by this factory rather than declared statically. + if omni_cls_name == "omni_renderer_cls": + OmniCls = OmniCls(UpCls) + omni_cls_name = OmniCls.__name__ except Exception as e: print(f"SKIP {omni_cls_name}: import error — {e}") continue @@ -233,6 +239,14 @@ def main() -> int: print(f"SKIP {omni_cls_name}: not a subclass of {up_cls_name}") continue + if up_cls_name == "EngineClient" and inspect.isabstract(OmniCls): + msg = ( + f"MISMATCH {omni_cls_name}: missing EngineClient implementations " + f"{sorted(OmniCls.__abstractmethods__)}; the engine cannot be instantiated" + ) + print(msg) + mismatches.append(msg) + for name in dir(OmniCls): if name.startswith("__") and name != "__init__": continue @@ -273,6 +287,11 @@ def main() -> int: else: print(f" OK {omni_cls_name}.{name}") + return mismatches + + +def main() -> int: + mismatches = _check_inheritance() constructor_call_checks = [ ( "vllm.entrypoints.openai.responses.serving", diff --git a/adapters/vllm_omni/rebase/prompt_data.yaml b/adapters/vllm_omni/rebase/prompt_data.yaml index 20b74de13..83bc9ed73 100644 --- a/adapters/vllm_omni/rebase/prompt_data.yaml +++ b/adapters/vllm_omni/rebase/prompt_data.yaml @@ -32,6 +32,15 @@ debug_prompt_template_live: | rebase regression. Separate assertion failures, latency-only failures, cancellation, and broken reporting steps. Record unknown attribution when evidence is insufficient; do not weaken thresholds or skip tests. + A local gated-model download denial is an access failure. Preserve the + original model source, tokenizer, configuration, geometry, selected + tests and timeout. Do not replace them with a synthetic local builder + or a different public tokenizer to make CI pass. If authorized access + is unavailable, finish with the exact unresolved failure so the parent + can continue the other jobs. A prototype is not the original job's proof. + Read run-directory operator evidence when present, including + codex_krea_scope_rejection.json and the remote main baseline. Do not + claim CI has no model access when its exact job has actually passed. 3. Read the target vLLM API and `git show origin/main:` for Omni intent. Fix the smallest confirmed rebase-specific cause. After a startup API fix, inspect adjacent constructor, override, result-field, dummy-run, @@ -39,7 +48,12 @@ debug_prompt_template_live: | 4. Reproduce with a targeted test using the real upstream API when possible, then run the owning suite. Report the exact validated commit and whether a new remote CI build has reached a terminal result. - 5. Write MODULE_DONE {module} to the signal file only after verification. + 5. Use the tool environment's IMX_LOG_DIR for this run's artifact directory; + form the absolute signal path as IMX_LOG_DIR/module.{module}.done. + Search only this run directory for signals, never / or /data. Write + MODULE_DONE {module} there only after verification. If the original + test remains failed, report the exact unresolved result and finish + without a success signal so the parent can continue the other jobs. MODULE={module} @@ -51,6 +65,20 @@ pytest_wrapper: "bash '{script_dir}/lib/run_module_pytest.sh'" # imx-omni-pytest instead of the retired shell script, and fix its quoting) pytest_wrapper_live: "imx-omni-pytest" +# Operational coverage may grow without changing the historical prompt maps. +# worker_v2 imports upstream worker/output APIs and must be ported with worker. +live_module_overrides: + scheduler: + omni_files: "vllm_omni/core/sched/" + test_paths: ["tests/core/sched/", "tests/distributed/", "tests/model_executor/test_cfg_pairing.py"] + input_output: + vllm_paths: "vllm/inputs/ vllm/renderers/ vllm/v1/engine/input_processor.py vllm/v1/engine/output_processor.py vllm/v1/request.py" + test_paths: ["tests/inputs/", "tests/entrypoints/test_omni_new_request_data.py", "tests/engine/", "tests/engine/test_omni_request_output.py"] + worker_runner: + omni_files: "vllm_omni/worker/ vllm_omni/worker_v2/" + test_paths: ["tests/worker/", "tests/worker_v2/", "tests/diffusion/test_diffusion_worker.py", "tests/e2e/features/custom_pipeline/"] + import_check: "from vllm_omni.worker.gpu_model_runner import *; from vllm_omni.worker.gpu_generation_model_runner import *; from vllm_omni.worker.gpu_ar_model_runner import *; from vllm_omni.worker.gpu_ar_worker import *; from vllm_omni.worker_v2.omni_ar_model_runner import *; from vllm_omni.worker_v2.omni_generation_model_runner import *; print('OK')" + module_vllm_paths: model_config: "vllm/config/ vllm/engine/arg_utils.py" input_output: "vllm/inputs/ vllm/v1/engine/input_processor.py vllm/v1/engine/output_processor.py vllm/v1/request.py" diff --git a/adapters/vllm_omni/rebase/shell_golden.json b/adapters/vllm_omni/rebase/shell_golden.json index 7ca8edae7..4d38acc82 100644 --- a/adapters/vllm_omni/rebase/shell_golden.json +++ b/adapters/vllm_omni/rebase/shell_golden.json @@ -601,7 +601,7 @@ "Cannot re-initialize CUDA in forked subprocess", "cuDNN error", "CUBLAS_STATUS", - "(? (already done)` = incremental relaunch. Do not assume all agents restart together. +**Orchestrator restart/resume:** `Skipping (already done)` = incremental relaunch. Inspect the current files and preserve work from completed modules. ### Shared log hygiene Anchor on the **current run**—match `Resuming run: rebase-* (phase: …)` for **this** session. `[claude_auth]` / `API key probe OK` is CLI init—**not** a module failure. Repeated `Using level-1 agent model:` banners are **noise**—diagnose from `ERROR:` / phase banners / task names. Git boilerplate (`detached HEAD`, `git switch -c`) is scaffold—not merge outcome unless paired with conflict/error lines. `SSH fetch failed; retrying via HTTPS token auth` followed by HTTPS success is normal infrastructure fallback—**not** a module failure. @@ -85,21 +112,15 @@ If a **required verification** fails and root cause is under `{OMNI_PATH}` but o 1. **Follow the root cause**—minimal targeted fixes anywhere under `{OMNI_PATH}`. 2. Keep fixes **small**—align with `{VLLM_PATH}`; no unrelated refactors. 3. **Document** every out-of-scope edit in the completion contract. -4. Prefer **additive** changes if another agent may touch the same file. +4. Read the current file first and preserve work from completed modules and resumed attempts. 5. After **two** failed attempts on that out-of-scope fix, **stop** and write `.fail` with traceback. ## Evaluation gates -**Truth = the latest checker output: exit code, final banner, every `MISMATCH`/`BROKEN`.** Ignore fix-agent tmux chatter and lines like `Import fix agent reported success` until a **full** `41`/`42` run after that path confirms (exit 0, zero `MISMATCH`/`BROKEN`). - -- **Chronology:** Anchor on the **latest** full run. Old failures are void after a later run exits 0 with no `MISMATCH`/`BROKEN`. -- **Success-before-failure:** Earlier green `41`/`42`/install lines before a later `ERROR:` still stand—do not blame install/Dockerfile unless that step also shows failure in the **same** segment. -- **42 after fix:** After `ERROR: N broken upstream import(s)`, **re-run** `tasks/42_check_upstream_imports.sh` before treating imports as green. -- **SKIP on green 41:** `SKIP` + exit **0** + `API drift check passed.` + no `MISMATCH` = pass. `SKIP` + non-zero exit or any `MISMATCH` = not green. -- **42 noise:** `All upstream imports resolve correctly.` may be followed by `Broken imports detected.`—anchor on the **last full `42` block**: exit 0 + no `BROKEN` = pass. If ambiguous, re-run 42. -- **SKIP when symbol absent:** Fix real imports/call sites per `{VLLM_PATH}`—no fake shims for drift only. -- **First pass alignment:** Match `{VLLM_PATH}` early. If `41` prints `FORBIDDEN` imports, fix in the same drift pass. +**Truth = the latest full checker output: exit code and every `MISMATCH`/`BROKEN`.** A completion message from another agent does not replace verification after its edits. -After edits, re-run **41** (zero `MISMATCH`) and **42** (zero `BROKEN`). ROCm/tokenizer/`TRANSFORMERS_CACHE` warnings are **noise** when checkers exit 0 green. +- Repeat both checks after a fix. Exit 0 with zero `MISMATCH`/`BROKEN` is required; an earlier green run does not validate later edits. +- Read skipped checks and explain any unverified path. Missing symbols need real imports/call-site ports matching `{VLLM_PATH}`, with no fake shims. +- ROCm/tokenizer/`TRANSFORMERS_CACHE` warnings alone do not fail a checker that exits 0 without mismatches. **Phase 1 ops (not your concern unless blocked):** Wheel/install halts—read `vllm_install.log` end-to-end. **`Failed modules: none`** + wheel failure ⇒ no module wave. **Phase 1 merge:** `CONFLICT`, `Automatic merge failed` → orchestrator halted **before** Phase 2—not your failure. **Path sync:** `tasks/35_sync_module_paths.sh failed`, missing `path_sync_final_*.json` → mapping drift, not pytest failure. @@ -108,11 +129,14 @@ After edits, re-run **41** (zero `MISMATCH`) and **42** (zero `BROKEN`). ROCm/to **Merge (your edits):** Resolve conflict markers **only** inside `{OMNI_FILES}`. No `git merge`/`git stash` for the rest of the tree. ## API drift (blocking before broad pytest) -`tasks/41_check_api_drift.sh` is **hard.** Any `MISMATCH` or non-zero exit → not green. Read output end-to-end; confirm on the **latest** full run after each fix. Open `api_drift_check.md` (or logged report) and fix **every** listed mismatch. +The old `tasks/41_check_api_drift.sh` and `tasks/42_check_upstream_imports.sh` were retired. Do not search the shared filesystem for them. Use these shipped equivalents with the target repository's Python: + +1. **API drift guard:** from `{OMNI_PATH}`, use `run_shell` to execute `python3 '{SCRIPT_DIR}/api_drift_guard.py'`. The runner puts the target venv on PATH; confirm `python3` resolves there. Read the entire output. Any `MISMATCH` or non-zero exit blocks completion. Repeat the full guard after fixes. +2. **Upstream import audit:** use `run_import_check` to resolve every explicit `vllm` module and named import found by AST in the scoped Omni files, including both worker implementations when assigned. Resolve names with the installed target modules; collect all failures as `BROKEN` and raise/exit non-zero if any exist. Also run the smoke-import command below. Inspect optional/hardware import failures rather than masking them; report any path that could not be verified. Repeat the full audit after fixes. There is no separate checker script or checker tool to locate. Typical fixes vs `{VLLM_PATH}`: `StageEngineCoreClient.shutdown` kwargs; `launch_core_engines` unpack/return-shape at the real omni↔upstream boundary. **`SKIP launch_core_engines unpack-check`** when the file does not inline `launch_core_engines` is fine on exit 0—no fake unpack. -**Infra:** `BROKEN`/`MISMATCH` followed by `tmux: command not found`—fix cited code first; `tmux` missing is environment. **`ImportError` from `{VLLM_PATH}/vllm/_C.abi3.so` with `undefined symbol`** → stale/incompatible binary—reinstall from `{VLLM_PATH}` (e.g. `VLLM_USE_PRECOMPILED=1 uv pip install -e .`). Never edit upstream source. Bad wheel index: same triage via `vllm_install.log` + `platform_tag`. +**Infra:** `BROKEN`/`MISMATCH` followed by `tmux: command not found`—fix cited code first; `tmux` missing is environment. An extension `ImportError` with `undefined symbol` indicates a stale/incompatible binary: inspect `vllm_install.log` and the pinned wheel/runtime first. Never edit upstream source. ## Inputs ### Module prompt source @@ -151,13 +175,13 @@ If `{COMMIT_LIST}` is empty while notes imply commits: do not invent churn—fla - Upstream vLLM target branch: {VLLM_PATH} Key paths: {UPSTREAM_FILES} - vllm-omni baseline intent: - `git show origin/main:` in {OMNI_PATH} + `git show {BASELINE_REF}:` in {OMNI_PATH} ## Hard rules 1. Edit only under {OMNI_PATH}. Never modify {VLLM_PATH}. 2. **Temporary files**: Create all helper scripts and scratch artifacts under `\${AGENT_TMP}` (outside the repos). Never create temp files in `{OMNI_PATH}` or `{VLLM_PATH}`. 3. Use only env from this prompt: `CUDA_VISIBLE_DEVICES={CUDA_DEVICES}`, `HF_HOME={HF_HOME}`. Ignore stale values in module prompt source. -4. Preserve omni-specific behavior from `origin/main` while adapting to upstream APIs. +4. Preserve omni-specific behavior from `{BASELINE_REF}` while adapting to upstream APIs. 5. Do not change copyright/license headers or license files. 6. Execute only valid shell commands. Never run bare narrative tokens (example: `Resume`). 7. **Never ask the user a question or wait for human input.** @@ -187,7 +211,7 @@ On its own line (parser-friendly): **Pytest truth:** `ERROR: FAILED: (rc=…)`/`FAILED`/`ERROR` **override** `[watchdog] … CONTINUE` when `rc≠0`. `rc=143` SIGTERM; `rc=124` timeout; `rc=2` collection error. -**Global timeout / stall:** `Build timed out` + frozen `Module progress` → parallel wave issue, not your `{OMNI_FILES}`. Confirm your module wrote the completion marker. +**Global timeout / stall:** `Build timed out` + frozen `Module progress` → inspect the active module's logs and owning test outcome. Confirm the completion marker only after required verification terminates successfully. **Other signals to filter:** - `[MODEL DOWNLOAD]`, `IGNORE (gated model access / HF 401)`, remote container deps (`installing vllm-omni[dev]`) → infra noise unless pytest after recovery fails on your files. @@ -202,20 +226,37 @@ On its own line (parser-friendly): - Post-green CI (`ECONNRESET`, push auth) → report exact line. Pre-commit: fix real Ruff/format in `{OMNI_PATH}`. ## Verification (must all pass) +- Every rendered full verification command below is required and must have an actual terminal pass with its original selected scope, timeout budget, and granted GPU geometry. Preserve the commands as rendered. +- Targeted diagnostics never substitute for a required full command. Write `MODULE_DONE` only after actual terminal evidence confirms that every required command passed; a live test, missing outcome, observation timeout, or diagnostic pass leaves full verification incomplete. **Module pytest wrapper:** Every verification line uses `imx-omni-pytest …`. **Do not replace with bare `pytest`.** It applies `test_watchdog.sh`, `TEST_TIMEOUT_SEC`, logs under `\${LOG_DIR}/tests/module_*.log`, and shared GPU lock for **every** pytest line from Phase-2 sessions. -**Long-running verification:** The MCP client timeout follows the harness -session budget; it is not a fixed 30-second limit. Use the module pytest wrapper -with `TEST_TIMEOUT_SEC={MODULE_TEST_TIMEOUT_SEC}` and allow terminal completion. -For short observation polls, guard missing paths before invoking commands that -would otherwise read stdin. Do not duplicate a live test or confuse an -observation timeout with a test result. -- All commands, assertions, selected scope and granted GPU geometry remain required. -- The engine cleans up this attempt's background processes when the attempt ends, - including after a normal return. Never return while verification is still live - or leave a detached chain for another module to inherit. -- If background verification is needed, redirect all three standard streams, - record the wrapper PID, poll it, and collect its terminal outcome before returning. +**Long-running verification and bridge call deadlines:** Every bridge tool call +has a client-side deadline set by the harness CLI, not by this prompt: zcode's +follows the session budget; cursor-agent's is fixed by the CLI and may be far +shorter than a test run. A call that outlives it is cut off at the client while +the shell keeps running, and the single-call stdio bridge then looks hung for +every later call. Therefore: +- Use `imx-omni-pytest …` with `TEST_TIMEOUT_SEC={MODULE_TEST_TIMEOUT_SEC}` for + every required verification command. Keep the selected scope, assertions and + granted GPU geometry; do not shrink or extend the test budget. +- Never block a tool call on a long wait (`sleep 480 && tail …`). For anything + longer than a short command, launch detached with stdin/stdout/stderr + redirected and a PID file, e.g. + `setsid -f sh -c 'echo $$ > "$1"; shift; exec "$@"' sh "$pidfile" imx-omni-pytest ... "$log" 2>&1` + (a plain `nohup ... &` is insufficient: the non-interactive bridge shell waits + for that job). Then poll the saved wrapper PID with short calls, at most one + check per tool call and under 20 seconds each including any sleep. +- Guard optional glob or command-substitution results before passing them to + stdin-reading commands (`tail`, `head`, `cat`, `grep`); an empty expansion + turns `tail -n 5 "$log"` into a bare stdin reader that holds the bridge. +- Observation or tool timeouts are not test results. While the owning wrapper or + its descendants are live, do not signal, restart or duplicate them; wait for + the wrapper's own terminal outcome and report incomplete when it is unavailable. +- The engine stops this attempt's background processes when the attempt ends, + including after a normal return. Never return while verification is still + live, and never leave a detached chain for another module to inherit. +- Produce incremental output (file writes, short commands, debug memory entries) + so the run keeps a heartbeat; a silent session is timed out as stalled. Run: {IMPORT_CHECK} @@ -242,14 +283,14 @@ If broken, find the new symbol in {VLLM_PATH} and update omni. Same semantics as ## Failure handling (up to {MAX_DEBUG_RETRIES} attempts) -### MANDATORY: When a test fails, FIRST check origin/main baseline -1. Call `git_show_test_baseline ""` — this shows how origin/main updated this test -2. If origin/main CHANGED the test (new params, imports, assertions) → apply the SAME changes to omni -3. If origin/main DELETED the test → check if omni still needs it -4. If origin/main RENAMED/SPLIT the test → find the new file and port omni modifications there -5. Only after aligning the test with origin/main, fix product code +### MANDATORY: When a test fails, FIRST check {BASELINE_REF} baseline +1. Call `git_show_test_baseline ""` — this shows how {BASELINE_REF} updated this test +2. If {BASELINE_REF} CHANGED the test (new params, imports, assertions) → apply the SAME changes to omni +3. If {BASELINE_REF} DELETED the test → check if omni still needs it +4. If {BASELINE_REF} RENAMED/SPLIT the test → find the new file and port omni modifications there +5. Only after aligning the test with {BASELINE_REF}, fix product code 6. Do NOT change product code to make a stale test pass -7. NEVER re-create or restore a test file that origin/main deleted or renamed +7. NEVER re-create or restore a test file that {BASELINE_REF} deleted or renamed just because the rebase harness (config.sh / a CI test command) still references the old path. Harness config is NOT ground truth: if a referenced test path no longer exists (pytest rc=4, "file or directory not found"), use @@ -261,7 +302,7 @@ If broken, find the new symbol in {VLLM_PATH} and update omni. Same semantics as 2. Classify: `unrecognized arguments` → setup. Ruff after green pytest → still blocking. Unmapped CI → mapping. **`41`/`MISMATCH`** → signature/unpack. **`42`/`BROKEN`** → import path/symbol vs `{VLLM_PATH}`. 3. **Time check:** If > 5 minutes of wall-clock work (not test wait) on the same failure without progress, write a `.fail` with what you tried and stop. Do not repeat the same fix pattern. - **"No code changes" trap:** If root cause is infra/config/environment, not omni code, you must still produce output. Document what you checked and why no fix is possible. Silence wastes the debug round. -4. Map failing omni call site ↔ upstream in `{VLLM_PATH}`; `git show origin/main` for intent. +4. Map failing omni call site ↔ upstream in `{VLLM_PATH}`; `git show {BASELINE_REF}` for intent. 5. Minimal compatible fix in `{OMNI_PATH}`. 6. Re-run failing check, then full verification. 7. Report: changed files, summary, latest results, `MODULE={MODULE_NAME}`. @@ -272,11 +313,19 @@ Infra (`rc=124`, GPU lock, OOM kill storms): one controlled retry with snapshot; Merge conflicts in your files: report paths, markers cleared, strategy—do not blame pytest until verification fails. -## Killing hung tests -Never `pgrep -f | kill` alone (broad pattern). Use: - bash {KILL_TEST_SCRIPT} "pytest.*" -or: - bash {KILL_TEST_SCRIPT} --pid +## Terminal orphan cleanup +Use `run_pytest` or `reproduce` for tests; their native runner owns actual +budget/watchdog cleanup. The old `lib/kill_test_tree.sh` is retired. +Observation or shell/tool timeouts never authorize manually terminating a live +owning wrapper or its descendants. Do not signal, restart, or duplicate them; +use short read-only checks and await the actual owning runner's terminal outcome. +Manual cleanup is allowed only AFTER that terminal outcome, for verified owned +orphans whose PID and birth identity match the owning attempt. Preserve foreign +processes. Replace both placeholders with the verified orphan PID and its +`/proc//stat` starttime ticks, then use the identity-checked helper: + {COPILOT_PYTHON} -c 'import sys; from infermatrix_copilot.testing.process_tree import kill_tree; pid=int(sys.argv[1]); kill_tree([pid], identity=dict([(pid, int(sys.argv[2]))]))' '' '' +Never terminate tests by a broad process-name pattern. Missing terminal evidence +means incomplete verification and does not authorize manual cleanup. ## Completion contract On success, write exactly: diff --git a/adapters/vllm_omni/testing/watchdog_patterns.yaml b/adapters/vllm_omni/testing/watchdog_patterns.yaml index 44d89691b..f6738eaec 100644 --- a/adapters/vllm_omni/testing/watchdog_patterns.yaml +++ b/adapters/vllm_omni/testing/watchdog_patterns.yaml @@ -24,7 +24,9 @@ critical: # with re.I, bare FATAL tier-1-killed a HEALTHY job on the benign # "Dropping uncorrelated non-fatal RPC error" WARNING (live full # run 2026-08-23); tier 1 sees the raw tail, so noise cannot save it - - "(?` (recorded on the day) · `archival_secret_allowlist` (PR7; may be empty). +## Release campaign startup and resume + +Choose the adapter's `upstream.target_branch` and freeze the upstream commit +before starting a campaign. The wheel picker fetches that branch explicitly, +including in scratch clones whose fetch configuration covers only the default +branch. A forced commit must be an ancestor of the fetched target branch; +a main-only commit cannot override a release target. Fetch errors stop the +run rather than using a stale tracking ref. + +Record the run id printed at startup. Resume that exact campaign with its +saved identity asserted: + +```bash +./infermatrix-copilot --resume \ + --repo --playbook repo-rebase-v3 \ + --task-param force_upstream_commit= \ + --task-param last_rebase_commit= +``` + +`--repo`, `--playbook`, and `--task-param` are assertions on resume. A mismatch +blocks before execution and leaves the saved task unchanged. Bare `--resume` +still selects the latest saved task; use an explicit id when several campaigns +share the run directory. Changing the target calls for a new run rather than +reusing another target's checkpoints or completion signals. + +Live module prompts take their target branch, upstream pin, assignment +baseline, and target-repository baseline ref from the current run. The +assignment baseline selects work; it does not prove the current tree's API +version or provide test evidence for a new target. The checkout SHA displayed +alongside the pin lets the agent check that its source and runtime agree. + +Agent shells and test shells disable Bash startup-file loading so SSH session +metadata cannot shadow the selected runtime's executables. Harness completion +requires a fresh module signal, no typed session/audit refusal, and the required +plan decision from that module's current attempt. Plain completion prose does +not replace these checks. + ## External checkout pin (EXT1-class guard) Before cutover, the EXTERNAL orchestrator's canonical checkout gains a diff --git a/doc/architecture/SPEC/app/core.md b/doc/architecture/SPEC/app/core.md index 017b2e56a..c8f17b1ab 100644 --- a/doc/architecture/SPEC/app/core.md +++ b/doc/architecture/SPEC/app/core.md @@ -1,6 +1,6 @@ # app/core.py —— 规范 - + `Copilot` is the headless workflow application. It resolves a TaskSpec, delegates reviewer verdict policy to `review.plan_gate`, confirms when needed, @@ -31,3 +31,11 @@ through the compatibility module. The application passes its reviewer to the plan gate, preserving the no-human fail-closed rule. Only orchestration belongs here; command argument parsing and stdio protocol registration remain transport concerns. + +## `--resume ` 断言式恢复（2026-10-27） + +`resume_last(run_id, *, repo, playbook, params)`：给定 run id 时只恢复那个 +run（`_contained_run_dir` 拒绝越界路径）；`--repo/--playbook/--task-param` +在恢复时是对已保存 task.json 的**断言**而非修改，任一不匹配即 BLOCKED。 +发布 launcher 因此能把 resume 绑定到不可变的 campaign，而不会误入更新的 +无关任务。损坏的 task.json 同样 BLOCKED 而非崩溃。 diff --git a/doc/architecture/SPEC/ci/buildkite.md b/doc/architecture/SPEC/ci/buildkite.md index 67dbc04e7..7d507cab5 100644 --- a/doc/architecture/SPEC/ci/buildkite.md +++ b/doc/architecture/SPEC/ci/buildkite.md @@ -1,6 +1,6 @@ # ci/buildkite.py —— 规范 - + `LOC ~253 · 边缘（CI provider 客户端） · refactor-status: ok` @@ -60,3 +60,10 @@ opt-in 默认关、BuildCreationRefused 只认 422+disabled、list_jobs/retry_jo 与最终 reconciliation 都押在它上面。对 `ci_loop.BuildCreationRefused` 的 惰性 import 是一条轻微的向内依赖（边缘 → rebase_engine）；接入第二个 provider 时把这个异常类型上提到共享的 CI 契约模块。 + +## jobs 列表分页（2026-10-27） + +`list_jobs` 跟随 `links.next` 游标翻页（同 scheme/host/path 才跟随，去重防 +环），`items`+`links` 形态与旧的裸列表/`jobs` 形态都接受；嵌入式 build 没有 +可读 jobs 列表时抛 `BuildkiteError` 而不是返回空列表（空列表曾被读成 +"没有失败的 job"）。 diff --git a/doc/architecture/SPEC/cli.md b/doc/architecture/SPEC/cli.md index e27c0197a..30ca547c0 100644 --- a/doc/architecture/SPEC/cli.md +++ b/doc/architecture/SPEC/cli.md @@ -1,6 +1,6 @@ # cli/ —— 规范 - + `LOC ~1420（6 个文件） · 接口 + 编排门面 · refactor-status: ok` @@ -109,3 +109,8 @@ RFC 子命令原样转发与共享入口：`test_rfc_integration.py`；服务契 ## 自进化接入（2026-10-04） `infermatrix-copilot improve …` 在通用 argparse 之前分派到 `improve.cli.main`；工作流检查、候选生成、实验、发布和账本查询使用该入口。 + +## 2026-10-27 `--resume [run-id]` + +`--resume` 现在接受可选 run id；`--repo`、`--playbook`、`--task-param` 与 +`--resume` 同用时是对保存任务的断言（见 `app/core`）。 diff --git a/doc/architecture/SPEC/engine/steps/rebase_v3.md b/doc/architecture/SPEC/engine/steps/rebase_v3.md index 04930e7b5..de9d5e3b7 100644 --- a/doc/architecture/SPEC/engine/steps/rebase_v3.md +++ b/doc/architecture/SPEC/engine/steps/rebase_v3.md @@ -51,6 +51,9 @@ step —— 薄的受治理 wrapper，substate-first、类型化失败、发布 lifecycle finalizer，每条退出路径都放。可变操作只碰 run_dir 内的 `git clone --shared` **scratch** upstream；canonical 路径绝不被采纳为可变 树；resume 采纳幸存 scratch 时重新注册 teardown。 +- full 模式的模块与 debug 入口也重新确保 scratch：wheel/assign 已 checkpoint + 时仍按 `upstream_commit` 恢复 checkout；不得改用 canonical HEAD，checkout + 失败返回 BLOCKED（debug 返回结构性 error）。 - **C3**（`_module_scope`）：repo 树 + run dir 是硬可写墙；模块 `local_paths` + `plans/` 是 primary；未知模块得到永不匹配的 primary —— 它的每次仓库写都被记录 out-of-scope，而非静默在圈内。 @@ -111,3 +114,22 @@ harness 会话通过 MCP tool bridge 拿到同一套工具面，plan gate 在派 的 `finally`，都调用 `rebase_engine.processes.cleanup_run_processes`： 上一个 attempt 留下的后台 pytest 链不能把 GPU 锁带进下一个 step （run-20261024 的 `GpuLockTimeout` 阻断即由此而来）。 + +## harness tier 直通、评审路由与 scratch 生命周期（2026-10-27，自 feat/rebase-main-v0320 移植） + +- `_tier_client`：`target.kind == "harness"` 时返回 `(transport, target)`，只检查 + CLI 可用与登录（`auth_gap`），不再因为空 api_key 而 BLOCKED—— + `STRICT_BACKEND=cursor` 的 run-20261027-173249 正是在此处的前身被挡。 +- `_harness_completion_client(settings, backend)`：把 harness `complete()` 适配成 + `messages.create` 形状，供计划评审与 wave-gate 评审在 `rebase_backend != api` + 时使用；`stop_reason != end_turn` 抛错而不是给出裁决。评审模型 = + `rebase_reviewer_model`，否则 `rebase_backend_model`/`strict_backend_model`。 +- `_run_debug_agent`：harness tier 走 `_harness_attempt`（与模块 attempt 相同的 + budgets、CUDA/HF、`target_venv`、state slice），API tier 保持 `run_agent_loop`。 +- remote_ci 的上游 commit 信号必须是 40 位 SHA，在边界拒绝。 +- scratch：只有干净 `done` 才丢弃一次性上游 checkout；blocked/崩溃的 run 可恢复， + 其 venv 的 editable vllm 解析进 scratch。full 模式的模块/debug 入口按 + `upstream_commit` 重新确保 scratch。 +- baseline venv（`ensure_baseline_venv`）：硬链接克隆 campaign venv 并换入 + baseline commit 的 wheel，让"omni main 对上次对齐的 wheel 是否也失败"可问； + 任何失败返回 None，保留保守的回归判定。 diff --git a/doc/architecture/SPEC/providers/base.md b/doc/architecture/SPEC/providers/base.md index dea613f35..abc13f3f0 100644 --- a/doc/architecture/SPEC/providers/base.md +++ b/doc/architecture/SPEC/providers/base.md @@ -1,6 +1,6 @@ # providers/base.py —— 规范 - + `LOC ~161 · provider 层契约 + 子进程环境白名单 · refactor-status: ok` @@ -60,3 +60,9 @@ stdlib + `..scopes.ToolScope`。它是一个叶子契约模块。 需要硬上限的调用方要预留"阈值 + 单个请求的最坏情况"。不支持的 transport 接受并忽略该参数，调用方必须先检查 `stops_at_spend`（知识服务的 gateway 会拒绝）。不得出现声称能硬性限制单次调用花费的属性。 `max_tokens` 对 CLI transport 不是花费上限。transport 报告的本次花费放在 `Reply.usage["cost_usd"]`（未知时不出现）。 + +## 2026-10-27 `AgentSessionRequest.read_roots` + +会话审计的额外**只读**包含根（rebase module agent 必须读上游 checkout 与 +目标 venv）。只放宽读包含判定，不放宽写；cursor 传输把它并进 +`audit_events(roots=...)`。 diff --git a/doc/architecture/SPEC/providers/claude_code.md b/doc/architecture/SPEC/providers/claude_code.md index cab70c0e1..ee56ed54d 100644 --- a/doc/architecture/SPEC/providers/claude_code.md +++ b/doc/architecture/SPEC/providers/claude_code.md @@ -1,6 +1,6 @@ # providers/claude_code.py —— 规范 - + `LOC ~205 · harness transport（Claude 订阅） · refactor-status: ok` @@ -50,3 +50,12 @@ x 必须为正。CLI 在每个 API 请求完成**之后**检查预算：它阻 `total_cost_usd` 以 `usage["cost_usd"]` 返回。 阈值参数由 `budget_arg()` 精确生成：浮点数的最短往返十进制、普通（非科学）记数法；非正、非有限值 → `ValueError`。 固定精度格式会移动边界（`.4f` 把 0.00001 变成 "0.0000" 即没有阈值，把 0.12345 变成 "0.1235" 即更高的阈值），所以不允许。 + +## 2026-10-27 会话契约 + +- 模块契约走 USER 通道（stdin）：`--system-prompt` argv 元素受 128KiB 单参数 + 上限约束，证据包会超出。 +- 以 `is_error` / refusal / `max_budget` 结束、或输出不可解析的会话一律 + truncated（绝不绿）；refusal/max_budget 另成为 typed refusal，标记文件 + 不能覆盖。 +- `-p json` 解析取最后一个带 `result` 的整行对象，容忍前后杂行。 diff --git a/doc/architecture/SPEC/providers/cursor.md b/doc/architecture/SPEC/providers/cursor.md index cc7bcc774..20218677f 100644 --- a/doc/architecture/SPEC/providers/cursor.md +++ b/doc/architecture/SPEC/providers/cursor.md @@ -1,6 +1,6 @@ # providers/cursor.py —— 规范 - + `LOC ~207 · harness transport（Cursor 订阅） · refactor-status: ok` @@ -62,3 +62,15 @@ worktree 物化失败而降级到 live checkout 的 PR run 都在这里。托管 worktree（repo+PR+sha 分键，run queue 不会让同 PR 的两个 run 并发）与 run 自己的 目录不加锁。等待上限为会话超时 + 60 s；超时则返回空文本、`truncated`、refusal， 并记 `capability_gap`（`cursor.exclusive_cwd`），绝不在别人的配置下静默开跑。 + +## 2026-10-27 会话失败不是干净退出；读审计根 + +- CLI 在发出 `result` 事件前退出（登录过期、未知模型、崩溃）曾表现为 + `text=""`、`truncated=False`，模块契约把它读成"会话完成"。现在 + `_session_failure` 把无 result 的会话标为 truncated 并附 typed refusal + （含退出码与 stderr 尾部）；`complete()` 返回 `stop_reason="error"`， + harness 评审适配器据此抛错而不是给出裁决。 +- 审计根 = cwd + run_dir + `req.read_roots`（上游 checkout、目标 venv）。 +- `.cursor/mcp.json` 不支持每工具调用超时（CLI 只认 `startupTimeoutSec`）， + `--print` 模式也没有对应开关；桥调用的客户端期限由 CLI 决定，prompt 不再 + 承诺"随会话预算"。 diff --git a/doc/architecture/SPEC/rebase_engine.md b/doc/architecture/SPEC/rebase_engine.md index 233107d8f..c423bdc8a 100644 --- a/doc/architecture/SPEC/rebase_engine.md +++ b/doc/architecture/SPEC/rebase_engine.md @@ -36,7 +36,9 @@ | `push_gate.py` | 推送闸裁决：结构性 vs 断言失败的确定性分类（Rev 8 §2.3） | | `push_to_ci.py` | commit+push-to-CI 编排：preflight、WAL 卫生、C4 双闸、单一传输 | | `push_wal.py` | 推送 WAL：先落盘的 intent、精确 OID 三分对账、回滚数据 | +| `remote_ci_owner.py` | 远端 CI 作业 → 模块归属解析（本地与远端作业名、pipeline 路由） | | `rebase_tools.py` | 父级 20 工具 + `doc_search`/`doc_read`（追加在父级顺序之后）作为 `ToolDef`；未接线后端**可见地**失败 | +| `wheel.py` | 预编译 wheel 选取与安装：目标分支游走、baseline rev-parse、venv 持有者释放、EBUSY 重试、scratch 自清理 | | `runctx.py` | `RebaseRuntime` + `CheckoutLock`（flock+卫生盾）+ 按事件循环的注册表 | | `substate.py` | 可持久、单写者、merge-not-overwrite 的 `state.json`（run_id 戳） | | `test_loop.py` | 本地测试环：逐测试恢复、baseline 复跑分流回归、类型化 skip | @@ -172,3 +174,20 @@ run-20261024 里这些链在 wave 1 完成后又占着 run 的 GPU 锁约 7 小 整套约 4 小时，旧的 7200s 包装器永远到不了终态），渲染为 prompt 的 `{MODULE_TEST_TIMEOUT_SEC}` 并写进每次 attempt 末尾的预算说明。 - `harness_timeout_s` 不再硬编码 7200，取 `settings.rebase_harness_timeout_sec`。 + +## harness 完成契约、读审计与宿主事实（2026-10-27，自 feat/rebase-main-v0320 移植） + +- `_harness_attempt` 的完成不再等于"会话未截断"：需要**本次 attempt** 重写的 + `signals/module..done`（首行 `MODULE_DONE `）、没有新的 `.fail`、未截断、 + 没有 typed refusal、且本模块的计划闸在本次 attempt 内打开。零事件的 CLI 崩溃 + 曾被读成 done 并通过 wave gate。 +- typed refusal 中，`audit: outside session roots: ...`（读包含）**不**否决 + 完成：那是原生读工具无法关闭的 harness 的侦测记录，写进 trace + （`harness_read_audit`）与结果文本。写审计与会话失败仍然否决。 +- 会话请求带 `read_roots=(vllm_path, target_venv)`；桥 spec 带 + `host_env={CUDA_VISIBLE_DEVICES, HF_HOME}`（见 `tool_bridge`）。 +- `module_pytest.py`：`rebase.testing.module_max_timeout_sec` 是 `TEST_TIMEOUT_SEC` + 的上限（vllm-omni 与计划预算同为 21600），`module_test_platform` 过滤非本平台 + 的 pytest 项。 +- `wheel.py`：`release_venv_holders`（仅 venv 结构存在时、排除调用者祖先链）、 + EBUSY/ETXTBSY 重试、baseline rev-parse 前置校验、分支重置前 `git clean -ffd`。 diff --git a/doc/architecture/SPEC/tool_bridge.md b/doc/architecture/SPEC/tool_bridge.md index c66cccfd4..d84413ebd 100644 --- a/doc/architecture/SPEC/tool_bridge.md +++ b/doc/architecture/SPEC/tool_bridge.md @@ -1,6 +1,6 @@ # tool_bridge.py —— 规范 - + `LOC ~308 · 面向 harness 会话的 scoped 工具 MCP server · refactor-status: ok` @@ -59,3 +59,12 @@ bridge 把 run 的 20 工具面 + `doc_search`/`doc_read` 暴露给 harness 会 ## 自进化接入（2026-10-04） bridge spec 序列化 `read_roots`、`deny_prefixes`、`strict_extras`、`executables`，并携带父进程 store 路径和单元上下文。`traced_call` 在每次工具调用时绑定上下文，使跨进程 trace 连续；不传递凭据。 + +## 宿主事实经 spec 进入桥进程（2026-10-27） + +harness CLI 以白名单 env 启动，桥进程的 `os.environ` 里没有调度器授予本 +run 的 `CUDA_VISIBLE_DEVICES`，也没有 `HF_HOME`——`_agent_shell_env` 与 +`_target_test_env` 都从 `os.environ` 读它们，于是桥内 run_pytest/reproduce +看到的是整机 GPU。spec 的 `rebase.host_env`（仅这两个非机密键，str→str， +否则 `TypeError`）在重建后端前写进桥进程 env。`rebase.target_venv` 同理 +携带已解析的目标 venv；凭据与完整子环境仍不进 spec。 diff --git a/knowledge/repos/vllm-omni/rebase/workflow.md b/knowledge/repos/vllm-omni/rebase/workflow.md index b57c88390..679718503 100644 --- a/knowledge/repos/vllm-omni/rebase/workflow.md +++ b/knowledge/repos/vllm-omni/rebase/workflow.md @@ -1,26 +1,29 @@ --- title: "Rebase 工作流：分支、波次与失败路由" created: 2026-07-16 -updated: 2026-09-04 +updated: 2026-10-13 type: guide tags: [vllm-omni, rebase] -sources: ["PR #6606", "vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh", ".buildkite/cuda/rebase-pipeline.yml", docker/Dockerfile.ci] +sources: ["PR #6606", "PR #8459", "vllm-omni-rebase-agent@122a9468:agent/config.py", "vllm-omni-rebase-agent@122a9468:config.sh", ".buildkite/cuda/rebase-pipeline.yml", docker/Dockerfile.ci] --- # Rebase 工作流：分支、波次与失败路由 运营事实来自 rebase-agent 配置快照（@122a9468，**可能漂移**）；仓库侧事实在 `main @ 19da2382` 复核。运营系统以 rebase-agent 仓库为准，本页是知识树快照； -upstream release baseline 为 `v0.28.0 @ 2cf0a6915ce5`。^[PR #6606] +upstream release baseline 为 `v0.31.0 @ db9527a46873454610df6dbedf79a36d6bf1a7f6` +（v0.28/v0.29/v0.30/v0.31 各轮 rebase 均已合入，v0.31 为 PR #8459）。^[PR #8459] ## 分支与管线 - 对齐分支 `dev/vllm-align`（目标合回 `main`）；专用 Buildkite 管线 `.buildkite/cuda/rebase-pipeline.yml`（仓库侧）+ 运营管线 `vllm-omni-rebase` （nightly 与 main CI）、`vllm-omni-release`（CI），org `vllm`；wheel 变体 - `cu130`；本轮 release-boundary rebase 的 vLLM pin 是 tag `v0.28.0` 的 - `2cf0a6915ce5`。CUDA/CI/ROCm base image 和安装文档须同步使用同一 release；只有 - 对齐未发布 SHA 时才启用额外 wheel reinstall/ABI repair。^[PR #6606] + `cu130`；v0.31 轮的 vLLM pin 是 tag `v0.31.0` 的 `db9527a468734`，v0.32 轮 + （对齐 upstream main，发布分支 cut 出前）改为 commit-pinned nightly wheel + （`VLLM_PRECOMPILED_WHEEL_COMMIT`）。CUDA/CI/ROCm base image 和安装文档须同步 + 使用同一 release；只有对齐未发布 SHA 时才启用额外 wheel reinstall/ABI repair。 + ^[PR #6606] - nightly 循环：定时 nightly 跑对齐分支全量套件；注意定时 nightly 与 API build 的互杀陷阱（[ci-environment-gotchas](../ci/guides/ci-environment-gotchas.md) 第 3 条）。 diff --git a/pyproject.toml b/pyproject.toml index ad6e63a11..6f2ecbcee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "openai>=1.50", "rich>=13.0", "markdown-it-py>=3", + "packaging>=24.0", ] [project.optional-dependencies] @@ -45,4 +46,7 @@ packages = ["src/infermatrix_copilot"] [tool.pytest.ini_options] testpaths = ["test"] +# fixture trees under test/fixtures contain real-looking test_*.py files and +# conftest.py copies of target repos; they are data, never collected +norecursedirs = ["fixtures", ".*", "build", "dist", "node_modules"] addopts = "-q" diff --git a/src/infermatrix_copilot/app/core.py b/src/infermatrix_copilot/app/core.py index cc747bbc3..98a3f47d5 100644 --- a/src/infermatrix_copilot/app/core.py +++ b/src/infermatrix_copilot/app/core.py @@ -293,17 +293,54 @@ def run_queue(self, specs: list[TaskSpec], *, assume_yes: bool = False, return code return 0 - def resume_last(self) -> int: - """Re-enter the most recent run at its first incomplete step.""" - runs = sorted(self.settings.run_root.glob("run-*")) \ - if self.settings.run_root.exists() else [] - for run_dir in reversed(runs): + def resume_last(self, run_id: str = "", *, repo: str = "", + playbook: str = "", params: dict | None = None) -> int: + """Resume a selected run, or the latest run when no id is supplied. + + Target arguments are assertions against the saved task, never edits to + it. Release launchers can therefore bind resume to an immutable campaign + without accidentally entering a newer, unrelated task. + """ + if run_id: + try: + runs = [self._contained_run_dir(run_id)] + except (OSError, ValueError, RuntimeError) as exc: + print(f"✋ cannot resume: {exc}") + return BLOCKED_EXIT + else: + runs = sorted(self.settings.run_root.glob("run-*"), reverse=True) \ + if self.settings.run_root.exists() else [] + for run_dir in runs: task_file = run_dir / "task.json" if not task_file.exists(): + if run_id: + print(f"✋ cannot resume {run_id}: task.json is missing") + return BLOCKED_EXIT continue - saved = json.loads(task_file.read_text(encoding="utf-8")) - spec = TaskSpec(**saved["spec"]) - playbook = parse_playbook(saved["playbook"], str(task_file)) + try: + saved = json.loads(task_file.read_text(encoding="utf-8")) + spec = TaskSpec(**saved["spec"]) + saved_playbook = parse_playbook(saved["playbook"], str(task_file)) + except (OSError, UnicodeError, ValueError, KeyError, TypeError, + AttributeError) as exc: + print(f"✋ cannot resume {run_dir.name}: invalid task.json ({exc})") + return BLOCKED_EXIT + mismatches = [] + if repo and repo != spec.repo: + mismatches.append("repo") + if playbook and playbook != saved_playbook.name: + mismatches.append("playbook") + for key, expected in (params or {}).items(): + if key not in spec.params or ( + json.dumps(spec.params[key], sort_keys=True) + != json.dumps(expected, sort_keys=True) + ): + mismatches.append(f"task param {key!r}") + if mismatches: + print(f"✋ cannot resume {run_dir.name}: saved task does not match " + + ", ".join(mismatches) + + "; resume preserves the recorded task") + return BLOCKED_EXIT try: # tier preflight holds on re-entry too (deployment state # may have changed since the run was created) self.settings.tier_target(spec.mode) @@ -311,7 +348,7 @@ def resume_last(self) -> int: print(style("✋ cannot resume: ", "red", "bold") + str(exc)) return BLOCKED_EXIT print(f"↻ resuming {run_dir.name}: {spec.describe()}") - return self._execute(playbook, spec, run_dir, resuming=True) + return self._execute(saved_playbook, spec, run_dir, resuming=True) print("no resumable run found") return 1 diff --git a/src/infermatrix_copilot/ci/buildkite.py b/src/infermatrix_copilot/ci/buildkite.py index ca7ab283b..17b6e5c07 100644 --- a/src/infermatrix_copilot/ci/buildkite.py +++ b/src/infermatrix_copilot/ci/buildkite.py @@ -183,22 +183,51 @@ def get_job_log(self, build_id: str, job_id: str) -> str: def list_jobs(self, build_id: str) -> list[dict]: """Authoritative job list: the dedicated /jobs endpoint when it - behaves, embedded jobs from a fresh build fetch otherwise (some + behaves, following its cursor links through every page; embedded + jobs from a fresh build fetch otherwise (some builds 404 the endpoint; it can also answer with the build object itself — parent-documented). RAISES when neither source is readable: retrieval failure must stay distinguishable from a genuinely empty job list, or a reconciliation during an API outage would silently pass (round-2 review).""" + jobs_path = self._build_path(build_id) + "/jobs" + url = jobs_path + "?per_page=100" + seen: set[str] = set() + jobs: list[dict] = [] try: - status, data = self._request( - "GET", self._build_path(build_id) + "/jobs?per_page=100", - None, False) - if status == 200: - if isinstance(data, list): - return [j for j in data if isinstance(j, dict)] - if isinstance(data, dict) and \ - isinstance(data.get("jobs"), list): - return [j for j in data["jobs"] if isinstance(j, dict)] + while url: + # Cursor links must remain on this build's jobs endpoint; + # repeated links cannot establish a complete job list. + parsed = urllib.parse.urlsplit(url) + expected = urllib.parse.urlsplit(jobs_path) + if url in seen or (parsed.scheme, parsed.netloc, parsed.path) != \ + (expected.scheme, expected.netloc, expected.path): + break + seen.add(url) + status, data = self._request("GET", url, None, False) + if status != 200: + break + if isinstance(data, dict) and "items" in data: + items, links = data["items"], data.get("links") + if not isinstance(items, list) or not isinstance(links, dict): + break + next_url = links.get("next") + if next_url is not None and not isinstance(next_url, str): + break + jobs.extend(j for j in items if isinstance(j, dict)) + if not next_url: + return jobs + url = next_url + continue + # Preserve the older unpaged response shapes only on the + # first request. A partial cursor traversal is never returned. + if len(seen) == 1: + if isinstance(data, list): + return [j for j in data if isinstance(j, dict)] + if isinstance(data, dict) and \ + isinstance(data.get("jobs"), list): + return [j for j in data["jobs"] if isinstance(j, dict)] + break except BuildkiteError: pass build = self.get_build(build_id) @@ -206,7 +235,10 @@ def list_jobs(self, build_id: str) -> list[dict]: raise BuildkiteError( "list_jobs failed: the jobs endpoint was unusable and the " "build itself is unreadable") - return [j for j in (build.get("jobs") or []) if isinstance(j, dict)] + if not isinstance(build.get("jobs"), list): + raise BuildkiteError( + "list_jobs failed: the build has no readable jobs list") + return [j for j in build["jobs"] if isinstance(j, dict)] def retry_job(self, build_id: str, job_id: str) -> tuple[str | None, bool]: diff --git a/src/infermatrix_copilot/cli/entry.py b/src/infermatrix_copilot/cli/entry.py index 56768d477..390a1d363 100644 --- a/src/infermatrix_copilot/cli/entry.py +++ b/src/infermatrix_copilot/cli/entry.py @@ -100,7 +100,8 @@ def main(argv: list[str] | None = None) -> int: "doc/RUNBOOK-rebase.md); rfc: portable RFC " "drafting and progress service") parser.add_argument("--repo", default="", - help="with migrate-knowledge: the target repo name") + help="with migrate-knowledge: target repo; " + "with --resume: assert the saved repo name") parser.add_argument("--dry-run", action="store_true", help="with migrate-knowledge: report only, write " "nothing else") @@ -115,16 +116,20 @@ def main(argv: list[str] | None = None) -> int: help="skip confirmation prompts (headless)") parser.add_argument("--plan-only", action="store_true", help="resolve and print the plan without executing") - parser.add_argument("--resume", action="store_true", - help="resume the most recent run at its first incomplete step") + parser.add_argument("--resume", nargs="?", const="", default=None, + metavar="RUN_ID", + help="resume RUN_ID, or the most recent run when omitted; " + "repo/playbook/task params assert its saved identity") parser.add_argument("--playbook", help="run a specific playbook by name " - "(incl. planner-invisible candidates)") + "(incl. planner-invisible candidates); " + "with --resume: assert the saved playbook") parser.add_argument("--report-only", action="store_true", help="with --playbook: read-only variant of the task") parser.add_argument("--task-param", action="append", default=[], metavar="KEY=VALUE", - help="with --playbook: task param (repeatable), " + help="with --playbook: task param; with --resume: " + "assert a saved task param (repeatable), " "e.g. --task-param local_ci_only=true") parser.add_argument("--no-chat", action="store_true", help="use the plain command REPL instead of the " @@ -207,8 +212,10 @@ def main(argv: list[str] | None = None) -> int: return copilot.execute_reserved(args.execute_reserved) if args.execute_strict_reserved: return copilot.execute_strict_reserved(args.execute_strict_reserved) - if args.resume: - return copilot.resume_last() + if args.resume is not None: + return copilot.resume_last(args.resume, repo=args.repo, + playbook=args.playbook or "", + params=parse_task_params(args.task_param)) if args.playbook: params = parse_task_params(args.task_param) return copilot.run_playbook(args.playbook, params=params, diff --git a/src/infermatrix_copilot/engine/steps/rebase_v3.py b/src/infermatrix_copilot/engine/steps/rebase_v3.py index 6855000ad..c8a362afe 100644 --- a/src/infermatrix_copilot/engine/steps/rebase_v3.py +++ b/src/infermatrix_copilot/engine/steps/rebase_v3.py @@ -12,6 +12,7 @@ from dataclasses import asdict as _dc_asdict import asyncio +import re import shlex import weakref from pathlib import Path @@ -28,6 +29,7 @@ from ...rebase_engine.push_gate import evaluate_push_gate from ...rebase_engine.substate import Substate from ...rebase_engine.test_manifest import ManifestSpec, build_manifest +from ...rebase_engine.remote_ci_owner import RemoteOwnerResolver from ...rebase_engine.testing_env import scrubbed_agent_env from ...scopes import PathScope, ToolScope from ..step import FailureKind, StepContext, StepResult @@ -95,6 +97,17 @@ def _tier_client(ctx: StepContext): target = ctx.settings.tier_target(mode) except TierNotConfiguredError as exc: return StepResult(False, FailureKind.BLOCKED, str(exc)) + if getattr(target, "kind", "api") == "harness": + from ...providers.registry import transport_for_id + + transport = transport_for_id(ctx.settings, target.provider_id) + gap = (None if transport.cli_path() else + f"{target.provider_id} CLI is unavailable") or transport.auth_gap() + if gap: + ctx.trace.record("capability_gap", capability="rebase.module_agent", + detail=gap) + return StepResult(False, FailureKind.BLOCKED, gap) + return transport, target if not target.api_key: ctx.trace.record("capability_gap", capability="rebase.module_agent", detail=f"no API key for tier target {target.source} " @@ -139,25 +152,42 @@ def _test_roots(manifest: dict) -> tuple: return tuple(tm.get("test_change_roots") or ("tests/",)) +def _host_facts(host_env: Mapping[str, str] | None = None) -> tuple[str | None, str | None]: + """(CUDA_VISIBLE_DEVICES, HF_HOME) for target subprocesses: the explicit + `host_env` (the bridge spec's copy of what the PARENT run was granted) + wins over this process's environment — a harness spawns the bridge with + an allowlisted env that carries neither.""" + import os + host = dict(host_env or {}) + return (host.get("CUDA_VISIBLE_DEVICES", os.environ.get("CUDA_VISIBLE_DEVICES")), + host.get("HF_HOME", os.environ.get("HF_HOME"))) + + def _target_test_env(settings, manifest: dict, - *, pythonpath_prepend: str | None = None) -> dict: + *, pythonpath_prepend: str | None = None, + venv_override: str | None = None, + host_env: Mapping[str, str] | None = None) -> dict: """The env for TARGET-repo subprocesses (tests, precommit, wheel installs): inherit-plus-overlay with the target venv on PATH, the host's CUDA selection, and HF_HOME — raw manifest commands like `pytest ...` must resolve inside the target repo's runtime, never the - copilot's own virtualenv.""" - import os + copilot's own virtualenv. `venv_override` swaps the runtime for + baseline runs (the last-rebase wheel — see `_ensure_baseline_venv`); + `host_env` supplies CUDA/HF when this process is a bridge.""" from ...testing.env_plan import build_subprocess_env - venv = _target_venv(manifest, extra=settings.expansion_env()) + venv = venv_override or _target_venv(manifest, + extra=settings.expansion_env()) + cuda, hf_home = _host_facts(host_env) return build_subprocess_env( venv=Path(venv) if venv else None, - cuda_visible_devices=os.environ.get("CUDA_VISIBLE_DEVICES"), - hf_home=os.environ.get("HF_HOME"), + cuda_visible_devices=cuda, + hf_home=hf_home, pythonpath_prepend=pythonpath_prepend) def _agent_shell_env(ctx: StepContext, manifest: dict, repo_root: str, - adapter_dir: Path, *, gpu_mutex: bool = True) -> dict: + adapter_dir: Path, *, gpu_mutex: bool = True, + host_env: Mapping[str, str] | None = None) -> dict: """The env for AGENT shells (`run_shell` + everything it spawns): the PR1 credential scrub over the process env, then the TARGET runtime overlay (venv PATH/VIRTUAL_ENV, host CUDA selection, HF_HOME) and the @@ -173,11 +203,11 @@ def _agent_shell_env(ctx: StepContext, manifest: dict, repo_root: str, # every other adapter keeps the token scrubbed) keep_hf = bool((manifest.get("validation") or {}) .get("requires_hf_token")) + cuda, hf_home = _host_facts(host_env) env = build_subprocess_env( base=scrubbed_agent_env(keep_hf_token=keep_hf), venv=Path(venv) if venv else None, - cuda_visible_devices=os.environ.get("CUDA_VISIBLE_DEVICES"), - hf_home=os.environ.get("HF_HOME")) + cuda_visible_devices=cuda, hf_home=hf_home) env["IMX_TARGET_REPO"] = str(repo_root) env["IMX_LOG_DIR"] = str(ctx.run_dir) env["IMX_ADAPTER_REBASE"] = str(adapter_dir / "rebase") @@ -202,9 +232,29 @@ def _agent_shell_env(ctx: StepContext, manifest: dict, repo_root: str, return env +def _harness_completion_client(settings, backend: str): + """Adapt shared tool-less harness replies for existing reviewer callers.""" + from types import SimpleNamespace + + from ...providers.registry import transport_for_id + + transport = transport_for_id(settings, backend) + + def create(**params): + reply = transport.complete( + system="", messages=params["messages"], + model=params["model"], max_tokens=params["max_tokens"]) + if reply.stop_reason != "end_turn": + raise RuntimeError("harness reviewer did not complete") + return SimpleNamespace(content=reply.blocks) + + return SimpleNamespace(messages=SimpleNamespace(create=create)) + + def build_backends(*, settings, state, run_dir: Path, trace, manifest: dict, repo: str, model: str, - base_url: str = "", api_key: str = ""): + base_url: str = "", api_key: str = "", + host_env: Mapping[str, str] | None = None): """The PRODUCTION `RebaseBackends`: plan review on the run's resolved tier backend, pytest/reproduce/precommit through the PR1 runner in the TARGET env, and the knowledge tools on the copilot stores (agents may @@ -224,16 +274,24 @@ def build_backends(*, settings, state, run_dir: Path, trace, from ...rebase_engine.rebase_tools import RebaseBackends from ...testing.runner import TestJob, TestRunner repo_name = (state.get("task_spec") or {}).get("repo", "") - reviewer_model = getattr(settings, "rebase_reviewer_model", "") \ - or model + review_backend = getattr(settings, "rebase_backend", "api") or "api" + reviewer_model = getattr(settings, "rebase_reviewer_model", "") or ( + (getattr(settings, "rebase_backend_model", "") + or getattr(settings, "strict_backend_model", "")) + if review_backend != "api" else model) def request_plan_review(**kw) -> dict: - from anthropic import Anthropic - ckw: dict = {"api_key": api_key} - if base_url: - ckw["base_url"] = base_url + if review_backend != "api": + client = _harness_completion_client(settings, review_backend) + else: + from anthropic import Anthropic + + ckw: dict = {"api_key": api_key} + if base_url: + ckw["base_url"] = base_url + client = Anthropic(**ckw) return review_plan( - Anthropic(**ckw), reviewer_model, + client, reviewer_model, plan_json_path=str(kw.get("plan_json_path", "")), plan_md_path=str(kw.get("plan_md_path", "") or ""), kind=str(kw.get("kind", "rebase"))) @@ -257,7 +315,7 @@ def _run_tests(kw: Mapping, key: str) -> dict: TestJob(key=f"{key}_{abs(hash(cmd)) % 10 ** 8}", command=cmd, timeout_sec=float(kw.get("timeout") or 1800), min_gpus=0, gpu_lock=True), - _target_test_env(settings, manifest)) + _target_test_env(settings, manifest, host_env=host_env)) tail = "" try: if outcome.log_file and Path(outcome.log_file).is_file(): @@ -300,7 +358,7 @@ def run_precommit(**kw) -> dict: TestJob(key="agent_precommit", command=command, timeout_sec=float(pc.get("timeout_sec") or 600), min_gpus=0, gpu_lock=False), - _target_test_env(settings, manifest)) + _target_test_env(settings, manifest, host_env=host_env)) return {"exit_code": outcome.rc, "passed": outcome.rc == 0, "log_file": outcome.log_file} @@ -480,7 +538,7 @@ def doc_search(**kw) -> dict: try: from ...knowledge_docs import KnowledgeDocs kn = manifest.get("knowledge") or {} - docs = KnowledgeDocs(ctx.settings.knowledge_dir, + docs = KnowledgeDocs(settings.knowledge_dir, kn.get("repo_subdir")) query = str(kw.get("query") or kw.get("keyword") or "").strip() if not query: @@ -497,7 +555,7 @@ def doc_read(**kw) -> dict: try: from ...knowledge_docs import KnowledgeDocs kn = manifest.get("knowledge") or {} - docs = KnowledgeDocs(ctx.settings.knowledge_dir, + docs = KnowledgeDocs(settings.knowledge_dir, kn.get("repo_subdir")) path = str(kw.get("path") or "").strip() if not path: @@ -598,8 +656,22 @@ def _register_scratch_teardown(ctx: StepContext, scratch: Path) -> None: _SCRATCH_REGISTERED.add(key) from ..lifecycle import register_finalizer - async def _teardown_scratch(_outcome, _path=scratch, _key=key) -> None: + async def _teardown_scratch(outcome, _path=scratch, _key=key) -> None: import shutil + # Only a clean terminal discards the disposable checkout. A blocked + # or crashed run is RESUMABLE, and the target venv's upstream package is an + # editable install resolving INTO this scratch — tearing it down on + # those exits silently breaks the upstream import for every later + # invocation until some step happens to rebuild the scratch + # (run-202601012: a resumed test loop died wholesale on + # "No module named " because the wheel step replayed its + # recorded outcome instead of re-validating the install). + status = getattr(outcome, "status", "") + if status != "done": + print(f"[rebase] upstream scratch kept for the resumable run " + f"(outcome={status or 'crash'}): {_path}", flush=True) + _SCRATCH_REGISTERED.discard(_key) + return shutil.rmtree(_path, ignore_errors=True) _SCRATCH_REGISTERED.discard(_key) @@ -621,14 +693,14 @@ def _ensure_upstream_scratch(ctx: StepContext) -> str | StepResult: # only a path INSIDE the run dir counts as an existing scratch — a # canonical path in `upstream_path` (older state, manual seeding) must # never be adopted as the mutable tree - if str(scratch).startswith(str(ctx.run_dir)) \ - and (scratch / ".git").exists(): - _register_scratch_teardown(ctx, scratch) - return str(scratch) - if not origin: - return StepResult(False, FailureKind.BLOCKED, - "no canonical upstream recorded — prelude gap") - scratch = ctx.run_dir / "upstream_scratch" + adopted = (str(scratch).startswith(str(ctx.run_dir)) + and (scratch / ".git").exists()) + created = False + if not adopted: + if not origin: + return StepResult(False, FailureKind.BLOCKED, + "no canonical upstream recorded — prelude gap") + scratch = ctx.run_dir / "upstream_scratch" if not (scratch / ".git").exists(): r = subprocess.run(["git", "clone", "--shared", "--no-checkout", str(origin), str(scratch)], @@ -637,11 +709,37 @@ def _ensure_upstream_scratch(ctx: StepContext) -> str | StepResult: return StepResult(False, FailureKind.BLOCKED, "upstream scratch clone failed: " + r.stderr.strip()[:300]) - subprocess.run(["git", "-C", str(scratch), "checkout", "--detach", - "HEAD"], capture_output=True, text=True, - timeout=300) + created = True ctx.trace.record("upstream_scratch_created", path=str(scratch)) _register_scratch_teardown(ctx, scratch) + if origin: + # The shared clone borrows local objects, but its fetches must reach + # the canonical repository's upstream. Fetching the local clone only + # exposes its local branches and can hide a configured release branch. + remote = ctx.state.get("upstream_remote", "origin") + source_remote = subprocess.run( + ["git", "-C", str(origin), "remote", "get-url", str(remote)], + capture_output=True, text=True, timeout=30, check=False) + if source_remote.returncode == 0 and source_remote.stdout.strip(): + r = subprocess.run( + ["git", "-C", str(scratch), "remote", "set-url", "origin", + source_remote.stdout.strip()], + capture_output=True, text=True, timeout=30, check=False) + if r.returncode != 0: + return StepResult(False, FailureKind.BLOCKED, + "upstream scratch remote configuration failed") + # A resumed wheel step is replayed, not re-run. Restore its recorded + # commit instead of silently using the canonical checkout's current HEAD. + # Plain checkout preserves local work or fails; never hard-reset it. + target = ctx.state.get("upstream_commit") + if target or created: + r = subprocess.run(["git", "-C", str(scratch), "checkout", "--detach", + str(target or "HEAD")], + capture_output=True, text=True, timeout=300) + if r.returncode != 0: + return StepResult(False, FailureKind.BLOCKED, + "upstream scratch checkout failed: " + + r.stderr.strip()[:300]) ctx.state["upstream_path"] = str(scratch) return str(scratch) @@ -849,6 +947,8 @@ async def _terminal_report(outcome, _run_dir=ctx.run_dir, _sub=sub, extra=ctx.settings.expansion_env()) if upstream: updates["upstream_origin_path"] = upstream + updates["upstream_remote"] = (manifest.get("upstream") or {}).get( + "remote", "origin") ctx.state.setdefault("upstream_origin_path", upstream) baseline = _task_params(ctx).get("last_rebase_commit", "") \ or ctx.state.get("last_rebase_upstream_commit", "") @@ -892,6 +992,17 @@ async def _terminal_report(outcome, _run_dir=ctx.run_dir, _sub=sub, "(--task-param upstream_commit=, or a " "resumed run's substate) — refusing to push " "an unprepared branch") + # The push path pins this value into the CI Dockerfile, whose pin + # forms only accept a full 40-hex SHA (push_to_ci validates the same + # way); a short SHA or tag here would pass the signal check and then + # fail every push round with "wheel pin does not match". Refuse it + # at the boundary instead. + if not re.fullmatch(r"[0-9a-f]{40}", upstream_commit): + return StepResult(False, FailureKind.BLOCKED, + f"remote_ci upstream-commit signal " + f"'{upstream_commit}' is not a full 40-hex SHA " + "— pass the full commit id the wheel step " + "selected") updates["upstream_commit"] = upstream_commit if mode in MUTATING_MODES: @@ -1023,6 +1134,7 @@ async def _v3_wheel(ctx: StepContext) -> StepResult: installed = wheel_mod.ensure_wheel_installed( Path(upstream), found, wheel_spec, python=str(Path(venv) / "bin" / "python"), + venv_root=Path(venv), install_log=ctx.run_dir / "wheel_install.log", import_check_log=ctx.run_dir / "wheel_import_check.log", pre_checkout_head=pre_head) @@ -1057,6 +1169,8 @@ async def _v3_assign(ctx: StepContext) -> StepResult: cfg = Phase1Config( upstream_repo=Path(upstream), target_repo=Path("."), log_dir=ctx.run_dir, baseline_commit=baseline, + target_branch=(manifest.get("upstream") or {}).get("target_branch") + or (manifest.get("repo") or {}).get("default_branch", "main"), base_class_watch_paths=tuple((manifest.get("rebase") or {}) .get("base_class_watch_paths") or ())) modules = manifest.get("modules") or {} @@ -1131,6 +1245,10 @@ async def _run_debug_agent(ctx: StepContext, manifest: dict, module: str, honestly gave up, or "error:" when the DEBUG MACHINERY itself failed (structural, per the §2.3 taxonomy — agent dispatch failures are never ordinary test failures).""" + if _task_params(ctx).get("rebase_mode") == "full": + upstream = _ensure_upstream_scratch(ctx) + if isinstance(upstream, StepResult): + return "error:" + upstream.summary client = _tier_client(ctx) if isinstance(client, StepResult): return "error:debug backend unavailable" @@ -1162,15 +1280,49 @@ async def _run_debug_agent(ctx: StepContext, manifest: dict, module: str, ctx.trace.record("debug_attempt", slug=slug, module=module, model=target.model) try: - result = await run_agent_loop( - client, prompt, model=target.model, tool_defs=defs, - extra_tools=tools, - scope=_module_scope(repo_root, module, manifest, - run_dir=ctx.run_dir), - trace=ctx.trace, require_plan_review=False, - model_aliases=ctx.settings.model_aliases, - model_mismatch_policy=ctx.settings.model_mismatch_policy, - agent_log=str(agent_log)) + scope = _module_scope(repo_root, module, manifest, run_dir=ctx.run_dir) + if getattr(target, "kind", "api") == "harness": + from ...rebase_engine.module_rebase import ( + ModuleRunConfig, + _harness_attempt, + ) + + import os as _os + config = ModuleRunConfig( + vllm_path=paths.vllm_path, omni_path=repo_root, + script_dir=str(adapter_dir / "rebase"), model=target.model, + log_dir=str(ctx.run_dir), settings=ctx.settings, + signal_dir=str(ctx.run_dir / "signals" / "debug" / slug), + # the same host facts and budgets a module attempt gets: the + # bridge forwards CUDA/HF from here and the debug prompt's + # budget note must not fall back to the dataclass defaults + cuda_devices=_os.environ.get("CUDA_VISIBLE_DEVICES", "0,1"), + hf_home=_os.environ.get("HF_HOME", "/model"), + harness_timeout_s=float(getattr( + ctx.settings, "rebase_harness_timeout_sec", 7200)), + test_timeout_sec=int(paths.env.get("TEST_TIMEOUT_SEC") or 7200), + backend=target.provider_id, + backend_model=getattr(ctx.settings, "rebase_backend_model", "") + or target.model, + manifest_path=str(adapter_dir / "manifest.yaml"), + paths_spec={k: val for k, val in _dc_asdict(paths).items() + if k != "env"}, repo=repo_name, + target_venv=paths.env.get("VIRTUAL_ENV", ""), + state_slice={"task_spec": ctx.state.get("task_spec") or {}, + "run_id": ctx.state.get("run_id", ""), + "upstream_commit": ctx.state.get("upstream_commit", "")}) + result = await _harness_attempt( + prompt, module=module or slug, config=config, scope=scope, + trace=ctx.trace, tool_defs=defs, plan_prefix="", + require_plan_review=False) + else: + result = await run_agent_loop( + client, prompt, model=target.model, tool_defs=defs, + extra_tools=tools, scope=scope, + trace=ctx.trace, require_plan_review=False, + model_aliases=ctx.settings.model_aliases, + model_mismatch_policy=ctx.settings.model_mismatch_policy, + agent_log=str(agent_log)) except Exception as exc: # noqa: BLE001 - a debug crash is STRUCTURAL ctx.trace.record("debug_attempt_error", slug=slug, error=str(exc)) return f"error:debug agent crashed: {exc}" @@ -1179,6 +1331,10 @@ async def _run_debug_agent(ctx: StepContext, manifest: dict, module: str, cleanup_run_processes(ctx.run_dir, ctx.run_dir.name) if result.get("done"): return "done" + if result.get("session_completed"): + # A harness can finish honestly without a verified success signal. + # That is an unresolved test, not broken dispatch machinery. + return "not_done" # done=False from the loop is ALWAYS machinery/budget (fatal auth, # stream error, model mismatch, truncation abort, max turns) — an agent # that finishes, even unsuccessfully, returns done=True. Structural. @@ -1228,15 +1384,20 @@ def report_fn(test_name: str, pattern: str, detail: str) -> None: def _client(): if not client_holder: - from anthropic import Anthropic - target = ctx.settings.tier_target("eco") - # max_retries=0: the 90 s cap is a WALL cap — the SDK's - # default retry loop would multiply it - ckw: dict = {"api_key": target.api_key, "max_retries": 0} - if target.base_url: - ckw["base_url"] = target.base_url - client_holder.append((Anthropic(**ckw), target.model)) + if getattr(target, "kind", "api") == "harness": + settings = ctx.settings.model_copy( + update={"strict_backend_timeout_s": 90.0}) + client = _harness_completion_client(settings, target.provider_id) + else: + from anthropic import Anthropic + + # max_retries=0: retries must not multiply the 90 s wall cap. + ckw: dict = {"api_key": target.api_key, "max_retries": 0} + if target.base_url: + ckw["base_url"] = target.base_url + client = Anthropic(**ckw) + client_holder.append((client, target.model)) return client_holder[0] def review_fn(test_name: str, snippet: str) -> str: @@ -1319,7 +1480,8 @@ async def _v3_test_loop(ctx: StepContext) -> StepResult: outputs={"state_updates": {"phase3_failed": []}}) import os - from ...testing.runner import TestJob, TestRunner + import subprocess + from ...testing.runner import TestJob, TestRunner, pass_marker_for from ...testing.watchdog import WatchdogPatterns if _venv_declared(manifest) and not _target_venv( manifest, extra=ctx.settings.expansion_env()): @@ -1387,6 +1549,38 @@ def run_fn(slug: str) -> tl.TestRunResult: worktree_path = ctx.run_dir / "main_worktree" + # A main-tracking campaign's wheel can remove symbols the target's main still + # imports, so a baseline sharing the campaign venv dies at conftest + # import and pre-existing failures read as regressions (run-202601012: + # tiny_model_tests' gated-model setup errors — the identical errors + # were PRE-EXISTING in the merged v0.30 campaign). Baseline runs get a + # venv carrying the LAST-REBASE wheel; None keeps the old shared-venv + # behavior (regression-preserving, never a silent pass). + baseline_commit = str(ctx.state.get("last_rebase_upstream_commit") + or "") + _baseline_venv_cache: list[Path | None] = [] + + def _ensure_baseline_venv() -> Path | None: + if _baseline_venv_cache: + return _baseline_venv_cache[0] + result: Path | None = None + rb_wheel = (manifest.get("rebase") or {}).get("wheel") + venv = _target_venv(manifest, extra=ctx.settings.expansion_env()) + upstream = str(ctx.state.get("upstream_path", "") or "") + if baseline_commit and rb_wheel and venv and upstream: + result = wheel_mod.ensure_baseline_venv( + Path(venv), wheel_mod.WheelSpec.from_manifest(rb_wheel), + Path(upstream), baseline_commit) + if result is None: + ctx.trace.record( + "baseline_venv_unavailable", + commit=baseline_commit[:12] or None, + detail="baseline runs share the campaign venv; pre-existing " + "classification may be impossible after upstream " + "import-breaking removals") + _baseline_venv_cache.append(result) + return result + def baseline_fn(slug: str) -> tl.TestRunResult | None: wt = tl.ensure_main_worktree(Path(repo), worktree_path, base_ref=_baseline_ref(manifest)) @@ -1398,12 +1592,15 @@ def baseline_fn(slug: str) -> tl.TestRunResult | None: cuda_visible_devices=runner.cuda, record_fn=record_fn, review_fn=review_fn, report_fn=report_fn, notify_download=notify_download) - # main's files must import main's code: prepend the worktree so the - # baseline never executes against rebase dependencies (parent's - # baseline PYTHONPATH override) — same TARGET env otherwise; infra - # outcomes propagate (a baseline timeout must never read as "fails - # on main too") - env = _target_test_env(ctx.settings, manifest, pythonpath_prepend=str(wt)) + # Prepend the worktree so baseline imports use its own target source. + # Native dependencies come from the BASELINE runtime when one could + # be built (last-rebase wheel); bootstrap failures cannot establish + # pre-existing test failures. Infra outcomes propagate (a baseline + # timeout must never read as "fails on main too") + bv = _ensure_baseline_venv() + env = _target_test_env(ctx.settings, manifest, + pythonpath_prepend=str(wt), + venv_override=str(bv) if bv else None) return _to_result(wt_runner.run(_job(slug), env, baseline=True)) async def debug_fn(slug: str, label: str, rc: int, @@ -1429,7 +1626,18 @@ async def debug_fn(slug: str, label: str, rc: int, # debug patch must NOT stay in the tree — assertion failures pass # the push gate by default, so leftover edits would eventually be # committed and pushed - snap, untracked = tl.snapshot_worktree(Path(repo)) + try: + snap, untracked = tl.snapshot_worktree(Path(repo)) + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + ctx.trace.record("local_debug_snapshot_unavailable", slug=slug, + detail=str(exc)) + return f"debug snapshot unavailable: {exc}" + try: + policy_before = capture_patch_policy(Path(repo)) + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + ctx.trace.record("local_debug_policy_unavailable", slug=slug, + detail=str(exc)) + return f"debug patch policy unavailable: {exc}" verdict = await _run_debug_agent( ctx, manifest, jobs_by_slug[slug].get("module", ""), slug, traceback_text or f"{label} failed with rc={rc}") @@ -1446,6 +1654,21 @@ async def debug_fn(slug: str, label: str, rc: int, if rerun.rc != 0: tl.restore_worktree(Path(repo), snap, untracked) return False + try: + policy = evaluate_debug_patch( + Path(repo), policy_before, local_verdict="passed") + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + tl.restore_worktree(Path(repo), snap, untracked) + pass_marker_for(runner.tests_dir, _job(slug)).unlink(missing_ok=True) + ctx.trace.record("local_debug_policy_unavailable", slug=slug, + detail=str(exc)) + return f"debug patch policy unavailable: {exc}" + if not policy.allowed: + tl.restore_worktree(Path(repo), snap, untracked) + pass_marker_for(runner.tests_dir, _job(slug)).unlink(missing_ok=True) + ctx.trace.record("local_debug_policy_rejected", slug=slug, + reason=policy.reason, paths=list(policy.paths)) + return False return True result = await tl.run_test_loop( @@ -1496,10 +1719,11 @@ def _halt_on_phase3(ctx: StepContext, sub: Substate) -> StepResult | None: def _precommit_failed_hooks(outcome) -> frozenset | None: - """The set of pre-commit hook ids that FAILED, parsed from the run's - log (`- hook id: ` lines follow each Failed hook), or None when - the log is missing/unparseable - callers must then treat the failure - as structural rather than guess.""" + """Return IDs following Failed headers, or None for unparseable red. + + Verbose Passed and Skipped hooks print IDs too. Counting those can + falsely classify a newly failing hook as pre-existing baseline debt. + """ global _PRECOMMIT_HOOK_ID_RE import re if _PRECOMMIT_HOOK_ID_RE is None: @@ -1511,8 +1735,21 @@ def _precommit_failed_hooks(outcome) -> frozenset | None: errors="replace") except OSError: return None - ids = frozenset(_PRECOMMIT_HOOK_ID_RE.findall(text)) - return ids or None + text = re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text) + status_re = re.compile( + r"^\S.*\.{3,}(?:\(no files to check\))?(Passed|Failed|Skipped)$") + ids = set() + awaiting_failed_id = False + for line in text.splitlines(): + if header := status_re.fullmatch(line): + if awaiting_failed_id: + return None + awaiting_failed_id = header.group(1) == "Failed" + elif hook := _PRECOMMIT_HOOK_ID_RE.fullmatch(line): + if awaiting_failed_id: + ids.add(hook.group(1)) + awaiting_failed_id = False + return frozenset(ids) if ids and not awaiting_failed_id else None def _precommit_baseline(ctx: StepContext, repo: str, pc: dict, @@ -1543,6 +1780,11 @@ def _precommit_baseline(ctx: StepContext, repo: str, pc: dict, timeout_sec=float(pc.get("timeout_sec") or 600), min_gpus=0, gpu_lock=False) outcome = runner.run(job, _target_test_env(ctx.settings, manifest)) + if not outcome.timed_out and outcome.rc != 0 \ + and pc.get("retry_once", True): + # Auto-fix hooks can change which hooks fail on the next pass. + # Compare the same configured retry on current and baseline trees. + outcome = runner.run(job, _target_test_env(ctx.settings, manifest)) if outcome.timed_out: # a timed-out probe proves nothing about the baseline - treat # it as unavailable so the original red stays STRUCTURAL @@ -1744,6 +1986,10 @@ async def _v3_module_rebase(ctx: StepContext) -> StepResult: ctx, manifest, _task_params(ctx).get("rebase_mode", "")) if blocked is not None: return blocked + if _task_params(ctx).get("rebase_mode") == "full": + upstream = _ensure_upstream_scratch(ctx) + if isinstance(upstream, StepResult): + return upstream client = _tier_client(ctx) if isinstance(client, StepResult): return client @@ -1804,6 +2050,7 @@ async def _v3_module_rebase(ctx: StepContext) -> StepResult: vllm_path=paths.vllm_path, omni_path=paths.omni_path, script_dir=str(adapter_dir / "rebase"), model=target.model, log_dir=str(ctx.run_dir), + signal_dir=str(ctx.run_dir / "signals"), last_rebase_vllm_commit=ctx.state.get("last_rebase_upstream_commit", ""), # the prompt's CUDA/HF facts must describe THIS host, not the @@ -1817,14 +2064,25 @@ async def _v3_module_rebase(ctx: StepContext) -> StepResult: backend=getattr(ctx.settings, "rebase_backend", "api") or "api", backend_model=getattr(ctx.settings, "rebase_backend_model", "") or "", harness_timeout_s=ctx.settings.rebase_harness_timeout_sec, - test_timeout_sec=int(paths.env["TEST_TIMEOUT_SEC"]), + # the env builder always sets TEST_TIMEOUT_SEC from the adapter's + # `rebase.testing.module_timeout_sec`; the fallback only serves + # stubbed env builders + test_timeout_sec=int(paths.env.get("TEST_TIMEOUT_SEC") or ( + (manifest.get("rebase") or {}).get("testing") or {} + ).get("module_timeout_sec", 7200)), settings=ctx.settings, manifest_path=str(adapter_dir / "manifest.yaml"), paths_spec={k: val for k, val in _dc_asdict(paths).items() if k != "env"}, + target_venv=paths.env.get("VIRTUAL_ENV", ""), repo=str((ctx.state.get("task_spec") or {}).get("repo") or ""), state_slice={"task_spec": ctx.state.get("task_spec") or {}, "run_id": ctx.state.get("run_id", ""), + "target_branch": (manifest.get("upstream") or {}) + .get("target_branch") or (manifest.get("repo") or {}) + .get("default_branch", "main"), + "last_rebase_upstream_commit": ctx.state.get( + "last_rebase_upstream_commit", ""), "upstream_commit": ctx.state.get("upstream_commit", "")}) async with _serial_lock(ctx.run_dir): outcome = await rebase_module( @@ -2125,6 +2383,9 @@ def _verify_locally(slug: str) -> str: from ...rebase_engine.test_manifest import _label_to_slug + owner_resolver = RemoteOwnerResolver( + Path(repo), manifest, tuple(ctx.state.get("active_modules") or ())) + async def debug_fn(jr) -> bool: """One serialized CI-debug dispatch (parent `_dispatch_sdk_ci_debug` parity): snapshot → agent → tree-change check → local harness @@ -2139,7 +2400,14 @@ async def debug_fn(jr) -> bool: f"{jr.name} recorded unfixed") return False slug = _label_to_slug(jr.name) - module = str((_local_jobs().get(slug) or {}).get("module", "")) + owner = owner_resolver.resolve(jr.name, _local_jobs()) + module = owner.module + # Routing coverage is independent of leaf identity and CI verdicts. + ctx.trace.record("ci_debug_owner_route", job_id=getattr(jr, "job_id", ""), + slug=slug, module=module, status=owner.status, + reason=owner.reason, + candidate_modules=list(owner.candidate_modules), + sources=list(owner.sources), path_count=len(owner.paths)) log_tail = "" try: if jr.log_file and Path(jr.log_file).is_file(): @@ -2153,7 +2421,12 @@ async def debug_fn(jr) -> bool: # file, a later agent's edit to that same file is invisible to # `status --porcelain` (round-1 review) before = _worktree_digest(repo) - snap, untracked = tl.snapshot_worktree(Path(repo)) + try: + snap, untracked = tl.snapshot_worktree(Path(repo)) + except (OSError, RuntimeError, subprocess.SubprocessError) as exc: + ctx.trace.record("ci_debug_snapshot_unavailable", job=jr.name, + detail=str(exc)) + return False try: policy_before = capture_patch_policy(Path(repo)) except (OSError, RuntimeError, subprocess.SubprocessError) as exc: diff --git a/src/infermatrix_copilot/providers/base.py b/src/infermatrix_copilot/providers/base.py index 943831cfb..8055fcf6f 100644 --- a/src/infermatrix_copilot/providers/base.py +++ b/src/infermatrix_copilot/providers/base.py @@ -85,6 +85,10 @@ class AgentSessionRequest: step_name: str = "" bridge_spec_path: Path | None = None trace: Any = None # RunTrace-shaped (record(kind, **fields)) or None + # Extra READ-containment roots for the session audit beyond the scope + # root and the run dir (a rebase module agent must read the upstream + # checkout and the installed target runtime). Never widens writes. + read_roots: tuple[str, ...] = () @dataclass diff --git a/src/infermatrix_copilot/providers/claude_code.py b/src/infermatrix_copilot/providers/claude_code.py index 13fbe0f46..0b645a148 100644 --- a/src/infermatrix_copilot/providers/claude_code.py +++ b/src/infermatrix_copilot/providers/claude_code.py @@ -101,8 +101,36 @@ def _run(self, prompt_text: str, *, system: str, cwd: str, @staticmethod def _parse(stdout: str) -> dict: - """The single JSON object from -p json output; tolerant of stray - warning lines before it.""" + """The single JSON result object from -p json output; tolerant of + stray warning lines before OR after it. Prefer the last whole-line + object carrying a "result" (telemetry trails more often than it + leads); a stray line containing a brace must never shadow the real + object or the session silently parses to {} — empty text, zero + turns, done.""" + text = stdout.strip() + if not text: + return {} + try: + data = json.loads(text) + if isinstance(data, dict): + return data + except json.JSONDecodeError: + pass + fallback: dict | None = None + for line in reversed(text.splitlines()): + line = line.strip() + if not line.startswith("{"): + continue + try: + data = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(data, dict): + if "result" in data: + return data + fallback = fallback if fallback is not None else data + if fallback is not None: + return fallback start = stdout.find("{") if start < 0: return {} @@ -178,10 +206,22 @@ def run_session(self, req: AgentSessionRequest) -> AgentOutcome: usage = self._usage(data) tool_calls, tools_used = self._bridge_activity(req.run_dir, before) is_error = bool(data.get("is_error")) + stop_reason = str(data.get("stop_reason") or "") + unparsed = not data + # A session the CLI ended badly (quota/auth/CLI error, refusal, budget + # stop) or whose output we could not parse is NOT a completed loop: + # the completion contract upstream treats truncated=True as "do not + # trust this attempt". refusal/max_budget additionally become typed + # refusals so a marker file can never override them. + truncated = (timed_out or is_error or unparsed + or stop_reason in ("max_turns", "max_budget", "refusal")) + refusals = ([f"claude-code stop_reason={stop_reason}"] + if stop_reason in ("refusal", "max_budget") else []) if req.trace is not None: req.trace.record( "harness_session", provider=self.spec.id, step=req.step_name, num_turns=data.get("num_turns"), is_error=is_error, + stop_reason=stop_reason or None, timed_out=timed_out, cost_usd=usage.cost_usd, bridge_tool_calls=tool_calls, served_model=usage.served_model) @@ -189,8 +229,8 @@ def run_session(self, req: AgentSessionRequest) -> AgentOutcome: text=str(data.get("result") or ""), iterations=int(data.get("num_turns") or 0), tool_calls=tool_calls, - truncated=timed_out or data.get("stop_reason") == "max_turns", - refusals=[], + truncated=truncated, + refusals=refusals, input_tokens=usage.input_tokens, output_tokens=usage.output_tokens, tools_used=tools_used[:40]) diff --git a/src/infermatrix_copilot/providers/cursor.py b/src/infermatrix_copilot/providers/cursor.py index ccf043df1..f94fa3c08 100644 --- a/src/infermatrix_copilot/providers/cursor.py +++ b/src/infermatrix_copilot/providers/cursor.py @@ -131,7 +131,9 @@ def _run(self, text: str, *, cwd: str, timeout_s: float, model: str = "") -> tuple[list[dict], bool]: """One CLI invocation → (parsed events, timed_out). A timeout kills the process but keeps the partial stream — a half-done investigation - is salvage material, not garbage.""" + is salvage material, not garbage. The exit code and stderr tail of + the last invocation are kept on `self._last_exit` for the session + contract to judge an empty stream.""" # --approve-mcps is load-bearing: headless runs do not auto-approve # configured MCP servers, and without it the tool bridge is silently # ignored (found in the live smoke — session ran on native tools only) @@ -141,12 +143,15 @@ def _run(self, text: str, *, cwd: str, timeout_s: float, if selected: cmd += ["--model", selected] timed_out = False + self._last_exit = (0, "") try: proc = subprocess.run( cmd, input=text, cwd=cwd, env=sanitized_env(), capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout_s, check=False) stdout = proc.stdout or "" + self._last_exit = (int(proc.returncode or 0), + (proc.stderr or "")[-400:].strip()) except subprocess.TimeoutExpired as exc: timed_out = True raw = exc.stdout or b"" @@ -163,6 +168,8 @@ def _run(self, text: str, *, cwd: str, timeout_s: float, continue return events, timed_out + _last_exit: tuple[int, str] = (0, "") + @staticmethod def _final_text(events: list[dict]) -> str: for event in reversed(events): @@ -170,6 +177,25 @@ def _final_text(events: list[dict]) -> str: return str(event.get("result") or "").strip() return "" + @staticmethod + def _has_result(events: list[dict]) -> bool: + return any(event.get("type") == "result" for event in events) + + def _session_failure(self, events: list[dict], timed_out: bool) -> str: + """Why a finished session is NOT a completed one, or "". A CLI that + exits before emitting its `result` event (login expired, unknown + model, crash) returns an empty or partial stream with a clean-looking + Python return; reading that as a done session let a crashed module + attempt pass the wave gate.""" + if timed_out or self._has_result(events): + return "" + rc, err = self._last_exit + detail = f" (exit code {rc})" if rc else "" + tail = f": {err}" if err else "" + return (f"cursor-agent ended without a result event{detail}{tail}" + if events or rc or err else + "cursor-agent produced no output") + @staticmethod def _usage(events: list[dict]) -> SessionUsage: """Best-effort: cursor-agent's usage/model reporting varies by @@ -265,9 +291,11 @@ def run_session(self, req: AgentSessionRequest) -> AgentOutcome: shutil.rmtree(path, ignore_errors=True) else: path.unlink(missing_ok=True) - audit = audit_events(events, roots=(str(cwd), str(req.run_dir)), - read_only=req.scope.read_only, cwd=str(cwd)) + audit = audit_events( + events, roots=(str(cwd), str(req.run_dir), *req.read_roots), + read_only=req.scope.read_only, cwd=str(cwd)) usage = self._usage(events) + failure = self._session_failure(events, timed_out) if req.trace is not None: req.trace.record( "harness_session", provider=self.spec.id, step=req.step_name, @@ -275,13 +303,14 @@ def run_session(self, req: AgentSessionRequest) -> AgentOutcome: shell_commands=audit.shell_commands, file_reads=audit.file_reads, writes=audit.writes, other_tool_calls=audit.other_tool_calls, timed_out=timed_out, - served_model=usage.served_model) + session_failure=failure, served_model=usage.served_model) return AgentOutcome( text=self._final_text(events), iterations=0, # the harness does not expose its round count tool_calls=audit.tool_calls, - truncated=timed_out, - refusals=[f"audit: {v}" for v in audit.violations], + truncated=timed_out or bool(failure), + refusals=[f"audit: {v}" for v in audit.violations] + + ([failure] if failure else []), input_tokens=usage.input_tokens, output_tokens=usage.output_tokens, tools_used=audit.tools_used[:40]) @@ -302,9 +331,14 @@ def complete(self, *, system: str, messages: list[dict], shutil.rmtree(scratch, ignore_errors=True) usage = self._usage(events) text = self._final_text(events) + failure = self._session_failure(events, timed_out) return Reply( blocks=[Block(type="text", text=text)] if text else [], - stop_reason="max_tokens" if timed_out else "end_turn", + # a session that died before its result is not an end_turn: the + # harness reviewer adapter raises on anything else, which is how + # a plan review on a crashed CLI stays an error, not a verdict + stop_reason=("max_tokens" if timed_out else + "error" if failure else "end_turn"), usage={"input_tokens": usage.input_tokens, "output_tokens": usage.output_tokens, "cache_read_input_tokens": 0, diff --git a/src/infermatrix_copilot/providers/zcode.py b/src/infermatrix_copilot/providers/zcode.py index 33093247b..5cc79a588 100644 --- a/src/infermatrix_copilot/providers/zcode.py +++ b/src/infermatrix_copilot/providers/zcode.py @@ -203,6 +203,15 @@ def auth_gap(self) -> str | None: def _write_mcp_config(session: Path, spec_path: Path, *, timeout_s: float = 30.0) -> None: package_root = Path(__file__).resolve().parents[2] + # The harness stays credential-free; its scoped bridge also needs + # the lease's GPU visibility and the target environment for tests. + bridge_env = {k: os.environ[k] for k in ( + "CUDA_VISIBLE_DEVICES", "LD_LIBRARY_PATH", "HF_HOME", + "OMP_NUM_THREADS", "MKL_NUM_THREADS", "ZCODE_DATA_BASE_DIR", + "STRICT_BACKEND", "STRICT_BACKEND_MODEL", "STRICT_BACKEND_CLI", + "ZCODE_PROVIDER_ID", "ZCODE_REASONING_LEVEL", + "REBASE_REVIEWER_MODEL", "NOTIFY_EMAIL") if k in os.environ} + bridge_env["PYTHONPATH"] = str(package_root) config = session / ".zcode" / "config.json" config.parent.mkdir() config.write_text(json.dumps({"features": {"memory": False}, "memory": {"use": False}, @@ -215,7 +224,7 @@ def _write_mcp_config(session: Path, spec_path: Path, *, "command": sys.executable, "args": ["-m", "infermatrix_copilot.tool_bridge", "--spec", str(spec_path)], - "env": {"PYTHONPATH": str(package_root)}, + "env": bridge_env, }}}}, indent=2), encoding="utf-8") @staticmethod @@ -296,7 +305,8 @@ def _write_model_config(self, session: Path, model: str) -> Path: return path def _run(self, text: str, *, session: Path, timeout_s: float, - tool_less: bool = False, model: str = "", native_event_sink=None) -> tuple[list[dict], bool]: + tool_less: bool = False, model: str = "", native_event_sink=None, + mode: str = "plan") -> tuple[list[dict], bool]: """One CLI invocation → (parsed events, timed_out). A timeout kills the process but keeps the partial stream as salvage material. `tool_less` also removes the native reads unless an oversized prompt @@ -310,15 +320,15 @@ def _run(self, text: str, *, session: Path, timeout_s: float, if oversized: prompt_file = session / "PROMPT.md" prompt_file.write_text(text, encoding="utf-8") - cmd += [f"--prompt=Your complete instructions are in the attached " + cmd += [(f"--prompt=Your complete instructions are in the attached " f"file {prompt_file}. Read ALL of it (page through it with " - f"offset) before acting, then follow it exactly.", + f"offset) before acting, then follow it exactly."), "--attach", str(prompt_file)] else: # `--prompt=` form: a prompt starting with "-" must not be # parsed as an option. cmd += [f"--prompt={text}"] - cmd += ["--output-format", "stream-json", "--mode", "plan", + cmd += ["--output-format", "stream-json", "--mode", mode, "--cwd", str(session), f"--disallowed-tools={','.join(disallowed)}"] env = sanitized_env() @@ -611,6 +621,12 @@ def _check_model(self, requested: str, served: str, trace=None) -> None: # -- transport contract -------------------------------------------------- def run_session(self, req: AgentSessionRequest) -> AgentOutcome: session = Path(tempfile.mkdtemp(prefix="imc-zcode-session-")) + if req.trace is not None: + req.trace.record( + "harness_session_started", provider=self.spec.id, + step=req.step_name, + requested_model=req.model or self.settings.strict_backend_model, + session_dir=str(session)) try: if req.bridge_spec_path is not None: self._write_mcp_config(session, req.bridge_spec_path, @@ -624,7 +640,10 @@ def run_session(self, req: AgentSessionRequest) -> AgentOutcome: events, timed_out = self._run( f"{header}{req.system}\n\n{req.prompt}", session=session, timeout_s=req.timeout_s, - model=req.model or self.settings.strict_backend_model) + model=req.model or self.settings.strict_backend_model, + # Mutating workflows write only through the scoped bridge; + # native writes/shell/subagents retain the same denylist. + mode="plan" if req.scope.read_only else "yolo") finally: shutil.rmtree(session, ignore_errors=True) usage = self._usage(events) diff --git a/src/infermatrix_copilot/rebase_engine/ci_loop.py b/src/infermatrix_copilot/rebase_engine/ci_loop.py index 98d67869a..73312b945 100644 --- a/src/infermatrix_copilot/rebase_engine/ci_loop.py +++ b/src/infermatrix_copilot/rebase_engine/ci_loop.py @@ -545,9 +545,11 @@ def monitor_build(client: CIClient, build_id: str, *, try: auth = client.list_jobs(build_id) except Exception: # noqa: BLE001 - degrade to the embedded snapshot - auth = [j for j in (final.get("jobs") or []) if isinstance(j, dict)] - if not final: - # neither the jobs endpoint nor the build itself was readable: + embedded = final.get("jobs") + auth = [j for j in embedded if isinstance(j, dict)] \ + if isinstance(embedded, list) else [] + if not isinstance(embedded, list): + # Neither source supplied a readable jobs list: # the reconciliation contract (nothing missed) cannot be # honored, so the outcome must not read clean reconciled = False diff --git a/src/infermatrix_copilot/rebase_engine/module_pytest.py b/src/infermatrix_copilot/rebase_engine/module_pytest.py index 873869f16..de6e0debf 100644 --- a/src/infermatrix_copilot/rebase_engine/module_pytest.py +++ b/src/infermatrix_copilot/rebase_engine/module_pytest.py @@ -18,19 +18,28 @@ Invocation parity: `imx-omni-pytest -vv -s tests/x/` runs pytest; `imx-omni-pytest python -c '...'` runs that command under the same gating -(the parent wrapper accepted both shapes).""" +(the parent wrapper accepted both shapes). An adapter may declare +``rebase.testing.module_test_platform`` so broad module runs do not execute +tests that CI routes exclusively to a different accelerator platform.""" from __future__ import annotations +import hashlib +import json import os +import re import shlex import sys +import tempfile from pathlib import Path -from ..testing.runner import TestJob, TestRunner +from ..testing.runner import TestJob, TestOutcome, TestRunner from ..testing.watchdog import WatchdogPatterns +_ACCELERATOR_PLATFORMS = frozenset({"cuda", "musa", "npu", "rocm", "xpu"}) + + def _needs_gpu_lock(args: list[str]) -> bool: if os.environ.get("IMX_GPU_MUTEX") == "1": return True @@ -45,6 +54,112 @@ def _job_key(args: list[str]) -> str: return "module_pytest" +def _platform_filtered_args(args: list[str], testing: dict) -> tuple[list[str], str | None]: + """Add the adapter's platform selector to otherwise unfiltered pytest. + + Main's hardware marks are CI routing metadata: an NPU-only item is not + expected to run merely because a CUDA job invokes a broad directory. + Keep platform-neutral and current-platform items, including tests marked + for several platforms. A caller-provided ``-m`` expression is explicit + and therefore remains authoritative. + """ + platform = testing.get("module_test_platform") + if platform is None or args[0] == "python" or args[0].startswith("python3"): + return args, None + if not isinstance(platform, str) or platform not in _ACCELERATOR_PLATFORMS: + choices = ", ".join(sorted(_ACCELERATOR_PLATFORMS)) + raise ValueError(f"rebase.testing.module_test_platform must be one of: {choices}") + if any(arg == "-m" or arg.startswith("-m=") for arg in args): + return args, None + others = sorted(_ACCELERATOR_PLATFORMS - {platform}) + expression = f"{platform} or (" + " and ".join(f"not {name}" for name in others) + ")" + return [*args, "-m", expression], platform + + +def _log_hash(log_file: str) -> tuple[str | None, str]: + """Hash bytes observed after runner return; later rotation is still possible.""" + if not log_file: + return None, "missing" + try: + with open(log_file, "rb") as stream: + before = os.fstat(stream.fileno()) + digest = hashlib.sha256() + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + after = os.fstat(stream.fileno()) + def identity(st): + return st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns + if identity(before) != identity(after): + return digest.hexdigest(), "changed_during_hash" + if identity(after) != identity(Path(log_file).stat()): + return digest.hexdigest(), "path_changed_during_hash" + return digest.hexdigest(), "stable_observation" + except FileNotFoundError: + return None, "missing" + except OSError: + return None, "unreadable" + + +def _write_outcome_receipt(log_dir: Path, *, args: list[str], run_id: str, + attempt: str, timeout: int, cuda: str, + outcome: TestOutcome, effective_args: list[str], + platform_marker: str | None) -> None: + """Atomic per-attempt runner outcome, without commands, paths, or env dumps. + + Requested argv is not effective inner argv/scope proof (pytest defaults + and the runner's existing coverage fallback still apply). A missing receipt + cannot certify success, including when a runner or receipt write raises. + """ + starttime = None + try: + # Linux self identity only; no other process or environment is read. + stat = Path("/proc/self/stat").read_text(encoding="utf-8") + starttime = int(stat.rsplit(")", 1)[1].split()[19]) + except (OSError, ValueError, IndexError): + pass + canonical = json.dumps(args, ensure_ascii=False, separators=(",", ":")) + effective_canonical = json.dumps(effective_args, ensure_ascii=False, + separators=(",", ":")) + cuda_subset = [part.strip() for part in cuda.split(",") if part.strip()] + # Arbitrary env strings must not become plaintext in the receipt. + valid_cuda = all(re.fullmatch( + r"-?\d+|GPU-[0-9a-fA-F-]+|MIG-[0-9a-fA-F-]+(?:/\d+/\d+)?", part + ) for part in cuda_subset) + log_sha, log_status = _log_hash(outcome.log_file) + receipt = { + "schema_version": 1, + "run_id": (run_id if re.fullmatch(r"run-\d{8}-\d{6}-[0-9a-f]{6}", run_id) + else None), + "run_id_sha256": hashlib.sha256(run_id.encode()).hexdigest(), + "attempt_id": attempt, + "wrapper_pid": os.getpid(), "wrapper_starttime_ticks": starttime, + "requested_argv_sha256": hashlib.sha256(canonical.encode()).hexdigest(), + "effective_argv_sha256": hashlib.sha256(effective_canonical.encode()).hexdigest(), + "platform_marker_injected": platform_marker, + "timeout_sec": timeout, + "cuda_visible_devices": cuda_subset if valid_cuda else None, + "cuda_selection_sha256": hashlib.sha256(cuda.encode()).hexdigest(), + "rc": outcome.rc, "watchdog_triggered": outcome.watchdog_triggered, + "timed_out": outcome.timed_out, "skipped": outcome.skipped, + "log_path_sha256": (hashlib.sha256(outcome.log_file.encode()).hexdigest() + if outcome.log_file else None), + "log_sha256": log_sha, "log_hash_status": log_status, + } + directory = log_dir / "tests" / "outcomes" + directory.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=f".{attempt}.", suffix=".tmp", + dir=directory) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(receipt, stream, ensure_ascii=False, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, directory / f"{attempt}.json") + finally: + Path(temporary).unlink(missing_ok=True) + + def main(argv: list[str] | None = None) -> int: args = list(sys.argv[1:] if argv is None else argv) repo = os.environ.get("IMX_TARGET_REPO", "") @@ -58,14 +173,9 @@ def main(argv: list[str] | None = None) -> int: file=sys.stderr) return 2 - if args[0] == "python" or args[0].startswith("python3"): - command = " ".join(shlex.quote(a) if i else a - for i, a in enumerate(args)) - else: - command = "python -m pytest " + " ".join(shlex.quote(a) for a in args) - patterns = None artifact_globs: list[str] = [] + testing: dict = {} rebase_dir = os.environ.get("IMX_ADAPTER_REBASE", "") # runtime overlay (design D4): the agent env contract may hand the # learned-noise overlay path down; absent ⇒ seed-only, as before @@ -80,11 +190,39 @@ def main(argv: list[str] | None = None) -> int: if manifest.is_file(): import yaml data = yaml.safe_load(manifest.read_text(encoding="utf-8")) or {} - artifact_globs = list(((data.get("rebase") or {}) - .get("testing") or {}) - .get("artifact_globs") or []) + testing = (data.get("rebase") or {}).get("testing") or {} + artifact_globs = list(testing.get("artifact_globs") or []) - timeout = int(os.environ.get("TEST_TIMEOUT_SEC", "7200")) + try: + effective_args, platform_marker = _platform_filtered_args(args, testing) + except ValueError as exc: + print(f"module test wrapper: {exc}", file=sys.stderr) + return 2 + if effective_args[0] == "python" or effective_args[0].startswith("python3"): + command = " ".join(shlex.quote(a) if i else a + for i, a in enumerate(effective_args)) + else: + command = "python -m pytest " + " ".join( + shlex.quote(a) for a in effective_args) + + if "module_max_timeout_sec" in testing: + cap = testing["module_max_timeout_sec"] + if type(cap) is not int or cap <= 0: + print("module test wrapper: rebase.testing.module_max_timeout_sec " + "must be a positive integer", file=sys.stderr) + return 2 + try: + timeout = int(os.environ.get("TEST_TIMEOUT_SEC", "7200")) + except ValueError: + timeout = 0 + if timeout <= 0 or timeout > cap: + print("module test wrapper: TEST_TIMEOUT_SEC must be a positive " + f"integer at most {cap}; keep the adapter's planned budget " + "and use short read-only polls for long tests", + file=sys.stderr) + return 2 + else: + timeout = int(os.environ.get("TEST_TIMEOUT_SEC", "7200")) cuda = os.environ.get("CUDA_VISIBLE_DEVICES", "") # two-tier watchdog wiring: learning + report artifacts land under the @@ -108,7 +246,7 @@ def main(argv: list[str] | None = None) -> int: # would wrongly dedup a real second decision — a collision-resistant # attempt id makes the identity process-unique (pid+seconds can # collide across same-second retries or pid reuse) - _attempt = f"wrap-{uuid.uuid4().hex[:12]}" + _attempt = f"wrap-{uuid.uuid4().hex}" def record_fn(pattern: str, verdict: str, test_name: str) -> None: # LogWatchdog's contract is (matched line, verdict, test name); @@ -145,6 +283,10 @@ def review_fn(test_name: str, snippet: str) -> str: job = TestJob(key=_job_key(args), command=command, timeout_sec=timeout, min_gpus=0, gpu_lock=_needs_gpu_lock(args), index=0) outcome = runner.run(job, dict(os.environ)) + _write_outcome_receipt(Path(log_dir), args=args, run_id=run_ident, + attempt=_attempt, timeout=timeout, cuda=cuda, + outcome=outcome, effective_args=effective_args, + platform_marker=platform_marker) if outcome.skipped: print(f"imx-omni-pytest: skipped — {outcome.skip_reason}", file=sys.stderr) diff --git a/src/infermatrix_copilot/rebase_engine/module_rebase.py b/src/infermatrix_copilot/rebase_engine/module_rebase.py index b85748c4c..4d568b90a 100644 --- a/src/infermatrix_copilot/rebase_engine/module_rebase.py +++ b/src/infermatrix_copilot/rebase_engine/module_rebase.py @@ -67,16 +67,24 @@ class ModuleRunConfig: # adapter baseline ref (repo.remote/default_branch) — reaches the # LIVE prompt's test-plan prose (2026-08-01 neutrality audit) baseline_ref: str = "origin/main" + target_venv: str = "" # resolved runtime path, never the full child env -def _plan_gate_opened(run_dir: Path) -> bool: +def _plan_gate_opened(run_dir: Path, *, plan_prefix: str = "", + since_ts: float | None = None) -> bool: """Read `plan_done` back out of the bridge trace. The gate runs in the bridge process, so the parent cannot observe it directly; `PlanGate` records `plan_gate_opened` when a decision file is - successfully written. Absent/unreadable trace ⇒ NOT opened (the same - fail-closed default the in-process loop starts from).""" + successfully written. Production callers restrict observations to this + module's plan directory and this attempt's start time: another module's + decision or an older attempt proves nothing about the current session. + Bridge restarts may restore the same module's durable decision, recording + a new event during this attempt. Absent/unreadable trace ⇒ NOT opened.""" import json as _json + import math as _math + + from .agent_loop import _under_plan_dir tp = Path(run_dir) / "bridge_trace.jsonl" try: for line in tp.read_text(encoding="utf-8").splitlines(): @@ -84,11 +92,23 @@ def _plan_gate_opened(run_dir: Path) -> bool: if not line: continue try: - if _json.loads(line).get("kind") == "plan_gate_opened": - return True + event = _json.loads(line) except ValueError: continue - except OSError: + if not isinstance(event, dict) or event.get("kind") != "plan_gate_opened": + continue + if since_ts is not None: + ts = event.get("ts") + if (not isinstance(ts, (int, float)) or not _math.isfinite(ts) + or ts < since_ts): + continue + if plan_prefix: + decision = event.get("decision") + if (not isinstance(decision, str) or ".decision.md" not in decision + or not _under_plan_dir(decision, plan_prefix)): + continue + return True + except (OSError, UnicodeError): return False return False @@ -130,6 +150,7 @@ def _native_writes(root: str, run_dir: Path, before: dict[str, int], contract rather than merely bypassing the bookkeeping. """ import json as _json + import math as _math bridged: set[str] = set() gate_ts: float | None = None @@ -143,13 +164,17 @@ def _native_writes(root: str, run_dir: Path, before: dict[str, int], d = _json.loads(line) except ValueError: continue - if d.get("ts", 0) < since_ts: + if not isinstance(d, dict): + continue + ts = d.get("ts") + if (not isinstance(ts, (int, float)) or not _math.isfinite(ts) + or ts < since_ts): continue if d.get("kind") == "plan_gate_opened" and gate_ts is None: - gate_ts = float(d.get("ts") or 0) or None + gate_ts = float(ts) if d.get("kind") == "tool_call" and d.get("path"): bridged.add(str(d["path"])) - except OSError: + except (OSError, UnicodeError): pass native, pre_gate = [], [] @@ -162,6 +187,23 @@ def _native_writes(root: str, run_dir: Path, before: dict[str, int], return sorted(native), sorted(pre_gate) +def _host_env(config) -> dict[str, str]: + """The non-secret host facts a bridge process must inherit from the + parent: the scheduler's GPU selection and the model cache location.""" + out: dict[str, str] = {} + for key, value in (("CUDA_VISIBLE_DEVICES", getattr(config, "cuda_devices", "")), + ("HF_HOME", getattr(config, "hf_home", ""))): + if value: + out[key] = str(value) + return out + + +def _is_read_audit_note(refusal: str) -> bool: + """A session-audit read-containment note (`audit: read outside session + roots: ...`), as opposed to a write audit or a session failure.""" + return refusal.startswith("audit: ") and "outside session roots" in refusal + + async def _harness_attempt(prompt: str, *, module: str, config, scope: ToolScope, trace: RunTrace, tool_defs: list[dict], plan_prefix: str, @@ -196,23 +238,70 @@ async def _harness_attempt(prompt: str, *, module: str, config, "tool_schemas": str(Path(config.script_dir) / "tool_schemas.json"), "manifest_path": config.manifest_path, "model": config.model, + "backend": config.backend, + "backend_model": config.backend_model, # prebuilt upstream: serializing the checkout paths HERE would # add repo-specific vocabulary to a neutral core module "paths": dict(config.paths_spec or {}), + "target_venv": getattr(config, "target_venv", ""), + # Host facts the bridge cannot recover from its own environment: + # the harness CLI is spawned with an allowlisted env, so the + # bridge's `os.environ` carries neither the GPU selection the + # scheduler granted this run nor the model cache. Without them + # bridge-run tests see every GPU on the host. + "host_env": _host_env(config), "state": dict(config.state_slice or {}), "plan_write_prefix": plan_prefix if require_plan_review else "", "gated_tools": list(GATED_TOOL_NAMES), }) + signal_dir = getattr(config, "signal_dir", "") + done_path = (Path(signal_dir) / f"module.{module}.done" + if signal_dir else None) + fail_path = (Path(signal_dir) / f"module.{module}.fail" + if signal_dir else None) + if done_path is not None: + # Debug templates may retain an older signal location. Supply the + # same authoritative path on every attempt, including retries. + prompt += ("\n\nAuthoritative completion signal for this attempt: " + f"write exactly `MODULE_DONE {module}` as the first line " + f"of `{done_path}` only after verification succeeds. " + "Rewrite it during this attempt; an older file does not " + "count. This path overrides other signal locations.") # NEVER forward the tier model: it names a raw-API model the harness # does not have. Empty lets the transport fall back to its own setting. + # The module contract rides the USER channel (stdin for claude-code): + # evidence packs (broken imports + test plans) exceed the 128KiB argv + # limit a --system-prompt element is bound by, and every other harness + # transport already folds the contract into the user turn. req = AgentSessionRequest( - system=prompt, prompt="", scope=scope, + system="", prompt=prompt, scope=scope, model=config.backend_model or "", max_iters=config.max_turns, timeout_s=config.harness_timeout_s, run_dir=run_dir, step_name=f"rebase.module.{module}", - bridge_spec_path=spec_path, trace=trace) + bridge_spec_path=spec_path, trace=trace, + # a module agent must read the upstream checkout and the installed + # target runtime; a session audit that only knows the edit root and + # the run dir would flag every one of those reads + read_roots=tuple(p for p in ( + getattr(config, "vllm_path", ""), + getattr(config, "target_venv", "")) if p)) import time as _time + # A normal harness exit is only transport success. The prompt requires + # an explicit completion file, rewritten by THIS attempt; older attempts + # may have left a signal even when the write audit rejected their work. + def signal_snapshot(path): + if path is None: + return None + try: + stat = path.stat() + return (stat.st_ino, stat.st_mtime_ns, stat.st_ctime_ns, + path.read_text(encoding="utf-8")) + except (OSError, UnicodeError): + return None + + prior_done = signal_snapshot(done_path) + prior_fail = signal_snapshot(fail_path) before = _changed_files(scope.root) started = _time.time() outcome = await asyncio.to_thread(transport.run_session, req) @@ -231,11 +320,57 @@ async def _harness_attempt(prompt: str, *, module: str, config, "text": ("harness wrote product files BEFORE the plan-review " "decision, bypassing the bridge: " + ", ".join(pre_gate[:10]))} - return {"done": not getattr(outcome, "truncated", False), - "text": getattr(outcome, "text", "") or "", + current_done = signal_snapshot(done_path) + current_fail = signal_snapshot(fail_path) + signaled = (current_done is not None and current_done != prior_done + and current_done[3].splitlines()[:1] + == [f"MODULE_DONE {module}"]) + failed = current_fail is not None and current_fail != prior_fail + truncated = bool(getattr(outcome, "truncated", False)) + # Refusals are typed provider outcomes (session failures, write audits), + # not words in the final prose. A marker cannot override them. Read- + # containment audit notes are different: they are the detective record + # of a harness whose native read tools cannot be disabled, and a read + # of, say, the stdlib must not void twelve hours of verified work. They + # stay disclosed (trace + result text) without vetoing completion. + all_refusals = [str(r) for r in (getattr(outcome, "refusals", ()) or ())] + audit_reads = [r for r in all_refusals if _is_read_audit_note(r)] + refusals = [r for r in all_refusals if not _is_read_audit_note(r)] + plan_done = (not require_plan_review or bool(plan_prefix) and + _plan_gate_opened(run_dir, plan_prefix=plan_prefix, + since_ts=started)) + done = signaled and not failed and not truncated and not refusals and plan_done + result_text = getattr(outcome, "text", "") or "" + if audit_reads: + if trace is not None: + trace.record("harness_read_audit", step=f"rebase.module.{module}", + count=len(audit_reads), notes=audit_reads[:10]) + result_text += ("\nSession read audit (not a completion veto): " + + "; ".join(audit_reads[:3])) + if not done: + reasons = [] + if not signaled: + reasons.append(f"a fresh exact MODULE_DONE {module} signal is required") + if failed: + reasons.append("a fresh failure signal was written") + if truncated: + reasons.append("the session was truncated") + if refusals: + reasons.append("provider refusal(s): " + "; ".join( + str(refusal) for refusal in refusals[:3])) + if not plan_done: + reasons.append("this attempt did not open its module's plan-review gate") + result_text += "\nHarness completion rejected: " + "; ".join(reasons) + "." + return {"done": done, + "session_completed": not truncated, + "text": result_text, "turns": int(getattr(outcome, "iterations", 0) or 0), - "plan_done": (not require_plan_review - or _plan_gate_opened(run_dir))} + "plan_done": plan_done} + + +def _append_agent_log(path: str, text: str) -> None: + with Path(path).open("a", encoding="utf-8") as log: + log.write(text) async def rebase_module( @@ -262,11 +397,21 @@ async def rebase_module( except Exception: # noqa: BLE001 - knowledge layer never blocks a rebase guidance = "" + state = config.state_slice or {} + params = (state.get("task_spec") or {}).get("params") or {} + assignment_baseline = (config.last_rebase_vllm_commit + or state.get("last_rebase_upstream_commit", "") + or params.get("last_rebase_commit", "")) + target_branch = (state.get("target_branch", "") + or params.get("target_branch", "")) + upstream_commit = (state.get("upstream_commit", "") + or params.get("upstream_commit", "") + or params.get("force_upstream_commit", "")) prompt = build_module_prompt( module, prompt_data, vllm_path=config.vllm_path, omni_path=config.omni_path, script_dir=config.script_dir, - last_rebase_vllm_commit=config.last_rebase_vllm_commit, + last_rebase_vllm_commit=assignment_baseline, cuda_devices=config.cuda_devices, hf_home=config.hf_home, log_dir=config.log_dir, signal_dir=config.signal_dir, rebase_run_id=substate.run_id, @@ -275,7 +420,8 @@ async def rebase_module( broken_imports=broken_imports, module_test_plan=module_test_plan, adaptive_guidance=guidance, live=True, baseline_ref=config.baseline_ref, - test_timeout_sec=config.test_timeout_sec) + test_timeout_sec=config.test_timeout_sec, + target_branch=target_branch, upstream_commit=upstream_commit) substate.update({"modules": {module: {"status": "running"}}}) agent_log = str(Path(config.log_dir) / "agents" / f"module-{module}.log") @@ -285,10 +431,17 @@ async def rebase_module( async def _attempt(p: str, *, require_plan_review: bool = True) -> dict: try: if config.backend and config.backend != "api": - return await _harness_attempt( + _append_agent_log( + agent_log, f"\nHarness agent started (provider={config.backend}, " + f"requested_model={config.backend_model})\n") + outcome = await _harness_attempt( p, module=module, config=config, scope=scope, trace=trace, tool_defs=tool_defs, plan_prefix=plan_prefix, require_plan_review=require_plan_review) + _append_agent_log( + agent_log, f"Harness result (done={outcome.get('done')}):\n" + f"{outcome.get('text', '')}\n") + return outcome return await run_agent_loop( client, p, model=config.model, tool_defs=tool_defs, extra_tools=extra_tools, scope=scope, trace=trace, diff --git a/src/infermatrix_copilot/rebase_engine/prompt_builder.py b/src/infermatrix_copilot/rebase_engine/prompt_builder.py index e54d7dbda..e9362db96 100644 --- a/src/infermatrix_copilot/rebase_engine/prompt_builder.py +++ b/src/infermatrix_copilot/rebase_engine/prompt_builder.py @@ -38,6 +38,7 @@ class ModulePromptData: module_omni_files: Mapping[str, str] = field(default_factory=dict) module_test_map: Mapping[str, Sequence[str]] = field(default_factory=dict) module_import_check: Mapping[str, str] = field(default_factory=dict) + live_module_overrides: Mapping[str, Mapping] = field(default_factory=dict) @classmethod def load(cls, rebase_data_dir: Path) -> "ModulePromptData": @@ -55,7 +56,8 @@ def load(cls, rebase_data_dir: Path) -> "ModulePromptData": module_vllm_paths=data.get("module_vllm_paths", {}), module_omni_files=data.get("module_omni_files", {}), module_test_map=data.get("module_test_map", {}), - module_import_check=data.get("module_import_check", {})) + module_import_check=data.get("module_import_check", {}), + live_module_overrides=data.get("live_module_overrides", {})) def _run_git(cmd: list[str], cwd: str, timeout: int = 15) -> str: @@ -214,6 +216,8 @@ def build_module_prompt( live: bool = False, baseline_ref: str = "origin/main", test_timeout_sec: int = 7200, + target_branch: str = "", + upstream_commit: str = "", run_git: Callable[[list[str], str], str] | None = None, ) -> str: """Byte-parity render of the parent's `build_module_prompt`. `script_dir` @@ -224,7 +228,8 @@ def build_module_prompt( vllm_commit = git(["rev-parse", "--short", "HEAD"], vllm_path) omni_commit = git(["rev-parse", "--short", "HEAD"], omni_path) - upstream_paths = data.module_vllm_paths.get(module, "") + overrides = data.live_module_overrides.get(module, {}) if live else {} + upstream_paths = overrides.get("vllm_paths", data.module_vllm_paths.get(module, "")) commit_list = "" if last_rebase_vllm_commit and upstream_paths: commit_list = git( @@ -242,10 +247,13 @@ def build_module_prompt( if live and data.pytest_wrapper_live else data.pytest_wrapper) wrapper = wrapper_template.format(script_dir=script_dir) + test_paths = overrides.get("test_paths", + data.module_test_map.get(module, [])) pytest_cmds = [f"{wrapper} -vv -s {test_path}" - for test_path in data.module_test_map.get(module, [])] + for test_path in test_paths] test_commands = "\n".join(pytest_cmds) - check = data.module_import_check.get(module, 'print("OK")') + check = overrides.get("import_check", + data.module_import_check.get(module, 'print("OK")')) if live: import shlex import_check_cmd = f"{wrapper} python -c {shlex.quote(check)}" @@ -270,7 +278,8 @@ def build_module_prompt( "CUDA_DEVICES": cuda_devices, "HF_HOME": hf_home, "COMMIT_LIST": commit_list, - "OMNI_FILES": data.module_omni_files.get(module, ""), + "OMNI_FILES": overrides.get("omni_files", + data.module_omni_files.get(module, "")), "UPSTREAM_FILES": upstream_paths, "IMPORT_CHECK": import_check_cmd, "PYTHON3_CMD": "python3", @@ -294,6 +303,25 @@ def build_module_prompt( "SCRIPT_DIR": script_dir, "SESSION": f"module-{module}", } + if live: + import sys + vars_map["COPILOT_PYTHON"] = shlex.quote(sys.executable) + # Use the run's pin even when the checkout has drifted; displaying + # both lets the agent verify the environment before making a port. + # Legacy callers without run context use observed checkout facts, + # never the release or prior-alignment SHA from an earlier campaign. + vars_map.update({ + "TARGET_BRANCH": (target_branch + or git(["symbolic-ref", "--quiet", "--short", "HEAD"], + vllm_path) + or "unspecified (detached checkout)"), + "UPSTREAM_COMMIT": (upstream_commit + or git(["rev-parse", "HEAD"], vllm_path) + or "unresolved (verify upstream checkout HEAD)"), + "LAST_REBASE_UPSTREAM_COMMIT": (last_rebase_vllm_commit + or "not supplied"), + "BASELINE_REF": baseline_ref, + }) for key, value in vars_map.items(): template = template.replace("{" + key + "}", value) return template diff --git a/src/infermatrix_copilot/rebase_engine/rebase_tools.py b/src/infermatrix_copilot/rebase_engine/rebase_tools.py index 04a85cbdb..661095499 100644 --- a/src/infermatrix_copilot/rebase_engine/rebase_tools.py +++ b/src/infermatrix_copilot/rebase_engine/rebase_tools.py @@ -23,7 +23,9 @@ from __future__ import annotations import json +import os import subprocess +import threading from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable, Mapping @@ -176,14 +178,55 @@ def handle_grep(pattern: str, path: str = "", include: str = "*.py", def handle_run_shell(command: str, workdir: str | None = None, timeout: int = 600, **_: Any) -> dict: + from ..testing.process_tree import collect_descendants, kill_tree + from ..testing.runner import TestRunner, _record_walk + cwd = workdir or paths.omni_path try: - proc = subprocess.run(["bash", "-c", command], capture_output=True, - text=True, errors="replace", timeout=timeout, - cwd=cwd, env=dict(paths.env) or None) - return {"exit_code": proc.returncode, - "stdout": proc.stdout[-5000:], - "stderr": proc.stderr[-5000:]} + with subprocess.Popen( + # Match TestRunner's runtime protection: inherited SSH + # metadata can make noninteractive Bash read ~/.bashrc + # and replace the target venv's PATH before verification. + ["bash", "--norc", "-c", command], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, errors="replace", + cwd=cwd, env=dict(paths.env) or None, + start_new_session=True) as proc: + # Killing only bash leaves pytest and setsid'd model workers + # alive. Reuse the test runner's identity-checked snapshots so + # cleanup also reaches children after their parent exits. + identity: dict[int, int] = {} + own_pid = os.getpid() + _record_walk(identity, [proc.pid], root=proc.pid, + root_parent=own_pid) + root_birth = identity.get(proc.pid) + stop = threading.Event() + + def snapshot_loop(): + while not stop.is_set() and proc.poll() is None: + _record_walk( + identity, collect_descendants(proc.pid), + root=proc.pid, root_parent=own_pid, + root_birth=root_birth) + stop.wait(TestRunner.SNAPSHOT_INTERVAL) + + snapshot_thread = threading.Thread( + target=snapshot_loop, daemon=True) + snapshot_thread.start() + try: + stdout, stderr = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + stop.set() + snapshot_thread.join(timeout=10) + kill_tree(sorted(identity), identity=identity) + proc.communicate(timeout=10) + return {"exit_code": -1, "stdout": "", + "stderr": f"Command timed out after {timeout}s"} + finally: + stop.set() + snapshot_thread.join(timeout=10) + return {"exit_code": proc.returncode, + "stdout": stdout[-5000:], + "stderr": stderr[-5000:]} except subprocess.TimeoutExpired: return {"exit_code": -1, "stdout": "", "stderr": f"Command timed out after {timeout}s"} diff --git a/src/infermatrix_copilot/rebase_engine/remote_ci_owner.py b/src/infermatrix_copilot/rebase_engine/remote_ci_owner.py new file mode 100644 index 000000000..2e8ea7d4b --- /dev/null +++ b/src/infermatrix_copilot/rebase_engine/remote_ci_owner.py @@ -0,0 +1,260 @@ +"""Remote CI debug ownership; never defines runnable jobs or pass criteria. + +Keep existing local routing. Missing remote labels use positive declared path +footprints, preferring specific matches over explicit shared test-root ownership. +Ancillary source imports never imply responsibility or causal attribution. +Unresolved routes remain available for the existing unscoped debug dispatch, +with explicit diagnostics. Provider job IDs and leaf/source identity are the +caller's separate evidence contract; a label is not a unique CI leaf ID. +""" +from __future__ import annotations + +from dataclasses import dataclass, replace +from pathlib import Path, PurePosixPath +import re +import shlex +from typing import Mapping, Sequence + +import yaml + +from .test_manifest import ManifestSpec, _extract_steps, _label_to_slug + + +@dataclass(frozen=True) +class RemoteOwner: + module: str = "" + status: str = "unmapped" + reason: str = "" + candidate_modules: tuple[str, ...] = () + sources: tuple[str, ...] = () + paths: tuple[str, ...] = () + + +class RemoteOwnerResolver: + """Read-only, routing-only view; the native local manifest is unchanged.""" + + def __init__(self, repo: Path, manifest: Mapping, + active_modules: Sequence[str]): + self.spec = None + try: + if not isinstance(manifest, Mapping): + raise TypeError("invalid_manifest") + modules = manifest.get("modules") + modules = {} if modules is None else modules + rb = manifest.get("rebase") + rb = {} if rb is None else rb + if not isinstance(modules, Mapping) or not isinstance(rb, Mapping): + raise TypeError("invalid_manifest_mapping") + tm = rb.get("test_manifest") + tm = {} if tm is None else tm + if not isinstance(tm, Mapping): + raise TypeError("invalid_manifest_mapping") + if isinstance(active_modules, (str, bytes)) or not isinstance(active_modules, Sequence): + raise TypeError("invalid_active_modules") + if not all(isinstance(m, str) for m in active_modules): + raise TypeError("invalid_active_modules") + for key in ("file_ref_prefixes", "file_tree_roots"): + if key in tm: + self._strings(tm[key]) + assignment = tm.get("assignment_paths") + assignment = {} if assignment is None else assignment + if not isinstance(assignment, Mapping): + raise TypeError("invalid_assignment_paths") + for module, patterns in assignment.items(): + if not isinstance(module, str): + raise TypeError("invalid_module_name") + self._strings(patterns) + for module, declared in modules.items(): + if not isinstance(module, str) or (declared is not None and not isinstance(declared, Mapping)): + raise TypeError("invalid_module_mapping") + for key in ("test_paths", "local_paths"): + self._strings((declared or {}).get(key)) + spec = replace(ManifestSpec.from_manifest(manifest), skip_patterns=()) + if not isinstance(spec.yaml_dir, str) or not spec.yaml_dir: + raise TypeError("invalid_yaml_directory") + if not all(isinstance(k, str) and isinstance(v, str) + for k, v in spec.pipelines.items()): + raise TypeError("invalid_pipeline_mapping") + self.repo = Path(repo).resolve() + self.active = tuple(m for m in active_modules if m in modules) + self.universal = {p.rstrip("/") for p in spec.file_tree_roots} + self.patterns = {} + for module in self.active: + declared = modules[module] or {} + patterns = [*(spec.assignment_paths.get(module) or ()), + *(declared.get("test_paths") or ()), + *(declared.get("local_paths") or ())] + self.patterns[module] = tuple(p for p in dict.fromkeys(patterns) if p) + roots = {*(spec.file_ref_prefixes or ()), + *(PurePosixPath(p).parts[0] for patterns in self.patterns.values() + for p in patterns if PurePosixPath(p).parts)} + roots = {p for p in roots if re.fullmatch(r"[A-Za-z_][\w-]*", p)} + self.ref_rx = re.compile( + r"^(?:%s)/" % "|".join(re.escape(p) for p in sorted(roots))) + # Publish availability only after all shape checks succeeded. + self.spec = spec + except (OSError, KeyError, TypeError, ValueError, AttributeError): + self.spec = None + + @staticmethod + def _strings(value): + if value is None: + return + if not isinstance(value, (list, tuple)) or not all(isinstance(p, str) for p in value): + raise TypeError("invalid_path_sequence") + + @staticmethod + def _validate_steps(steps): + pending = list(steps) + count = 0 + while pending: + item = pending.pop() + count += 1 + if count > 10_000: + raise ValueError("too_many_pipeline_steps") + if not isinstance(item, Mapping): + continue # ordinary scalar wait/barrier steps are not test leaves + if "label" in item and not isinstance(item["label"], str): + raise TypeError("invalid_remote_label") + if "group" in item and "steps" in item: + if not isinstance(item["steps"], list): + raise TypeError("invalid_nested_steps") + pending.extend(item["steps"]) + + def _paths(self, command: str) -> tuple[str, ...]: + refs = [] + lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|<>") + lexer.whitespace_split = True + lexer.commenters = "" + for token in lexer: + # Handle complete literal path words and --option=path, never + # a prefix truncated before a variable, quote concat or expansion. + if token.startswith("--") and "=" in token: + token = token.split("=", 1)[1] + if not self.ref_rx.match(token): + continue + ref, separator, node = token.partition("::") + if not re.fullmatch(r"[\w/.*-]+", ref): + raise ValueError("unsupported_path_reference") + if separator and not re.fullmatch(r"[\w.:\[\],=+-]+", node): + raise ValueError("unsupported_pytest_node_suffix") + refs.append(ref.rstrip("/")) + refs = list(dict.fromkeys(refs)) + if len(refs) > 128: + raise ValueError("too_many_path_references") + paths = set() + for ref in refs: + parts = PurePosixPath(ref).parts + if ".." in parts or "**" in ref: + raise ValueError("unsafe_or_recursive_path_reference") + matches = self.repo.glob(ref) if "*" in ref else (self.repo / ref,) + found = False + for path in matches: + if len(paths) >= 512: + raise ValueError("too_many_glob_matches") + resolved = path.resolve() + if not resolved.is_relative_to(self.repo): + raise ValueError("path_reference_outside_repo") + if path.exists(): + found = True + paths.add(path.relative_to(self.repo).as_posix()) + if not found: + raise ValueError("missing_path_reference") + return tuple(sorted(paths)) + + def _owner(self, paths: tuple[str, ...]) -> RemoteOwner: + if not paths: + return RemoteOwner(reason="no_repository_path_references") + owners, unknown, shared = set(), False, False + for path in paths: + hits = [(len(pattern.rstrip("/")), module, pattern) + for module, patterns in self.patterns.items() + for pattern in patterns + if (path == pattern.rstrip("/") or + (pattern.endswith("/") and path.startswith(pattern)))] + if not hits: + unknown = True + continue + length = max(length for length, _, _ in hits) + selected = [(module, pattern) for size, module, pattern in hits + if size == length] + owners.update(module for module, _ in selected) + shared |= any(pattern.rstrip("/") in self.universal + for _, pattern in selected) + candidates = tuple(sorted(owners)) + if len(owners) > 1: + return RemoteOwner(status="ambiguous", + reason="multiple_specific_declared_owners", + candidate_modules=candidates, paths=paths) + if unknown or not owners: + return RemoteOwner(reason="path_outside_specific_owner_footprints", + candidate_modules=candidates, paths=paths) + return RemoteOwner(module=candidates[0], + status="declared_shared" if shared else "declared", + reason="explicit_declared_shared_root" if shared + else "specific_declared_path_footprint", + candidate_modules=candidates, paths=paths) + + def resolve(self, label: str, local_jobs: Mapping[str, Mapping]) -> RemoteOwner: + if not isinstance(label, str): + return RemoteOwner(reason="invalid_remote_label") + slug = _label_to_slug(label) + if not isinstance(local_jobs, Mapping): + return RemoteOwner(reason="invalid_local_routing_view") + local = local_jobs.get(slug) or {} + if not isinstance(local, Mapping): + return RemoteOwner(reason="invalid_local_route") + if local.get("module") and not isinstance(local["module"], str): + return RemoteOwner(reason="invalid_local_route") + if local.get("module"): + return RemoteOwner(module=str(local["module"]), status="local", + reason="preserved_existing_local_route") + if self.spec is None: + return RemoteOwner(reason="remote_routing_view_unavailable") + candidates = [] + try: + if len(self.spec.pipelines) > 32: + raise ValueError("too_many_pipeline_files") + for filename, source in self.spec.pipelines.items(): + path = self.repo / self.spec.yaml_dir / filename + if not path.resolve().is_relative_to(self.repo): + raise ValueError("pipeline_outside_repo") + if not path.is_file(): + raise ValueError("missing_pipeline_file") + if path.stat().st_size > 2_000_000: + raise ValueError("pipeline_file_too_large") + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(data, Mapping) or not isinstance(data.get("steps"), list): + raise ValueError("invalid_pipeline_shape") + self._validate_steps(data["steps"]) + for job in _extract_steps(data["steps"], source, self.spec): + if job.slug == slug: + owner = self._owner(self._paths(job.command)) + candidates.append(replace(owner, sources=(source,))) + if not candidates: + result = RemoteOwner(reason="no_declared_remote_label") + else: + owners = {c.module for c in candidates if c.module} + statuses = {c.status for c in candidates} + merged = dict( + sources=tuple(sorted({s for c in candidates for s in c.sources})), + paths=tuple(sorted({p for c in candidates for p in c.paths})), + candidate_modules=tuple(sorted({m for c in candidates + for m in c.candidate_modules}))) + if len(owners) == 1 and statuses <= {"declared", "declared_shared"}: + result = RemoteOwner(module=next(iter(owners)), + status="declared_shared" if "declared_shared" in statuses + else "declared", + reason="explicit_declared_shared_root" if "declared_shared" in statuses + else "specific_declared_path_footprint", + **merged) + elif "ambiguous" in statuses or len(owners) > 1: + result = RemoteOwner(status="ambiguous", + reason="remote_label_owner_conflict", + **merged) + else: + result = RemoteOwner(reason="path_outside_specific_owner_footprints", + **merged) + except (OSError, ValueError, TypeError, AttributeError, RecursionError, yaml.YAMLError): + result = RemoteOwner(reason="remote_routing_view_unavailable") + return result diff --git a/src/infermatrix_copilot/rebase_engine/test_loop.py b/src/infermatrix_copilot/rebase_engine/test_loop.py index 7307eb496..4d639dd71 100644 --- a/src/infermatrix_copilot/rebase_engine/test_loop.py +++ b/src/infermatrix_copilot/rebase_engine/test_loop.py @@ -3,8 +3,9 @@ Control-flow parity, pinned by tests: per-test resume from checkpointed progress (this run only — substate is run-stamped); GPU-count skips; a -failure re-runs the SAME test on the main-baseline worktree to split -pre-existing failures from rebase regressions (an unavailable worktree must +failure re-runs the SAME test on the main-baseline worktree and requires +matching failure nodes and causes to establish pre-existing failures +(an unavailable worktree must not mask a regression behind a git error — it is treated as a regression); regressions go to the debug loop; every transition checkpoints. @@ -28,7 +29,7 @@ from .substate import Substate -from .ci_loop import extract_failed_test_ids +from .ci_loop import extract_exception_lines, extract_failed_test_ids log = logging.getLogger(__name__) @@ -53,6 +54,23 @@ def _log_text(output: str) -> str: return output +def _baseline_reproduces(current: str, baseline: str) -> bool: + """Require baseline evidence for every current failure and its cause. + + A baseline that cannot import the old source against the new runtime + has not executed the failing tests and cannot establish old failures. + """ + current_log, baseline_log = _log_text(current), _log_text(baseline) + current_tests = extract_failed_test_ids(current_log) + current_errors = extract_exception_lines(current_log) + return bool( + current_tests + and current_tests <= extract_failed_test_ids(baseline_log) + and current_errors + and current_errors <= extract_exception_lines(baseline_log) + ) + + @dataclass(frozen=True) @@ -205,15 +223,20 @@ def checkpoint(current: str = "") -> None: # regression; regression-preserving, like an absent worktree log.warning(" baseline INFRA FAILURE (%s) on %s — treating as " "REGRESSION", baseline.infra, label) - elif baseline.rc != 0: + elif baseline.rc != 0 and _baseline_reproduces( + result.output, baseline.output): log.info(" [PRE-EXISTING] %s fails on main too (rebase rc=%d, " "main rc=%d) — skipping, not a regression", label, result.rc, baseline.rc) skipped_tests.append(slug) checkpoint(slug) continue + elif baseline.rc != 0: + log.warning(" baseline failed without reproducing every current " + "failure and cause on %s — treating as REGRESSION", + label) - log.info(" [REGRESSION] %s passes on main but fails on rebase — " + log.info(" [REGRESSION] %s has no matching baseline failure — " "debugging", label) verdict = await debug_fn(slug, label, result.rc, result.output) if verdict is True: @@ -296,10 +319,42 @@ def snapshot_worktree(repo: Path) -> tuple[str, dict[str, dict]]: except OSError: pass untracked[f] = entry - snap = _git_out(repo, ["stash", "create", - "pre-debug-attempt snapshot"]).strip() - if not snap: - snap = _git_out(repo, ["rev-parse", "HEAD"]).strip() + # ``stash create`` must be authoritative for a dirty tracked tree. A + # failed command also writes no stdout, which used to look identical to a + # clean tree and silently fell back to HEAD. Restoring that fallback + # erased every accepted edit that predated the rejected debug attempt. + tracked_status = subprocess.run( + ["git", "-C", str(repo), "status", "--porcelain=v1", + "--untracked-files=no"], + capture_output=True, text=True, timeout=60) + if tracked_status.returncode != 0: + raise RuntimeError( + "cannot inspect tracked worktree before debug snapshot: " + f"{tracked_status.stderr.strip() or 'git status failed'}") + + stash = subprocess.run( + ["git", "-C", str(repo), "stash", "create", + "pre-debug-attempt snapshot"], + capture_output=True, text=True, timeout=120) + snap = stash.stdout.strip() + if tracked_status.stdout.strip(): + if stash.returncode != 0 or not snap: + raise RuntimeError( + "cannot snapshot dirty tracked worktree before debug attempt: " + f"{stash.stderr.strip() or 'git stash create returned no commit'}") + elif stash.returncode != 0: + raise RuntimeError( + "cannot snapshot clean worktree before debug attempt: " + f"{stash.stderr.strip() or 'git stash create failed'}") + elif not snap: + head = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + capture_output=True, text=True, timeout=60) + if head.returncode != 0 or not head.stdout.strip(): + raise RuntimeError( + "cannot resolve HEAD for clean debug snapshot: " + f"{head.stderr.strip() or 'git rev-parse returned no commit'}") + snap = head.stdout.strip() return snap, untracked diff --git a/src/infermatrix_copilot/rebase_engine/wheel.py b/src/infermatrix_copilot/rebase_engine/wheel.py index 4d9e880df..bc2210842 100644 --- a/src/infermatrix_copilot/rebase_engine/wheel.py +++ b/src/infermatrix_copilot/rebase_engine/wheel.py @@ -11,14 +11,22 @@ variant listing mentions our arch wins; on reaching the last-known-good baseline without a hit, fall back to the baseline itself if it has a wheel, else keep searching older commits. -- A forced commit must resolve and (outside release mode) must have a wheel. +- A forced commit must belong to the target branch and (outside release mode) + must have a wheel. - Release mode uses the branch tip and skips the wheel probe. -- Install failures abort immediately; only *import-validation* failures retry - (with a stale-artifact clean between attempts). +- The last-rebase baseline is resolved to a full commit id before the walk; + a baseline that does not resolve is a configuration error (hard fail), not + a silent walk past it. +- Install failures abort immediately, except a busy-target failure (EBUSY/ + ETXTBSY — another process holds venv files open, e.g. a leftover test + python from a crashed run): those first release the holders, then retry + within the same bounded attempts; only *import-validation* failures retry + with a stale-artifact clean between attempts. """ from __future__ import annotations +import json import os import re import subprocess @@ -27,6 +35,10 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Mapping +from urllib.parse import unquote, urldefrag, urlsplit + +from packaging.utils import parse_wheel_filename +from packaging.version import Version from ..testing.process_tree import _pgrep, kill_tree @@ -62,6 +74,9 @@ class WheelSpec: stale_artifact_globs: tuple[str, ...] = () install_env: Mapping[str, str] = field(default_factory=dict) reinstall_retries: int = 3 + # Complete published wheels avoid editable installs tied to a disposable + # source checkout. Optional: existing adapters retain editable installs. + install_url_template: str = "" @classmethod def from_manifest(cls, data: Mapping) -> "WheelSpec": @@ -75,6 +90,7 @@ def from_manifest(cls, data: Mapping) -> "WheelSpec": stale_artifact_globs=tuple(data.get("stale_artifact_globs", ())), install_env=dict(data.get("install_env", {})), reinstall_retries=int(data.get("reinstall_retries", 3)), + install_url_template=str(data.get("install_url_template", "")), ) @@ -145,34 +161,76 @@ def pick_wheel_commit(repo: Path, target_branch: str, spec: WheelSpec, *, the newest commit with a usable wheel, and leave the checkout AT that commit (detached). Returns the selected full SHA.""" repo = Path(repo) - rc, _, err = run(["git", "fetch", "origin"], cwd=repo) + rc, _, err = run(["git", "check-ref-format", + f"refs/heads/{target_branch}"], cwd=repo) + if rc != 0: + raise WheelPickError(f"invalid upstream target branch {target_branch!r}") + # Scratch clones can inherit a single-branch fetch refspec. Fetch the + # requested branch explicitly, including its tracking ref, rather than + # depending on remote.origin.fetch (the v0.31.0 campaign hit this). + # A failed fetch must never fall through to a stale local tracking ref. + remote_ref = f"refs/remotes/origin/{target_branch}" + rc, _, err = run(["git", "fetch", "origin", + f"+refs/heads/{target_branch}:{remote_ref}"], cwd=repo) if rc != 0: - raise WheelPickError(f"git fetch origin failed in {repo}: {err.strip()}") + raise WheelPickError( + f"remote branch origin/{target_branch} does not exist or could " + f"not be fetched in {repo}: {err.strip()}") - rc, _, _ = run(["git", "rev-parse", "--verify", f"origin/{target_branch}"], + rc, _, _ = run(["git", "rev-parse", "--verify", f"{remote_ref}^{{commit}}"], cwd=repo) if rc != 0: raise WheelPickError( f"remote branch origin/{target_branch} does not exist in {repo}; " "check the configured target branch") + # A resumed run's scratch can carry build artifacts from an interrupted + # wheel install (an editable install builds in-tree and drops untracked + # files, e.g. rust parser benches); untracked files then block the branch + # reset with "would be removed by checkout". The scratch is disposable by + # contract (Rev 8 §4) — clean it instead of asking a human to. + run(["git", "clean", "-ffd"], cwd=repo) log(f"Checking out {target_branch} (resetting to origin/{target_branch})...") - rc, _, err = run(["git", "checkout", "-B", target_branch, + rc, _, err = run(["git", "checkout", "-f", "-B", target_branch, f"origin/{target_branch}"], cwd=repo) if rc != 0: raise WheelPickError( f"failed to reset {target_branch} to origin/{target_branch}: " f"{err.strip()} — working tree may be dirty; clean {repo} and retry") + if baseline: + # The walk compares `rev == baseline` against `git log --format=%H` + # output (always full SHAs). An un-resolved baseline (short SHA, tag, + # branch name, typo) would never match, silently walking BELOW the + # last-known-good commit and possibly pin an older wheel — a backward + # rebase. Resolve first; anything that does not resolve is a + # configuration error, not a search hint. + rc, out, _ = run(["git", "rev-parse", "--verify", "--end-of-options", + f"{baseline}^{{commit}}"], cwd=repo) + if rc != 0 or not out.strip(): + raise WheelPickError( + f"last-rebase baseline '{baseline}' does not resolve to a " + f"commit in {repo}; pass --task-param last_rebase_commit= that exists in the upstream checkout") + baseline = out.strip() + found = "" if force_commit: log(f"Using forced commit override: {force_commit}") - rc, out, _ = run(["git", "rev-parse", f"{force_commit}^{{commit}}"], + rc, out, _ = run(["git", "rev-parse", "--verify", "--end-of-options", + f"{force_commit}^{{commit}}"], cwd=repo) if rc != 0 or not out.strip(): raise WheelPickError( f"forced commit '{force_commit}' could not be resolved in {repo}") found = out.strip() + rc, _, err = run(["git", "merge-base", "--is-ancestor", found, + remote_ref], cwd=repo) + if rc != 0: + raise WheelPickError( + f"forced commit {found} is not proven to belong to " + f"origin/{target_branch}; choose a commit from the target " + f"branch: {err.strip()}") if release_mode: log(f"Release mode: skipping wheel check for forced commit {found[:12]}") elif probe(found): @@ -295,6 +353,33 @@ def read_installed_version(spec: WheelSpec, *, python: str, return out.strip() if rc == 0 else "" +def installed_wheel_matches(spec: WheelSpec, commit: str, version: str, *, + python: str, run: RunFn = _run) -> bool: + """Release versions omit git hashes: verify the wheel URL and checksum.""" + selected = spec.install_url_template.format( + commit=commit, variant=spec.variant, arch=spec.arch, + package=spec.package) + url, fragment = urldefrag(selected) + snippet = ("import importlib.metadata, sys; " + "print(importlib.metadata.distribution(sys.argv[1])" + ".read_text('direct_url.json') or '{}')") + rc, out, _ = run([python, "-c", snippet, spec.package]) + if rc != 0: + return False + try: + provenance = json.loads(out) + _, expected_version, _, _ = parse_wheel_filename( + unquote(urlsplit(url).path.rsplit("/", 1)[-1])) + checksum = re.fullmatch(r"sha256=([0-9a-f]{64})", fragment) + return (Version(version) == expected_version + and provenance.get("url") == url + and checksum is not None + and provenance.get("archive_info", {}).get("hashes", {}) + .get("sha256") == checksum.group(1)) + except (ValueError, TypeError, AttributeError): + return False + + def _append(path: Path, text: str) -> None: with open(path, "a", encoding="utf-8", errors="replace") as f: f.write(text) @@ -334,6 +419,138 @@ def release_editable_install_locks( return matches +# uv/pip surface a busy install target as EBUSY ("Device or resource busy", +# "os error 16") or ETXTBSY ("Text file busy") — e.g. renaming +# site-packages/cv2/.tmp*/cv2.abi3.so over a file a leftover test python +# still has mapped (run-202601012-181847). Everything else is a real failure. +_BUSY_INSTALL_RE = re.compile( + r"device or resource busy|text file busy|os error 16\b|os error 26\b", + re.IGNORECASE) + + +def derive_venv_root(python: str, venv_root: Path = Path("")) -> Path: + """The venv owning `python` — the LOGICAL parent dir, never resolved. + + `Path("")` is truthy, so a naive falsy-guard silently keeps it and a + later resolve() turns it into the PROCESS CWD — the wrong root then + scopes the holder scan (and its kills) to whatever directory the copilot + happens to run in (run-202601012's third resume killed its own launcher + this way). The interpreter itself is typically a SYMLINK to a system + python, so resolving it escapes the venv entirely (`/usr` matches half + the machine); the un-resolved `/bin/python` spelling is the root + that owns the site-packages uv renames into. NOTE `str(Path(""))` is + "." — truthy! — so the empty default needs an explicit string check.""" + if str(venv_root).strip() not in ("", "."): + return Path(venv_root) + return Path(os.path.abspath(python)).parent.parent + + +def venv_root_is_plausible(root: Path) -> bool: + """A root we are willing to kill processes over. Wrong roots are far + worse than no scan: require venv anatomy (pyvenv.cfg, or a lib/ with + site-packages) before believing it.""" + root = Path(root) + if (root / "pyvenv.cfg").is_file(): + return True + lib = root / "lib" + return lib.is_dir() and any(lib.glob("python*/site-packages")) + + +def _read_status_ppid(path: str) -> str: + with open(path, encoding="utf-8", errors="replace") as f: + for line in f: + if line.startswith("PPid:"): + return line.split()[1] + return "0" + + +def _proc_ancestors(pid: int, *, proc_root: str = "/proc", + read: Callable[[str], str] = _read_status_ppid) -> set[int]: + """The pid's ancestor chain (its launcher/supervisor lineage). The wheel + step must never be able to kill the very processes that supervise it, + whatever the scan matched.""" + chain: set[int] = set() + while pid and pid > 1 and pid not in chain: + chain.add(pid) + try: + pid = int(read(f"{proc_root}/{pid}/status") or 0) + except (OSError, ValueError): + break + return chain + + +def scan_venv_holders( + venv: Path, *, + listdir: Callable[[str], list[str]] = os.listdir, + readlink: Callable[[str], str] = os.readlink, + self_pid: int = 0, + skip_pids: frozenset[int] = frozenset(), + proc_root: str = "/proc") -> dict[int, str]: + """Processes with an open fd or cwd under `venv` — best-effort /proc scan. + + Unreadable entries (other users' processes, exited races) are skipped, + never fatal: the goal is releasing THIS campaign's leftover holders, not + an exhaustive audit. Returns {pid: first matching path}.""" + venv = str(Path(venv).resolve()) + holders: dict[int, str] = {} + try: + entries = listdir(proc_root) + except OSError: + return holders + for name in entries: + if not name.isdigit(): + continue + pid = int(name) + if pid == self_pid or pid in skip_pids: + continue + for kind in ("cwd", "fd"): + base = f"{proc_root}/{name}/{kind}" + try: + if kind == "cwd": + targets = [readlink(base)] + else: + targets = [readlink(f"{base}/{fd}") + for fd in listdir(base)] + except OSError: + continue + for target in targets: + if target == venv or target.startswith(venv + "/"): + holders.setdefault(pid, target) + break + return holders + + +def release_venv_holders( + venv: Path, *, + scan: Callable[..., dict[int, str]] = scan_venv_holders, + kill: Callable[[list[int]], list[int]] = kill_tree, + log: Callable[[str], None] = _log) -> list[int]: + """Terminate processes still holding files under the TARGET venv open. + + The venv belongs to this campaign alone: any holder at wheel-install + time is a leftover from a crashed prior run (test pythons, stray + import checks) — exactly the processes whose mapped .so files make uv's + atomic renames fail EBUSY. Returns the pids killed. The caller's own + ancestor chain (launcher, gpu wrapper, the copilot itself) is exempt no + matter what the scan matched.""" + if not venv_root_is_plausible(venv): + log(f"Refusing venv-holder release: {venv} does not look like a " + "virtualenv (no pyvenv.cfg / site-packages) — check " + "the adapter's declared venv variable; a wrong root here kills " + "bystanders.") + return [] + exempt = _proc_ancestors(os.getpid()) | {os.getpid()} + holders = scan(venv, self_pid=os.getpid(), skip_pids=frozenset(exempt)) + # belt and suspenders: filter here too, so even a scan that ignores + # skip_pids can never route a supervisor into the kill list + pids = sorted(p for p in holders if p not in exempt) + if pids: + detail = ", ".join(f"{pid} ({holders[pid]})" for pid in pids[:8]) + log(f"Releasing {len(pids)} process(es) holding the target venv: {detail}") + kill(pids) + return pids + + def _clean_stale_artifacts(repo: Path, spec: WheelSpec, log: Callable[[str], None]) -> None: """Remove stale compiled extensions so a clean reinstall can place a single @@ -359,6 +576,7 @@ def ensure_wheel_installed(repo: Path, commit: str, spec: WheelSpec, *, import_check_log: Path, pre_checkout_head: str = "", release_mode: bool = False, + venv_root: Path = Path(""), run: RunFn = _run, sleep: Callable[[float], None] = time.sleep, log: Callable[[str], None] = _log) -> bool: @@ -368,17 +586,27 @@ def ensure_wheel_installed(repo: Path, commit: str, spec: WheelSpec, *, Skip requires all three: the checkout was already at `commit` before the pick, the import check passes, and the installed version embeds the commit - hash (checkout alone lies when the package was never reinstalled).""" + hash (checkout alone lies when the package was never reinstalled). + + `venv_root` defaults to the interpreter's LOGICAL grandparent + (`/bin/python` → ``); holders of files under it are released + before each install attempt, and a busy (EBUSY/ETXTBSY) install failure + retries within the same bounded attempts after releasing them again.""" repo = Path(repo) + venv_root = derive_venv_root(python, venv_root) check_snippet = build_import_check_snippet(spec) def import_ok() -> bool: - rc, out, err = run([python, "-c", check_snippet], cwd=repo) + rc, out, err = run([python, "-c", check_snippet], + cwd=None if spec.install_url_template else repo) import_check_log.write_text(out + err, encoding="utf-8") return rc == 0 installed_version = read_installed_version(spec, python=python, run=run) - matches = version_matches_commit(installed_version, commit) + matches = (installed_wheel_matches( + spec, commit, installed_version, python=python, run=run) + if spec.install_url_template + else version_matches_commit(installed_version, commit)) healthy = import_ok() if pre_checkout_head == commit and healthy and matches: @@ -419,6 +647,10 @@ def import_ok() -> bool: # A concurrent editable install holds the project lock forever if its # parent died; release it before uninstalling — scoped to THIS repo release_editable_install_locks(repo) + # Leftover processes from a crashed run keep venv files mapped, which + # makes uv's atomic site-packages renames fail EBUSY — release them + # too (scoped to THIS venv) before every install attempt. + release_venv_holders(venv_root, log=log) # Every uv call must name the TARGET venv explicitly: without # --python, uv resolves the environment from VIRTUAL_ENV / a .venv in @@ -431,13 +663,46 @@ def import_ok() -> bool: _append(install_log, f"--- uv pip uninstall {spec.package} ---\n{out}{err}") env = dict(os.environ) - env.update(spec.install_env) - rc, out, err = run([uv, "pip", "install", "--python", python, - "-e", "."], cwd=repo, env=env, - timeout=3600) - _append(install_log, "--- uv pip install -e . (precompiled) ---\n" + # The upstream installer otherwise resolves a base commit on main, + # which can differ from the selected release-branch commit. Carry + # both selection inputs through to the precompiled extension install. + env.update({key: value.replace("{commit}", commit) + .replace("{variant}", spec.variant) + for key, value in spec.install_env.items()}) + install_args = ([spec.install_url_template.format( + commit=commit, variant=spec.variant, arch=spec.arch, + package=spec.package)] if spec.install_url_template else ["-e", "."]) + if spec.install_url_template: + # uv omits archive hashes from direct_url.json in some versions. + # pip enforces the URL's sha256 and records PEP 610 provenance. + rc, _, _ = run([python, "-m", "pip", "--version"]) + if rc != 0: + rc, out, err = run([uv, "pip", "install", "--python", + python, "pip"], timeout=600) + _append(install_log, "--- seed target pip ---\n" + out + err) + if rc != 0: + raise WheelInstallError("target pip setup failed; " + f"see {install_log}") + install_cmd = [python, "-m", "pip", "install", *install_args] + else: + install_cmd = [uv, "pip", "install", "--python", python, + *install_args] + rc, out, err = run(install_cmd, cwd=repo, env=env, timeout=3600) + _append(install_log, "--- install selected precompiled artifact ---\n" f"{out}{err}") if rc != 0: + if _BUSY_INSTALL_RE.search(out + err): + # A leftover holder raced our release (or mapped the file + # between the scan and the rename). Release again and use + # the loop's bounded attempts; a busy target is transient, + # every other failure class stays fail-fast. + holders = release_venv_holders(venv_root, log=log) + log(f"Install attempt {attempt}/{retries} hit a busy target " + f"({len(holders)} holder(s) released); " + + ("retrying..." if attempt < retries + else "no attempts left.")) + if attempt < retries: + continue if release_mode: # The release-branch tip may have no precompiled wheel: fall # back to a source build (only path where that is expected). @@ -457,6 +722,13 @@ def import_ok() -> bool: f"see {install_log}") if import_ok(): + if spec.install_url_template and not installed_wheel_matches( + spec, commit, read_installed_version(spec, python=python, + run=run), + python=python, run=run): + raise WheelInstallError( + f"installed wheel provenance does not match {commit[:12]}; " + f"see {install_log}") log(f"{spec.package} install completed and import validation passed.") return True log(f"Import validation failed (attempt {attempt}/{retries}); " @@ -475,6 +747,95 @@ def import_ok() -> bool: f"for {commit[:12]}{hint}; see {import_check_log}") +# -- baseline runtime ---------------------------------------------------------- + +def _clone_tree_hardlinked(src: Path, dst: Path) -> bool: + """Hardlink clone (near-zero disk, same filesystem). uv replaces files + by rename, which breaks the links — the campaign venv is never modified + through the clone.""" + rc, _, err = _run(["cp", "-al", str(src), str(dst)], timeout=1800) + if rc != 0: + _log(f"baseline-venv clone failed: {err.strip()}") + return False + return True + + +def ensure_baseline_venv( + target_venv: Path, spec: WheelSpec, upstream: Path, + baseline_commit: str, *, + uv: str = "uv", + run: RunFn = _run, + clone: Callable[[Path, Path], bool] = _clone_tree_hardlinked, + log: Callable[[str], None] = _log) -> Path | None: + """A runtime clone of the TARGET venv carrying the BASELINE commit's + wheel, for main-worktree baseline runs. + + A main-tracking campaign's wheel routinely removes symbols the target + repo's main still imports, so a baseline sharing the campaign venv dies at conftest + import — and every pre-existing failure then reads as a regression + (run-202601012: tiny_model_tests' gated-model setup errors). The + baseline question has always been "does the target repo's main fail this test + against the wheel main was LAST aligned to"; this builds exactly that + runtime: hardlink-clone the campaign venv (identical deps), swap in + the baseline wheel from a throwaway worktree of the upstream at the + baseline commit. Returns None on any failure — a missing baseline + runtime keeps the loop's regression-preserving default and must never + read as pre-existing.""" + import shutil + + target_venv = Path(target_venv) + if not baseline_commit: + return None + bv = target_venv.with_name( + f"{target_venv.name}-baseline-{baseline_commit[:12]}") + python = str(bv / "bin" / "python") + if (bv / "pyvenv.cfg").is_file(): + installed = read_installed_version(spec, python=python, run=run) + if installed and version_matches_commit(installed, baseline_commit): + log(f"baseline venv already healthy at {bv}") + return bv + if not (Path(upstream) / ".git").exists(): + log(f"baseline venv: upstream {upstream} is not a git checkout") + return None + if bv.exists(): + shutil.rmtree(bv, ignore_errors=True) + if not clone(target_venv, bv): + return None + # the editable install builds metadata from the tree it installs, so + # give it a worktree of the upstream AT the baseline commit + wt = bv.parent / (bv.name + "-src") + rc, _, err = run(["git", "-C", str(upstream), "worktree", "add", + "--detach", str(wt), baseline_commit], timeout=600) + if rc != 0: + log(f"baseline worktree add failed: {err.strip()}") + shutil.rmtree(bv, ignore_errors=True) + return None + try: + run([uv, "pip", "uninstall", "--python", python, spec.package], + cwd=wt) + env = dict(os.environ) + env.update({key: value.replace("{commit}", baseline_commit) + .replace("{variant}", spec.variant) + for key, value in spec.install_env.items()}) + rc, _, err = run([uv, "pip", "install", "--python", python, + "-e", "."], cwd=wt, env=env, timeout=3600) + if rc != 0: + log(f"baseline wheel install failed: {err.strip()[-300:]}") + shutil.rmtree(bv, ignore_errors=True) + return None + rc, _, err = run([python, "-c", build_import_check_snippet(spec)], + cwd=wt) + if rc != 0: + log(f"baseline venv import check failed: {err.strip()[-200:]}") + shutil.rmtree(bv, ignore_errors=True) + return None + finally: + run(["git", "-C", str(upstream), "worktree", "remove", + "--force", str(wt)], timeout=600) + log(f"baseline venv ready at {bv} (wheel {baseline_commit[:12]})") + return bv + + # -- Dockerfile pin ----------------------------------------------------------- def is_pinned(text: str, commit: str, pin: PinSpec) -> bool: diff --git a/src/infermatrix_copilot/testing/cpu_ci_resources.py b/src/infermatrix_copilot/testing/cpu_ci_resources.py new file mode 100644 index 000000000..97fa8e5d3 --- /dev/null +++ b/src/infermatrix_copilot/testing/cpu_ci_resources.py @@ -0,0 +1,46 @@ +"""Match the target repo's simple CPU jobs' single-card Buildkite allocation. + +The parent campaign holds four cards for later GPU jobs. Exposing all four +to a Simple CPU job enables extra GPU parity tests beyond its l4_1 mirror. +The temporary target pytest entrypoint calls this helper; GPU and mixed +marker expressions retain their full allocation. +""" + +from __future__ import annotations + +import os +import re +import sys +from collections.abc import Mapping, Sequence + + +def cpu_ci_env(argv: Sequence[str], env: Mapping[str, str]) -> dict[str, str]: + result = dict(env) + expression = None + for index, arg in enumerate(argv): + if arg in ("-m", "--markexpr") and index + 1 < len(argv): + expression = argv[index + 1] + elif arg.startswith(("-m=", "--markexpr=")): + expression = arg.split("=", 1)[1] + if expression is None or not re.fullmatch( + r"\s*core_model\s+and\s+cpu\s*", expression + ): + return result + devices = [ + value.strip() + for value in result.get("CUDA_VISIBLE_DEVICES", "").split(",") + if value.strip() + ] + if len(devices) > 1: + result["CUDA_VISIBLE_DEVICES"] = devices[0] + return result + + +def main(target_python: str) -> None: + argv = sys.argv[1:] + env = cpu_ci_env(argv, os.environ) + if env.get("CUDA_VISIBLE_DEVICES") != os.environ.get("CUDA_VISIBLE_DEVICES"): + print( + "[CPU CI resources] exposing one card to match the l4_1 mirror", flush=True + ) + os.execve(target_python, [target_python, "-m", "pytest", *argv], env) diff --git a/src/infermatrix_copilot/testing/runner.py b/src/infermatrix_copilot/testing/runner.py index a2f0c9ddc..eb6cb7b56 100644 --- a/src/infermatrix_copilot/testing/runner.py +++ b/src/infermatrix_copilot/testing/runner.py @@ -328,7 +328,7 @@ def run(self, job: TestJob, env: dict[str, str], *, baseline: bool = False, dry_run: bool = False) -> TestOutcome: log_file = log_file_for(self.tests_dir, job, baseline=baseline) plan = RunPlan( - argv=["bash", "-c", _exec_wrap(job.command)], + argv=["bash", "--norc", "-c", _exec_wrap(job.command)], env_overlay=dict(job.env), timeout_sec=job.timeout_sec, needs_gpu_lock=(job.gpu_lock if job.gpu_lock is not None else job.min_gpus > 0), @@ -450,7 +450,7 @@ def _run_setup(self, job: TestJob, env: dict[str, str], note = "" try: with open(log_file, "a", encoding="utf-8") as lf: - proc = subprocess.Popen(["bash", "-c", job.setup], + proc = subprocess.Popen(["bash", "--norc", "-c", job.setup], cwd=self.repo_root, env=env, stdout=lf, stderr=lf, start_new_session=True) @@ -496,7 +496,11 @@ def _spawn(self, exec_cmd: str, job: TestJob, env: dict[str, str], except OSError: attempt_offset = 0 with open(log_file, "a" if append else "w", encoding="utf-8") as lf: - proc = subprocess.Popen(["bash", "-c", exec_cmd], + # Bash can source ~/.bashrc for a noninteractive SSH-launched + # shell at SHLVL=0. That can put a foreign CLI ahead of the + # explicitly selected runtime. Keep setup and test environments + # authoritative; explicit sourcing in a job still works. + proc = subprocess.Popen(["bash", "--norc", "-c", exec_cmd], cwd=self.repo_root, env=env, stdout=lf, stderr=lf, start_new_session=True) diff --git a/src/infermatrix_copilot/tool_bridge.py b/src/infermatrix_copilot/tool_bridge.py index e779699f4..3eb51d839 100644 --- a/src/infermatrix_copilot/tool_bridge.py +++ b/src/infermatrix_copilot/tool_bridge.py @@ -145,7 +145,20 @@ def __init__(self, plan_prefix: str, gated: tuple[str, ...], self.plan_prefix = plan_prefix self.gated = tuple(gated) self.trace = trace - self.open = False + # A harness may restart its stdio MCP process while preserving the + # model session. The decision file is the durable gate state; starting + # every replacement bridge closed forces the agent to rewrite an + # already-reviewed decision before it can finish or emit its signal. + try: + restored = next( + (path for path in Path(plan_prefix).rglob("*.decision.md") + if path.is_file()), None) + except OSError: + restored = None + self.open = restored is not None + if restored is not None and self.trace is not None: + self.trace.record("plan_gate_opened", decision=str(restored), + restored=True) def refusal(self, name: str, args: dict) -> str | None: """The refusal text for a call that must not run yet, else None.""" @@ -184,27 +197,42 @@ def _rebase_extra(spec: dict, trace: RunTrace, scope: ToolScope) -> dict: """Rebuild the adapter's rebase tool pack in the bridge process. Credentials are NOT in the spec: `api_key` comes from this process's - settings, and the child env for shell/pytest handlers is this process's - own environment (inherited from the harness), so no secret is written to - `run_dir/bridge/`. + settings. Shell handlers reconstruct the same scrubbed target env as the + API agent; the spec carries only the resolved venv path, not credentials + or a full child environment. """ + from types import SimpleNamespace + from .config import Settings - from .engine.steps.rebase_v3 import build_backends - from .rebase_engine.rebase_tools import (RebasePaths, build_rebase_tools, - load_tool_schemas) + from .engine.steps.rebase_v3 import _agent_shell_env, build_backends + from .rebase_engine.rebase_tools import ( + RebasePaths, + build_rebase_tools, + load_tool_schemas, + ) rb = spec["rebase"] - settings = Settings() + settings = Settings(**{ + key: rb[field] for key, field in ( + ("rebase_backend", "backend"), + ("rebase_backend_model", "backend_model")) if field in rb}) # generic reconstruction: the spec's payload IS the dataclass's fields, # so naming them here would put repo vocabulary in a neutral module payload = dict(rb.get("paths") or {}) if "test_roots" in payload: payload["test_roots"] = tuple(payload["test_roots"]) - paths = RebasePaths(env=dict(os.environ), **payload) manifest: dict = {} mpath = rb.get("manifest_path", "") if mpath and Path(mpath).is_file(): import yaml manifest = yaml.safe_load(Path(mpath).read_text(encoding="utf-8")) or {} + # The harness may not inherit a launcher-only ${VAR} override. Preserve + # the parent's resolved runtime without serializing its environment. + target_venv = rb.get("target_venv", "") + if not isinstance(target_venv, str): + raise TypeError("bridge target_venv must be a path string") + if target_venv: + manifest = {**manifest, "repo": {**(manifest.get("repo") or {}), + "venv": target_venv}} # `repo` here is the repo ROOT PATH — build_backends feeds it to # TestRunner(repo_root=...). The spec's "repo" is the repo NAME and # belongs in `state.task_spec.repo`; passing the name here made every @@ -212,6 +240,25 @@ def _rebase_extra(spec: dict, trace: RunTrace, scope: ToolScope) -> dict: # FileNotFoundError on a relative path. state = dict(rb.get("state") or {}) state.setdefault("task_spec", {"repo": spec.get("repo", "")}) + # The harness spawned this process with an allowlisted env, so the GPU + # selection and model cache the parent run was granted are not here; + # the spec carries those two non-secret facts, handed EXPLICITLY to the + # env builders (agent shells and run_pytest/reproduce alike) — the + # process env itself is never mutated (env guardrail census). + host_env = rb.get("host_env") or {} + if not isinstance(host_env, dict) or not all( + isinstance(k, str) and isinstance(v, str) + for k, v in host_env.items()): + raise TypeError("bridge host_env must map str to str") + host_env = {k: host_env[k] for k in ("CUDA_VISIBLE_DEVICES", "HF_HOME") + if k in host_env} + ctx = SimpleNamespace(settings=settings, state=state, + run_dir=Path(spec["run_dir"])) + adapter_dir = (Path(mpath).parent if mpath else + Path(rb["tool_schemas"]).parent.parent) + paths = RebasePaths( + env=_agent_shell_env(ctx, manifest, scope.root, adapter_dir, + host_env=host_env), **payload) backends = build_backends( settings=settings, state=state, run_dir=Path(spec["run_dir"]), trace=trace, manifest=manifest, @@ -221,6 +268,7 @@ def _rebase_extra(spec: dict, trace: RunTrace, scope: ToolScope) -> dict: model=rb.get("model", ""), base_url=rb.get("base_url", ""), api_key=getattr(settings, "anthropic_api_key", "") or "", + host_env=host_env, ) defs = load_tool_schemas(Path(rb["tool_schemas"])) return build_rebase_tools(defs, paths, backends), defs diff --git a/test/fixtures/remote_ci_owner/adapter_manifest.yaml b/test/fixtures/remote_ci_owner/adapter_manifest.yaml new file mode 100644 index 000000000..ee871327b --- /dev/null +++ b/test/fixtures/remote_ci_owner/adapter_manifest.yaml @@ -0,0 +1,430 @@ +# Adapter zero — vllm-omni repo knowledge extracted to the edge. +name: vllm_omni +status: active +created_by: human +schema_version: 1 +# explicit capabilities beyond the derived ones (repo.path, ci.provider, ...) +capabilities: [] + +repo: + # Portable: expanded via ${VAR}/~ at read time. REPO_PATHS in .env (gitignored) + # takes precedence; this is only the fallback when no .env is in reach. Never + # hardcode a machine-specific absolute path in this committed file. + path: ${VLLM_OMNI_REPO} + full_name: vllm-project/vllm-omni + aliases: [vllm-omni, vllm_omni] + default_branch: main + rebase_branch: dev/vllm-align + language: python + # the TARGET repo's virtualenv — test jobs and wheel installs run in it, + # never in the copilot's own venv (env-expanded; empty = not configured, + # which BLOCKS the modes that need it rather than silently using ours) + venv: ${VLLM_OMNI_VENV} + +upstream: + kind: fork_tracking + repo_path: ${VLLM_UPSTREAM_REPO} + remote: origin + # v0.31.0 alignment targets this release branch. Wheel selection and + # module assignment must use its commits, including release-only fixes. + target_branch: releases/v0.31.0 + +# Repo-specific slice of the SHARED, human-curated knowledge base (the vendored +# community docs at the repo-root `knowledge/` = settings.knowledge_dir; +# cross-repo general/ knowledge is shared, NOT nested here). Replaces the +# retired AI profile/. Layout and consumption rules are in +# doc/architecture/KNOWLEDGE.md. Paths below are relative to settings.knowledge_dir. +knowledge: + source: zuiho-kai/claude-workflow-starter + repo_subdir: repos/vllm-omni # this repo's slice within the shared base + briefing_docs: # always-on repo-specific slice: rules + nav + - repos/vllm-omni/rules.md + - repos/vllm-omni/_index.md + # Extended slice for ALL tiers (plan v2: the eco/performance boundary is + # model-only): a compact owner router; deep component/model pages remain + # on-demand through doc_read/doc_search. + briefing_docs_extra: + - repos/vllm-omni/review/guides/maintainer-pattern-routing.md + # repo-specific review checklist injected (first 4k chars) into the Strict + # reviewer's system prompt — triggered checks distilled from maintainer + # review patterns; the page lives in the shared knowledge tree so Direct + # and knowledge maintenance see the same source + review_checklist: repos/vllm-omni/review/guides/strict-review-checklist.md + +# upstream_paths/test_paths are the parent orchestrator's authoritative module +# maps (config.sh §11 MODULE_VLLM_PATHS / MODULE_TEST_MAP, verbatim — shell +# test selection). Job→module ROUTING deliberately uses a SEPARATE map (the +# parent test_manifest.py inline flavor, rebase.test_manifest +# .assignment_paths below): scoring over a different pattern set produces +# different winners, so the two flavors must not be merged (DRIFT_TRIAGE #6). +# The rebase path-sync step retargets the *_paths lists against the live +# tree each run. +# local_paths is the UNION of the original coarse scoping prefixes and every +# parent MODULE_OMNI_FILES entry not already covered by them — module rebases +# must see the parent's full scope, and prefix-matching consumers (review +# scoping) may only ever gain coverage here, never lose it. +modules: + worker_runner: + local_paths: [vllm_omni/worker/, vllm_omni/worker_v2/] + upstream_paths: [vllm/v1/worker/] + test_paths: [tests/worker/, tests/worker_v2/, tests/diffusion/test_diffusion_worker.py, tests/e2e/features/custom_pipeline/, tests/e2e/online_serving/test_qwen3_tts_base.py, tests/entrypoints/test_omni_sleep_mode.py] + import_check: "from vllm_omni.worker.gpu_model_runner import *; from vllm_omni.worker.gpu_generation_model_runner import *; from vllm_omni.worker.gpu_ar_model_runner import *; from vllm_omni.worker.gpu_ar_worker import *; from vllm_omni.worker_v2.omni_ar_model_runner import *; from vllm_omni.worker_v2.omni_generation_model_runner import *; print('Worker/Runner imports OK')" + wave: 1 + risk: high + model_executor: + local_paths: [vllm_omni/model_executor/, vllm_omni/experimental/, vllm_omni/model_extras/, vllm_omni/transformers_utils/] + upstream_paths: [vllm/model_executor/] + test_paths: [tests/model_executor/, tests/diffusion/, tests/e2e/offline_inference/] + import_check: "from vllm_omni.model_executor.models.registry import *; print('Model Executor imports OK')" + wave: 1 + risk: high + input_output: + local_paths: [vllm_omni/inputs/, vllm_omni/outputs/, vllm_omni/utils/mm_outputs.py, vllm_omni/data_entry_keys.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/serialization.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_engine_core_client.py, vllm_omni/engine/output_processor.py, vllm_omni/request.py] + upstream_paths: [vllm/inputs/, vllm/renderers/, vllm/v1/engine/input_processor.py, vllm/v1/engine/output_processor.py, vllm/v1/request.py] + test_paths: [tests/inputs/, tests/entrypoints/test_omni_new_request_data.py, tests/engine/test_async_omni_engine_input.py, tests/test_outputs.py] + import_check: "from vllm_omni.inputs.data import *; from vllm_omni.inputs.preprocess import *; from vllm_omni.engine.async_omni_engine import AsyncOmniEngine; from vllm_omni.engine.orchestrator import Orchestrator; from vllm_omni.engine.serialization import serialize_additional_information; print('Input/Output imports OK')" + wave: 1 + scheduler: + local_paths: [vllm_omni/core/] + upstream_paths: [vllm/v1/core/sched/] + test_paths: [tests/core/sched/, tests/distributed/, tests/model_executor/test_cfg_pairing.py, tests/diffusion/distributed/test_ulysses_uaa_perf.py] + import_check: "from vllm_omni.core.sched.omni_ar_scheduler import *; from vllm_omni.core.sched.omni_generation_scheduler import *; print('Scheduler imports OK')" + wave: 1 + risk: high + online_serving: + local_paths: [vllm_omni/entrypoints/, vllm_omni/engine/] + upstream_paths: [vllm/entrypoints/] + test_paths: [tests/entrypoints/test_async_omni.py, tests/entrypoints/test_async_omni_diffusion_config.py, tests/entrypoints/openai_api/] + import_check: "from vllm_omni.entrypoints.omni import Omni; from vllm_omni.entrypoints.async_omni import AsyncOmni; from vllm_omni.entrypoints.openai.api_server import omni_run_server; from vllm_omni.entrypoints.openai.serving_speech import ServingSpeech; from vllm_omni.entrypoints.cli.serve import OmniServeCommand; print('Online Serving imports OK')" + wave: 1 + model_config: + local_paths: [vllm_omni/config/, vllm_omni/deploy/, vllm_omni/engine/arg_utils.py] + upstream_paths: [vllm/config/, vllm/engine/arg_utils.py] + test_paths: [tests/entrypoints/test_stage_utils.py, tests/metrics/test_stats.py] + import_check: "from vllm_omni.config.model import OmniModelConfig; from vllm_omni.engine.arg_utils import OmniEngineArgs; print('Model Config imports OK')" + wave: 1 + # risk high: platform code sets process-wide defaults (op priorities, + # capability gates) inherited by EVERY model on that platform — a 3-file + # 61-line default flip here is a repo-wide behavior change (PR 4964/5009 + # class), so small diffs must still get the full review ensemble + platform: + local_paths: [vllm_omni/platforms/] + upstream_paths: [vllm/platforms/] + test_paths: [tests/] + import_check: "from vllm_omni.platforms.interface import OmniPlatform; from vllm_omni.platforms.cuda.platform import CudaOmniPlatform; print('Platform imports OK')" + wave: 1 + risk: high + # shared diffusion pipeline: denoise loop, attention backends, DiT/VAE — + # multi-model shared code where one-model-validated changes have cross-model + # blast radius (kernel-provider and hub-loading changes live here) + diffusion: + local_paths: [vllm_omni/diffusion/] + wave: 1 + risk: high + # deploy YAMLs are the serving registry: defaults and per-platform blocks + # (cudagraph modes, capacity) that decide runtime behavior for CLI and yaml + # launches alike. NOT risk-tagged: a 2-line deploy deletion measured as a + # precision WIN at light depth (pr4970), so tiny deploy diffs keep the + # cheap tier; deploy changes bundled with code still escalate via the + # code-side signals + deploy: + local_paths: [vllm_omni/deploy/] + wave: 1 + distributed: + local_paths: [vllm_omni/distributed/] + wave: 1 + risk: high + engine: + local_paths: [vllm_omni/engine/] + wave: 1 + attention: + local_paths: [vllm_omni/attention/] + wave: 1 + quantization: + local_paths: [vllm_omni/quantization/] + wave: 1 + runtime_patch: + local_paths: [vllm_omni/patch.py] + test_paths: [tests/diffusion/test_inductor_divisibility_patch.py] + wave: 1 + risk: high + # Filesystem-backed immutable host artifacts have process-wide lock, lease, + # durability, and fallback contracts; keep this owner explicit rather than + # letting generic runtime scopes silently omit it. + host_weight_runtime: + local_paths: [vllm_omni/host_weight_runtime/] + test_paths: [tests/host_weight_runtime/] + wave: 1 + risk: high + sampling: + local_paths: [vllm_omni/sample/] + wave: 1 + tokenizers: + local_paths: [vllm_omni/tokenizers/] + wave: 1 + metrics: + local_paths: [vllm_omni/metrics/] + wave: 2 + benchmarks: + local_paths: [benchmarks/, vllm_omni/benchmarks/, vllm_omni/entrypoints/cli/benchmark/] + upstream_paths: [vllm/benchmarks/, vllm/entrypoints/cli/benchmark/] + test_paths: [tests/benchmarks/test_serve_cli.py] + import_check: "from vllm_omni.benchmarks.serve import main; from vllm_omni.entrypoints.cli.benchmark.serve import OmniBenchmarkServingSubcommand; print('Benchmarks imports OK')" + wave: 2 + fullduplex: + local_paths: [vllm_omni/clients/, vllm_omni/experimental/fullduplex/] + test_paths: [tests/clients/, tests/benchmarks/duplex/, tests/e2e/features/fullduplex/] + import_check: "from vllm_omni.clients.duplex import DuplexClient; print('Full-duplex client imports OK')" + wave: 2 + +review_routes: +- prefix: vllm_omni/patch.py + owner: tooling + doc: repos/vllm-omni/tooling/rules.md +- prefix: vllm_omni/config/ + owner: config + doc: repos/vllm-omni/components/configuration/rules.md +- prefix: vllm_omni/core/ + owner: scheduler + doc: repos/vllm-omni/components/scheduler/rules.md +- prefix: vllm_omni/clients/ + owner: serving + doc: repos/vllm-omni/components/serving/rules.md +- prefix: vllm_omni/entrypoints/ + owner: serving + doc: repos/vllm-omni/components/serving/rules.md +- prefix: vllm_omni/model_executor/ + owner: model-executor + doc: repos/vllm-omni/components/model-executor/rules.md +- prefix: vllm_omni/models/ + owner: models + doc: repos/vllm-omni/models/_index.md +- prefix: vllm_omni/diffusion/ + owner: diffusion + doc: repos/vllm-omni/components/diffusion/rules.md + +# Phase-1 rebase machinery data (consumed by rebase_engine — repo specifics +# live here so the engine stays neutral). Sourced from the parent's config.sh +# + tasks 10/20/01/40/35. +rebase: + # behavioral hooks (plan §1.2): declared explicitly, loaded only while the + # adapter is active; this whole `rebase:` section is HIGH-RISK (human-only + # writes) so an agent can never self-install hooks + hooks: rebase/hooks.py + # The READ-ONLY parent-store read-compat `knowledge:` block (Rev 8 §9 + # PR4c) is removed (owner decision 2026-10-01): the parent rebase-agent + # checkout no longer exists, so v3 runs on its own runtime + adapter seed + # knowledge only. An absent declaration attests nothing and blocks nothing. + # the shared checkout flock name: v3, the v1 backend, EXT1's external + # startup guard, and PR7 archival ALL take locks/.lock inside + # the checkout, so exclusion is mutual (repo data — the core stays neutral) + lock_name: omni + # phase-3.2 precommit workflow (parent config.sh §10 __precommit__ entry: + # command, 600s timeout, 0 GPUs; one retry — auto-fix hooks often pass on + # the second run) + precommit: + command: pre-commit run --all-files + timeout_sec: 600 + retry_once: true + wheel: + package: vllm + index_url_template: "https://wheels.vllm.ai/{commit}/{variant}/{package}/" + variant: cu130 + arch: x86_64 + install_url_template: "https://wheels.vllm.ai/{commit}/vllm-0.31.0rc3-cp38-abi3-manylinux_2_28_{arch}.whl#sha256=c6d27b7da5cc6642c3f50eb1c9e93009fcf56fedd582e09a8181ed5e7d137843" + import_check_modules: [vllm._C_stable_libtorch, vllm._C] + ops_namespace: _C + stale_artifact_globs: ["_C*.so", "*.abi3.so", "_moe_C*.so"] + install_env: + VLLM_USE_PRECOMPILED: "1" + VLLM_PRECOMPILED_WHEEL_COMMIT: "{commit}" + VLLM_PRECOMPILED_WHEEL_VARIANT: "{variant}" + # Release wheels have an HTML index but no nightly metadata.json. + VLLM_PRECOMPILED_WHEEL_LOCATION: "https://wheels.vllm.ai/{commit}/vllm-0.31.0rc3-cp38-abi3-manylinux_2_28_x86_64.whl" + reinstall_retries: 3 + pin: + dockerfile: docker/Dockerfile.ci + url_pattern: 'wheels\.vllm\.ai/[0-9a-f]{40}' + url_template: "wheels.vllm.ai/{commit}" + commit_env_var: VLLM_PRECOMPILED_WHEEL_COMMIT + ci: + # remote-CI pipeline identities (parent config: BUILDKITE_CI_PIPELINE + # for the build-under-test; the rebase pipeline is where the nightly / + # gate baseline of pre-existing main failures lives) + pipeline: vllm-omni-release + baseline_pipeline: vllm-omni-rebase + # trigger env (parent create_build env + trigger_and_monitor extra_env, + # verbatim): opts the FULL suite in — a webhook build without these + # runs fewer tests, which is why pushed rounds create their own build + build_env: {NIGHTLY: "1", RUN_HUNYUAN_IMAGE3_PERF: "1", EMAIL_DISTRIBUTION: "0"} + # monitor classification data (parent monitor.py constants — EXCEPT + # the pipeline upload/load lanes, moved to structural: their failure + # suppresses every dynamic test job, so ignoring it was a false-green + # vector in the parent; deliberate divergence, 2026-08-01 review) + # the org runs this pipeline SCHEDULE-ONLY (provider branch builds + # disabled, 2026-08); API creates need the documented + # ignore_pipeline_branch_filters override (write_builds scope only) + ignore_branch_filters: true + ignorable_name_patterns: ["(?i)testcase statistics", + "(?i)collection & email"] + structural_name_patterns: ["(?i):pipeline:\\s*(upload|load)", + "(?i)(upload|load).*pipeline"] + ignorable_log_patterns: ["401 Unauthorized", "gated model", + "HFValidationError", "RepositoryNotFoundError"] + extra_exception_names: [ChatTemplateResolutionError, OmniEngineDeadError] + test_manifest: + # dynamic manifest building (parent test_manifest.py constants — EXCEPT + # yaml_dir: the live tree moved the pipelines under .buildkite/cuda/; + # the parent's hardcoded `.buildkite` is stale against it and would + # find ZERO jobs today (recorded in DRIFT_TRIAGE) + yaml_dir: .buildkite/cuda + pipelines: {test-merge.yml: merge, test-ready.yml: ready, test-nightly.yml: nightly} + priority_source: ready + # tree roots whose baseline diffs count as TEST changes + test_change_roots: [tests/] + default_queue: gpu_1_queue + queue_map: + gpu_1_queue: [1, any] + gpu_4_queue: [4, any] + mithril-h100-pool: [2, h100] + skip_patterns: ["(?i)perf", "(?i)accuracy", "(?i)testcase statistics", + "(?i)collection & email", "(?i)benchmark.*nightly"] + # upstream test-layout refactor rename families (longest-first) + rename_suffixes: [_tts_expansion, _expansion, _tts] + file_ref_prefixes: [tests, vllm_omni] + file_tree_roots: [tests, vllm_omni] + # slug -> pre-run setup command (parent CI_TEST_SETUP verbatim; feeds the + # runner's model-download notification hook). The parent's LIVE map is + # currently empty — its only entries are commented STALE — so this stays + # {} until a job declares one. + setup_map: {} + # job→module ROUTING patterns — the parent test_manifest.py inline map + # VERBATIM (a third flavor, distinct from §11 MODULE_TEST_MAP above): + # `_assign_modules` scores file refs against these (dir prefix = 2, + # file = 1, best>0 wins) and the winner feeds module test plans, prompts, + # and debug scoping. Golden-pinned (shell_golden.json assignment_map + + # behavioral routing replay). + assignment_paths: + model_config: [tests/entrypoints/test_stage_utils.py, tests/metrics/test_stats.py] + input_output: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/, tests/test_outputs.py] + scheduler: [tests/distributed/] + worker_runner: [tests/worker/, tests/worker_v2/, tests/diffusion/test_diffusion_worker.py, tests/e2e/offline_inference/custom_pipeline/, tests/e2e/features/custom_pipeline/] + model_executor: [tests/model_executor/, tests/diffusion/, tests/e2e/offline_inference/] + online_serving: [tests/entrypoints/, tests/e2e/online_serving/] + benchmarks: [tests/benchmarks/test_serve_cli.py] + platform: [tests/] + # ordered change-path -> module classification (parent heuristic order: + # entrypoints beats engine) + change_path_rules: + - [/entrypoints/, online_serving] + - [/engine/, input_output] + - [/model_executor/, model_executor] + - [/diffusion/, model_executor] + - [/worker/, worker_runner] + - [/distributed/, scheduler] + - [/benchmarks/, benchmarks] + testing: + module_max_timeout_sec: 7200 + # root-level artifacts tests leak between runs (test_artifact_cleanup.sh + # TEST_ARTIFACT_GLOBS, verbatim) — gitignored upstream but pile up on disk + artifact_globs: ["audio_chatcmpl-*.wav", "audio_[0-9]*ch_*.wav", + "test_*.wav", "output.wav", "output_*.wav", + "diffusion_result_test_*.json", + "diffusion_result_qwen_*.json"] + guard: + # untracked pytest-copied stage configs are auto-discarded before the + # dirty-tree verdict (regexes over the repo-relative path) + discard_untracked_patterns: + - '^tests/e2e/stage_configs/.*_[0-9]{6,}\.yaml$' + - '^tests/dfx/[^/]+/stage_configs/.*_[0-9]{6,}\.yaml$' + push: + commit_message_template: "rebase: align vllm-omni with vLLM {short}" + # generated outputs never reach a rebase commit (git_helpers.sh + # unstage_generated_outputs case patterns, fnmatch form) + unstage_globs: [".pytest_cache/*", ".ruff_cache/*", "__pycache__/*", + "*/__pycache__/*", "htmlcov/*", "coverage.xml", + ".coverage", ".coverage.*", "pytest.xml", "junit.xml", + "junit-*.xml", "rebase_logs/*", "logs/*", "*.log", + "*.out", "*.tmp", "*.pid", "tests/*/artifacts/*", + "tests/*/outputs/*", "tests/*/tmp/*", "tests/*/.tmp/*"] + # upstream files whose changes signal base-class inheritance churn + base_class_watch_paths: [vllm/entrypoints/llm.py, vllm/v1/engine/core_client.py, vllm/engine/protocol.py, vllm/inputs/preprocess.py, vllm/v1/core/sched/scheduler.py, vllm/v1/worker/gpu_model_runner.py] + path_sync: + # curated candidate overlays (task 35): candidates are filtered against + # the live tree; fallback survives when none exist + curated: + local_paths: + input_output: + candidates: [vllm_omni/inputs/data.py, vllm_omni/inputs/preprocess.py, vllm_omni/engine/async_omni_engine.py, vllm_omni/engine/orchestrator.py, vllm_omni/engine/serialization.py, vllm_omni/engine/stage_init_utils.py, vllm_omni/engine/stage_engine_core_client.py, vllm_omni/engine/input_processor.py, vllm_omni/engine/output_processor.py, vllm_omni/request.py] + fallback: vllm_omni/inputs/data.py + online_serving: + candidates: [vllm_omni/entrypoints/omni_base.py, vllm_omni/entrypoints/omni.py, vllm_omni/entrypoints/async_omni.py, vllm_omni/entrypoints/async_omni_diffusion.py, vllm_omni/entrypoints/openai/api_server.py, vllm_omni/entrypoints/openai/serving_chat.py, vllm_omni/entrypoints/openai/serving_speech.py, vllm_omni/entrypoints/openai/utils.py] + fallback: vllm_omni/entrypoints/openai/api_server.py + test_paths: + model_config: + candidates: [tests/entrypoints/test_stage_utils.py, tests/metrics/test_stats.py, tests/entrypoints/test_omni_llm.py] + fallback: tests/entrypoints/test_stage_utils.py + input_output: + candidates: [tests/entrypoints/test_omni_new_request_data.py, tests/engine/test_async_omni_engine_input.py, tests/entrypoints/test_omni_input_preprocessor.py, tests/test_outputs.py] + fallback: tests/entrypoints/test_omni_new_request_data.py + online_serving: + candidates: [tests/entrypoints/test_async_omni.py, tests/entrypoints/test_async_omni_diffusion_config.py, tests/entrypoints/test_omni_diffusion.py, tests/entrypoints/test_omni_stage_diffusion_config.py, tests/entrypoints/openai_api/] + fallback: tests/entrypoints/openai_api/ + +validation: + test_manifest_sources: [.buildkite/] + precommit: true + # gated-model tests (TTS/omni checkpoints) need the HF token inside agent + # shells — the Rev 8 §4 EXPLICIT opt-in; without it the credential scrub + # strips HF_TOKEN and gated-model verification fails + requires_hf_token: true + +ci: + provider: buildkite + org: vllm + +push: + default_remote: origin + protected_branches: [main] + allowed: false # deliver via PR; never direct-push main + # plan §5.3 (human-gated HIGH-RISK edit): the v3 rebase pipeline may push + # to the REBASE BRANCH only; main stays protected regardless, guard_push + # remains the choke point, and ALLOW_PUSH=1 is still required at execution + rebase_branch_allowed: true + # the ONE branch the rebase pipeline may push (parent REBASE_BRANCH) — + # the push scope is data; v3_ci refuses any other checkout branch + rebase_branch: dev/vllm-align + # rebase-commit author identity (parent config SIGNOFF_NAME/EMAIL) + signoff: {name: tzhouam, email: tzhouam@connect.ust.hk} + +# Knowledge service (doc/features/knowledge-service.md). Human-only section: +# switching `mode` to auto_merge is a reviewed code change, never an agent write. +knowledge_lifecycle: + enabled: true + mode: shadow # shadow | auto_merge + knowledge_dir: repos/vllm-omni + intake: + merged_prs: true + copilot_runs: true + human_prs: true + daily_model_budget_usd: 50 + release: + trigger: github_release + auditor: release_audit.py # adapters/vllm_omni/release_audit.py + sweep: + fallback_interval_days: 30 + upstream_visibility: public + circuit_breaker: + retire_ratio: 0.10 + max_files: 50 + protected_rules: [] + calibration_set: kb-calibration # adapters/vllm_omni/kb-calibration/cases/*.json + model_dir: repos/vllm-omni/models + +# Unit-test coverage candidates for new public functions (#164). Sources are +# the package; tests live under tests/. Entry points and generated/registry +# files are skipped: their coverage is end-to-end by design. +ut_coverage: + source_roots: [vllm_omni/] + test_globs: ["tests/*", "*/test_*.py", "*/conftest.py"] + exclude: ["*/__init__.py", "vllm_omni/entrypoints/cli/*", "vllm_omni/version.py"] diff --git a/test/fixtures/remote_ci_owner/conftest.py b/test/fixtures/remote_ci_owner/conftest.py new file mode 100644 index 000000000..1f18555f8 --- /dev/null +++ b/test/fixtures/remote_ci_owner/conftest.py @@ -0,0 +1 @@ +collect_ignore = ["repo"] diff --git a/test/fixtures/remote_ci_owner/expected106_routes.json b/test/fixtures/remote_ci_owner/expected106_routes.json new file mode 100644 index 000000000..5fb490062 --- /dev/null +++ b/test/fixtures/remote_ci_owner/expected106_routes.json @@ -0,0 +1,1207 @@ +{ + "target_branch": "releases/v0.31.0", + "target_commit": "f42629247d0efcd4f7fd9d0a1cf6fb2060909fc9", + "active_modules": [ + "worker_runner", + "model_executor", + "input_output", + "scheduler", + "online_serving", + "model_config", + "platform", + "benchmarks" + ], + "missing_leaf_ids": [ + "ready:024", + "nightly:005", + "nightly:006", + "nightly:008", + "nightly:009", + "nightly:010", + "nightly:011", + "nightly:015", + "nightly:016", + "nightly:022", + "nightly:023", + "nightly:024", + "nightly:025", + "nightly:026", + "nightly:036", + "nightly:037", + "nightly:038", + "nightly:039", + "nightly:040", + "nightly:041", + "nightly:042", + "nightly:043" + ], + "leaves": [ + { + "expected_leaf_id": "ready:000", + "source": "ready", + "name_template": "Simple · Model Executor Test", + "original_command_sha256": "79faec301b62a087e70e54f77cfaf5a13d6da781aaa10542e740d9203c828694", + "rendered_command_sha256": "5998b08b0e8f0e52030459b62202a8ea48dd502ec7c388babaa7bbe31804cb99", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:001", + "source": "ready", + "name_template": "Simple · CosyVoice3 Packed Flow Test", + "original_command_sha256": "035324b722c07b60c9d9bc5f65fafe068cb591fa5b6a5e1c2c768cd41b6d0a61", + "rendered_command_sha256": "1d14724ebd1a81c62c65bced73767aea983d15286d7ba5d49451d18c99d5bc6f", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:002", + "source": "ready", + "name_template": "Simple · Diffusion Test", + "original_command_sha256": "b02536970c506eef713c98f179aba982e22ff989b8463a18e69983d948383394", + "rendered_command_sha256": "8f3fac6a49ca302ac74efef38753a62f0ed7ce81cc6fdb704de70e949e677312", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:003", + "source": "ready", + "name_template": "Simple · Engine&Entrypoints Test", + "original_command_sha256": "9dd19813764f64ccef3d765d952208526cb383c0f206b3dc374545ccdba49dc8", + "rendered_command_sha256": "a5edd03570c8aeb7db41b68db220ae0d95b27ab8ab7145627ef0ea744765d3a6", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:004", + "source": "ready", + "name_template": "Simple · Other Test", + "original_command_sha256": "697c03294f78a15ef3052a8134c8af79c8fbcd18a4c2d00e8c00f7181d986287", + "rendered_command_sha256": "852f685907066e17cad84fb1f986512d08e7eadfc7c1dfa9954dd3b7efcbe847", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:005", + "source": "ready", + "name_template": "Diffusion · FlashAttention Test", + "original_command_sha256": "01fd2da0e0e2a815de293aa4fa8dba2bede43140a8e1d57c76e2bca3cc205f70", + "rendered_command_sha256": "c6c783074eb4aeccbe839bb158ecde33cdfd6f33dde52c3fd7d35910a2f12068", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:006", + "source": "ready", + "name_template": "Diffusion · Offloader Test", + "original_command_sha256": "74377c5598432b187c417a46136929fd4a6cf66c94b5f82fb56683afe7ee0300", + "rendered_command_sha256": "bb3745110c3750b9eeac5d76b54162392b000478f4833fb64085a2c401287b31", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:007", + "source": "ready", + "name_template": "Diffusion · Cache Test", + "original_command_sha256": "e2681bb8c99f8c3e00805c6c15f2087682465300c19d0ed47eed3ff58dc2576a", + "rendered_command_sha256": "932f93774830b6a3adaea1b142d11025b448a28c0c056a7c27fee41ad04d5025", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:008", + "source": "ready", + "name_template": "Diffusion · Batch Test", + "original_command_sha256": "4f70051edea538dd06c6d76342dea5e862841785a9abaa2cecc4e7fd9b3a063d", + "rendered_command_sha256": "23fedfea361608713073e88b7a3a1843f5ffcef2a58e736c0fd8be2bd053edaf", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:009", + "source": "ready", + "name_template": "Diffusion · Distributed Test · Single-GPU", + "original_command_sha256": "01c779e2a7c278c92de8dbaf1dbb5a60528d09e8332e4fc66066963970daba92", + "rendered_command_sha256": "0d7d68951ad30532c5c5db7c61b92e312089b6d77ef3b0998e9594fcb479b60e", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:010", + "source": "ready", + "name_template": "Diffusion · Distributed Test · 2-GPU", + "original_command_sha256": "5dd031fc9f97c6bd3206cbb97285c64a94710c983b7a72258e7a4e407d135936", + "rendered_command_sha256": "b8e3edad4f787272cb42f899d3ce93b03ee8ad68a734d318eba15325c6c0f68f", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:011", + "source": "ready", + "name_template": "Diffusion · Distributed & Attention Test · 4-GPU", + "original_command_sha256": "87e3cc98fe7d43d5b3e9d371ea4f2ed3451a734f5279ac3680888fa5472ad988", + "rendered_command_sha256": "d15ab88ed56453d76ce6d8f800c15310584ccc50841bec4e4a514857947434c4", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:012", + "source": "ready", + "name_template": "Diffusion · Model Test", + "original_command_sha256": "b2428ab3584dee774c302ff7f27191cf0bf3cae6bd336cabb27f1d07b935187a", + "rendered_command_sha256": "847bb59ef3bde5928e71c2c14cf887ad05521b2a1e07a884d21836a4271f6c09", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:013", + "source": "ready", + "name_template": "Diffusion · Other Test", + "original_command_sha256": "40f8372ae22bfcf67992ddbc99a988ae337d618d61f25ea4f2331b7a4995423c", + "rendered_command_sha256": "fe640e2069de8fd1924ad3638a3835ab6604271a28ae18a6e4904f36fbfaa174", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:014", + "source": "ready", + "name_template": "Tiny Model Tests · Diffusion (base)", + "original_command_sha256": "27e463994eaaaa33ac0762d9479e301309eb2b6712f886ad4459b8315b02f39c", + "rendered_command_sha256": "8876ed3f5eb4e3fb1244f1c9db717a141d96f48c0ed95c9b72c76659d677c5d6", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:015", + "source": "ready", + "name_template": "Engine&Model Executor Test", + "original_command_sha256": "83ab5648ecc9d8cfcbaeadfe224e23170f8510bbfe7417bbf54cf64bd5942f9c", + "rendered_command_sha256": "a55916712a15cc09b4b7e290304a6f039752f8b7245bfac65b603898cf54d107", + "parent_dependencies": [], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:016", + "source": "ready", + "name_template": "Distributed Test", + "original_command_sha256": "947b586a66bb3cb07b42116d33d2b5ce8ed6c1eeef0e6fb8bd3cf1c1aa25d904", + "rendered_command_sha256": "6e38518a4b6408bae217e072a591883be790613f39d8378863b611536440fa21", + "parent_dependencies": [], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:017", + "source": "ready", + "name_template": "Entrypoints Test", + "original_command_sha256": "5973d6295aed0b46852f73635215e1198a3b455a15596a3d62b4af3bcf9342a7", + "rendered_command_sha256": "7fa60e23748f3182f7dcd554ea7d955194afe98e88503d3c7cc7e78fa13cb39f", + "parent_dependencies": [], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:018", + "source": "ready", + "name_template": "Custom Pipeline Test", + "original_command_sha256": "f8c1e3d6741dbae4244cdb510837a68161d9c19936e32ba091a76ba454b2d759", + "rendered_command_sha256": "2c2f7dfd2801df17ab6682fb4904132e3d640b441ecc0f84867beea6b8f4a2da", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:019", + "source": "ready", + "name_template": "RLHF · VeRL-Omni E2E Test", + "original_command_sha256": "48a2319e6709b0c6fff53509932ccfa0371316f6170594d889d2eb410d0e5961", + "rendered_command_sha256": "2a07fa11c8f46d269c4be712eda5d205b5ae2ed0cae6475d7e5d7a8cdd123bb7", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:020", + "source": "ready", + "name_template": "Omni · Qwen3-Omni Test", + "original_command_sha256": "b01c8b488039ffb55305fdd18f49d80ae66e73749689c1a1071a3481bf534e4e", + "rendered_command_sha256": "e89b6c7c8c4f9d8ac258faa6b7d34fb5904b1ed39e31abafed0c2dbd3078ac15", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:021", + "source": "ready", + "name_template": "Omni · MiniCPM-o 4.5 Offline Test", + "original_command_sha256": "4d16271f187984493a2d08b082539805b6a5279dbc3eef9ce602ffb62df9b05e", + "rendered_command_sha256": "20deba619a908c54aa12c950566381c9015cb0f8955781559a21283de114901c", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:022", + "source": "ready", + "name_template": "Omni · MiniCPM-o 4.5 Online Test", + "original_command_sha256": "2327729368ad440927462e0303261bcf3bc7732b4a8dcdb8c96a3be5256d0251", + "rendered_command_sha256": "1ab4a6cfa741c5e0e271b0beb7f2ac0de4007cb308111bbe2258b31070605f48", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:023", + "source": "ready", + "name_template": "Omni · MiniCPM-o 4.5 Duplex Test", + "original_command_sha256": "4816bf279e52db4d42d628efee229908ffa16ba18f147fe64b419c25864bca88", + "rendered_command_sha256": "1ed1178c4621bc917d089de5e25ebf76b9f1cef8599be89e62f78fddf769b4bf", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:024", + "source": "ready", + "name_template": "Omni · MiniCPM-o 4.5 Perf Test", + "original_command_sha256": "a887839b14ea0c77f8ace593a8bdbc4eee5cd5db23bff57de3b01c4ad5465354", + "rendered_command_sha256": "57f54ced3b19bbb3a522964cdaafdb3a96716e0df134f5f39cbcce405a306291", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:025", + "source": "ready", + "name_template": "TTS · Higgs v3 Full Response Test", + "original_command_sha256": "405b4e26663e9be5a3707a310c9ee5a67ccffbb7b6d45f62cfc963fb9fe82357", + "rendered_command_sha256": "a35b4c17134709a00253a754c7adde22f7c1c1c978ff71b664f76c20b4153143", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:026", + "source": "ready", + "name_template": "TTS · Qwen3-TTS CustomVoice Test", + "original_command_sha256": "6aaef03a87e621a48841f8c8b317625d458fdd0854866893b41f14868488d862", + "rendered_command_sha256": "c4a5c9b607c1bf8eb63f843bfcb6a504941a40c0cd5572b4f74af9a6e49a0362", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:027", + "source": "ready", + "name_template": "Diffusion · Wan22 Test", + "original_command_sha256": "73662fcf7acc32ca0da6a936470827966231cc7b2a10dbd21f548d8049ede7cf", + "rendered_command_sha256": "c42f98b636b4b38bdf24d1e6de0bc3780ffb4cf3c02d5adf3f041e73e934d7ba", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:028", + "source": "ready", + "name_template": "Diffusion · Cosmos3 Test", + "original_command_sha256": "d2860a3e3eed5e20d5174f093142fa5f4b5b152d29b36971c71fe542c7f7912d", + "rendered_command_sha256": "7b232701e7e8ab1d0f4b9e19a55410d5534b6ed1b8c10475f4dd50cb16d37358", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:029", + "source": "ready", + "name_template": "Diffusion · MiniMax-H3 DLO DP2 T2VA Test", + "original_command_sha256": "a090fd8bb6f475a1e1b0145caec10c0653b4a5e32d7b089c9aa1cbdf26ac57c0", + "rendered_command_sha256": "ffea7c82d1f8a1093f5578013c976841f8ba43cc6c1f3d250664d1cf1018a235", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:030", + "source": "ready", + "name_template": "Diffusion · Qwen Image Test", + "original_command_sha256": "654f25291cc36e9f5795f70c779ee76fe751f5dd925c9125fe5fbf5104ef0457", + "rendered_command_sha256": "501a11908d2c8e6ab6a937566bb9a1b94df69228478a30ce9b85c0341b9dd36a", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:031", + "source": "ready", + "name_template": "Diffusion · HunyuanVideo-1.5 Test", + "original_command_sha256": "4bd8099d0be72ee8ca20082f97736fcbe50b08e589338013de608337d68166c0", + "rendered_command_sha256": "4f788cb7f11f96ac04e4da768c402febdb51c0940c54a77d84cb80b0cd2922b4", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "ready:032", + "source": "ready", + "name_template": "Diffusion · LingBot-Video Test", + "original_command_sha256": "f38ed5a4897d346eb7656b25cc378c28bc602798ab7912e086df5cc27b98fe2c", + "rendered_command_sha256": "9ae60d69c0298d2fa0626c4b73e9a44cf49a826d4cc0a352a56d35408bc80fe5", + "parent_dependencies": [ + "upload-ready-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:000", + "source": "merge", + "name_template": "Simple · Model Executor Test", + "original_command_sha256": "79faec301b62a087e70e54f77cfaf5a13d6da781aaa10542e740d9203c828694", + "rendered_command_sha256": "5998b08b0e8f0e52030459b62202a8ea48dd502ec7c388babaa7bbe31804cb99", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:001", + "source": "merge", + "name_template": "Simple · Diffusion Test", + "original_command_sha256": "b02536970c506eef713c98f179aba982e22ff989b8463a18e69983d948383394", + "rendered_command_sha256": "8f3fac6a49ca302ac74efef38753a62f0ed7ce81cc6fdb704de70e949e677312", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:002", + "source": "merge", + "name_template": "Simple · Engine&Entrypoints Test", + "original_command_sha256": "9dd19813764f64ccef3d765d952208526cb383c0f206b3dc374545ccdba49dc8", + "rendered_command_sha256": "a5edd03570c8aeb7db41b68db220ae0d95b27ab8ab7145627ef0ea744765d3a6", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:003", + "source": "merge", + "name_template": "Simple · Other Test", + "original_command_sha256": "697c03294f78a15ef3052a8134c8af79c8fbcd18a4c2d00e8c00f7181d986287", + "rendered_command_sha256": "852f685907066e17cad84fb1f986512d08e7eadfc7c1dfa9954dd3b7efcbe847", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:004", + "source": "merge", + "name_template": "Diffusion · LoRA Test", + "original_command_sha256": "b05e923b42847aadb01898fe0421ec49bcfb77744102bc1e18a8a357fbed43bc", + "rendered_command_sha256": "d28893f0b4864ce8f67e2dc4b74486fd32d2029ba6b6e096af96ec9785843f85", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:005", + "source": "merge", + "name_template": "Distributed Test", + "original_command_sha256": "27ab1bd1f2676c4d4d573980891fa70372a077a0f3e063d853b82ff3eb463591", + "rendered_command_sha256": "06f10b908174fc8e70e7bcf9c8812168964e70652bf1f36a98e4a734aff29247", + "parent_dependencies": [], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:006", + "source": "merge", + "name_template": "Entrypoint Test with H100", + "original_command_sha256": "1074ac8cf5c0a7efd93e1dce86039fdee56431cadf546a4a868ff0b528130ea5", + "rendered_command_sha256": "4f86636e0086e86b321f60e72ece8db9d0436093867e5b6e3d1b0b2e0f23cf63", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:007", + "source": "merge", + "name_template": "Entrypoint Test with L4", + "original_command_sha256": "3f39c1035f8ef278a295babd67244c0c32e1889aae07d62691d0aa30aae3c819", + "rendered_command_sha256": "c873271e7bdd6eaf4793987c47522bbd6f579ff9d3f6727b5348e39f78c04220", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:008", + "source": "merge", + "name_template": "Entrypoint Test with L4 · 2-GPU", + "original_command_sha256": "ccb17eafcb7332af99f141bf69210d1352641a464eeca6e40ccac58e794294ad", + "rendered_command_sha256": "0335fc81b8ae7e530a43e88f763ae31e9f988349aeeaf1d0502910cdfd9f95c7", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:009", + "source": "merge", + "name_template": "TTS · Qwen3-TTS CustomVoice Test", + "original_command_sha256": "c5109d2c909ed3936ad6b1a23afc45444d664a571d2a19d7ebd2b0d0d8e6dfc0", + "rendered_command_sha256": "9572bbd61031b8929447cee013ceb3510ba922de4e4d726b207c6bf1aa3fc98e", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:010", + "source": "merge", + "name_template": "TTS · Qwen3-TTS Base Test", + "original_command_sha256": "cbb1d42f72631c106d941e7a0e1864baa4169a6746bd2af44ed99454160422a0", + "rendered_command_sha256": "63ac8193b278140c9e4811587909f86733f6cd7b68d8d5055462789e10147bdb", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:011", + "source": "merge", + "name_template": "Omni · Qwen3-Omni Test", + "original_command_sha256": "36ce9d924d7a40cbf4e51170e9bfce40d8662aa6312d15aea2fe2174ab479bd4", + "rendered_command_sha256": "bf157f9268da5878db4417a51396fedf0976438b786222e53d5bb50d13a3eb0d", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:012", + "source": "merge", + "name_template": "Omni · MiniCPM-o 4.5 Test", + "original_command_sha256": "0fc80ab5d86065d12bdf95804b4c92d067b7e1bbd9239f7edace589fe14543be", + "rendered_command_sha256": "6d7a91420b701ecac3b5bacbaf0c03d0b2b59f9ff69a42ae9a78c096311b5e4d", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:013", + "source": "merge", + "name_template": "Omni · MiniCPM-o 4.5 Duplex Test", + "original_command_sha256": "97e413cfdc6f301f5c39ddbd01cf9f9b79a9c959a12ed7daf2a862b011d9395d", + "rendered_command_sha256": "2b3209e48003d9fc6bc66c26e08f07b4a1bdbc7745a167d3e191452a253c3f80", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:014", + "source": "merge", + "name_template": "Diffusion · Qwen Image Test", + "original_command_sha256": "35ea1a0c17f71d0f7d07dc7921d959e67405e3e9b1c28893d3988a4928208414", + "rendered_command_sha256": "2510f38a8aa7f39fd1fee33c4c2074caf2a7a18c86b62136a566e2300460cc2c", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:015", + "source": "merge", + "name_template": "Diffusion · Wan22 Test", + "original_command_sha256": "f8193cfaae1f84d376fb395ec5c60d8126f92c5746dc8a91f9b8f176b7727f01", + "rendered_command_sha256": "223ca11e8b70873f8ec9af56abfd1e79bcd085a6b630d55fd48dcf9b8a473cfa", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:016", + "source": "merge", + "name_template": "Diffusion · HunyuanVideo-1.5 Test", + "original_command_sha256": "5278ed4c44675722ccacec3682f1f81fc4d70afdc38c6a392d5583efb6ad58db", + "rendered_command_sha256": "91d0e50591bf17b1e7471e3ce12738fa19d7badda6fc075f1076ed125a2d06d5", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:017", + "source": "merge", + "name_template": "Diffusion · LingBot-Video Test", + "original_command_sha256": "e026e14d68d150188fe01080ca28d95445c3fdd3e8a74419cc45c45618260c11", + "rendered_command_sha256": "cc8763c7cc37cecbf5056d27fa45b9c4d904725ae9bad24a6af3fae69b5718c1", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:018", + "source": "merge", + "name_template": "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3", + "original_command_sha256": "4a5e7ce1eda59300946ea2491c09470ef6cdc12ca62636f7bf112047253cc647", + "rendered_command_sha256": "72927324f691b687fbc89e8154f9c79f7974265c46a5edb636f25a62cc593b1e", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "merge:019", + "source": "merge", + "name_template": "Diffusion · MiniMax-H3 Latent Upscale and Refine", + "original_command_sha256": "52cbabdb7cd434aa8e1d7f49618d0ed0ea5ba32a1a4b31f8939a8e3ae1db90ec", + "rendered_command_sha256": "e21ee8c56d70c6b1fb83ffba828e51301c6f963e305ea8dea7b970b023c34895", + "parent_dependencies": [ + "upload-merge-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:000", + "source": "nightly", + "name_template": ":full_moon: Omni · Function Test with H100 · Single-GPU", + "original_command_sha256": "6e9f497ffc63756ccdff00623a53af1b3bb07ce2c354112143bac2ce9a499e6d", + "rendered_command_sha256": "9ad0b6bd60b4fe816be5a91bb64f78c785fe5ee1781243725ce60bdbd9075d0f", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:001", + "source": "nightly", + "name_template": ":full_moon: Omni · Function Test with H100 · 2-GPU", + "original_command_sha256": "12d9341d4b1fc20a431de5a31a172503cde8c3e080b20b9ea8bdac31913b33bc", + "rendered_command_sha256": "e7fdc619160947454061e40299c7c18541675f8c5f2b271dc20b314615c66d45", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:002", + "source": "nightly", + "name_template": ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test", + "original_command_sha256": "f1234049eb2c5a567ec3565618f52ff9fb0272171977d1790eb65a18722e7981", + "rendered_command_sha256": "a68b1aab0ed1db2abc309a056a26aacc3986d26907260fc86ea0cd7fa956e5c4", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:003", + "source": "nightly", + "name_template": ":full_moon: Omni · Doc Test with L4 · 4-GPU", + "original_command_sha256": "4b11ce65aefc0d8fd30d5be0c6d8383894cfafdb8146944bd8b05cc1bae61ba6", + "rendered_command_sha256": "32340770f96e4fe3d2ed8e213a1bf9615247a075a857fcf92776966b9ad2cb5c", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:004", + "source": "nightly", + "name_template": ":full_moon: Omni · Doc Test with H100 · 2-GPU", + "original_command_sha256": "ae7dad6cec575d0d2408f0689f6db9a21e7e03b59e7560df76ae35640b2b6a45", + "rendered_command_sha256": "1ef16c4ac893f4d156f12db41af2dcb9ca66989e14b5f02921ac569d84278167", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:005", + "source": "nightly", + "name_template": ":full_moon: Omni · Accuracy Test", + "original_command_sha256": "5d42362224b19e74b923da3fea32c0b8203ccb12708933cbb258426697985fb7", + "rendered_command_sha256": "4c505c5a1c567ee57d1128e640553efa61d2bf04d91656cb553738cc22974ebf", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:006", + "source": "nightly", + "name_template": ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test", + "original_command_sha256": "178b56499595f3af8884ccb54538a5d3b335699906be8d571204d379aaf85fab", + "rendered_command_sha256": "301d792f551690b3dd17a978b2774bbd4d33b82dd8619a71abaf82d3ce3eb644", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:007", + "source": "nightly", + "name_template": ":full_moon: Omni · Omni-DuplexEval CI Guard", + "original_command_sha256": "22fa36754cf0fb0980f06d51a27a6260621b9f1c374b03f7ca8806d4d7489cf3", + "rendered_command_sha256": "3e002e7c5db537f01f7d66ee1921510f296f099a9b26bd6cdfd868d4165ead11", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:008", + "source": "nightly", + "name_template": ":full_moon: Omni · Perf Test · No Async Chunk", + "original_command_sha256": "ba3eb175c806616f72b1bd343fbb2ab0e886d7f8ec77b2f1621c1eac58d7f8a2", + "rendered_command_sha256": "ad012394a329b39674569c1ec5e733dd17d69c4ecc82c091741bf2729bad0430", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:009", + "source": "nightly", + "name_template": ":full_moon: Omni · Perf Test · Async Chunk", + "original_command_sha256": "4e142f7227f2ec8f49e2c076fe4abf7b3e4684367ffb43bb48ac4cb8dd2a31c9", + "rendered_command_sha256": "87f305c8d0e4de9956618b9bffd2ceabeca799cea6acf5c8536f790d0d038ae1", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:010", + "source": "nightly", + "name_template": ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test", + "original_command_sha256": "4a46fe6f4cc260ce212218a2a9d3b386c0c16fdd7abd7a1e551b48720d065a2a", + "rendered_command_sha256": "d0ae0fe1883a8728d00086f4e5a954fba74d067b89bdbb11157655d6ed873c0a", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:011", + "source": "nightly", + "name_template": ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test", + "original_command_sha256": "bacd1de059f3fd7a18571d0bd6fae71173080acea332a9c706c6614b2226b15c", + "rendered_command_sha256": "30123a181b309adb785fd022ebb4b4daead4b436ef932183fd779f2c3937f4fe", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:012", + "source": "nightly", + "name_template": ":full_moon: Omni · Multi-Replica Startup Test with 4x H100", + "original_command_sha256": "4a5642b81ac5ebc11fe2fd7855b9ea87bc5febca418637cc1ea64b1e99e97a85", + "rendered_command_sha256": "562fd704dd4dafa0658e99ed99f084f97945263dd582b9be861164b0155bd521", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:013", + "source": "nightly", + "name_template": ":full_moon: Entrypoint Test with H100", + "original_command_sha256": "8cd65b1c0609c5e044e1a9cda4a44dc05cc62b86175a6bf0877992830c359d38", + "rendered_command_sha256": "d8109f61e7c7464eb4723f632db843100fed2d745b31c259bd6e962a2d696c1e", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:014", + "source": "nightly", + "name_template": ":full_moon: TTS · Function Test with L4", + "original_command_sha256": "9916565ac09f042ecccbd236856e82d556b8007b1c2aa603238ddac1af007087", + "rendered_command_sha256": "d2aacbbfb38972e79a525557a2ed15384f0e8406149b61519ee3edc0e7dfaa7d", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:015", + "source": "nightly", + "name_template": ":full_moon: TTS · Perf Test · Single-GPU", + "original_command_sha256": "f02abd3e31374257d05053ea122358f6a40963ca95512604d9be6a885194ce6d", + "rendered_command_sha256": "8e74932ec0eef74283035c3ba2585ba3afb6113b9aacb92c12ae2cabd6b52395", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:016", + "source": "nightly", + "name_template": ":full_moon: TTS · Perf Test · 2-GPU", + "original_command_sha256": "5a57bee8ac2ea890c12063bd77405f940b54a371877af0d9c98323789960fb31", + "rendered_command_sha256": "33762cdd06825ddc77ce9ce3f9f0b95dd15e6afdd736564fb477add225e2090b", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:017", + "source": "nightly", + "name_template": ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU", + "original_command_sha256": "a5f299b5628ff7e9c9dde94f138e4d4b28e9683c530809b7b4bce5f45fa73f77", + "rendered_command_sha256": "d5e90d7716ca85505968637472f53f4d49ed63388bd33517f949bc1a50a1be66", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:018", + "source": "nightly", + "name_template": ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU", + "original_command_sha256": "790ff6be8d11dccbdc197d98d09e1ae75fd6be5f3d13ce5da7ad3e84049c4ec1", + "rendered_command_sha256": "3aaba4cfe6c078f8b76b4be94e23509c6e38141e843bf58b16354685d16deda4", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:019", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU", + "original_command_sha256": "20c70cbc9384e224bf5d0b05e9ca60b0432319dd719e3ed1fd8d8a1fcc72ce95", + "rendered_command_sha256": "5e9ef1133e3d50bd6871d6ed7c0b309d1a9010002e2a738b9553dd3d6d15d7b0", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:020", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU", + "original_command_sha256": "9513ee60be1311801f8dd5018e6b723bddad1daf3d1fe79e52553338349aa21a", + "rendered_command_sha256": "537b962e7a0876c868ae6df5ff48a80891fa47b4a3c75b6ae5ff05fe35391f5a", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:021", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Single-GPU", + "original_command_sha256": "8d11d2e19e3e04386838d79a3292b35a5ae82bda8ba7aa00d883d07b01e31230", + "rendered_command_sha256": "f0c0bd2156e5c34b8e6511516f212f46fc00a3713f99835bbf04f441306dc453", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:022", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2I(&A&T) · Doc Test", + "original_command_sha256": "c3cc39c8187259ab40846e77fa211b3a557c4888e6163de6a0c18593f184f2b1", + "rendered_command_sha256": "b543dd67ed1d36708702f09803db604ee38759ef9eb0f5a04572a078accdc0df", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:023", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2I(&A&T) · Accuracy Test", + "original_command_sha256": "93698921abd81c9ce689e041a05cb61e37bd97c6a1912d3b2bcb412a751e550d", + "rendered_command_sha256": "08ac73b3ee8912a89d62ae1c365dadcb26ec9ae81813187c7240819558db3c5e", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:024", + "source": "nightly", + "name_template": ":full_moon: HunyuanImage3-DIT · Accuracy Test", + "original_command_sha256": "078dcea1a5d189ef31abd356bb22465c8fba4c133dcdc698527723e2ce801c18", + "rendered_command_sha256": "f3816500f04a1f2fae268fdde8d12630a54bac849a00e525884e0f26f0317d0e", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:025", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU", + "original_command_sha256": "aca5e654e69063d2850b05282c65a72f0cf56f6bcd1b9b48531a911bff710889", + "rendered_command_sha256": "e84e9c4ca477ecf227d12d337519916b0d22bcf6869042f2a045a824aa818521", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:026", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU", + "original_command_sha256": "3575b58c519207a70185c19b3ae75a117ac70ec718ba80ff8225915d7eb3b75e", + "rendered_command_sha256": "3e0c6a9a2e6d3c014b6c04739cdcf8da69b0ded67008bb01de8def933710b306", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:027", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU", + "original_command_sha256": "292f1b8a8137dc7a5ffe00b5aa209260354d5205b16dd04dce1754e002b2307d", + "rendered_command_sha256": "6be14d3d6312eb4f92f4a3f2afd767998c64d2b142cc19d203bb2fe43e5b778a", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:028", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU", + "original_command_sha256": "c1a1352de99b8c3b1f94ab176a7e8ae89c57210082d3c7d71d7c39543f4d5489", + "rendered_command_sha256": "67d36cc83e11cf571a131fee4f34c00867c79e5fc03e7619c8197c11379ba936", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:029", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU", + "original_command_sha256": "a60cbced1da1d071378f7968856e0169bf5c78199042c25f09b4e96c01e130fd", + "rendered_command_sha256": "bbb1c184de66d849623544e0386d10a129d24b745ebae24d3e20e1f07b644979", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:030", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU", + "original_command_sha256": "3f6772d8697b0b0028e6b41d3d8740817ba078a0c11ad27d61a44da04e3b242f", + "rendered_command_sha256": "0636f469957d568aeb9b818c03cee3c9353c2b9a5238ebe74e9988bd0eb44569", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:031", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test", + "original_command_sha256": "28ef7151f133a81cff5bda97617e5030af9236a5f335a45803bc0805d335d20d", + "rendered_command_sha256": "2aa2d2553b1971daa22fb0db5cc5e4f61c27f78fd0a1687d7c32ac7d2581d4ca", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:032", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · LingBot Function Test", + "original_command_sha256": "5571f0874c8d25171963d35d57a4c2e4b4a501fe0a367bddfb314cb93a8d43e7", + "rendered_command_sha256": "961fdb6b3caa2bd1ba59a0b6aa12c3e865071d150ece0ea71c0894eb83b2fc57", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:033", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Doc Test", + "original_command_sha256": "e4047bcb51809021bf5e72a95c2be712f0c95acafad928b09b92992cafc5a912", + "rendered_command_sha256": "ebf2dc82cadf9d6631cab86eb10e91628eeb5a06729124bc531fa1a38e25c914", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:034", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Wan2.2 Autoround Function Test", + "original_command_sha256": "84709626ea3751563251f988ea8195383c8234376269777a81978d2a3ff7156a", + "rendered_command_sha256": "a1852c757f87531bf5f405ec3e96a921ff3f67ca6731c441fa3f9cb7b6db166f", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:035", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Wan2.2 Device-Postprocess Equivalence Test", + "original_command_sha256": "3d3616ab502b9bea0856f88348457e047ff1237a9a951e4ea5a0c261dd71866f", + "rendered_command_sha256": "30b0bc30cf89969c8b959a2c7d48b5acb6cdb17e67cf92fb870721329ec6764b", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:036", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test", + "original_command_sha256": "041a2ec1385238acd9766a77f2784055d8cd08dc88865c3f43581924ad09ac4e", + "rendered_command_sha256": "fce83d98e23d3b0bd0dc007f5aa0baa1e9e5835ff4f2008c2c8b49557cd14223", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:037", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test", + "original_command_sha256": "2300c429a866177d2b8f714bd6b3a02a1f63a0c0fb36ac33eee38b24c65213ab", + "rendered_command_sha256": "a4045d043c401269a6048a3d728b66e3f39eea8ea613a4913d8aab0b699f6b7d", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:038", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Accuracy Test", + "original_command_sha256": "55bb50ededa0fe3d3b18b1b666f7301de6aa4470b6eeb060fbc4545fb1bfc6f4", + "rendered_command_sha256": "ec0bec7c17869e2853ba6472206777b9f6f6b62ab12e3a5370f6e29fe392a62c", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:039", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Perf Test · Single-GPU", + "original_command_sha256": "92768159d84f80b87230f4dc745c58c7499e5d384200bef40bf0e6fa1c3a8e91", + "rendered_command_sha256": "433aa72c2dd5209d9e3bced6ffa87c46680af9977246f59c585c504f1b798780", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:040", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Perf Test · 2-GPU", + "original_command_sha256": "ee144208d545a2b5b8fe5b88e179f1e876ae46e0d21ff425b9c92a5e12ace7b2", + "rendered_command_sha256": "d06273df766860d29638dd6e8769d079e31a46447bf0876a004c16d8e0e5661c", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:041", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5", + "original_command_sha256": "1aa53dacc60c6fde30f32cbc3061eda1476c4880d91ac3e1495c87f4e7e7cb9f", + "rendered_command_sha256": "f2674652f4020f06fa46cbd3e91d81824f87a528e6dd58fc11b6e91b6ac2e5b5", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:042", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3", + "original_command_sha256": "2d59a88ba93fdc830a90f754f05628e83c02794f5cd7594ef5934accb84058ea", + "rendered_command_sha256": "ae72ee413d638033a7e9ff9174df34bb292de665a33616a92f45ead076957e72", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:043", + "source": "nightly", + "name_template": ":full_moon: Diffusion X2V · Perf Test · LingBot", + "original_command_sha256": "f5a4dbee2e37a1161061fdbf61ad9b69a3f10e1dba3177953936f795605eb1a4", + "rendered_command_sha256": "22e812fbe76137db7e48e3e93de48b9c1f313d345618ff0eef94c8ef6b4dd82a", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:044", + "source": "nightly", + "name_template": ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU", + "original_command_sha256": "133fa34aa808adf95157a9bb7706075bd2ffa0ee8199a1131e39339e8881ab07", + "rendered_command_sha256": "96bae51b1ab0afe75828cfa237836c711bf4b27b3482d869928c5d1c4624bc86", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:045", + "source": "nightly", + "name_template": ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU", + "original_command_sha256": "cffc92f84a03dae305da96156fe0884dfce44a2e2b2b8b4f6f289bbdf3dcaccd", + "rendered_command_sha256": "28d1cd08dc22b8f2bc68026b0733bae6a3384ae8dc053762df6a1f3d96ea475c", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:046", + "source": "nightly", + "name_template": ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU", + "original_command_sha256": "c44890b440542cb478445eaf2129f9a982cff77129c8cc5dd3953abe3e6360cc", + "rendered_command_sha256": "d0d23f6bea13af24c8924f7080abfa5e470bbb32d16b633b077daff5a4577173", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:047", + "source": "nightly", + "name_template": ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU", + "original_command_sha256": "53f3489c9b24b7e078addb0782368cd9341b3aa0943dbe3374ef489ae6ebd039", + "rendered_command_sha256": "ac31a565dd52f3ceeda0457a2a55714d9abe389e4a4737454ae9755a1863db18", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:048", + "source": "nightly", + "name_template": ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU", + "original_command_sha256": "713b205389bc55b67d592e3fbb3d5f614dfa73867b0c1aa1bca3a82b9b40404a", + "rendered_command_sha256": "27186112630c6e9bf9e93de08f46ff81e5c4d38b8c6579091efb569914818e7e", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:049", + "source": "nightly", + "name_template": ":full_moon: Diffusion · Offloader Test", + "original_command_sha256": "6ac3fde8a922dd4e8fd36c28fb3654b76ef8bc9425ffee2874fbc8d88725aa08", + "rendered_command_sha256": "b8618e5f2aed7c2caf5fc07f691e57247b559ee0c65ac7f2730f57b8abf9c906", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:050", + "source": "nightly", + "name_template": ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU", + "original_command_sha256": "4818fd901c7c8136e742da594ae9be4c2b60a598e3e4ae80d382dd5f0668b5ff", + "rendered_command_sha256": "e3012f07e54bfcff619edaf41153149c3a4f71cf1a00d8ca81dc5749da9e981d", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:051", + "source": "nightly", + "name_template": ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU", + "original_command_sha256": "7f2ab66d7558eb07f946198d310d33da48096037ed4d673e4ac9e62f381c0685", + "rendered_command_sha256": "fa71789b3d8b7a1674b3b180c2b3184dff4c97170074e0fde4663dbd4cf7f1ad", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:052", + "source": "nightly", + "name_template": ":full_moon: Diffusion · Quantization Test with L4", + "original_command_sha256": "248656fc3042194306104d894f4a3e0508fa7909c992d7b22dcd804be9c3cbde", + "rendered_command_sha256": "e17851caa4c24669a06071eb448a6208e4db35e66776ff016cd8ecb3101964d3", + "parent_dependencies": [ + "upload-nightly-pipeline" + ], + "expected_enabled_under_campaign_env": true + }, + { + "expected_leaf_id": "nightly:053", + "source": "nightly", + "name_template": ":email: Nightly Collection & Email", + "original_command_sha256": "f9ea18a97d78ddc0881b1e034ed8ac3ac9dc688a00b8aeb39afc5d4219d7293b", + "rendered_command_sha256": "f9ea18a97d78ddc0881b1e034ed8ac3ac9dc688a00b8aeb39afc5d4219d7293b", + "parent_dependencies": [], + "expected_enabled_under_campaign_env": false + } + ] +} diff --git a/test/fixtures/remote_ci_owner/native73_manifest.json b/test/fixtures/remote_ci_owner/native73_manifest.json new file mode 100644 index 000000000..0796e5348 --- /dev/null +++ b/test/fixtures/remote_ci_owner/native73_manifest.json @@ -0,0 +1,1635 @@ +{ + "jobs": [ + { + "slug": "simple_model_executor_test", + "label": "Simple \u00b7 Model Executor Test", + "source": "ready", + "command": "pytest -sv tests/model_executor -m 'core_model and cpu'", + "timeout_sec": 1200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "simple_diffusion_test", + "label": "Simple \u00b7 Diffusion Test", + "source": "ready", + "command": "pytest -sv tests/diffusion -m 'core_model and cpu'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "simple_engineentrypoints_test", + "label": "Simple \u00b7 Engine&Entrypoints Test", + "source": "ready", + "command": "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'", + "timeout_sec": 1200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "simple_other_test", + "label": "Simple \u00b7 Other Test", + "source": "ready", + "command": "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine", + "timeout_sec": 1200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "diffusion_lora_test", + "label": "Diffusion \u00b7 LoRA Test", + "source": "merge", + "command": "pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level \"advanced_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "distributed_test", + "label": "Distributed Test", + "source": "ready", + "command": "pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "scheduler", + "setup": "" + }, + { + "slug": "entrypoint_test_with_h100", + "label": "Entrypoint Test with H100", + "source": "merge", + "command": "pytest -s -v tests/entrypoints/ -m \"advanced_model and cuda and H100\" --run-level \"advanced_model\" --deselect tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_server_vad_multi_turn_without_client_commit", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "entrypoint_test_with_l4", + "label": "Entrypoint Test with L4", + "source": "merge", + "command": "set -eu\npytest -s -v tests/entrypoints/test_omni_sleep_mode.py::TestOmniDiffusionSleepMode::test_diffusion_sleep_handshake -m \"advanced_model and cuda and L4 and cards_1\" --run-level \"advanced_model\"\npytest -s -v tests/entrypoints/ -m \"advanced_model and cuda and L4 and cards_1\" --run-level \"advanced_model\" --deselect tests/entrypoints/test_omni_sleep_mode.py::TestOmniDiffusionSleepMode::test_diffusion_sleep_handshake\n", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "entrypoint_test_with_l4_2-gpu", + "label": "Entrypoint Test with L4 \u00b7 2-GPU", + "source": "merge", + "command": "pytest -s -v tests/entrypoints/ -m \"advanced_model and cuda and L4 and cards_2\" --run-level \"advanced_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "tts_qwen3-tts_customvoice_test", + "label": "TTS \u00b7 Qwen3-TTS CustomVoice Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_ALLOW_LONG_MAX_MODEL_LEN=1", + "module": "online_serving", + "setup": "" + }, + { + "slug": "tts_qwen3-tts_base_test", + "label": "TTS \u00b7 Qwen3-TTS Base Test", + "source": "merge", + "command": ".buildkite/common/scripts/run_cov_split.sh \\\n --model-id qwen3_tts \\\n --offline tests/e2e/offline_inference/test_qwen3_tts_base.py \\\n --online tests/e2e/online_serving/test_qwen3_tts_base.py \\\n -- -m 'advanced_model and cuda' --run-level 'advanced_model'\n", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_LOGGING_LEVEL=DEBUG VLLM_ALLOW_LONG_MAX_MODEL_LEN=1", + "module": "platform", + "setup": "" + }, + { + "slug": "omni_qwen3-omni_test", + "label": "Omni \u00b7 Qwen3-Omni Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "omni_minicpm-o_45_test", + "label": "Omni \u00b7 MiniCPM-o 4.5 Test", + "source": "merge", + "command": "pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'", + "timeout_sec": 3000, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "omni_minicpm-o_45_duplex_test", + "label": "Omni \u00b7 MiniCPM-o 4.5 Duplex Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_duplex_client_live.py -m 'core_model and cuda' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "diffusion_qwen_image_test", + "label": "Diffusion \u00b7 Qwen Image Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_IMAGE_FETCH_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "diffusion_wan22_test", + "label": "Diffusion \u00b7 Wan22 Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model and cuda' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_IMAGE_FETCH_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "diffusion_hunyuanvideo-15_test", + "label": "Diffusion \u00b7 HunyuanVideo-1.5 Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_IMAGE_FETCH_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "diffusion_lingbot-video_test", + "label": "Diffusion \u00b7 LingBot-Video Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'core_model and cuda' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_IMAGE_FETCH_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "diffusion_minimax-h3_l3_dlo_dp2_and_fasth3", + "label": "Diffusion \u00b7 MiniMax-H3 L3 DLO DP2 and FastH3", + "source": "merge", + "command": "set -eu\nif [ -f vllm_omni/diffusion/models/minimax_h3/lora.py ]; then\n export VLLM_TEST_MINIMAX_H3_TURBO_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id=\"lightx2v/Minimax-h3-Turbo\", filename=\"minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors\"))')\nfi\nif [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then\n export VLLM_TEST_MINIMAX_H3_FASTH3_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id=\"FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA\", filename=\"dense-datafree/adapter_model.safetensors\"))')\nfi\ntimeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_fl2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'\ntimeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_ref2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'\ntimeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'\ntimeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'\n", + "timeout_sec": 9000, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_IMAGE_FETCH_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "diffusion_minimax-h3_latent_upscale_and_refine", + "label": "Diffusion \u00b7 MiniMax-H3 Latent Upscale and Refine", + "source": "merge", + "command": "set -eu\nVLLM_TEST_MINIMAX_H3_LATENT_UPSCALER=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id=\"LBH-123-AI/Minimax_h3_latent_Upscaler\", filename=\"minimax_h3_latent_upscaler_3d_conv_v1/minimax_h3_latent_upscaler_3d_conv_v1_bf16.safetensors\"))')\nexport VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER\ntimeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'\n", + "timeout_sec": 2400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "simple_cosyvoice3_packed_flow_test", + "label": "Simple \u00b7 CosyVoice3 Packed Flow Test", + "source": "ready", + "command": "pytest -sv tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py -k packed -m 'core_model and cuda' --run-level=core_model", + "timeout_sec": 900, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_flashattention_test", + "label": "Diffusion \u00b7 FlashAttention Test", + "source": "ready", + "command": "timeout 15m pytest -s -v tests/diffusion/attention/ -m 'core_model and cuda and L4 and cards_1' --run-level \"core_model\"", + "timeout_sec": 1200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_offloader_test", + "label": "Diffusion \u00b7 Offloader Test", + "source": "ready", + "command": "timeout 40m pytest -s -v tests/diffusion/offloader -m \"core_model and cuda\" --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_cache_test", + "label": "Diffusion \u00b7 Cache Test", + "source": "ready", + "command": "timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_batch_test", + "label": "Diffusion \u00b7 Batch Test", + "source": "ready", + "command": "timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_distributed_test_single-gpu", + "label": "Diffusion \u00b7 Distributed Test \u00b7 Single-GPU", + "source": "ready", + "command": "timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_distributed_test_2-gpu", + "label": "Diffusion \u00b7 Distributed Test \u00b7 2-GPU", + "source": "ready", + "command": "timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level \"core_model\"\ntimeout 10m pytest -sv tests/diffusion/attention/test_ulysses_uaa.py -m 'core_model and cuda and L4 and cards_2' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_distributed_attention_test_4-gpu", + "label": "Diffusion \u00b7 Distributed & Attention Test \u00b7 4-GPU", + "source": "ready", + "command": "timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level \"core_model\"\ntimeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_model_test", + "label": "Diffusion \u00b7 Model Test", + "source": "ready", + "command": "timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "diffusion_other_test", + "label": "Diffusion \u00b7 Other Test", + "source": "ready", + "command": "timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion -m 'core_model and cuda' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "tiny_model_tests_diffusion_base", + "label": "Tiny Model Tests \u00b7 Diffusion (base)", + "source": "ready", + "command": "timeout 15m pytest -sv -n 4 tests/model_tests/diffusion -m 'core_model and cuda' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "enginemodel_executor_test", + "label": "Engine&Model Executor Test", + "source": "ready", + "command": "pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "entrypoints_test", + "label": "Entrypoints Test", + "source": "ready", + "command": "pytest -sv tests/entrypoints/test_async_omni.py::test_diffusion_generate_request_id -m 'core_model and cuda' --run-level 'core_model'\n pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model' --deselect tests/entrypoints/test_async_omni.py::test_diffusion_generate_request_id", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_WORKER_MULTIPROC_METHOD=spawn", + "module": "online_serving", + "setup": "" + }, + { + "slug": "custom_pipeline_test", + "label": "Custom Pipeline Test", + "source": "ready", + "command": "timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "worker_runner", + "setup": "" + }, + { + "slug": "rlhf_verl-omni_e2e_test", + "label": "RLHF \u00b7 VeRL-Omni E2E Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "omni_minicpm-o_45_offline_test", + "label": "Omni \u00b7 MiniCPM-o 4.5 Offline Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "omni_minicpm-o_45_online_test", + "label": "Omni \u00b7 MiniCPM-o 4.5 Online Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "tts_higgs_v3_full_response_test", + "label": "TTS \u00b7 Higgs v3 Full Response Test", + "source": "ready", + "command": "timeout 20m pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py::test_full_response_v1_preserves_audio -m 'core_model and cuda' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "diffusion_cosmos3_test", + "label": "Diffusion \u00b7 Cosmos3 Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_IMAGE_FETCH_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "diffusion_minimax-h3_dlo_dp2_t2va_test", + "label": "Diffusion \u00b7 MiniMax-H3 DLO DP2 T2VA Test", + "source": "ready", + "command": "pytest -s -v tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py -m 'core_model and cuda and diffusion' --run-level 'core_model'", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_IMAGE_FETCH_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_omni_function_test_with_h100_single-gpu", + "label": ":full_moon: Omni \u00b7 Function Test with H100 \u00b7 Single-GPU", + "source": "nightly", + "command": "pytest -sv tests/e2e --run-level \"full_model\" -m \"full_model and H100 and B200 and omni and cards_1\" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_window.py", + "timeout_sec": 7200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_omni_function_test_with_h100_2-gpu", + "label": ":full_moon: Omni \u00b7 Function Test with H100 \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -sv tests/e2e --run-level \"full_model\" -m \"full_model and H100 and B200 and omni and cards_2\" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_omni_minicpm-o_45_duplex_test", + "label": ":full_moon: Omni \u00b7 MiniCPM-o 4.5 Duplex Test", + "source": "nightly", + "command": "pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m \"full_model and cuda and H100 and B200 and omni and cards_1\" --run-level \"full_model\"\npytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_window.py -m \"full_model and cuda and H100 and B200 and omni and cards_1\" --run-level \"full_model\"", + "timeout_sec": 7200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_omni_doc_test_with_l4_4-gpu", + "label": ":full_moon: Omni \u00b7 Doc Test with L4 \u00b7 4-GPU", + "source": "nightly", + "command": "pytest -s -v tests/examples/ -m \"full_model and omni and L4 and B200 and cards_4\" --run-level \"full_model\"", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_ALLOW_LONG_MAX_MODEL_LEN=\"1\"", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_omni_doc_test_with_h100_2-gpu", + "label": ":full_moon: Omni \u00b7 Doc Test with H100 \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -s -v tests/examples/ -m \"full_model and omni and H100 and B200 and cards_2\" --run-level \"full_model\"", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_omni_omni-duplexeval_ci_guard", + "label": ":full_moon: Omni \u00b7 Omni-DuplexEval CI Guard", + "source": "nightly", + "command": "set +e\npytest -s -v tests/e2e/accuracy/omni_duplex_eval/test_omni_duplex_eval_ci.py \\\n -m \"full_model and H100 and B200 and cuda and cards_2\" \\\n --run-level full_model \\\n --basetemp /tmp/omni_duplex_ci\nEXIT=$$?\nbuildkite-agent artifact upload \"tests/e2e/accuracy/omni_duplex_eval/results/*.json\"\nexit $$EXIT\n", + "timeout_sec": 14400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_omni_multi-replica_startup_test_with_4x_h100", + "label": ":full_moon: Omni \u00b7 Multi-Replica Startup Test with 4x H100", + "source": "nightly", + "command": "pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m \"full_model and H100 and B200 and cards_4\" --run-level \"core_model\"", + "timeout_sec": 2700, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_entrypoint_test_with_h100", + "label": ":full_moon: Entrypoint Test with H100", + "source": "nightly", + "command": "pytest -s -v tests/entrypoints/ -m \"full_model and cuda and H100 and B200 and cards_2\" --run-level \"full_model\"", + "timeout_sec": 10800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_tts_function_test_with_l4", + "label": ":full_moon: TTS \u00b7 Function Test with L4", + "source": "nightly", + "command": "pytest -s -v tests/e2e/ -m \"full_model and L4 and B200 and tts and cards_1\" --run-level \"full_model\" --ignore=tests/e2e/accuracy", + "timeout_sec": 7200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60 VLLM_ALLOW_LONG_MAX_MODEL_LEN=\"1\" VLLM_USE_DEEP_GEMM=\"0\" VLLM_MOE_USE_DEEP_GEMM=\"0\"", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_tiny_model_tests_diffusion_multi-gpu_accelerations_2-gpu", + "label": ":full_moon: Tiny Model Tests \u00b7 Diffusion (Multi-GPU accelerations) \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_2' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_tiny_model_tests_diffusion_multi-gpu_accelerations_4-gpu", + "label": ":full_moon: Tiny Model Tests \u00b7 Diffusion (Multi-GPU accelerations) \u00b7 4-GPU", + "source": "nightly", + "command": "pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_4' --run-level \"core_model\"", + "timeout_sec": 1800, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2iat_function_test_with_h100_single-gpu", + "label": ":full_moon: Diffusion X2I(&A&T) \u00b7 Function Test with H100 \u00b7 Single-GPU", + "source": "nightly", + "command": "pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py -m \"full_model and diffusion and H100 and B200 and cards_1\" --run-level \"full_model\"", + "timeout_sec": 7200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2iat_function_test_with_h100_2-gpu", + "label": ":full_moon: Diffusion X2I(&A&T) \u00b7 Function Test with H100 \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -sv tests/e2e/online_serving/test_qwen_image_expansion.py -m \"full_model and diffusion and H100 and B200 and cards_2\" --run-level \"full_model\"", + "timeout_sec": 7200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2iat_function_test_with_l4_single-gpu", + "label": ":full_moon: Diffusion X2I(&A&T) \u00b7 Function Test with L4 \u00b7 Single-GPU", + "source": "nightly", + "command": "pytest -sv tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py -m \"full_model and diffusion and L4 and B200 and cards_1\" --run-level \"full_model\"", + "timeout_sec": 7200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2iat_doc_test", + "label": ":full_moon: Diffusion X2I(&A&T) \u00b7 Doc Test", + "source": "nightly", + "command": "pytest -s -v tests/examples/*/test_text_to_image.py -m \"full_model and example and H100 and B200 and (cards_1 or cards_2)\" --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_wan22_t2v_function_test_single-gpu", + "label": ":full_moon: Diffusion X2V \u00b7 Wan2.2 T2V Function Test \u00b7 Single-GPU", + "source": "nightly", + "command": "pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k \"t2v\" -m \"full_model and cuda and H100 and B200 and cards_1\" --run-level \"full_model\"", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_wan22_t2v_function_test_2-gpu", + "label": ":full_moon: Diffusion X2V \u00b7 Wan2.2 T2V Function Test \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k \"t2v\" -m \"full_model and cuda and H100 and B200 and cards_2\" --run-level \"full_model\"", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_wan22_i2vti2v_function_test_single-gpu", + "label": ":full_moon: Diffusion X2V \u00b7 Wan2.2 I2V/TI2V Function Test \u00b7 Single-GPU", + "source": "nightly", + "command": "pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k \"i2v\" -m \"full_model and cuda and H100 and B200 and cards_1\" --run-level \"full_model\"", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_wan22_i2vti2v_function_test_2-gpu", + "label": ":full_moon: Diffusion X2V \u00b7 Wan2.2 I2V/TI2V Function Test \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k \"i2v\" -m \"full_model and cuda and H100 and B200 and cards_2\" --run-level \"full_model\"", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_hunyuanvideo-15_function_test", + "label": ":full_moon: Diffusion X2V \u00b7 HunyuanVideo-1.5 Function Test", + "source": "nightly", + "command": "pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m \"full_model and H100 and B200 and cuda and (cards_1 or cards_2)\" --run-level \"full_model\"", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "online_serving", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_lingbot_function_test", + "label": ":full_moon: Diffusion X2V \u00b7 LingBot Function Test", + "source": "nightly", + "command": "pytest -s -v tests/e2e/online_serving/test_lingbot_video_expansion.py tests/e2e/online_serving/test_lingbot_video_moe.py tests/e2e/offline_inference/test_lingbot_world_v2.py -m \"full_model and diffusion and H100 and B200 and cards_1\" --run-level \"full_model\"", + "timeout_sec": 7200, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_doc_test", + "label": ":full_moon: Diffusion X2V \u00b7 Doc Test", + "source": "nightly", + "command": "pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m \"full_model and example and H100 and B200 and cards_1\" --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "platform", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_wan22_autoround_function_test", + "label": ":full_moon: Diffusion X2V \u00b7 Wan2.2 Autoround Function Test", + "source": "nightly", + "command": "pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m \"full_model and H100 and B200 and cuda and (cards_1 or cards_2)\" --run-level \"full_model\"", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_x2v_wan22_device-postprocess_equivalence_test", + "label": ":full_moon: Diffusion X2V \u00b7 Wan2.2 Device-Postprocess Equivalence Test", + "source": "nightly", + "command": "VLLM_OMNI_DEVICE_POSTPROCESS_MODEL=Wan-AI/Wan2.2-TI2V-5B-Diffusers \\\n pytest -s -v tests/e2e/offline_inference/test_device_postprocess_equivalence.py -m \"full_model and H100 and B200 and cuda and cards_1\" --run-level \"full_model\"\n", + "timeout_sec": 5400, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_distributed_test_with_l4_2-gpu", + "label": ":full_moon: Diffusion \u00b7 Distributed Test with L4 \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_2' --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_distributed_test_with_l4_3-gpu", + "label": ":full_moon: Diffusion \u00b7 Distributed Test with L4 \u00b7 3-GPU", + "source": "nightly", + "command": "pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_3' --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_distributed_test_with_l4_4-gpu", + "label": ":full_moon: Diffusion \u00b7 Distributed Test with L4 \u00b7 4-GPU", + "source": "nightly", + "command": "pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_4' --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_distributed_test_with_h100_2-gpu", + "label": ":full_moon: Diffusion \u00b7 Distributed Test with H100 \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_2' --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_distributed_test_with_h100_4-gpu", + "label": ":full_moon: Diffusion \u00b7 Distributed Test with H100 \u00b7 4-GPU", + "source": "nightly", + "command": "pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_4' --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_offloader_test", + "label": ":full_moon: Diffusion \u00b7 Offloader Test", + "source": "nightly", + "command": "pytest -sv tests/diffusion/offloader -m \"full_model and L4 and B200 and cuda and cards_1\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_quantization_test_with_h100_single-gpu", + "label": ":full_moon: Diffusion \u00b7 Quantization Test with H100 \u00b7 Single-GPU", + "source": "nightly", + "command": "pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100 and B200 and cards_1' --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_minimax_h3_fp8_quality_test_with_h100_2-gpu", + "label": ":full_moon: Diffusion \u00b7 MiniMax H3 FP8 Quality Test with H100 \u00b7 2-GPU", + "source": "nightly", + "command": "pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100 and B200 and cards_2' --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + }, + { + "slug": "full_moon_diffusion_quantization_test_with_l4", + "label": ":full_moon: Diffusion \u00b7 Quantization Test with L4", + "source": "nightly", + "command": "pip install \"vllm-gguf-plugin==0.0.4\"\npytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4 and B200 and cards_1' --run-level \"full_model\"", + "timeout_sec": 3600, + "min_gpus": 1, + "hw": "any", + "env": "VLLM_WORKER_MULTIPROC_METHOD=spawn HF_HUB_DOWNLOAD_TIMEOUT=300 HF_HUB_ETAG_TIMEOUT=60", + "module": "model_executor", + "setup": "" + } + ], + "changes": [ + { + "path": "tests/buildkite/test_rocm_dockerfile.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/buildkite/test_upload_pipeline.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_generation_scheduler_restore.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_aborted_queue_sweep.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_logprobs.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_stale_drain.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_streaming.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_unified_decode_deferral.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_cfg_ar_scheduler.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_generation_scheduler_update_session.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_sched_deferred_free_fence.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_finish_requests_purge.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_input_coordinator_cleanup.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_mixin_shared.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_ready_inbox.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_streaming_input_counter.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/cache/test_seacache.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/diffusion_kv/test_kv_cache_utils.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/diffusion_kv/test_mooncake_cfg_fanout.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/diffusion_kv/test_paged_attention_adapter.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/distributed/test_comm.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/distributed/test_pipeline_parallel.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/models/flux2/test_flux2_quantization.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/quantization/test_fp8_config.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/quantization/test_online_fp8_gpu.py", + "type": "added", + "new_path": "" + }, + { + "path": "tests/distributed/omni_connectors/test_chunk_transfer_adapter.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_arg_utils_shared_fields.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_async_omni_engine_input.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_async_omni_engine_stage_init.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_omni_request.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_omni_request_output.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_orchestrator_error_handling.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_output_processor_mrv2_text.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_stage_engine_args.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_stage_engine_core_client.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_stage_pool_collective_rpc.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/entrypoints/test_async_omni.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/entrypoints/test_async_omni_pause_sleep_routing.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/helpers/cpu_collective_server.py", + "type": "added", + "new_path": "" + }, + { + "path": "tests/helpers/distributed.py", + "type": "added", + "new_path": "" + }, + { + "path": "tests/helpers/runtime.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/helpers/tests/test_runtime_omni_runner.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/inputs/test_mm_processor.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/inputs/test_preprocess.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/metrics/test_emit_calls.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/metrics/test_stat_logger.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/model_executor/models/omnivoice/test_processor_prompt_tokenization.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/model_executor/models/test_omni_processing.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/model_executor/test_cfg_pairing.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/worker/test_omni_v2_profile_dispatch.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/worker_v2/test_capture_query_bound_gpu.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/worker_v2/test_omni_ar_model_runner.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/worker_v2/test_omni_generation_model_runner.py", + "type": "modified", + "new_path": "" + } + ], + "dropped": [], + "module_plans": { + "platform": { + "ci_tests": [ + "simple_model_executor_test", + "simple_diffusion_test", + "simple_engineentrypoints_test", + "simple_other_test", + "tts_qwen3-tts_base_test", + "omni_minicpm-o_45_test", + "tiny_model_tests_diffusion_base", + "enginemodel_executor_test", + "rlhf_verl-omni_e2e_test", + "full_moon_omni_function_test_with_h100_single-gpu", + "full_moon_omni_function_test_with_h100_2-gpu", + "full_moon_omni_doc_test_with_l4_4-gpu", + "full_moon_omni_doc_test_with_h100_2-gpu", + "full_moon_omni_omni-duplexeval_ci_guard", + "full_moon_tts_function_test_with_l4", + "full_moon_tiny_model_tests_diffusion_multi-gpu_accelerations_2-gpu", + "full_moon_tiny_model_tests_diffusion_multi-gpu_accelerations_4-gpu", + "full_moon_diffusion_x2v_lingbot_function_test", + "full_moon_diffusion_x2v_doc_test" + ], + "upstream_changes": [], + "omni_specific": [] + }, + "model_executor": { + "ci_tests": [ + "diffusion_lora_test", + "simple_cosyvoice3_packed_flow_test", + "diffusion_flashattention_test", + "diffusion_offloader_test", + "diffusion_cache_test", + "diffusion_batch_test", + "diffusion_distributed_test_single-gpu", + "diffusion_distributed_test_2-gpu", + "diffusion_distributed_attention_test_4-gpu", + "diffusion_model_test", + "diffusion_other_test", + "omni_minicpm-o_45_offline_test", + "full_moon_diffusion_x2iat_function_test_with_l4_single-gpu", + "full_moon_diffusion_x2v_wan22_autoround_function_test", + "full_moon_diffusion_x2v_wan22_device-postprocess_equivalence_test", + "full_moon_diffusion_distributed_test_with_l4_2-gpu", + "full_moon_diffusion_distributed_test_with_l4_3-gpu", + "full_moon_diffusion_distributed_test_with_l4_4-gpu", + "full_moon_diffusion_distributed_test_with_h100_2-gpu", + "full_moon_diffusion_distributed_test_with_h100_4-gpu", + "full_moon_diffusion_offloader_test", + "full_moon_diffusion_quantization_test_with_h100_single-gpu", + "full_moon_diffusion_minimax_h3_fp8_quality_test_with_h100_2-gpu", + "full_moon_diffusion_quantization_test_with_l4" + ], + "upstream_changes": [ + { + "path": "tests/diffusion/cache/test_seacache.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/diffusion_kv/test_kv_cache_utils.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/diffusion_kv/test_mooncake_cfg_fanout.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/diffusion_kv/test_paged_attention_adapter.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/distributed/test_comm.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/distributed/test_pipeline_parallel.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/models/boogu_image/test_pipeline_boogu_image.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/models/flux2/test_flux2_quantization.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/quantization/test_fp8_config.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/diffusion/quantization/test_online_fp8_gpu.py", + "type": "added", + "new_path": "" + }, + { + "path": "tests/model_executor/models/omnivoice/test_processor_prompt_tokenization.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/model_executor/models/qwen2_5_omni/test_per_video_audio_mask.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/model_executor/models/test_omni_processing.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/model_executor/test_cfg_pairing.py", + "type": "modified", + "new_path": "" + } + ], + "omni_specific": [] + }, + "scheduler": { + "ci_tests": [ + "distributed_test" + ], + "upstream_changes": [ + { + "path": "tests/distributed/omni_connectors/test_chunk_transfer_adapter.py", + "type": "modified", + "new_path": "" + } + ], + "omni_specific": [] + }, + "online_serving": { + "ci_tests": [ + "entrypoint_test_with_h100", + "entrypoint_test_with_l4", + "entrypoint_test_with_l4_2-gpu", + "tts_qwen3-tts_customvoice_test", + "omni_qwen3-omni_test", + "omni_minicpm-o_45_duplex_test", + "diffusion_qwen_image_test", + "diffusion_wan22_test", + "diffusion_hunyuanvideo-15_test", + "diffusion_lingbot-video_test", + "diffusion_minimax-h3_l3_dlo_dp2_and_fasth3", + "diffusion_minimax-h3_latent_upscale_and_refine", + "entrypoints_test", + "omni_minicpm-o_45_online_test", + "tts_higgs_v3_full_response_test", + "diffusion_cosmos3_test", + "diffusion_minimax-h3_dlo_dp2_t2va_test", + "full_moon_omni_minicpm-o_45_duplex_test", + "full_moon_omni_multi-replica_startup_test_with_4x_h100", + "full_moon_entrypoint_test_with_h100", + "full_moon_diffusion_x2iat_function_test_with_h100_single-gpu", + "full_moon_diffusion_x2iat_function_test_with_h100_2-gpu", + "full_moon_diffusion_x2v_wan22_t2v_function_test_single-gpu", + "full_moon_diffusion_x2v_wan22_t2v_function_test_2-gpu", + "full_moon_diffusion_x2v_wan22_i2vti2v_function_test_single-gpu", + "full_moon_diffusion_x2v_wan22_i2vti2v_function_test_2-gpu", + "full_moon_diffusion_x2v_hunyuanvideo-15_function_test" + ], + "upstream_changes": [ + { + "path": "tests/entrypoints/test_async_omni.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/entrypoints/test_async_omni_pause_sleep_routing.py", + "type": "modified", + "new_path": "" + } + ], + "omni_specific": [] + }, + "worker_runner": { + "ci_tests": [ + "custom_pipeline_test" + ], + "upstream_changes": [ + { + "path": "tests/worker/test_omni_v2_profile_dispatch.py", + "type": "modified", + "new_path": "" + } + ], + "omni_specific": [] + }, + "unknown": { + "ci_tests": [ + "full_moon_diffusion_x2iat_doc_test" + ], + "upstream_changes": [ + { + "path": "tests/buildkite/test_rocm_dockerfile.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/buildkite/test_upload_pipeline.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_generation_scheduler_restore.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_aborted_queue_sweep.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_logprobs.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_stale_drain.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_streaming.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_ar_scheduler_unified_decode_deferral.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_cfg_ar_scheduler.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_generation_scheduler_update_session.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_sched_deferred_free_fence.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_finish_requests_purge.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_input_coordinator_cleanup.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_mixin_shared.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_ready_inbox.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/core/sched/test_omni_scheduler_streaming_input_counter.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/helpers/cpu_collective_server.py", + "type": "added", + "new_path": "" + }, + { + "path": "tests/helpers/distributed.py", + "type": "added", + "new_path": "" + }, + { + "path": "tests/helpers/runtime.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/helpers/tests/test_runtime_omni_runner.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/inputs/test_mm_processor.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/inputs/test_preprocess.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/metrics/test_emit_calls.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/metrics/test_stat_logger.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/worker_v2/test_capture_query_bound_gpu.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/worker_v2/test_omni_ar_model_runner.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/worker_v2/test_omni_generation_model_runner.py", + "type": "modified", + "new_path": "" + } + ], + "omni_specific": [] + }, + "input_output": { + "ci_tests": [], + "upstream_changes": [ + { + "path": "tests/engine/test_arg_utils_shared_fields.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_async_omni_engine_input.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_async_omni_engine_stage_init.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_omni_request.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_omni_request_output.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_orchestrator_error_handling.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_output_processor_mrv2_text.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_stage_engine_args.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_stage_engine_core_client.py", + "type": "modified", + "new_path": "" + }, + { + "path": "tests/engine/test_stage_pool_collective_rpc.py", + "type": "modified", + "new_path": "" + } + ], + "omni_specific": [] + }, + "model_config": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "diffusion": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "deploy": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "distributed": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "engine": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "attention": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "quantization": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "runtime_patch": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "host_weight_runtime": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "sampling": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "tokenizers": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "metrics": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "benchmarks": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + }, + "fullduplex": { + "ci_tests": [], + "upstream_changes": [], + "omni_specific": [] + } + } +} \ No newline at end of file diff --git a/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-merge.yml b/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-merge.yml new file mode 100644 index 000000000..104d6df26 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-merge.yml @@ -0,0 +1,221 @@ +# CUDA CI L3 (`advanced_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`). +# Uploaded by Upload Merge Pipeline: PR label `merge-test`, or post-merge on main +# (both diff-aware via source_file_dependencies). + +env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HF_HUB_DOWNLOAD_TIMEOUT: 300 + HF_HUB_ETAG_TIMEOUT: 60 + +steps: + - group: ":card_index_dividers: Simple Test" + depends_on: upload-merge-pipeline + steps: + - label: "Simple · Model Executor Test" + timeout_in_minutes: 20 + commands: + - "pytest -sv tests/model_executor -m 'core_model and cpu'" + mirror_hardwares: l4_1 + + - label: "Simple · Diffusion Test" + timeout_in_minutes: 30 + commands: + - "pytest -sv tests/diffusion -m 'core_model and cpu'" + mirror_hardwares: l4_1 + + - label: "Simple · Engine&Entrypoints Test" + timeout_in_minutes: 20 + commands: + - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'" + mirror_hardwares: l4_1 + + - label: "Simple · Other Test" + timeout_in_minutes: 20 + commands: + - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine" + mirror_hardwares: l4_1 + + - group: ":card_index_dividers: Diffusion Test" + depends_on: upload-merge-pipeline + steps: + - label: "Diffusion · LoRA Test" + timeout_in_minutes: 30 + commands: + - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" + mirror_hardwares: l4_1 + + - label: "Distributed Test" + depends_on: upload-merge-pipeline + timeout_in_minutes: 60 + commands: + - | + timeout 55m bash -c " + set -e + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model' + " + mirror_hardwares: h100_1 + + - group: ":card_index_dividers: Entrypoints Test" + depends_on: upload-merge-pipeline + steps: + - label: "Entrypoint Test with H100" + timeout_in_minutes: 60 + commands: + # Turn-based server VAD over /v1/realtime is deferred to RFC #7181 PR 5 + # (see the P0 thread on #7413); the test stays in the tree and runs again there. + - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model" --deselect tests/entrypoints/openai_api/test_qwen3_omni_realtime_websocket.py::TestQwen3OmniRealtimeWebSocket::test_server_vad_multi_turn_without_client_commit + mirror_hardwares: h100_2 + + - label: "Entrypoint Test with L4" + timeout_in_minutes: 60 + # Workaround https://github.com/vllm-project/vllm-omni/issues/8016: shutdown() + # leaves inline UniProc DiffusionWorker VRAM mapped in the pytest parent. + # This job does not collect the ready leftover + # (test_diffusion_generate_request_id, core_model); isolate the + # advanced_model DiT handshake instead. Revert to a single sweep after + # the product shutdown fix. + commands: + - | + set -eu + pytest -s -v tests/entrypoints/test_omni_sleep_mode.py::TestOmniDiffusionSleepMode::test_diffusion_sleep_handshake -m "advanced_model and cuda and L4 and cards_1" --run-level "advanced_model" + pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4 and cards_1" --run-level "advanced_model" --deselect tests/entrypoints/test_omni_sleep_mode.py::TestOmniDiffusionSleepMode::test_diffusion_sleep_handshake + mirror_hardwares: l4_1 + + - label: "Entrypoint Test with L4 · 2-GPU" + timeout_in_minutes: 60 + commands: + - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4 and cards_2" --run-level "advanced_model" + mirror_hardwares: l4_2 + + - group: ":card_index_dividers: E2E Test" + depends_on: upload-merge-pipeline + steps: + - label: "TTS · Qwen3-TTS CustomVoice Test" + source_file_dependencies: tts_qwen3_tts_function + commands: + - | + timeout 40m bash -c " + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py tests/e2e/features/prefill_phase/test_chunked_prefill_tail.py -m 'advanced_model and cuda' --run-level 'advanced_model' + " + mirror_hardwares: l4_1 + + - label: "TTS · Qwen3-TTS Base Test" + source_file_dependencies: tts_qwen3_tts_cov + # 40 -> 60: the coverage split runs the model twice. + timeout_in_minutes: 60 + commands: + - | + export VLLM_LOGGING_LEVEL=DEBUG + export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + .buildkite/common/scripts/run_cov_split.sh \ + --model-id qwen3_tts \ + --offline tests/e2e/offline_inference/test_qwen3_tts_base.py \ + --online tests/e2e/online_serving/test_qwen3_tts_base.py \ + -- -m 'advanced_model and cuda' --run-level 'advanced_model' + mirror_hardwares: l4_1 + + - label: "Omni · Qwen3-Omni Test" + source_file_dependencies: omni_qwen3_omni_function + timeout_in_minutes: 50 + commands: + - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model' + mirror_hardwares: h100_2 + + - label: "Omni · MiniCPM-o 4.5 Test" + source_file_dependencies: omni_minicpmo_4_5_function + timeout_in_minutes: 50 + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model' + # h100_2, not h100_1: advanced_model asserts transcribe every audio + # response with Whisper, and tests/helpers/media.py requires at least + # 16 GiB free VRAM on a device to run Whisper on GPU. When the model + # server leaves insufficient memory, Whisper falls back to CPU: + # ~8-10 min per clip vs ~30 s, which overruns the 50-minute job timeout + # (build 2956). h100_2 guarantees a second GPU with ample free VRAM. + mirror_hardwares: h100_2 + + - label: "Omni · MiniCPM-o 4.5 Duplex Test" + source_file_dependencies: omni_minicpmo_4_5_duplex_function + timeout_in_minutes: 50 + commands: + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_minicpmo_4_5_window.py -m 'advanced_model and cuda' --run-level 'advanced_model' + mirror_hardwares: h100_1 + + - label: "Diffusion · Qwen Image Test" + source_file_dependencies: diffusion_qwen_image_function + commands: + - | + timeout 40m bash -c ' + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model' + ' + mirror_hardwares: h100_1 + + - label: "Diffusion · Wan22 Test" + source_file_dependencies: diffusion_wan22_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + " + mirror_hardwares: h100_1 + + - label: "Diffusion · HunyuanVideo-1.5 Test" + source_file_dependencies: diffusion_hunyuan_video_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + " + mirror_hardwares: h100_1 + + - label: "Diffusion · LingBot-Video Test" + source_file_dependencies: diffusion_lingbot_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + " + mirror_hardwares: h100_1 + + - label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3" + source_file_dependencies: diffusion_minimax_h3_function + # Keep four server lifecycles in one Buildkite job, but never in one + # pytest process. DLO uses multiprocessing shared-memory queues; the + # parent's resource_tracker can retain a stale /psm_* entry during + # teardown and strand the next Ref2VA collective. + # A per-file process plus a hard timeout also makes a regression fail + # instead of leaving the merge queue stuck indefinitely. + timeout_in_minutes: 150 + commands: + - | + set -eu + export VLLM_IMAGE_FETCH_TIMEOUT=60 + if [ -f vllm_omni/diffusion/models/minimax_h3/lora.py ]; then + export VLLM_TEST_MINIMAX_H3_TURBO_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="lightx2v/Minimax-h3-Turbo", filename="minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"))') + fi + if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then + export VLLM_TEST_MINIMAX_H3_FASTH3_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA", filename="dense-datafree/adapter_model.safetensors"))') + fi + timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_fl2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_ref2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + mirror_hardwares: h100_2 + + - label: "Diffusion · MiniMax-H3 Latent Upscale and Refine" + source_file_dependencies: diffusion_minimax_h3_function + timeout_in_minutes: 40 + commands: + - | + set -eu + VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="LBH-123-AI/Minimax_h3_latent_Upscaler", filename="minimax_h3_latent_upscaler_3d_conv_v1/minimax_h3_latent_upscaler_3d_conv_v1_bf16.safetensors"))') + export VLLM_TEST_MINIMAX_H3_LATENT_UPSCALER + timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + mirror_hardwares: h100_2 diff --git a/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-nightly.yml b/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-nightly.yml new file mode 100644 index 000000000..376e322ed --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-nightly.yml @@ -0,0 +1,547 @@ +# CUDA CI L4 (`full_model`): nightly and release share this file. +# Uploaded by Upload Nightly Pipeline: NIGHTLY=1, or PR labels (nightly-test / omni-test / ...). +# Hardware: omit mirror_hardwares to compose from -m SKU + cards_n; or set a preset string to force. + +env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HF_HUB_DOWNLOAD_TIMEOUT: 300 + HF_HUB_ETAG_TIMEOUT: 60 + +steps: + # Group: collapses under one heading in the Buildkite UI; child steps still run in parallel. + - group: ":card_index_dividers: Omni Model Test" + key: nightly-omni-test-group + depends_on: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" + steps: + + - label: ":full_moon: Omni · Function Test with H100 · Single-GPU" + source_file_dependencies: + - omni_qwen3_omni_function + - omni_minicpmo_4_5_function + timeout_in_minutes: 120 + commands: + - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_window.py + + - label: ":full_moon: Omni · Function Test with H100 · 2-GPU" + source_file_dependencies: + - omni_qwen3_omni_function + - omni_minicpmo_4_5_function + timeout_in_minutes: 90 + commands: + - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and B200 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py + + - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test" + source_file_dependencies: omni_minicpmo_4_5_duplex_function + timeout_in_minutes: 120 + commands: + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model" + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_window.py -m "full_model and cuda and H100 and B200 and omni and cards_1" --run-level "full_model" + + - label: ":full_moon: Omni · Doc Test with L4 · 4-GPU" + source_file_dependencies: omni_qwen2_5_omni_doc + timeout_in_minutes: 90 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - pytest -s -v tests/examples/ -m "full_model and omni and L4 and B200 and cards_4" --run-level "full_model" + + - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU" + source_file_dependencies: omni_qwen3_omni_doc + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/examples/ -m "full_model and omni and H100 and B200 and cards_2" --run-level "full_model" + + - label: ":full_moon: Omni · Accuracy Test" + source_file_dependencies: omni_qwen3_omni_accuracy + timeout_in_minutes: 180 + artifact_paths: + - tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json + commands: + - export SEED_TTS_WER_EVAL=1 + - export SEED_TTS_EVAL_DEVICE=cuda:1 + - pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model and H100 and B200 and cards_2" --run-level full_model + + - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test" + source_file_dependencies: omni_minicpmo_4_5_accuracy + timeout_in_minutes: 180 + artifact_paths: + - tests/e2e/accuracy/minicpmo_4_5/results/*.json + commands: + - export SEED_TTS_WER_EVAL=1 + - export SEED_TTS_EVAL_DEVICE=cuda:1 + - pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level full_model + + - label: ":full_moon: Omni · Omni-DuplexEval CI Guard" + source_file_dependencies: omni_duplex_eval_ci + timeout_in_minutes: 240 + env: + VLLM_DUPLEX_EVAL_JUDGE_URL: "http://127.0.0.1:8001" + commands: + - | + set +e + pytest -s -v tests/e2e/accuracy/omni_duplex_eval/test_omni_duplex_eval_ci.py \ + -m "full_model and H100 and B200 and cuda and cards_2" \ + --run-level full_model \ + --basetemp /tmp/omni_duplex_ci + EXIT=$$? + buildkite-agent artifact upload "tests/e2e/accuracy/omni_duplex_eval/results/*.json" + exit $$EXIT + + - label: ":full_moon: Omni · Perf Test · No Async Chunk" + source_file_dependencies: omni_qwen3_omni_perf + key: nightly-omni-performance-no-async-chunk + timeout_in_minutes: 300 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json -m "H100 and B200 and cards_2" + + - label: ":full_moon: Omni · Perf Test · Async Chunk" + source_file_dependencies: omni_qwen3_omni_perf + key: nightly-omni-performance-async-chunk + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and B200 and full_model and cards_2" + + - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test" + source_file_dependencies: omni_minicpmo_4_5_perf + key: nightly-omni-performance-minicpmo-4-5 + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json -m "H100 and B200 and cards_1" + + - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test" + source_file_dependencies: omni_minicpmo_4_5_duplex_perf + key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json -m "H100 and B200 and cards_1" + + - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100" + source_file_dependencies: omni_qwen3_omni_function + timeout_in_minutes: 45 + commands: + - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model and H100 and B200 and cards_4" --run-level "core_model" + + - group: ":card_index_dividers: Entrypoints Test" + key: nightly-entrypoints-test-group + depends_on: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" + steps: + + - label: ":full_moon: Entrypoint Test with H100" + source_file_dependencies: entrypoints_function + # Merge collects advanced_model (5 duplex interruptions). Nightly + # collects full_model (20 interruptions + BAGEL sleep). + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/entrypoints/ -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model" + + - group: ":card_index_dividers: TTS Model Test" + key: nightly-tts-test-group + depends_on: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" + steps: + + - label: ":full_moon: TTS · Function Test with L4" + source_file_dependencies: tts_qwen3_tts_function + timeout_in_minutes: 120 + commands: + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - export VLLM_USE_DEEP_GEMM="0" + - export VLLM_MOE_USE_DEEP_GEMM="0" + - pytest -s -v tests/e2e/ -m "full_model and L4 and B200 and tts and cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy + + - label: ":full_moon: TTS · Perf Test · Single-GPU" + source_file_dependencies: tts_qwen3_tts_perf + key: nightly-tts-performance-single + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_1" + + - label: ":full_moon: TTS · Perf Test · 2-GPU" + source_file_dependencies: tts_qwen3_tts_perf + key: nightly-tts-performance-2gpu + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and B200 and cards_2" + + - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]" + key: nightly-tiny-model-test-group + depends_on: upload-nightly-pipeline + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" + steps: + + - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 2-GPU" + source_file_dependencies: diffusion_tiny_model + timeout_in_minutes: 30 + # NOTE: Single-GPU (core_model) tiny model tests run on the ready label, + # so nightly only adds the multi-GPU parallelism configurations. + # + # This is a bit confusing, but `full_model` is used to mark the multigpu tests, and + # --run-level `core_model` is used to indicate that we should use tiny model weights + # for the CI, since these tests should pass irrespective of whether or not we use + # the full weights or not. + commands: + - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_2' --run-level "core_model" + + - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations) · 4-GPU" + source_file_dependencies: diffusion_tiny_model + timeout_in_minutes: 30 + # CFG+TP extra_test_groups request 2×2=4 devices (see get_required_device_count). + commands: + - pytest -sv tests/model_tests/diffusion -m 'full_model and L4 and B200 and cuda and cards_4' --run-level "core_model" + + - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" + key: nightly-diffusion-x2iat-group + depends_on: upload-nightly-pipeline + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" + steps: + + - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU" + source_file_dependencies: diffusion_qwen_image_function + timeout_in_minutes: 120 + commands: + - >- + pytest -sv + tests/e2e/online_serving/test_qwen_image_expansion.py + -m "full_model and diffusion and H100 and B200 and cards_1" + --run-level "full_model" + + - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU" + source_file_dependencies: diffusion_qwen_image_function + timeout_in_minutes: 120 + commands: + - >- + pytest -sv + tests/e2e/online_serving/test_qwen_image_expansion.py + -m "full_model and diffusion and H100 and B200 and cards_2" + --run-level "full_model" + + - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Single-GPU" + source_file_dependencies: diffusion_qwen_image_function + timeout_in_minutes: 120 + commands: + - pytest -sv tests/e2e/offline_inference/test_qwen_image_autoround_w4a16_expansion.py -m "full_model and diffusion and L4 and B200 and cards_1" --run-level "full_model" + + + - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test" + source_file_dependencies: diffusion_text_to_image_doc + timeout_in_minutes: 60 + commands: + - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100 and B200 and (cards_1 or cards_2)" --run-level "full_model" + + - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test" + source_file_dependencies: diffusion_qwen_image_accuracy + timeout_in_minutes: 180 + commands: + - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120 + - pytest -s -v tests/e2e/accuracy/test_qwen_image.py -m "H100 and B200 and cards_1" --run-level full_model + - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model + + - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test" + source_file_dependencies: diffusion_hunyuan_image3_accuracy + timeout_in_minutes: 180 + commands: + - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct" + - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3" + - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py tests/e2e/accuracy/test_hunyuan_image3_lora.py -m "full_model and H100 and B200 and cards_4" --run-level full_model + + - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU" + source_file_dependencies: diffusion_qwen_image_perf + key: nightly-diffusion-x2iat-performance-qwen-image-single + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/diffusion_result_*.json + - tests/dfx/perf/results/logs/*.log + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export CACHE_DIT_VERSION=1.5.0 + - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_1" + + - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU" + source_file_dependencies: diffusion_qwen_image_perf + key: nightly-diffusion-x2iat-performance-qwen-image-4gpu + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/diffusion_result_*.json + - tests/dfx/perf/results/logs/*.log + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export CACHE_DIT_VERSION=1.5.0 + # Do not pin DIFFUSION_ATTENTION_BACKEND: H100 auto-selects FLASH_ATTN; + # B200 rejects explicit FLASH_ATTN without FA4 and uses CUDNN/TRTLLM. + - pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and B200 and cards_4" + + - group: ":card_index_dividers: Diffusion X2V Model Test" + key: nightly-diffusion-x2v-group + depends_on: upload-nightly-pipeline + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" + steps: + + - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU" + source_file_dependencies: diffusion_wan22_function + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU" + source_file_dependencies: diffusion_wan22_function + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU" + source_file_dependencies: diffusion_wan22_function + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_1" --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU" + source_file_dependencies: diffusion_wan22_function + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and B200 and cards_2" --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test" + source_file_dependencies: diffusion_hunyuan_video_function + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · LingBot Function Test" + source_file_dependencies: diffusion_lingbot_function + timeout_in_minutes: 120 + commands: + - >- + pytest -s -v + tests/e2e/online_serving/test_lingbot_video_expansion.py + tests/e2e/online_serving/test_lingbot_video_moe.py + tests/e2e/offline_inference/test_lingbot_world_v2.py + -m "full_model and diffusion and H100 and B200 and cards_1" + --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · Doc Test" + source_file_dependencies: diffusion_image_to_video_doc + timeout_in_minutes: 60 + commands: + - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100 and B200 and cards_1" --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · Wan2.2 Autoround Function Test" + source_file_dependencies: diffusion_wan22_function + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and H100 and B200 and cuda and (cards_1 or cards_2)" --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · Wan2.2 Device-Postprocess Equivalence Test" + source_file_dependencies: diffusion_wan22_function + timeout_in_minutes: 90 + commands: + - | + VLLM_OMNI_DEVICE_POSTPROCESS_MODEL=Wan-AI/Wan2.2-TI2V-5B-Diffusers \ + pytest -s -v tests/e2e/offline_inference/test_device_postprocess_equivalence.py -m "full_model and H100 and B200 and cuda and cards_1" --run-level "full_model" + + - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test" + source_file_dependencies: diffusion_wan22_accuracy + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model + - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m "full_model and H100 and B200 and cards_1" --run-level full_model + + - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test" + source_file_dependencies: diffusion_minimax_h3_accuracy + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m "full_model and H100 and B200 and cards_4" --run-level full_model + + - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Accuracy Test" + source_file_dependencies: diffusion_hunyuan_video_accuracy + timeout_in_minutes: 180 + commands: + - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model + - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m "full_model and H100 and B200 and (cards_1 or cards_2)" --run-level full_model + + - label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU" + source_file_dependencies: diffusion_wan22_perf + key: nightly-diffusion-x2v-performance-single + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_1" + + - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU" + source_file_dependencies: + - diffusion_wan22_perf + - diffusion_cosmos3_perf + key: nightly-diffusion-x2v-performance-2gpu + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and B200 and cards_2" + EXIT1=$$? + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and B200 and cards_2" + EXIT2=$$? + if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi + exit $$EXIT2 + + - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5" + source_file_dependencies: diffusion_hunyuan_video_perf + key: nightly-diffusion-x2v-performance-hunyuanvideo15 + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)" + EXIT1=$$? + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json -m "H100 and B200 and (cards_1 or cards_2)" + EXIT2=$$? + if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi + exit $$EXIT2 + + - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3" + source_file_dependencies: diffusion_minimax_h3_perf + key: nightly-diffusion-x2v-performance-minimax-h3 + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json -m "H100 and B200 and cards_4" + + - label: ":full_moon: Diffusion X2V · Perf Test · LingBot" + source_file_dependencies: diffusion_lingbot_perf + key: nightly-diffusion-x2v-performance-lingbot + timeout_in_minutes: 180 + artifact_paths: + - tests/dfx/perf/results/*.json + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json -m "H100 and B200 and cards_1" + + - group: ":card_index_dividers: Diffusion Test" + key: nightly-diffusion-test-group + depends_on: upload-nightly-pipeline + if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" + steps: + - label: ":full_moon: Diffusion · Distributed Test with L4 · 2-GPU" + source_file_dependencies: diffusion_distributed_attention + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_2' --run-level "full_model" + + - label: ":full_moon: Diffusion · Distributed Test with L4 · 3-GPU" + source_file_dependencies: diffusion_distributed_attention + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_3' --run-level "full_model" + + - label: ":full_moon: Diffusion · Distributed Test with L4 · 4-GPU" + source_file_dependencies: diffusion_distributed_attention + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4 and B200 and cards_4' --run-level "full_model" + + - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU" + source_file_dependencies: diffusion_distributed_attention + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model" + + - label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU" + source_file_dependencies: diffusion_distributed_attention + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and B200 and cards_4' --run-level "full_model" + + - label: ":full_moon: Diffusion · Offloader Test" + source_file_dependencies: diffusion_offloader + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/offloader -m "full_model and L4 and B200 and cuda and cards_1" + + - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU" + source_file_dependencies: diffusion_quantization + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100 and B200 and cards_1' --run-level "full_model" + + - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU" + source_file_dependencies: diffusion_minimax_h3_function + timeout_in_minutes: 60 + commands: + - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100 and B200 and cards_2' --run-level "full_model" + + - label: ":full_moon: Diffusion · Quantization Test with L4" + source_file_dependencies: diffusion_quantization + timeout_in_minutes: 60 + commands: + - pip install "vllm-gguf-plugin==0.0.4" + - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4 and B200 and cards_1' --run-level "full_model" + + # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon. + # Do not add new depends_on entries or extend this step. + - label: ":email: Nightly Collection & Email" + key: nightly-perf-distribution + depends_on: + - nightly-omni-performance-no-async-chunk + - nightly-omni-performance-async-chunk + - nightly-tts-performance-single + - nightly-tts-performance-2gpu + - nightly-diffusion-x2iat-performance-qwen-image-single + - nightly-diffusion-x2iat-performance-qwen-image-4gpu + - nightly-diffusion-x2v-performance-single + - nightly-diffusion-x2v-performance-2gpu + if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1" + artifact_paths: + - tests/dfx/perf/results/*.xlsx + - tests/dfx/perf/results/*.html + commands: + - pip install openpyxl + - export DEFAULT_INPUT_DIR=tests/dfx/perf/results + - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-single + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-2gpu + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-single + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-4gpu + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-single + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-2gpu + - python tools/nightly/generate_nightly_perf_excel.py + - python tools/nightly/generate_nightly_perf_html.py + - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html" + agents: + queue: "cpu_queue_premerge" diff --git a/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-ready.yml b/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-ready.yml new file mode 100644 index 000000000..ee93db4f5 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/.buildkite/cuda/test-ready.yml @@ -0,0 +1,287 @@ +# CUDA CI L2 (`core_model` e2e) plus L1 CPU Simple Test (`core_model and cpu`). +# Uploaded by Upload Ready Pipeline: PR label `ready`, or main+WEEKLY=1 with --e2e. + +env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HF_HUB_DOWNLOAD_TIMEOUT: 300 + HF_HUB_ETAG_TIMEOUT: 60 + +steps: + - group: ":card_index_dividers: Simple Test" + depends_on: upload-ready-pipeline + steps: + - label: "Simple · Model Executor Test" + timeout_in_minutes: 20 + commands: + - "pytest -sv tests/model_executor -m 'core_model and cpu'" + mirror_hardwares: l4_1 + + - label: "Simple · CosyVoice3 Packed Flow Test" + timeout_in_minutes: 15 + source_file_dependencies: + - tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py + - vllm_omni/model_executor/models/cosyvoice3/ + commands: + - "pytest -sv tests/model_executor/models/cosyvoice3/test_cosyvoice3_full_batch.py -k packed -m 'core_model and cuda' --run-level=core_model" + mirror_hardwares: h100_1 + + - label: "Simple · Diffusion Test" + timeout_in_minutes: 30 + commands: + - "pytest -sv tests/diffusion -m 'core_model and cpu'" + mirror_hardwares: l4_1 + + - label: "Simple · Engine&Entrypoints Test" + timeout_in_minutes: 20 + commands: + - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'" + mirror_hardwares: l4_1 + + - label: "Simple · Other Test" + timeout_in_minutes: 20 + commands: + - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine" + mirror_hardwares: l4_1 + + - group: ":card_index_dividers: Diffusion Test" + depends_on: upload-ready-pipeline + steps: + + - label: "Diffusion · FlashAttention Test" + timeout_in_minutes: 20 + commands: + - timeout 15m pytest -s -v tests/diffusion/attention/ -m 'core_model and cuda and L4 and cards_1' --run-level "core_model" + mirror_hardwares: l4_1 + + - label: "Diffusion · Offloader Test" + commands: + - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model" + mirror_hardwares: l4_1 + + - label: "Diffusion · Cache Test" + commands: + - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model" + mirror_hardwares: l4_1 + + - label: "Diffusion · Batch Test" + commands: + - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model" + mirror_hardwares: l4_1 + + - label: "Diffusion · Distributed Test · Single-GPU" + source_file_dependencies: diffusion_distributed_attention + commands: + - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_1' --run-level "core_model" + mirror_hardwares: l4_1 + + - label: "Diffusion · Distributed Test · 2-GPU" + source_file_dependencies: diffusion_distributed_attention + commands: + - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_2' --run-level "core_model" + - timeout 10m pytest -sv tests/diffusion/attention/test_ulysses_uaa.py -m 'core_model and cuda and L4 and cards_2' --run-level "core_model" + mirror_hardwares: l4_2 + + - label: "Diffusion · Distributed & Attention Test · 4-GPU" + source_file_dependencies: diffusion_distributed_attention + commands: + - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda and L4 and cards_4' --run-level "core_model" + - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda and L4 and cards_4' --run-level "core_model" + mirror_hardwares: l4_4 + + - label: "Diffusion · Model Test" + commands: + - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model" + mirror_hardwares: l4_1 + + - label: "Diffusion · Other Test" + commands: + - timeout 15m pytest -sv tests/diffusion/*.py tests/diffusion/ar_diffusion -m 'core_model and cuda' --run-level "core_model" + mirror_hardwares: l4_1 + + - group: ":card_index_dividers: Tiny Model Tests [Single GPU]" + depends_on: upload-ready-pipeline + steps: + + - label: "Tiny Model Tests · Diffusion (base)" + commands: + - timeout 15m pytest -sv -n 4 tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model" + mirror_hardwares: l4_1 + + - label: "Engine&Model Executor Test" + depends_on: upload-ready-pipeline + commands: + - | + timeout 15m bash -c " + pytest -sv tests/engine/ tests/model_executor/ tests/worker/test_batched_omni_output.py tests/worker_v2/ -m 'core_model and cuda' --run-level 'core_model' + " + mirror_hardwares: l4_1 + + - label: "Distributed Test" + depends_on: upload-ready-pipeline + commands: + - | + timeout 20m bash -c " + pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model' + " + mirror_hardwares: l4_1 + + - label: "Entrypoints Test" + depends_on: upload-ready-pipeline + # Workaround https://github.com/vllm-project/vllm-omni/issues/8016: shutdown() + # leaves inline UniProc DiffusionWorker VRAM mapped in the pytest parent. + # Isolate that leftover case in its own process so the driver drops the + # mapping; the rest stays a directory sweep with --deselect. Revert to a + # single sweep after the product shutdown fix. + commands: + - | + timeout 30m bash -c " + set -eu + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -sv tests/entrypoints/test_async_omni.py::test_diffusion_generate_request_id -m 'core_model and cuda' --run-level 'core_model' + pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model' --deselect tests/entrypoints/test_async_omni.py::test_diffusion_generate_request_id + " + mirror_hardwares: l4_1 + + - group: ":card_index_dividers: E2E Test" + depends_on: upload-ready-pipeline + steps: + - label: "Custom Pipeline Test" + source_file_dependencies: custom_pipeline + commands: + - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model" + mirror_hardwares: l4_1 + + + - label: "RLHF · VeRL-Omni E2E Test" + source_file_dependencies: rlhf_verl_omni + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py + " + mirror_hardwares: l4_1 + + - label: "Omni · Qwen3-Omni Test" + source_file_dependencies: omni_qwen3_omni_function + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model' + " + mirror_hardwares: h100_2 + + - label: "Omni · MiniCPM-o 4.5 Offline Test" + source_file_dependencies: omni_minicpmo_4_5_function + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + " + mirror_hardwares: h100_1 + + - label: "Omni · MiniCPM-o 4.5 Online Test" + source_file_dependencies: omni_minicpmo_4_5_function + commands: + - | + timeout 20m bash -c " + pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + " + mirror_hardwares: h100_1 + + - label: "Omni · MiniCPM-o 4.5 Duplex Test" + source_file_dependencies: omni_minicpmo_4_5_duplex_function + commands: + - | + timeout 35m bash -c " + pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py tests/e2e/online_serving/test_duplex_client_live.py -m 'core_model and cuda' --run-level 'core_model' + " + mirror_hardwares: h100_1 + + - label: "Omni · MiniCPM-o 4.5 Perf Test" + timeout_in_minutes: 60 + source_file_dependencies: omni_minicpmo_4_5_perf + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda' + mirror_hardwares: h100_1 + + - label: "TTS · Higgs v3 Full Response Test" + source_file_dependencies: + - tests/e2e/online_serving/test_higgs_audio_v3.py + - vllm_omni/model_executor/models/higgs_audio_v3/ + - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py + - vllm_omni/deploy/higgs_multimodal_qwen3.yaml + commands: + - timeout 20m pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py::test_full_response_v1_preserves_audio -m 'core_model and cuda' --run-level 'core_model' + mirror_hardwares: h100_1 + + - label: "TTS · Qwen3-TTS CustomVoice Test" + source_file_dependencies: tts_qwen3_tts_function + commands: + - | + timeout 20m bash -c " + export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 + pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model' + " + mirror_hardwares: l4_1 + + - label: "Diffusion · Wan22 Test" + source_file_dependencies: diffusion_wan22_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model and cuda' --run-level 'core_model' + " + mirror_hardwares: h100_1 + + - label: "Diffusion · Cosmos3 Test" + source_file_dependencies: diffusion_cosmos3_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model' + " + mirror_hardwares: h100_1 + + - label: "Diffusion · MiniMax-H3 DLO DP2 T2VA Test" + timeout_in_minutes: 90 + source_file_dependencies: diffusion_minimax_h3_function + commands: + - | + timeout 80m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py -m 'core_model and cuda and diffusion' --run-level 'core_model' + " + mirror_hardwares: h100_2 + + - label: "Diffusion · Qwen Image Test" + source_file_dependencies: diffusion_qwen_image_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model' + " + mirror_hardwares: h100_1 + + - label: "Diffusion · HunyuanVideo-1.5 Test" + source_file_dependencies: diffusion_hunyuan_video_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model' + " + mirror_hardwares: h100_1 + + - label: "Diffusion · LingBot-Video Test" + source_file_dependencies: diffusion_lingbot_function + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_lingbot_video.py -m 'core_model and cuda' --run-level 'core_model' + " + mirror_hardwares: h100_1 diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/scripts/run_benchmark.py b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/scripts/run_benchmark.py new file mode 100644 index 000000000..67689e136 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/scripts/run_benchmark.py @@ -0,0 +1,669 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import json +import math +import os +import threading +from collections.abc import Callable +from contextlib import AbstractContextManager, ExitStack, contextmanager +from pathlib import Path +from typing import Any + +import pytest + +from tests.dfx.conftest import ( + create_paired_omni_benchmark_pytest_params, + create_test_parameter_mapping, + get_benchmark_params_for_server, + get_runtime_resource_label, + is_diffusion_perf_config, + load_benchmark_configs, + run_benchmark, +) +from tests.helpers.runtime import OmniServer + +# Optional JSON field ``mark`` is applied as pytest marks via +# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks": +# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``). + + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + + +def _get_config_file_from_argv() -> str | None: + """Read ``--test-config-file`` from ``sys.argv`` at import time so parametrization can use it.""" + import sys + + for i, arg in enumerate(sys.argv): + if arg == "--test-config-file" and i + 1 < len(sys.argv): + return sys.argv[i + 1] + if arg.startswith("--test-config-file="): + return arg.split("=", 1)[1] + return None + + +_PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests" + +CONFIG_FILE_PATH = _get_config_file_from_argv() +if CONFIG_FILE_PATH is None: + _all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR) + BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)] + print( + f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s) from " + f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; " + f"use -m to filter, e.g. -m tts)" + ) +else: + _loaded = load_benchmark_configs(CONFIG_FILE_PATH) + BENCHMARK_CONFIGS = [cfg for cfg in _loaded if not is_diffusion_perf_config(cfg)] + skipped = len(_loaded) - len(BENCHMARK_CONFIGS) + if skipped: + print( + f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s); " + f"skipped {skipped} remaining diffusion case(s) (chat completions / custom jsonl)" + ) + +DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy" +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) +paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) + +_omni_server_lock = threading.Lock() + + +class _SingleActiveContext: + """Reuse one active context while its configuration key is unchanged.""" + + def __init__(self) -> None: + self._key: Any = None + self._stack: ExitStack | None = None + self._value: Any = None + + def acquire(self, key: Any, factory: Callable[[], AbstractContextManager[Any]]) -> Any: + if self._stack is not None and key == self._key: + return self._value + + self.close() + stack = ExitStack() + value = stack.enter_context(factory()) + self._key = key + self._stack = stack + self._value = value + return value + + def close(self) -> None: + stack = self._stack + self._key = None + self._stack = None + self._value = None + if stack is not None: + stack.close() + + +def _config_for_test(test_name: str) -> dict[str, object] | None: + for config in BENCHMARK_CONFIGS: + if isinstance(config, dict) and config.get("test_name") == test_name: + return config + return None + + +def _judge_server_params_for_test(test_name: str) -> dict[str, object] | None: + config = _config_for_test(test_name) + if config is None: + return None + raw = config.get("judge_server_params") + if raw is None: + return None + if not isinstance(raw, dict): + raise TypeError(f"judge_server_params for {test_name} must be an object") + return raw + + +def _resolve_judge_cuda_visible_devices(configured: object) -> str: + """Map a judge card index onto the process-visible CUDA device list. + + ``judge_server_params.cuda_visible_devices`` is an index into + ``CUDA_VISIBLE_DEVICES`` when that env is set (``1`` with + ``CUDA_VISIBLE_DEVICES=2,3`` selects physical GPU 3). Without the env it + is used as a raw device list. + """ + if not isinstance(configured, str) or not configured: + raise ValueError("judge_server_params.cuda_visible_devices must be a non-empty string") + parent = os.environ.get("CUDA_VISIBLE_DEVICES") + if parent is None or not parent.strip(): + return configured + visible = [part.strip() for part in parent.split(",") if part.strip()] + if configured.isdigit(): + index = int(configured) + if index < 0 or index >= len(visible): + raise ValueError( + f"judge_server_params.cuda_visible_devices={configured!r} is outside CUDA_VISIBLE_DEVICES={parent!r}" + ) + return visible[index] + return configured + + +@contextmanager +def _start_judge_server(judge_params: dict[str, object]): + model = judge_params.get("model") + if not isinstance(model, str) or not model: + raise ValueError("judge_server_params.model must be a non-empty string") + extra = judge_params.get("extra_cli_args") or () + if not isinstance(extra, list | tuple): + raise TypeError("judge_server_params.extra_cli_args must be a list") + visible = _resolve_judge_cuda_visible_devices(judge_params.get("cuda_visible_devices", "1")) + print(f"Starting OmniInteract judge with model: {model} on CUDA_VISIBLE_DEVICES={visible}") + with OmniServer( + model, + [str(item) for item in extra], + use_omni=bool(judge_params.get("use_omni", False)), + env_dict={"CUDA_VISIBLE_DEVICES": visible}, + ) as judge: + print(f"OmniInteract judge started on {judge.host}:{judge.port}") + yield judge + print("OmniInteract judge stopping...") + print("OmniInteract judge stopped") + + +# OmniServer defaults for flags not already present in JSON ``serve_args`` / +# ``extra_cli_args``. Add new (flag, value) pairs here rather than special-casing. +_OMNI_DEFAULT_SERVER_ARGS: tuple[tuple[str, str], ...] = ( + ("--stage-init-timeout", "600"), + ("--init-timeout", "900"), +) + + +def _cli_flag_names(cli_args: tuple[str, ...] | list[str]) -> set[str]: + """Return long-option names present in a flat CLI argv list.""" + names: set[str] = set() + for item in cli_args: + token = str(item) + if not token.startswith("--"): + continue + names.add(token.split("=", 1)[0]) + return names + + +def _merge_omni_default_server_args( + extra_cli_args: tuple[str, ...] | list[str], + *, + use_omni: bool, + defaults: tuple[tuple[str, str], ...] = _OMNI_DEFAULT_SERVER_ARGS, +) -> list[str]: + """Fill Omni defaults for flags not already set in JSON-derived CLI args. + + JSON ``serve_args`` / ``extra_cli_args`` win; only missing flags are appended. + """ + if not use_omni: + return [] + present = _cli_flag_names(extra_cli_args) + args: list[str] = [] + for flag, value in defaults: + if flag not in present: + args += [flag, value] + return args + + +def _omni_server_env() -> dict[str, str]: + """Writable video/image storage for ``/v1/videos`` and related generation APIs.""" + result_dir = Path(os.environ.get("BENCHMARK_DIR", "tests/dfx/perf/results")) + storage_path = Path(os.environ.get("VLLM_OMNI_STORAGE_PATH", str(result_dir / "storage"))) + storage_path.mkdir(parents=True, exist_ok=True) + return {"VLLM_OMNI_STORAGE_PATH": str(storage_path)} + + +def _resolve_offline_model(model: str) -> str: + """Resolve HF ids / MiniMax env overrides the same way as the diffusion runner.""" + import huggingface_hub + + from vllm_omni.transformers_utils.repo_utils import hf_api + + if not model or os.path.isdir(model): + return model + + model_env_overrides = { + "MiniMaxAI/MiniMax-H3": "VLLM_TEST_MINIMAX_H3_MODEL", + "MiniMaxAI/MiniMax-H3/FL2VA": "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL", + "MiniMaxAI/MiniMax-H3/Ref2VA": "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL", + } + env_name = model_env_overrides.get(model) + if env_name: + env_model = os.environ.get(env_name) + if env_model: + return env_model + + parts = model.split("/") + if len(parts) >= 3: + repo_id = "/".join(parts[:2]) + subfolder = "/".join(parts[2:]) + snapshot_root = hf_api().snapshot_download( + repo_id, + allow_patterns=[f"{subfolder}/**"], + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ) + return str(Path(snapshot_root) / subfolder) + + if not huggingface_hub.constants.HF_HUB_OFFLINE: + return model + return hf_api().snapshot_download(model, local_files_only=True) + + +@contextmanager +def _start_omni_server(server_param): + test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param + extra = tuple(extra_cli_args or ()) + model = _resolve_offline_model(model) + + print(f"Starting OmniServer with test: {test_name}, model: {model}") + + server_args: list[str] = _merge_omni_default_server_args(extra, use_omni=use_omni) + # --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py): + # deploy-config sets the base; stage-overrides are applied on top. Both can be set. + if stage_config_path: + server_args = ["--deploy-config", stage_config_path] + server_args + if stage_overrides: + server_args = ["--stage-overrides", stage_overrides] + server_args + if extra: + server_args = list(extra) + server_args + judge_params = _judge_server_params_for_test(test_name) + with OmniServer(model, server_args, use_omni=use_omni, env_dict=_omni_server_env()) as server: + server.test_name = test_name + server.judge_base_url = None + print("OmniServer started successfully") + if judge_params is None: + yield server + else: + with _start_judge_server(judge_params) as judge: + server.judge_base_url = f"http://{judge.host}:{judge.port}" + yield server + print("OmniServer stopping...") + + print("OmniServer stopped") + + +@pytest.fixture(scope="module") +def omni_server_context(): + """Start vLLM-Omni server as a subprocess with actual model weights. + Reuse it for adjacent benchmark cases with the same server configuration. + Multi-stage initialization can take 10-20+ minutes. + """ + with _omni_server_lock: + active_context = _SingleActiveContext() + try: + yield active_context + finally: + active_context.close() + + +@pytest.fixture +def omni_server(request, omni_server_context): + return omni_server_context.acquire(request.param, lambda: _start_omni_server(request.param)) + + +@pytest.fixture +def benchmark_params(request): + """Benchmark parameters fixture; paired with ``omni_server`` via parametrization.""" + test_name, param_index = request.param + + all_params = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping) + + if not all_params: + raise ValueError(f"No benchmark parameters found for test: {test_name}") + + if param_index >= len(all_params): + raise ValueError(f"No benchmark parameters found for index {param_index} in test: {test_name}") + + current = param_index + 1 + total = len(all_params) + print(f"\n Running benchmark {current}/{total} for {test_name}") + + return { + "test_name": test_name, + "params": all_params[param_index], + } + + +def _resolve_num_warmups(params: dict[str, Any], *, default: int) -> int: + value = params.get("num_warmups") + if value is None: + return default + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError("num_warmups must be a non-negative integer") + return value + + +def _assert_fixed_stage_workloads(result: dict[str, Any], params: dict[str, Any], num_prompts: int) -> None: + """Reject successful responses that did less work than a fixed-length benchmark requested.""" + extra_body = params.get("extra_body") or params.get("extra-body") or {} + fixed_lengths = { + str(stage): sampling["max_tokens"] + for stage, sampling in enumerate(extra_body.get("sampling_params_list", [])) + if isinstance(sampling, dict) + and isinstance(sampling.get("max_tokens"), int) + and sampling["max_tokens"] > 0 + and sampling.get("min_tokens") == sampling["max_tokens"] + } + if not fixed_lengths: + return + + snapshots = result.get("request_stage_metrics") + assert isinstance(snapshots, list) and len(snapshots) == num_prompts, ( + "Fixed stage workload: missing per-request stage metrics" + ) + for request_index, snapshot in enumerate(snapshots): + for stage, expected in fixed_lengths.items(): + metrics = snapshot.get(stage) if isinstance(snapshot, dict) else None + actual = metrics.get("num_tokens_out") if isinstance(metrics, dict) else None + assert actual == expected, ( + f"Fixed stage workload: request {request_index}, stage {stage}: " + f"expected {expected} output tokens, got {actual!r}" + ) + stages = ", ".join(f"stage {stage}={expected}" for stage, expected in fixed_lengths.items()) + print(f"Fixed stage workload OK: {num_prompts} requests, num_tokens_out {stages}") + + +def _is_finite_number(value: object) -> bool: + return isinstance(value, int | float) and not isinstance(value, bool) and math.isfinite(value) + + +_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1 = "omniinteract_aggregate_min_ia_qtf1" +_OMNIINTERACT_AGGREGATE_SUBSETS = "omniinteract_aggregate_subsets" +_OMNIINTERACT_AGGREGATE_GROUP = "omniinteract_aggregate_group" +_OMNIINTERACT_AGGREGATE_COUNTS: dict[tuple[str, str], tuple[float, float, float]] = {} +_OMNIINTERACT_MIN_EVALUATED_CASES = 1 + + +def _reset_omniinteract_aggregate_counts() -> None: + _OMNIINTERACT_AGGREGATE_COUNTS.clear() + + +def _ia_qtf1_from_counts(tp: float, fp: float, fn: float) -> float: + precision = tp / (tp + fp) if tp + fp > 0 else 0.0 + recall = tp / (tp + fn) if tp + fn > 0 else 0.0 + return (2.0 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0 + + +def _assert_subset_evaluated(accuracy: dict[str, object]) -> None: + """Reject a subset whose clipped or cancelled cases left nothing to score. + + Individual ineligible cases may be skipped. A subset with fewer than + ``_OMNIINTERACT_MIN_EVALUATED_CASES`` evaluated cases must not pass, or its + zero TP/FP/FN would be pooled as a successful accuracy result. + """ + + evaluated = accuracy.get("evaluated") + if not isinstance(evaluated, int) or isinstance(evaluated, bool): + raise AssertionError("OmniInteract accuracy evaluated count is missing") + if evaluated < _OMNIINTERACT_MIN_EVALUATED_CASES: + raise AssertionError( + f"OmniInteract subset evaluated {evaluated} cases (skipped={accuracy.get('skipped')}); " + f"at least {_OMNIINTERACT_MIN_EVALUATED_CASES} evaluated case is required" + ) + + +def _finite_accuracy_count(value: object, name: str) -> float: + assert _is_finite_number(value), f"OmniInteract accuracy summary {name} is missing" + return float(value) + + +def _aggregate_accuracy_params_for_test(test_name: str) -> dict[str, object]: + config = _config_for_test(test_name) + if config is None: + return {} + min_ia_qtf1 = config.get(_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1) + if min_ia_qtf1 is None: + return {} + if not _is_finite_number(min_ia_qtf1): + raise ValueError("omniinteract_aggregate_min_ia_qtf1 must be a finite number") + subsets: list[str] = [] + raw_params = config.get("benchmark_params") or [] + if not isinstance(raw_params, list): + raise TypeError(f"benchmark_params for {test_name} must be a list") + for item in raw_params: + if not isinstance(item, dict) or not item.get("omniinteract_evaluate"): + continue + subset = item.get("omniinteract_subsets") + if not isinstance(subset, str) or not subset: + raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_subsets on each evaluated case") + subsets.append(subset) + if not subsets: + raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires at least one evaluated OmniInteract subset") + if len(set(subsets)) != len(subsets): + raise ValueError("omniinteract_aggregate_subsets must not contain duplicates") + return { + _OMNIINTERACT_AGGREGATE_MIN_IA_QTF1: float(min_ia_qtf1), + _OMNIINTERACT_AGGREGATE_SUBSETS: subsets, + _OMNIINTERACT_AGGREGATE_GROUP: test_name, + } + + +def _maybe_assert_aggregate_ia_qtf1(params: dict[str, object], acc_summary: dict[str, object]) -> None: + min_ia_qtf1 = params.get(_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1) + if min_ia_qtf1 is None: + return + assert _is_finite_number(min_ia_qtf1), "omniinteract_aggregate_min_ia_qtf1 must be a finite number" + subsets = params.get(_OMNIINTERACT_AGGREGATE_SUBSETS) + if not isinstance(subsets, list | tuple) or not subsets: + raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_aggregate_subsets") + if len(set(subsets)) != len(subsets): + raise ValueError("omniinteract_aggregate_subsets must not contain duplicates") + if not all(isinstance(item, str) and item for item in subsets): + raise ValueError("omniinteract_aggregate_subsets must be non-empty strings") + subset = params.get("omniinteract_subsets") + if not isinstance(subset, str) or not subset: + raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_subsets") + if subset not in subsets: + raise ValueError(f"omniinteract_subsets {subset!r} is not in omniinteract_aggregate_subsets") + group = params.get(_OMNIINTERACT_AGGREGATE_GROUP) + if not isinstance(group, str) or not group: + raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_aggregate_group") + tp = _finite_accuracy_count(acc_summary.get("Global_TP"), "Global_TP") + fp = _finite_accuracy_count(acc_summary.get("Global_FP"), "Global_FP") + fn = _finite_accuracy_count(acc_summary.get("Global_FN"), "Global_FN") + _OMNIINTERACT_AGGREGATE_COUNTS[(group, subset)] = (tp, fp, fn) + recorded = [_OMNIINTERACT_AGGREGATE_COUNTS.get((group, name)) for name in subsets] + present = [counts for counts in recorded if counts is not None] + if len(present) != len(subsets): + return + total_tp = sum(counts[0] for counts in present) + total_fp = sum(counts[1] for counts in present) + total_fn = sum(counts[2] for counts in present) + ia_qtf1 = _ia_qtf1_from_counts(total_tp, total_fp, total_fn) + print( + f"OmniInteract aggregate All Global IA-QTF1: {ia_qtf1:.6f} " + f"(TP={total_tp:.6f} / FP={total_fp:.6f} / FN={total_fn:.6f})" + ) + assert ia_qtf1 >= float(min_ia_qtf1), f"OmniInteract aggregate All Global IA-QTF1 {ia_qtf1} is below {min_ia_qtf1}" + + +def assert_result(result, params, num_prompt) -> None: + assert result["completed"] == num_prompt, "Request failures exist" + _assert_fixed_stage_workloads(result, params, num_prompt) + if params.get("dataset_name") == "omniinteract": + summary = result.get("omniinteract") + assert isinstance(summary, dict), "OmniInteract summary is missing" + assert (summary.get("total"), summary.get("success"), summary.get("failed")) == ( + num_prompt, + num_prompt, + 0, + ), "OmniInteract requests did not all succeed" + assert summary.get("artifacts_complete") is True, "OmniInteract artifacts are incomplete" + if params.get("omniinteract_evaluate"): + accuracy = summary.get("accuracy") + assert isinstance(accuracy, dict), "OmniInteract accuracy is missing" + assert accuracy.get("status") == "ok", "OmniInteract accuracy did not complete" + assert accuracy.get("failed") == 0, "OmniInteract accuracy reported failed cases" + _assert_subset_evaluated(accuracy) + acc_summary = accuracy.get("summary") + assert isinstance(acc_summary, dict), "OmniInteract accuracy summary is missing" + ia_qtf1 = acc_summary.get("IA_QTF1") + assert _is_finite_number(ia_qtf1), "OmniInteract All Global IA-QTF1 is missing" + min_ia_qtf1 = params.get("omniinteract_min_ia_qtf1") + if min_ia_qtf1 is not None: + assert _is_finite_number(min_ia_qtf1), "omniinteract_min_ia_qtf1 must be a finite number" + assert float(ia_qtf1) >= float(min_ia_qtf1), ( + f"OmniInteract All Global IA-QTF1 {ia_qtf1} is below {min_ia_qtf1}" + ) + _maybe_assert_aggregate_ia_qtf1(params, acc_summary) + baseline = params.get("baseline") + hardware = result.get("Hardware") + hardware_baseline = baseline.get(hardware) if isinstance(baseline, dict) and isinstance(hardware, str) else None + if isinstance(hardware_baseline, dict) and "mean_tpot_ms" in hardware_baseline: + num_tpot_samples = result.get("num_tpot_samples") + mean_tpot_ms = result.get("mean_tpot_ms") + assert isinstance(num_tpot_samples, int) and not isinstance(num_tpot_samples, bool) and num_tpot_samples > 0, ( + "TPOT baseline is configured, but no measurable TPOT samples were produced" + ) + assert ( + isinstance(mean_tpot_ms, int | float) and not isinstance(mean_tpot_ms, bool) and math.isfinite(mean_tpot_ms) + ), "TPOT baseline is configured, but mean_tpot_ms is not finite" + expected_audio_turns = params.get("expected_duplex_audio_turns_per_session") + if expected_audio_turns is not None: + session_metrics = result.get("duplex_session_metrics") + assert isinstance(session_metrics, list), "Duplex session metrics are missing" + assert len(session_metrics) == num_prompt, ( + f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}" + ) + assert all( + isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns + for metric in session_metrics + ), f"Not every duplex session emitted {expected_audio_turns} audio turns" + + +@pytest.mark.benchmark +@pytest.mark.parametrize( + "omni_server,benchmark_params", + paired_benchmark_params, + indirect=["omni_server", "benchmark_params"], +) +def test_performance_benchmark(omni_server, benchmark_params): + test_name = benchmark_params["test_name"] + params = dict(benchmark_params["params"]) + dataset_name = params.get("dataset_name", "") + if params.get("omniinteract_evaluate"): + judge_base_url = getattr(omni_server, "judge_base_url", None) + if not isinstance(judge_base_url, str) or not judge_base_url: + raise ValueError("omniinteract_evaluate requires top-level judge_server_params to start a judge") + params["omniinteract_judge_base_url"] = judge_base_url + + host = omni_server.host + port = omni_server.port + model = omni_server.model + + print(f"Running benchmark for model: {model}") + print(f"Benchmark parameters: {benchmark_params}") + + resource_label = get_runtime_resource_label() + + def to_list(value, default=None): + if value is None: + return [] if default is None else [default] + return [value] if not isinstance(value, (list, tuple)) else list(value) + + qps_list = to_list(params.get("request_rate")) + num_prompt_list = to_list(params.get("num_prompts")) + max_concurrency_list = to_list(params.get("max_concurrency")) + + max_len = max(len(qps_list), len(max_concurrency_list)) + if len(num_prompt_list) == 1 and max_len > 1: + num_prompt_list = num_prompt_list * max_len + elif max_len == 1 and len(num_prompt_list) > 1: + if len(qps_list) == 1: + qps_list = qps_list * len(num_prompt_list) + if len(max_concurrency_list) == 1: + max_concurrency_list = max_concurrency_list * len(num_prompt_list) + max_len = max(len(qps_list), len(max_concurrency_list)) + elif len(num_prompt_list) != max_len and max_len > 0: + raise ValueError("The number of prompts does not match the QPS or max_concurrency") + + args = ["--host", host, "--port", str(port)] + exclude_keys = { + "request_rate", + "baseline", + "num_prompts", + "max_concurrency", + "num_warmups", + "task", + "enabled", + "eval_phase", + "trust_remote_code", + "expected_duplex_audio_turns_per_session", + "omniinteract_min_ia_qtf1", + _OMNIINTERACT_AGGREGATE_MIN_IA_QTF1, + _OMNIINTERACT_AGGREGATE_SUBSETS, + _OMNIINTERACT_AGGREGATE_GROUP, + "judge_server_params", + "name", + "enable_negative_prompt", + "random_request_config", + "num_input_images", + "warmup_requests", + "warmup_concurrency", + "warmup_num_inference_steps", + } + + param_keys = {str(key).replace("-", "_") for key in params} + if "model" not in param_keys: + args.extend(["--model", str(model)]) + + for key, value in params.items(): + if key in exclude_keys or value is None: + continue + if key in {"extra_body", "extra-body"} and value == {}: + continue + + arg_name = f"--{key.replace('_', '-')}" + + if isinstance(value, bool) and value: + args.append(arg_name) + elif isinstance(value, dict): + json_str = json.dumps(value, ensure_ascii=False, separators=(",", ":")) + args.extend([arg_name, json_str]) + elif not isinstance(value, bool): + args.extend([arg_name, str(value)]) + + for config in BENCHMARK_CONFIGS: + if config.get("test_name") != test_name: + continue + server_params = config.get("server_params") or {} + if server_params.get("trust_remote_code") or params.get("trust_remote_code"): + args.append("--trust-remote-code") + break + + # QPS / request-rate sweep + for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)): + args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)] + result = run_benchmark( + args=args, + test_name=test_name, + flow=qps, + dataset_name=dataset_name, + num_prompt=num_prompt, + baseline_config=params.get("baseline"), + sweep_index=sweep_index, + random_input_len=params.get("random_input_len"), + random_output_len=params.get("random_output_len"), + resource_label=resource_label, + num_warmups=_resolve_num_warmups(params, default=2), + benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None, + ) + assert_result(result, {**params, **_aggregate_accuracy_params_for_test(test_name)}, num_prompt) + + # concurrency test + for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)): + args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"] + result = run_benchmark( + args=args, + test_name=test_name, + flow=concurrency, + dataset_name=dataset_name, + num_prompt=num_prompt, + baseline_config=params.get("baseline"), + sweep_index=sweep_index, + random_input_len=params.get("random_input_len"), + random_output_len=params.get("random_output_len"), + resource_label=resource_label, + num_warmups=_resolve_num_warmups(params, default=max(2, int(concurrency))), + benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None, + ) + assert_result(result, {**params, **_aggregate_accuracy_params_for_test(test_name)}, num_prompt) diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/scripts/run_diffusion_benchmark.py new file mode 100644 index 000000000..0523b9b3a --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -0,0 +1,1055 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +Performance benchmark CI runner for diffusion models. + +This runner separates two concepts: + +1. ``server_type``: how the serving process is started. + Currently only ``vllm-omni`` is supported here. +2. ``benchmark_endpoint``: which serving API the benchmark client calls. + Examples: ``/v1/chat/completions`` and ``/v1/videos``. + +A config JSON file may be passed via --test-config-file. If omitted, every ``*.json`` under +``tests/dfx/perf/tests/`` is loaded and pytest ``-m`` filters by each case's ``mark``: + pytest run_diffusion_benchmark.py -m "diffusion" + pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json + +Optional JSON field ``mark`` is applied as pytest marks on that case via +``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``). + +All benchmark results are written under BENCHMARK_RESULT_DIR (override via the +DIFFUSION_BENCHMARK_DIR environment variable). Each source JSON file gets one +aggregated ``diffusion_result_{config_stem}_{hardware}_{timestamp}.json`` (JSON array +of all runs from cases in that file). Bulk load without ``--test-config-file`` uses +the same per-file aggregation; ``-m`` only selects which cases run. +""" + +import json +import os +import socket +import subprocess +import sys +import tempfile +import threading +import time +from datetime import datetime +from pathlib import Path +from typing import Any, cast + +import psutil +import pytest + +from benchmarks.diffusion.backends import endpoint_filename_token, normalize_endpoint +from tests.dfx.conftest import ( + create_paired_benchmark_pytest_params, + get_runtime_resource_label, + hardware_json_value, + is_diffusion_perf_config, + resolve_baseline_for_sweep, + resolve_pytest_marks, + resource_label_for_filename, +) +from tests.helpers.runtime import get_open_port + + +# --------------------------------------------------------------------------- +# Inline field processing +# --------------------------------------------------------------------------- +def _process_inline_fields(obj: Any, parent_key: str = "") -> None: + """Recursively process '*-inline' fields into temp files.""" + + if isinstance(obj, list): + for item in obj: + _process_inline_fields(item, parent_key) + return + + if not isinstance(obj, dict): + return + + import atexit + + import yaml + + for key in list(obj.keys()): + value = obj[key] + + if not key.endswith("-inline"): + _process_inline_fields(value, key) + continue + + base_key = key[:-7] + full_key = f"{parent_key}.{key}" if parent_key else key + + try: + if not isinstance(value, dict): + raise ValueError("must be a dict") + + file_type = value.get("type") + content = value.get("content") + + if file_type not in {"yaml", "jsonl"}: + raise ValueError(f"invalid type: {file_type}") + + fd, path = tempfile.mkstemp( + suffix=f".{file_type}", + prefix=f"{base_key}_", + ) + + atexit.register(Path(path).unlink, missing_ok=True) + + with os.fdopen(fd, "w", encoding="utf-8") as f: + if file_type == "jsonl": + items = content if isinstance(content, list) else [content] + f.writelines(json.dumps(x, ensure_ascii=False) + "\n" for x in items) + else: + yaml.dump( + content, + f, + allow_unicode=True, + sort_keys=False, + indent=2, + ) + + obj[base_key] = path + del obj[key] + + except Exception as e: + print(f"Warning: failed processing '{full_key}': {e}") + + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- + +_DEFAULT_RESULT_DIR = Path(__file__).parent.parent / "results" +BENCHMARK_RESULT_DIR = Path(os.environ.get("DIFFUSION_BENCHMARK_DIR", str(_DEFAULT_RESULT_DIR))) + +BENCHMARK_SCRIPT = str( + Path(__file__).parent.parent.parent.parent.parent / "benchmarks" / "diffusion" / "diffusion_benchmark_serving.py" +) + +# Single aggregated result file for the entire benchmark session. +# Populated lazily after CONFIG_FILE_PATH is resolved. +_SESSION_TIMESTAMP = datetime.now().strftime("%Y%m%d-%H%M%S") +_RESULT_LOCK = threading.Lock() +_BRANCHPOINT_COMMIT_SHA: str | None = None +DIFFUSION_RESULT_TEMPLATE_PATH = Path(__file__).parent / "diffusion_result_template.json" + + +_DIFFUSION_SOURCE_CONFIG_KEY = "_source_config_file" +_PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests" + + +def _get_config_file_from_argv() -> str | None: + """Read --test-config-file from sys.argv at import time so pytest parametrize can use it. + + pytest_addoption (below) registers the same flag so pytest does not reject it. + Supports both ``--test-config-file path`` and ``--test-config-file=path`` forms. + Returns None if the flag is not present; callers must handle the missing case. + """ + for i, arg in enumerate(sys.argv): + if arg == "--test-config-file" and i + 1 < len(sys.argv): + return sys.argv[i + 1] + if arg.startswith("--test-config-file="): + return arg.split("=", 1)[1] + return None + + +CONFIG_FILE_PATH = _get_config_file_from_argv() + +# --------------------------------------------------------------------------- +# Config loading +# --------------------------------------------------------------------------- + + +def _resolve_refs(configs: list[dict[str, Any]], config_dir: Path) -> list[dict[str, Any]]: + """Resolve {"$ref": "filename.json"} in benchmark_params fields.""" + for cfg in configs: + bp = cfg.get("benchmark_params") + if isinstance(bp, dict) and "$ref" in bp: + ref_path = config_dir / bp["$ref"] + try: + with open(ref_path, encoding="utf-8") as f: + cfg["benchmark_params"] = json.load(f) + except FileNotFoundError: + raise ValueError(f"benchmark_params $ref not found: {ref_path}") + except json.JSONDecodeError as e: + raise ValueError(f"JSON parsing error in {ref_path}: {e}") + return configs + + +def load_configs(config_path: str) -> list[dict[str, Any]]: + """Load benchmark configs from JSON file and process inline fields.""" + try: + abs_path = Path(config_path).resolve() + with open(abs_path, encoding="utf-8") as f: + configs = json.load(f) + configs = _resolve_refs(configs, abs_path.parent) + _process_inline_fields(configs) + return configs + except json.JSONDecodeError as e: + raise ValueError(f"JSON parsing error: {str(e)}") + except FileNotFoundError: + raise ValueError(f"Configuration file not found: {config_path}") + except Exception as e: + raise RuntimeError(f"Failed to load configuration file: {str(e)}") + + +def load_diffusion_benchmark_configs( + config_path: str | None = None, + *, + config_dir: Path | None = None, +) -> list[dict[str, Any]]: + """Load one diffusion benchmark JSON, or merge all ``*.json`` under *config_dir*.""" + if config_path is not None: + loaded = load_configs(config_path) + source = str(Path(config_path).resolve()) + for cfg in loaded: + cfg.setdefault(_DIFFUSION_SOURCE_CONFIG_KEY, source) + return loaded + if config_dir is None: + raise ValueError("load_diffusion_benchmark_configs requires config_path or config_dir") + configs: list[dict[str, Any]] = [] + for path in sorted(config_dir.glob("*.json")): + source = str(path.resolve()) + for cfg in load_configs(str(path)): + cfg[_DIFFUSION_SOURCE_CONFIG_KEY] = source + configs.append(cfg) + if not configs: + raise ValueError(f"No benchmark JSON files found under {config_dir}") + return configs + + +if CONFIG_FILE_PATH is None: + _all_configs = load_diffusion_benchmark_configs(config_dir=_PERF_TESTS_DIR) + BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if is_diffusion_perf_config(cfg)] + print( + f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} diffusion case(s) from " + f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} omni/tts; " + f"use -m to filter, e.g. -m diffusion)" + ) +else: + _loaded = load_diffusion_benchmark_configs(CONFIG_FILE_PATH) + BENCHMARK_CONFIGS = [cfg for cfg in _loaded if is_diffusion_perf_config(cfg)] + skipped = len(_loaded) - len(BENCHMARK_CONFIGS) + if skipped: + print( + f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} diffusion case(s); " + f"skipped {skipped} omni-bench generation case(s) " + f"(/v1/images/edits, /v1/images/generations, /v1/videos → run_benchmark.py)" + ) + +_AGGREGATED_RESULT_FILES_BY_SOURCE: dict[str, Path] = {} + + +def _normalized_source_path(source_file: str) -> str: + return str(Path(source_file).resolve()) + + +def _aggregated_result_file_for_source(source_file: str) -> Path: + """One session aggregate per source JSON (same naming as single ``--test-config-file``).""" + key = _normalized_source_path(source_file) + if key not in _AGGREGATED_RESULT_FILES_BY_SOURCE: + stem = Path(key).stem + resource = resource_label_for_filename(get_runtime_resource_label()) + if resource: + result_name = f"diffusion_result_{stem}_{resource}_{_SESSION_TIMESTAMP}.json" + else: + result_name = f"diffusion_result_{stem}_{_SESSION_TIMESTAMP}.json" + _AGGREGATED_RESULT_FILES_BY_SOURCE[key] = BENCHMARK_RESULT_DIR / result_name + return _AGGREGATED_RESULT_FILES_BY_SOURCE[key] + + +def _write_result_record(record: dict[str, Any]) -> Path: + """Append one benchmark record to the aggregate file for its source JSON.""" + source_file = record.get("source_file") or CONFIG_FILE_PATH + if not source_file: + raise ValueError("benchmark record missing source_file") + target = _aggregated_result_file_for_source(str(source_file)) + with _RESULT_LOCK: + BENCHMARK_RESULT_DIR.mkdir(parents=True, exist_ok=True) + if target.exists(): + with open(target, encoding="utf-8") as f: + records: list[dict] = json.load(f) + else: + records = [] + records.append(record) + with open(target, "w", encoding="utf-8") as f: + json.dump(records, f, indent=2, ensure_ascii=False) + return target + + +_server_lock = threading.Lock() + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def _wait_for_port(host: str, port: int, timeout: int = 1200, proc: subprocess.Popen | None = None) -> None: + """Block until the given host:port accepts connections or timeout expires. + + If *proc* is provided, also monitors the process; raises RuntimeError + immediately if the server process exits before the port becomes available. + """ + start = time.time() + while time.time() - start < timeout: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(1) + if s.connect_ex((host, port)) == 0: + return + except Exception: + pass + if proc is not None: + ret = proc.poll() + if ret is not None: + raise RuntimeError(f"Server process exited with code {ret} before port {host}:{port} became ready") + time.sleep(2) + raise RuntimeError(f"Server did not start on {host}:{port} within {timeout}s") + + +def _kill_process_tree(pid: int) -> None: + try: + parent = psutil.Process(pid) + children = parent.children(recursive=True) + all_pids = [pid] + [c.pid for c in children] + + for child in children: + try: + child.terminate() + except psutil.NoSuchProcess: + pass + + gone, alive = psutil.wait_procs(children, timeout=10) + for child in alive: + try: + child.kill() + except psutil.NoSuchProcess: + pass + + try: + parent.terminate() + parent.wait(timeout=10) + except (psutil.NoSuchProcess, psutil.TimeoutExpired): + try: + parent.kill() + except psutil.NoSuchProcess: + pass + + time.sleep(1) + still_alive = [p for p in all_pids if psutil.pid_exists(p)] + if still_alive: + print(f"Warning: processes still alive after shutdown: {still_alive}") + for p in still_alive: + try: + subprocess.run(["kill", "-9", str(p)], timeout=2) + except Exception: + pass + except psutil.NoSuchProcess: + pass + + +# --------------------------------------------------------------------------- +# Server classes +# --------------------------------------------------------------------------- + + +def _resolve_offline_model(model: str) -> str: + """Resolve a model string to a local directory. + + Local paths pass through unchanged. HF repo ids are resolved to their local + snapshot dir when ``HF_HUB_OFFLINE`` is set. ``repo_id/subfolder`` (e.g. + ``MiniMaxAI/MiniMax-H3/FL2VA``) is resolved to the matching partition + subfolder of the snapshot, downloading only that subfolder when online. + """ + import huggingface_hub + + from vllm_omni.transformers_utils.repo_utils import hf_api + + if not model or os.path.isdir(model): + return model + + # MiniMax-H3 custom code uses relative imports across files, which breaks + # when the HF cache snapshot is symlinked (``get_class_from_dynamic_module`` + # resolves ``realpath`` into the blobs dir). Mirror the accuracy test's env + # overrides so CI / local runs can point at a materialized partition. The + # repo-root key (used with ``--task-type fl2va|ref2va``) expects a + # materialized root containing the FL2VA/Ref2VA subfolders. + model_env_overrides = { + "MiniMaxAI/MiniMax-H3": "VLLM_TEST_MINIMAX_H3_MODEL", + "MiniMaxAI/MiniMax-H3/FL2VA": "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL", + "MiniMaxAI/MiniMax-H3/Ref2VA": "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL", + } + env_name = model_env_overrides.get(model) + if env_name: + env_model = os.environ.get(env_name) + if env_model: + return env_model + + parts = model.split("/") + if len(parts) >= 3: + repo_id = "/".join(parts[:2]) + subfolder = "/".join(parts[2:]) + snapshot_root = hf_api().snapshot_download( + repo_id, + allow_patterns=[f"{subfolder}/**"], + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ) + return str(Path(snapshot_root) / subfolder) + + if not huggingface_hub.constants.HF_HUB_OFFLINE: + return model + return hf_api().snapshot_download(model, local_files_only=True) + + +class DiffusionServer: + """Start a vLLM-Omni diffusion model server as a subprocess. + + Launched via vllm_omni.entrypoints.cli.main with the diffusion-specific + parallelism flags (--usp, --ring, --cfg-parallel-size, etc.) passed directly + on the CLI. Minimum hardware: 4× NVIDIA H100 80 GB. + """ + + server_type = "vllm-omni" + + def __init__( + self, + server_cfg: dict[str, Any], + *, + port: int | None = None, + ) -> None: + self.server_cfg: dict[str, Any] = server_cfg + self.model = _resolve_offline_model(server_cfg["model"]) + self.serve_args = server_cfg["serve_args"] + self.host = "127.0.0.1" + self.port = port if port is not None else get_open_port(self.host) + self.proc: subprocess.Popen | None = None + self.test_name: str = "" + + def _start_server(self) -> None: + env = os.environ.copy() + env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + # Give each run its own writable video-storage dir. The server default + # (/tmp/storage) may be missing or not writable on shared hosts / CI. + storage_path = Path(os.environ.get("VLLM_OMNI_STORAGE_PATH", str(BENCHMARK_RESULT_DIR / "storage"))) + storage_path.mkdir(parents=True, exist_ok=True) + env["VLLM_OMNI_STORAGE_PATH"] = str(storage_path) + + cmd = [ + sys.executable, + "-m", + "vllm_omni.entrypoints.cli.main", + "serve", + self.model, + "--omni", + "--host", + self.host, + "--port", + str(self.port), + ] + self.serve_args + + print(f"Launching DiffusionServer: {' '.join(cmd)}") + self.proc = subprocess.Popen( + cmd, + env=env, + cwd=str(Path(__file__).parent.parent.parent.parent), + ) + _wait_for_port(self.host, self.port, proc=self.proc) + print(f"DiffusionServer ready on {self.host}:{self.port}") + + def __enter__(self): + self._start_server() + return self + + def __exit__(self, *_): + if self.proc: + _kill_process_tree(self.proc.pid) + + +# --------------------------------------------------------------------------- +# Config helpers +# --------------------------------------------------------------------------- + + +def _build_serve_args(serve_args_dict: dict[str, Any]) -> list[str]: + """Convert a serve_args dict from test.json into a flat CLI argument list.""" + args: list[str] = [] + for key, value in serve_args_dict.items(): + flag = f"--{key}" + if isinstance(value, bool): + if value: + args.append(flag) + elif isinstance(value, dict): + args.extend([flag, json.dumps(value, separators=(",", ":"))]) + else: + args.extend([flag, str(value)]) + return args + + +def _get_branchpoint_commit_sha() -> str: + """Return the branch-point commit SHA against main. + + Uses git command: ``git merge-base HEAD origin/main``. + """ + global _BRANCHPOINT_COMMIT_SHA + if _BRANCHPOINT_COMMIT_SHA is not None: + return _BRANCHPOINT_COMMIT_SHA + + repo_root = Path(__file__).parent.parent.parent.parent + try: + sha = ( + subprocess.check_output( + ["git", "merge-base", "HEAD", "origin/main"], + cwd=str(repo_root), + stderr=subprocess.STDOUT, + text=True, + ) + .strip() + .splitlines()[0] + ) + _BRANCHPOINT_COMMIT_SHA = sha + except Exception as e: + print(f"Warning: failed to get branch-point commit SHA: {e}") + _BRANCHPOINT_COMMIT_SHA = "" + return _BRANCHPOINT_COMMIT_SHA + + +def _to_resolution_string(params: dict[str, Any]) -> str: + width = params.get("width", "unknown width") + height = params.get("height", "unknown height") + return f"{width}x{height}" + + +def _to_parallelism_string(framework: str, serve_args_dict: dict[str, Any]) -> str: + parts: list[str] = [] + if framework == "vllm-omni": + keys = [ + "num-gpus", + "usp", + "ulysses-degree", + "ring", + "ring-degree", + "cfg-parallel-size", + "vae-patch-parallel-size", + "vae-use-tiling", + "tensor-parallel-size", + ] + for key in keys: + if key in serve_args_dict: + parts.append(f"{key}={serve_args_dict[key]}") + return ",".join(parts) if parts else "none" + + +def _to_cache_string(framework: str, serve_args_dict: dict[str, Any]) -> str: + if framework == "vllm-omni": + if "cache-backend" in serve_args_dict: + return str(serve_args_dict["cache-backend"]) + return "disabled" + + +def _to_offload_string(framework: str, serve_args_dict: dict[str, Any]) -> str: + selected: list[str] = [] + if framework == "vllm-omni": + offload_keys = [ + "enable-cpu-offload", + "enable-layerwise-offload", + ] + for key in offload_keys: + if key in serve_args_dict: + selected.append(key) + return f"enabled({';'.join(selected)})" if selected else "disabled" + + +def _to_compile_value(framework: str, serve_args_dict: dict[str, Any]) -> str: + if framework == "vllm-omni": + if "enforce-eager" in serve_args_dict: + return "disabled" + return "enabled" + return "disabled" + + +def _to_quantization_value(framework: str, serve_args_dict: dict[str, Any]) -> str: + if framework == "vllm-omni": + quant = serve_args_dict.get("quantization") + return str(quant) if quant else "disabled" + return "disabled" + + +def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return one server-config dict per unique test_name.""" + seen: set[str] = set() + result: list[dict[str, Any]] = [] + for cfg in configs: + test_name = cfg["test_name"] + if test_name in seen: + continue + seen.add(test_name) + server_type = cfg.get("server_type", "vllm-omni") + if server_type != "vllm-omni": + raise ValueError(f"Unsupported server_type in config: {server_type}") + serve_args_dict = cfg["server_params"].get("serve_args", {}) + result.append( + { + "test_name": test_name, + "server_type": server_type, + "model": cfg["server_params"]["model"], + "serve_args_dict": serve_args_dict, + "serve_args": _build_serve_args(serve_args_dict), + "benchmark_endpoint": cfg.get("benchmark_endpoint", cfg.get("benchmark_backend")), + "server_params": cfg["server_params"], + "mark": cfg.get("mark"), + } + ) + return result + + +def _test_param_mapping(configs: list[dict[str, Any]]) -> dict[str, list[dict]]: + mapping: dict[str, list[dict]] = {} + for cfg in configs: + name = cfg["test_name"] + mapping.setdefault(name, []) + mapping[name].extend(cfg["benchmark_params"]) + return mapping + + +def _marks_by_test_name(configs: list[dict[str, Any]]) -> dict[str, list[pytest.MarkDecorator]]: + return {str(cfg["test_name"]): resolve_pytest_marks(cfg.get("mark")) for cfg in configs} + + +def _paired_diffusion_benchmark_pytest_params(configs: list[dict[str, Any]]) -> list[Any]: + """Paired params for ``run_diffusion_benchmark.py``; same shape as omni runner.""" + test_param_map = _test_param_mapping(configs) + server_entries = [(cfg, cfg["test_name"]) for cfg in _unique_server_params(configs)] + return create_paired_benchmark_pytest_params(server_entries, test_param_map, _marks_by_test_name(configs)) + + +# --------------------------------------------------------------------------- +# Parametrize data +# --------------------------------------------------------------------------- + +test_param_map = _test_param_mapping(BENCHMARK_CONFIGS) +paired_benchmark_params = _paired_diffusion_benchmark_pytest_params(BENCHMARK_CONFIGS) + + +def _make_server(server_cfg: dict[str, Any]) -> DiffusionServer: + """Factory: return a vLLM-Omni diffusion server instance for the config.""" + return DiffusionServer(server_cfg=server_cfg) + + +# --------------------------------------------------------------------------- +# Pytest fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def diffusion_server(request): + """Start one vLLM-Omni server per unique test configuration.""" + with _server_lock: + server_cfg: dict[str, Any] = request.param + test_name = server_cfg["test_name"] + server_type = server_cfg["server_type"] + + print(f"\nStarting {server_type} server for test: {test_name}") + with _make_server(server_cfg) as server: + server.test_name = test_name + print(f"{server_type} server started successfully") + yield server + print(f"{server_type} server stopping…") + + print(f"{server_type} server stopped") + + +@pytest.fixture +def benchmark_params(request): + """Benchmark params for the paired server/index parametrization.""" + test_name, param_index = request.param + + params_list = test_param_map.get(test_name, []) + if not params_list: + raise ValueError(f"No benchmark params for test: {test_name}") + + current = param_index + 1 + total = len(params_list) + print(f"\n Running benchmark {current}/{total} for {test_name}") + return {"test_name": test_name, "params": params_list[param_index]} + + +# --------------------------------------------------------------------------- +# Benchmark runner +# --------------------------------------------------------------------------- + + +_STAGE_METRICS_ENDPOINTS = {"/v1/chat/completions"} +_DIFFUSION_PIPELINE_PROFILER_ARG = "enable-diffusion-pipeline-profiler" + + +def run_benchmark( + host: str, + port: int, + model: str, + params: dict[str, Any], + test_name: str, + endpoint: str = "/v1/chat/completions", + server_cfg: dict[str, Any] | None = None, + source_file: str = "", +) -> dict[str, Any]: + """Run diffusion_benchmark_serving.py as a subprocess and return parsed metrics. + + The raw metrics are written to a temporary file by the subprocess. After + the run completes the metrics are merged with full metadata (test_name, + endpoint, benchmark_params, timestamp, flat reporting fields) and appended + to ``diffusion_result_{config_stem}_{hardware}_{timestamp}.json`` for the + source JSON file. The temporary file is removed afterwards. Subprocess stdout/stderr are tee'd + to a .log file under BENCHMARK_RESULT_DIR/logs/; its path is stored in + the record. + """ + timestamp = datetime.now().strftime("%Y%m%d-%H%M%S") + endpoint = normalize_endpoint(endpoint) + + log_dir = BENCHMARK_RESULT_DIR / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + endpoint_label = endpoint_filename_token(endpoint) + resource_label = get_runtime_resource_label() + hw_for_filename = resource_label_for_filename(resource_label) + if hw_for_filename: + log_file = log_dir / f"{test_name}_{hw_for_filename}_{endpoint_label}_{timestamp}.log" + else: + log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp: + tmp_result_file = Path(tmp.name) + + exclude_keys = {"baseline", "dataset", "task", "name"} + + cmd = [ + sys.executable, + BENCHMARK_SCRIPT, + "--host", + host, + "--port", + str(port), + "--model", + model, + "--endpoint", + endpoint, + "--dataset", + params.get("dataset", "random"), + "--task", + params.get("task", "t2i"), + "--output-file", + str(tmp_result_file), + ] + + serve_args_dict = (server_cfg or {}).get("serve_args_dict") + profiler_enabled = isinstance(serve_args_dict, dict) and bool(serve_args_dict.get(_DIFFUSION_PIPELINE_PROFILER_ARG)) + if endpoint in _STAGE_METRICS_ENDPOINTS and profiler_enabled: + cmd.append("--return-stage-metrics") + + for key, value in params.items(): + if key in exclude_keys or value is None: + continue + flag = f"--{key}" + if isinstance(value, bool): + if value: + cmd.append(flag) + elif isinstance(value, (dict, list)): + cmd.extend([flag, json.dumps(value, separators=(",", ":"))]) + else: + cmd.extend([flag, str(value)]) + + # Insert -u so the subprocess runs with unbuffered stdout/stderr, ensuring + # all print() output is flushed to the pipe immediately instead of being + # held in Python's internal block-buffer until process exit (which can + # cause truncated or out-of-order log output when stdout is piped). + cmd = [cmd[0], "-u"] + cmd[1:] + + print(f"\nRunning benchmark (endpoint={endpoint}): {' '.join(cmd)}") + print(f" Log file: {log_file}") + + # Redirect stdout + stderr directly to the log file at the OS level + # (equivalent to `cmd > log 2>&1`), so no output is ever lost regardless + # of how the subprocess exits. The log is echoed to the terminal afterwards. + with open(log_file, "w", encoding="utf-8") as log_fh: + log_fh.write(f"cmd: {' '.join(cmd)}\n\n") + log_fh.flush() + + process = subprocess.Popen( + cmd, + stdout=log_fh, + stderr=log_fh, + cwd=str(Path(__file__).parent.parent.parent.parent), + ) + process.wait() + + with open(log_file, encoding="utf-8") as log_fh: + print(log_fh.read(), end="") + + if process.returncode != 0: + tmp_result_file.unlink(missing_ok=True) + print(f"ERROR:Benchmark script exited with code {process.returncode}") + + if not tmp_result_file.exists(): + with open(DIFFUSION_RESULT_TEMPLATE_PATH, encoding="utf-8") as f: + template_payload = json.load(f) + # Template schema is fixed and owned by this repo: + # ``diffusion_result_template.json`` is a one-item list and metrics live at [0]["result"]. + template_metrics: dict[str, Any] = template_payload[0]["result"] + with open(tmp_result_file, "w", encoding="utf-8") as f: + json.dump(template_metrics, f, ensure_ascii=False, indent=2) + print(f"Benchmark result file not generated, fallback to template: {tmp_result_file}") + + try: + with open(tmp_result_file, encoding="utf-8") as f: + metrics: dict[str, Any] = json.load(f) + finally: + tmp_result_file.unlink(missing_ok=True) + + server_cfg = server_cfg or {} + server_type = cast(str, server_cfg.get("server_type", "vllm-omni")) + serve_args_dict = server_cfg.get("serve_args_dict", {}) + if not isinstance(serve_args_dict, dict): + serve_args_dict = {} + + completed = metrics.get("completed_requests", metrics.get("completed", 0)) + failed = metrics.get("failed_requests", metrics.get("failed", 0)) + + # Persist sweep-resolved baseline from params (already narrowed in _build_run_params). + baseline = params.get("baseline") or {} + metrics["baseline"] = baseline + + record: dict[str, Any] = { + "test_name": test_name, + "endpoint": endpoint, + "timestamp": timestamp, + "server_params": server_cfg.get("server_params"), + "benchmark_params": params, + "baseline": baseline, + "result": metrics, + "log_file": str(log_file), + "Model": model, + "Framework": server_type, + "API Endpoint": endpoint, + "Hardware": hardware_json_value(resource_label), + "Deployment": "", + "Task": params.get("task", "t2i"), + "Dataset": params.get("dataset", "random"), + "resolution": _to_resolution_string(params), + "Parallelism": _to_parallelism_string(server_type, serve_args_dict), + "max_concurrency": params.get("max-concurrency", ""), + "Cache": _to_cache_string(server_type, serve_args_dict), + "Quantization": _to_quantization_value(server_type, serve_args_dict), + "offload": _to_offload_string(server_type, serve_args_dict), + "compile": _to_compile_value(server_type, serve_args_dict), + "Attn_backend": str(serve_args_dict.get("diffusion-attention-backend") or ""), + "num_inference_steps": params.get("num-inference-steps", ""), + "completed": completed, + "failed": failed, + "throughput_qps": metrics.get("throughput_qps"), + "latency_mean": metrics.get("latency_mean"), + "latency_median": metrics.get("latency_median"), + "latency_p99": metrics.get("latency_p99"), + "latency_p95": metrics.get("latency_p95"), + "latency_p50": metrics.get("latency_p50"), + "peak_memory_mb_max": metrics.get("peak_memory_mb_max"), + "peak_memory_mb_mean": metrics.get("peak_memory_mb_mean"), + "peak_memory_mb_median": metrics.get("peak_memory_mb_median"), + "stage_durations_mean": metrics.get("stage_durations_mean"), + "stage_durations_p50": metrics.get("stage_durations_p50"), + "stage_durations_p99": metrics.get("stage_durations_p99"), + "commit_sha": _get_branchpoint_commit_sha(), + "build_id": os.environ.get("BUILDKITE_BUILD_ID", ""), + "build_url": os.environ.get("BUILDKITE_BUILD_URL", ""), + "source_file": source_file, + } + result_path = _write_result_record(record) + print(f"\n Result saved to: {result_path}") + print(f" Log saved to: {log_file}") + + return metrics + + +# --------------------------------------------------------------------------- +# Assertions +# --------------------------------------------------------------------------- + + +def assert_result( + result: dict[str, Any], + num_prompts: int, +) -> None: + """Assert that the expected number of requests completed.""" + completed = result.get("completed_requests", result.get("completed", 0)) + assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}" + + +def _default_benchmark_endpoint_for_task(task: str) -> str: + """Return the default client-side benchmark endpoint for a diffusion task.""" + if task in {"t2v", "i2v", "ti2v", "v2v"}: + return "/v1/videos" + if task in {"t2i", "i2i", "ti2i"}: + return "/v1/chat/completions" + raise ValueError(f"Unsupported task for benchmark endpoint resolution: {task}") + + +def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, Any]) -> str: + """Resolve which serving API the benchmark client should call.""" + configured = server_cfg.get("benchmark_endpoint") + if configured: + return normalize_endpoint(cast(str, configured)) + return _default_benchmark_endpoint_for_task(cast(str, params.get("task", "t2i"))) + + +def _to_list(value: Any) -> list[Any]: + if value is None: + return [] + return [value] if not isinstance(value, (list, tuple)) else list(value) + + +def _build_run_params( + params: dict[str, Any], + *, + num_prompts: int, + sweep_index: int | None = None, + request_rate: Any | None = None, + max_concurrency: Any | None = None, +) -> dict[str, Any]: + run_params = { + key: value for key, value in params.items() if key not in {"request-rate", "max-concurrency", "num-prompts"} + } + run_params["num-prompts"] = num_prompts + if request_rate is not None: + run_params["request-rate"] = request_rate + if max_concurrency is not None: + run_params["max-concurrency"] = max_concurrency + if "baseline" in params: + # Keep all hardware buckets; pick the metric value for this sweep step only. + run_params["baseline"] = resolve_baseline_for_sweep( + params.get("baseline"), + sweep_index=sweep_index, + ) + return run_params + + +def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]: + request_rate_list = _to_list(params.get("request-rate")) + num_prompt_list = _to_list(params.get("num-prompts", 10)) + max_concurrency_list = _to_list(params.get("max-concurrency")) + + max_len = max(len(request_rate_list), len(max_concurrency_list)) + if len(num_prompt_list) == 1 and max_len > 1: + num_prompt_list = num_prompt_list * max_len + elif max_len == 1 and len(num_prompt_list) > 1: + if len(request_rate_list) == 1: + request_rate_list = request_rate_list * len(num_prompt_list) + if len(max_concurrency_list) == 1: + max_concurrency_list = max_concurrency_list * len(num_prompt_list) + max_len = max(len(request_rate_list), len(max_concurrency_list)) + elif len(num_prompt_list) != max_len and max_len > 0: + raise ValueError("The number of prompts does not match the request-rate or max-concurrency") + + sweep_runs: list[dict[str, Any]] = [] + + for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)): + sweep_runs.append( + { + "params": _build_run_params( + params, + request_rate=request_rate, + num_prompts=num_prompts, + sweep_index=sweep_index, + ), + "num_prompts": num_prompts, + } + ) + + for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)): + sweep_runs.append( + { + "params": _build_run_params( + params, + max_concurrency=max_concurrency, + num_prompts=num_prompts, + request_rate="inf", + sweep_index=sweep_index, + ), + "num_prompts": num_prompts, + } + ) + + if not sweep_runs: + default_num_prompts = num_prompt_list[0] + sweep_runs.append( + { + "params": _build_run_params( + params, + num_prompts=default_num_prompts, + sweep_index=0, + ), + "num_prompts": default_num_prompts, + } + ) + + return sweep_runs + + +# --------------------------------------------------------------------------- +# Test entry point +# --------------------------------------------------------------------------- +@pytest.mark.benchmark +@pytest.mark.parametrize( + "diffusion_server,benchmark_params", + paired_benchmark_params, + indirect=["diffusion_server", "benchmark_params"], +) +def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): + """Run the diffusion performance benchmark and verify request completion. + + One server is started per unique parallel configuration (module scope). + For each server, all benchmark parameter sets defined in the config JSON + are executed sequentially; metrics are recorded to the aggregated result file. + """ + test_name = benchmark_params["test_name"] + params = benchmark_params["params"] + server_cfg = getattr(diffusion_server, "server_cfg", {}) + sweep_runs = _iter_sweep_runs(params) + + for sweep_run in sweep_runs: + endpoint = _resolve_benchmark_endpoint(server_cfg, sweep_run["params"]) + result = run_benchmark( + host=diffusion_server.host, + port=diffusion_server.port, + model=diffusion_server.model, + params=sweep_run["params"], + test_name=test_name, + endpoint=endpoint, + server_cfg=server_cfg, + source_file=server_cfg.get( + _DIFFUSION_SOURCE_CONFIG_KEY, + CONFIG_FILE_PATH or f"{_PERF_TESTS_DIR}/*.json", + ), + ) + + print(f"\n{'=' * 60}") + print(f"Results for {test_name} (server={diffusion_server.server_type}, endpoint={endpoint}):") + for key in ( + "throughput_qps", + "latency_mean", + "latency_median", + "latency_p50", + "latency_p99", + "peak_memory_mb_max", + "peak_memory_mb_mean", + "peak_memory_mb_median", + ): + if key in result: + print(f" {key}: {result[key]:.4f}") + + source = server_cfg.get(_DIFFUSION_SOURCE_CONFIG_KEY) or CONFIG_FILE_PATH + if source: + print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}") + print("=" * 60) + + assert_result(result, sweep_run["num_prompts"]) diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json new file mode 100644 index 000000000..662f845f6 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_cosmos3_vllm_omni.json @@ -0,0 +1,288 @@ +[ + { + "test_name": "test_cosmos3_t2i_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano text-to-image official demo workload on 2 GPUs", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps4", + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/images/generations", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 1024, + "height": 1024, + "num_inference_steps": 4, + "seed": 42, + "guidance_scale": 7.0, + "flow_shift": 3.0, + "guardrails": false + }, + "baseline": { + "H100": { + "request_throughput": 1.1958, + "mean_e2el_ms": 843.0, + "mean_peak_memory_mb": 80000.0 + } + } + } + ] + }, + { + "test_name": "test_cosmos3_t2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano text-to-video official demo workload on 2 GPUs", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "seed": 42, + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false, + "use_resolution_template": false, + "use_duration_template": false + }, + "baseline": { + "H100": { + "request_throughput": 0.0307, + "mean_e2el_ms": 32766.5, + "mean_peak_memory_mb": 28762.4762 + } + } + } + ] + }, + { + "test_name": "test_cosmos3_i2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano image-to-video official demo workload on 2 GPUs", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(720, 1280, 1)": 1.0 + }, + "extra_body": { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "seed": 42, + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false + }, + "baseline": { + "H100": { + "request_throughput": 0.0305, + "mean_e2el_ms": 33100.9, + "mean_peak_memory_mb": 29441.1429 + } + } + } + ] + }, + { + "test_name": "test_cosmos3_v2v_official_demo_2gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "Cosmos3-Nano video-to-video official demo workload on 2 GPUs", + "server_params": { + "model": "nvidia/Cosmos3-Nano", + "serve_args": { + "cfg-parallel-size": 2, + "use-hsdp": true, + "hsdp-shard-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "model-class-name": "Cosmos3OmniDiffusersPipeline", + "no-guardrails": true, + "stage-init-timeout": 900, + "init-timeout": 1200, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1280x720_frames189_steps4", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "video": 1 + }, + "random_mm_bucket_config": { + "(720, 1280, 189)": 1.0 + }, + "extra_body": { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 189, + "fps": 24, + "seed": 42, + "guidance_scale": 6.0, + "flow_shift": 10.0, + "max_sequence_length": 4096, + "guardrails": false, + "condition_frame_indexes_vision": [ + 0, + 1 + ], + "condition_video_keep": "first" + }, + "baseline": { + "H100": { + "request_throughput": 0.0299, + "mean_e2el_ms": 33925.0, + "mean_peak_memory_mb": 29443.1429 + } + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json new file mode 100644 index 000000000..f14e77bcb --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json @@ -0,0 +1,124 @@ +[ + { + "test_name": "test_hunyuanvideo15_i2v_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", + "serve_args": { + "flow-shift": 5.0, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames33_steps4", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 1)": 1.0 + }, + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "seed": 42, + "guidance_scale": 6.0, + "flow_shift": 5.0, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } + } + ] + }, + { + "test_name": "test_hunyuanvideo15_i2v_cachedit_tp2_vae_patch2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", + "serve_args": { + "flow-shift": 5.0, + "cache-backend": "cache_dit", + "tensor-parallel-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames33_steps4", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 1)": 1.0 + }, + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "seed": 42, + "guidance_scale": 6.0, + "flow_shift": 5.0, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json new file mode 100644 index 000000000..3a2f66d82 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json @@ -0,0 +1,104 @@ +[ + { + "test_name": "test_hunyuanvideo15_t2v_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + "serve_args": { + "flow-shift": 5.0, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames33_steps4", + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "seed": 42, + "guidance_scale": 6.0, + "flow_shift": 5.0, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } + } + ] + }, + { + "test_name": "test_hunyuanvideo15_t2v_cachedit_tp2_vae_patch2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"], + "npu": "A3" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + "serve_args": { + "flow-shift": 5.0, + "cache-backend": "cache_dit", + "tensor-parallel-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames33_steps4", + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "seed": 42, + "guidance_scale": 6.0, + "flow_shift": 5.0, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json new file mode 100644 index 000000000..9f85a14a2 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json @@ -0,0 +1,50 @@ +[ + { + "test_name": "test_lingbot_video_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device LingBot dense T2V baseline at 320x192, 9 frames, 2 steps.", + "server_params": { + "model": "robbyant/lingbot-video-dense-1.3b", + "serve_args": { + "model-class-name": "LingBotVideoPipeline", + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "320x192_frames9_steps2", + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 320, + "height": 192, + "num_frames": 9, + "fps": 24, + "num_inference_steps": 2, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5.json new file mode 100644 index 000000000..510715b22 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5.json @@ -0,0 +1,172 @@ +[ + { + "test_name": "test_minicpmo_4_5", + "description": "H100 baseline refreshed 2026-09-29: equal-weight nightly means for 2026-09-22 through 2026-09-28 (7 runs each for C1/P32, C4/P64, C8/P128), rounded to 4 decimals without an added margin. Uses request reference audio after #6628; duplex_seed_tts excluded. A3 unchanged.", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"], + "npu": "A3" + }, + "num_cards": 1 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8 + ], + "no_oversample": true, + "disable_shuffle": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", + "baseline": { + "H100": { + "request_throughput": [ + 0.9615, + 1.4882, + 1.6245 + ], + "mean_ttft_ms": [ + 156.5006, + 272.8529, + 283.2613 + ], + "mean_e2el_ms": [ + 1042.7617, + 2669.8806, + 4882.2845 + ], + "mean_audio_ttfp_ms": [ + 585.7387, + 1396.1575, + 2508.4384 + ], + "mean_audio_rtf": [ + 0.2504, + 0.6281, + 1.1787 + ] + }, + "A3": { + "request_throughput": [ + 0.4813, + 0.6381, + 0.8728 + ], + "mean_ttft_ms": [ + 328.182, + 470.0053, + 506.3193 + ], + "mean_e2el_ms": [ + 2086.7028, + 6480.6866, + 9103.7395 + ], + "mean_audio_ttfp_ms": [ + 1023.3301, + 2734.1486, + 3151.4020 + ], + "mean_audio_rtf": [ + 0.4659, + 1.4737, + 2.1758 + ] + } + } + } + ] + }, + { + "test_name": "test_minicpmo_4_5_a2", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A2" + }, + "num_cards": 1 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8 + ], + "no_oversample": true, + "disable_shuffle": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json new file mode 100644 index 000000000..a145459ed --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json @@ -0,0 +1,50 @@ +[ + { + "test_name": "test_minicpmo_4_5_duplex_seed_tts", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"], + "npu": "A3" + }, + "num_cards": 1 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-realtime-tts", + "endpoint": "/v1/realtime", + "num_prompts": [ + 1 + ], + "max_concurrency": [ + 1 + ], + "no_oversample": true, + "disable_shuffle": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "seed_tts_turns_per_session": 4, + "expected_duplex_audio_turns_per_session": 4, + "extra_body": { + "save_duplex_request_metrics": true + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_ttfp,audio_rtf,audio_duration" + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json new file mode 100644 index 000000000..26dc54aab --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minicpmo_4_5_ready.json @@ -0,0 +1,81 @@ +[ + { + "test_name": "test_minicpmo_4_5_ready", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100", + "npu": "A3" + }, + "num_cards": 1 + } + }, + "core_model", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10, + 40 + ], + "max_concurrency": [ + 1, + 4 + ], + "no_oversample": true, + "disable_shuffle": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration", + "baseline": { + "H100": { + "request_throughput": [ + 0.9731, + 1.1986 + ], + "mean_ttft_ms": [ + 152.5665, + 270.2559 + ], + "mean_e2el_ms": [ + 1027.2953, + 3306.3735 + ], + "mean_audio_ttfp_ms": [ + 458.1553, + 2030.2335 + ], + "mean_audio_rtf": [ + 0.2357, + 0.791 + ] + } + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json new file mode 100644 index 000000000..3bf68064e --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json @@ -0,0 +1,300 @@ +[ + { + "test_name": "test_minimax_h3_t2v_4gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "MiniMax-H3 T2VA text-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "stage-init-timeout": 1800, + "init-timeout": 1800, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1344x768_frames209_steps8", + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 1344, + "height": 768, + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "seed": 42, + "flow_shift": 12.0, + "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" + }, + "baseline": { + "H100": { + "request_throughput": 0.0211, + "mean_e2el_ms": 38325.2, + "mean_peak_memory_mb": 54219.6 + } + } + } + ] + }, + { + "test_name": "test_minimax_h3_ti2v_4gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "MiniMax-H3 FL2VA first-frame image-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "stage-init-timeout": 1800, + "init-timeout": 1800, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1344x768_frames209_steps8", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(768, 1344, 1)": 1.0 + }, + "extra_body": { + "width": 1344, + "height": 768, + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "seed": 42, + "flow_shift": 12.0, + "extra_params": "{\"task\":\"fl2va\",\"duration\":8.7,\"audio_flow_shift\":3.0}" + }, + "baseline": { + "H100": { + "request_throughput": 0.0212, + "mean_e2el_ms": 38138.5, + "mean_peak_memory_mb": 54622.8 + } + } + } + ] + }, + { + "test_name": "test_minimax_h3_v2v_4gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "MiniMax-H3 Ref2VA video-reference-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "ref2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "stage-init-timeout": 1800, + "init-timeout": 1800, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1344x768_frames209_steps8", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "video": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 209)": 1.0 + }, + "extra_body": { + "width": 1344, + "height": 768, + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "seed": 42, + "flow_shift": 12.0, + "extra_params": "{\"task\":\"ref2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" + }, + "baseline": { + "H100": { + "request_throughput": 0.0077, + "mean_e2el_ms": 105750.8, + "mean_peak_memory_mb": 55180.8 + } + } + } + ] + }, + { + "test_name": "test_minimax_h3_t2v_dlo_4gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "MiniMax-H3 T2VA with 4-card distributed layerwise offload (DLO), 8 denoise steps (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "stage-init-timeout": 1800, + "init-timeout": 1800, + "enable-diffusion-pipeline-profiler": true, + "enable-distributed-layerwise-offload": true, + "dlo-no-use-allgather": true, + "dlo-resident-layers": 8, + "enforce-eager": true + } + }, + "benchmark_params": [ + { + "name": "1344x768_frames209_steps8", + "dataset_name": "random", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 3, + "max_concurrency": 1, + "num_warmups": 1, + "random_input_len": 8, + "random_output_len": 1, + "percentile_metrics": "e2el", + "extra_body": { + "width": 1344, + "height": 768, + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "seed": 42, + "flow_shift": 12.0, + "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" + }, + "baseline": { + "H100": { + "request_throughput": 0.0212, + "mean_e2el_ms": 38208.9, + "mean_peak_memory_mb": 36218.4 + } + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json new file mode 100644 index 000000000..1864bf027 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -0,0 +1,325 @@ +[ + { + "test_name": "test_qwen3_omni_chunk", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"], + "npu": "A3" + }, + "num_cards": 2 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "extra_cli_args": [ + "--async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "mean_audio_rtf": [ + 0.1098 + ] + }, + "A3": { + "mean_audio_rtf": [ + 0.7014 + ] + } + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "mean_audio_rtf": [ + 0.1335 + ] + }, + "A3": { + "mean_audio_rtf": [ + 1.2352 + ] + } + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "mean_audio_rtf": [ + 0.1798 + ] + }, + "A3": { + "mean_audio_rtf": [ + 2.5023 + ] + } + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "H100": { + "mean_e2el_ms": [ + 756.8415 + ] + }, + "A3": { + "mean_e2el_ms": [ + 2436.1048 + ] + } + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "H100": { + "mean_e2el_ms": [ + 694.9777 + ] + }, + "A3": { + "mean_e2el_ms": [ + 2534.4575 + ] + } + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "H100": { + "mean_e2el_ms": [ + 1006.1663 + ] + }, + "A3": { + "mean_e2el_ms": [ + 3460.0385 + ] + } + } + } + ] + }, + { + "test_name": "test_qwen3_omni_chunk_cuda", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"] + }, + "num_cards": 2 + } + }, + "omni", + "slow" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "extra_cli_args": [ + "--async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 32, + 128 + ], + "max_concurrency": [ + 1, + 8, + 32 + ], + "random_input_len": 100, + "random_output_len": 100, + "ignore_eos": true, + "extra_body": { + "modalities": [ + "text" + ] + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "H100": { + "mean_e2el_ms": [ + 4354.8945, + 9861.1687, + 18716.709 + ] + } + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json new file mode 100644 index 000000000..012532fef --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -0,0 +1,186 @@ +[ + { + "test_name": "test_qwen3_omni", + "description": "random row: fixed workload (Thinker 900 / Talker 1536 tokens, min_tokens == max_tokens) for comparable audio RTF; H100 baseline pending recalibration, see #8100", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"], + "npu": "A3" + }, + "num_cards": 2 + } + }, + "full_model", + "omni" + ], + "server_params": { + "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", + "extra_cli_args": [ + "--no-async-chunk" + ] + }, + "benchmark_params": [ + { + "dataset_name": "random", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], + "random_input_len": 2500, + "random_output_len": 900, + "ignore_eos": true, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "seed": 0, + "random_range_ratio": 0.0, + "extra_body": { + "return_stage_metrics": true, + "sampling_params_list": [ + { + "temperature": 0.0, + "min_tokens": 900, + "max_tokens": 900, + "ignore_eos": true + }, + { + "temperature": 0.9, + "top_k": 50, + "repetition_penalty": 1.05, + "seed": 0, + "min_tokens": 1536, + "max_tokens": 1536 + } + ] + }, + "baseline": {} + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "mean_audio_rtf": [ + 0.138 + ] + }, + "A3": { + "mean_audio_rtf": [ + 0.6604 + ] + } + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "mean_audio_rtf": [ + 0.1609 + ] + }, + "A3": { + "mean_audio_rtf": [ + 1.0521 + ] + } + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "mean_audio_rtf": [ + 0.2048 + ] + }, + "A3": { + "mean_audio_rtf": [ + 1.9804 + ] + } + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json new file mode 100644 index 000000000..5483ffd16 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -0,0 +1,408 @@ +[ + { + "test_name": "test_qwen_image_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"] + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline (no parallelism)", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "H100": { + "throughput_qps": 0.4153, + "latency_mean": 2.4065, + "peak_memory_mb_mean": 55062.0 + } + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "H100": { + "throughput_qps": 0.0423, + "latency_mean": 23.5991, + "peak_memory_mb_mean": 64546.0 + } + } + } + ] + }, + { + "test_name": "test_qwen_image_single_device_a3", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline (no parallelism)", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_qwen_image_single_device_step_execution", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"] + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device step execution: sequential 512/1536 plus 512 concurrency sweep. One server (max-num-seqs 8).", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "step-execution": true, + "max-num-seqs": 8 + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "H100": { + "throughput_qps": 0.4175, + "latency_mean": 2.3953, + "peak_memory_mb_mean": 54952.0 + } + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "H100": { + "throughput_qps": 0.0423, + "latency_mean": 23.5768, + "peak_memory_mb_mean": 64500.0 + } + } + }, + { + "name": "512x512_steps20_high_concurrency", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": [20, 40, 80, 160], + "max-concurrency": [1, 2, 4, 8], + "warmup-requests": 8, + "warmup-concurrency": 8, + "warmup-num-inference-steps": 20, + "enable-negative-prompt": true, + "baseline": { + "H100": { + "throughput_qps": [0.3937, 0.6606, 0.6188, 0.7752], + "latency_mean": [2.5466, 3.0235, 6.3948, 10.1944], + "peak_memory_mb_mean": [55149.0, 55150.0, 55153.1458, 55156.325] + } + } + } + ] + }, + { + "test_name": "test_qwen_image_single_device_step_execution_a3", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device step execution: sequential 512 plus 512 concurrency sweep. One server (max-num-seqs 8).", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "enable-diffusion-pipeline-profiler": true, + "step-execution": true, + "max-num-seqs": 8 + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true + }, + { + "name": "512x512_steps20_high_concurrency", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": [20, 40, 80, 160], + "max-concurrency": [1, 2, 4, 8], + "warmup-requests": 8, + "warmup-concurrency": 8, + "warmup-num-inference-steps": 20, + "enable-negative-prompt": true + } + ] + }, + { + "test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"], + "npu": "A3" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "H100": { + "throughput_qps": 0.1214, + "latency_mean": 8.2211, + "peak_memory_mb_mean": 54474.0 + } + } + } + ] + }, + { + "test_name": "test_qwen_image_ulysses2_cfg2_cache_dit", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"] + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "cache-backend": "cache_dit", + "cache-config": { + "Fn_compute_blocks": 1, + "Bn_compute_blocks": 0, + "max_warmup_steps": 4, + "residual_diff_threshold": 0.24, + "max_continuous_cached_steps": 3, + "enable_taylorseer": false, + "taylorseer_order": 1, + "scm_steps_mask_policy": null, + "scm_steps_policy": "dynamic" + }, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "H100": { + "throughput_qps": 0.559, + "latency_mean": 1.794, + "peak_memory_mb_mean": 55174.0 + } + } + }, + { + "name": "1536x1536_steps35", + "dataset": "random", + "task": "t2i", + "width": 1536, + "height": 1536, + "num-inference-steps": 35, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true, + "baseline": { + "H100": { + "throughput_qps": 0.1918, + "latency_mean": 5.2022, + "peak_memory_mb_mean": 64750.0 + } + } + } + ] + }, + { + "test_name": "test_qwen_image_ulysses2_cfg2_cache_dit_a3", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration", + "server_type": "vllm-omni", + "server_params": { + "model": "Qwen/Qwen-Image", + "serve_args": { + "ulysses-degree": 2, + "cfg-parallel-size": 2, + "cache-backend": "cache_dit", + "cache-config": { + "Fn_compute_blocks": 1, + "Bn_compute_blocks": 0, + "max_warmup_steps": 4, + "residual_diff_threshold": 0.24, + "max_continuous_cached_steps": 3, + "enable_taylorseer": false, + "taylorseer_order": 1, + "scm_steps_mask_policy": null, + "scm_steps_policy": "dynamic" + }, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "512x512_steps20", + "dataset": "random", + "task": "t2i", + "width": 512, + "height": 512, + "num-inference-steps": 20, + "num-prompts": 10, + "max-concurrency": 1, + "enable-negative-prompt": true + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_tts.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_tts.json new file mode 100644 index 000000000..5fac5d9ef --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_tts.json @@ -0,0 +1,551 @@ +[ + { + "test_name": "test_qwen3_tts_base", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"], + "npu": "A3" + }, + "num_cards": 2 + } + }, + "full_model", + "tts" + ], + "server_params": { + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", + "stage_overrides": { + "1": { + "devices": "1" + } + } + }, + "benchmark_params": [ + { + "task": "voice_clone", + "eval_phase": "latency", + "dataset_name": "seed-tts", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "linyueqian/seed-tts-eval-subset", + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], + "seed_tts_locale": "en", + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 141.2791 + ], + "median_audio_rtf": [ + 0.1591 + ] + }, + "A3": { + "median_audio_ttfp_ms": [ + 220.3239 + ], + "median_audio_rtf": [ + 0.3995 + ] + } + } + }, + { + "task": "voice_clone", + "eval_phase": "throughput", + "dataset_name": "seed-tts", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "linyueqian/seed-tts-eval-subset", + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], + "seed_tts_locale": "en", + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 226.747, + 647.1272, + 4777.2714 + ], + "median_audio_rtf": [ + 0.3313, + 0.3642, + 1.2282 + ], + "audio_throughput": [ + 24.2296, + 40.3552, + 43.2572 + ] + }, + "A3": { + "median_audio_ttfp_ms": [ + 345.5784, + 451.6758, + 1844.1087 + ], + "median_audio_rtf": [ + 0.6414, + 0.7981, + 1.7064 + ], + "audio_throughput": [ + 12.2961, + 18.9478, + 31.6908 + ] + } + } + }, + { + "task": "voice_clone", + "eval_phase": "quality", + "enabled": false, + "dataset_name": "seed-tts", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "linyueqian/seed-tts-eval-subset", + "num_prompts": [ + 200 + ], + "max_concurrency": [ + 4 + ], + "seed_tts_locale": "en", + "seed_tts_wer_eval": true, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "mean_audio_rtf": [ + 0.45 + ] + } + } + } + ] + }, + { + "test_name": "test_qwen3_tts_customvoice", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"] + }, + "num_cards": 2 + } + }, + "full_model", + "tts" + ], + "server_params": { + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "stage_overrides": { + "1": { + "devices": "1" + } + } + }, + "benchmark_params": [ + { + "task": "default_voice", + "eval_phase": "latency", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 46.8434 + ], + "median_audio_rtf": [ + 0.1367 + ] + } + } + }, + { + "task": "default_voice", + "eval_phase": "throughput", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 75.3341, + 713.7727, + 5941.7179 + ], + "median_audio_rtf": [ + 0.1859, + 0.3148, + 1.1662 + ], + "audio_throughput": [ + 38.3858, + 55.3523, + 68.7032 + ] + } + } + }, + { + "task": "default_voice", + "eval_phase": "quality", + "enabled": false, + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 200 + ], + "max_concurrency": [ + 4 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, + "seed_tts_wer_eval": true, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "mean_audio_rtf": [ + 0.35 + ] + } + } + }, + { + "task": "default_voice", + "eval_phase": "stress", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 56.746 + ], + "median_audio_rtf": [ + 0.1571 + ], + "audio_throughput": [ + 11.1226 + ] + } + } + } + ] + }, + { + "test_name": "test_qwen3_tts_voicedesign", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"] + }, + "num_cards": 2 + } + }, + "full_model", + "tts" + ], + "server_params": { + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", + "stage_overrides": { + "1": { + "devices": "1" + } + } + }, + "benchmark_params": [ + { + "task": "voice_design", + "eval_phase": "latency", + "dataset_name": "seed-tts-design", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_design", + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], + "seed_tts_locale": "en", + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 46.3479 + ], + "median_audio_rtf": [ + 0.1352 + ] + } + } + }, + { + "task": "voice_design", + "eval_phase": "throughput", + "dataset_name": "seed-tts-design", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_design", + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], + "seed_tts_locale": "en", + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 75.5967, + 587.5877, + 4960.3726 + ], + "median_audio_rtf": [ + 0.1871, + 0.3143, + 1.1703 + ], + "audio_throughput": [ + 38.9378, + 55.1846, + 71.5708 + ] + } + } + }, + { + "task": "voice_design", + "eval_phase": "stress", + "dataset_name": "seed-tts-design", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_design", + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], + "seed_tts_locale": "en", + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 55.5029 + ], + "median_audio_rtf": [ + 0.1508 + ], + "audio_throughput": [ + 9.4885 + ] + } + } + } + ] + }, + { + "test_name": "test_voxtral_tts", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H100", "B200"] + }, + "num_cards": 1 + } + }, + "full_model", + "tts" + ], + "server_params": { + "model": "mistralai/Voxtral-4B-TTS-2603" + }, + "benchmark_params": [ + { + "task": "default_voice", + "eval_phase": "latency", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "casual_male" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 82.6344 + ], + "median_audio_rtf": [ + 0.1277 + ] + } + } + }, + { + "task": "default_voice", + "eval_phase": "throughput", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 80 + ], + "max_concurrency": [ + 8 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "casual_male" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 170.3737 + ], + "median_audio_rtf": [ + 0.2076 + ], + "audio_throughput": [ + 37.0834 + ] + } + } + }, + { + "task": "default_voice", + "eval_phase": "stress", + "dataset_name": "seed-tts-text", + "backend": "openai-audio-speech", + "endpoint": "/v1/audio/speech", + "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], + "seed_tts_locale": "en", + "extra_body": { + "voice": "casual_male" + }, + "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "H100": { + "median_audio_ttfp_ms": [ + 101.0293 + ], + "median_audio_rtf": [ + 0.1498 + ], + "audio_throughput": [ + 10.1186 + ] + } + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json new file mode 100644 index 000000000..4ef4ecbed --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -0,0 +1,179 @@ +[ + { + "test_name": "test_wan22_i2v_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ] + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames81_steps4", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 1)": 1.0 + }, + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "seed": 42, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + }, + "baseline": { + "H100": { + "request_throughput": 0.0361, + "mean_e2el_ms": 27620.6, + "mean_peak_memory_mb": 80548.0 + } + } + } + ] + }, + { + "test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": [ + "H100", + "B200" + ], + "npu": "A3" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing", + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "usp": 2, + "vae-patch-parallel-size": 2, + "use-hsdp": true, + "vae-use-slicing": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames81_steps4", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(480, 832, 1)": 1.0 + }, + "extra_body": { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "seed": 42, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + }, + "baseline": { + "H100": { + "request_throughput": 0.05263, + "mean_e2el_ms": 18941.5, + "mean_peak_memory_mb": 50053.7 + }, + "A3": { + "request_throughput": 0.0155, + "mean_e2el_ms": 64626.4676, + "mean_peak_memory_mb": 59932.5714 + } + } + }, + { + "name": "1280x720_frames121_steps4", + "dataset_name": "random-mm", + "tokenizer": "gpt2", + "endpoint": "/v1/videos", + "num_prompts": 10, + "max_concurrency": 1, + "random_input_len": 8, + "random_output_len": 1, + "random_range_ratio": 0.0, + "percentile_metrics": "e2el", + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0, + "random_mm_limit_mm_per_prompt": { + "image": 1 + }, + "random_mm_bucket_config": { + "(720, 1280, 1)": 1.0 + }, + "extra_body": { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 121, + "fps": 16, + "seed": 42, + "negative_prompt": "Negative prompt for benchmarking diffusion models" + }, + "baseline": { + "H100": { + "request_throughput": 0.009595, + "mean_e2el_ms": 104105.8, + "mean_peak_memory_mb": 59718.0 + }, + "A3": { + "request_throughput": 0.0021, + "mean_e2el_ms": 469604.3521, + "mean_peak_memory_mb": 59889.7714 + } + } + } + ] + } +] diff --git a/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py b/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py new file mode 100644 index 000000000..f5df4004f --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py @@ -0,0 +1,266 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import torch + +from tests.e2e.accuracy.helpers import ( + assert_video_metadata, + assert_video_similarity_metrics, + build_online_image_reference, + materialize_image_source, + probe_binary, + probe_video, + send_video_request_with_timeout, + validate_image_source, + video_artifact_dir, +) +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +MODEL_NAME = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v" +IMAGE_URL = "https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/wan_i2v_input.JPG" +PROMPT = ( + "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard. " + "The fluffy-furred feline gazes directly at the camera with a relaxed expression. " + "Blurred beach scenery forms the background featuring crystal-clear waters, distant green hills, " + "and a blue sky dotted with white clouds. The cat assumes a naturally relaxed posture, " + "as if savoring the sea breeze and warm sunlight. A close-up shot highlights the feline's " + "intricate details and the refreshing atmosphere of the seaside." +) +WIDTH = 832 +HEIGHT = 480 +FPS = 24 +NUM_FRAMES = 33 +NUM_INFERENCE_STEPS = 50 +GUIDANCE_SCALE = 6.0 +FLOW_SHIFT = 5.0 +SEED = 42 +SSIM_THRESHOLD = 0.94 +PSNR_THRESHOLD = 28.0 + +REPO_ROOT = Path(__file__).resolve().parents[4] +WORKSPACE_ROOT = REPO_ROOT.parent +RUNNER_PATH = REPO_ROOT / "examples" / "offline_inference" / "image_to_video" / "image_to_video.py" +RESULT_ROOT = Path(__file__).parent / "result" +VIDEO_TIMEOUT_SECONDS = 60 * 60 +SERVER_CASES = [ + pytest.param( + OmniServerParams( + model=MODEL_NAME, + server_args=[ + "--flow-shift", + str(FLOW_SHIFT), + "--enforce-eager", + ], + env_dict={"VLLM_OMNI_STORAGE_PATH": str(RESULT_ROOT / "storage")}, + use_omni=True, + ), + id="hunyuanvideo15_i2v_default", + ) +] + + +def _runner_env() -> dict[str, str]: + env = os.environ.copy() + pythonpath_parts = [ + str(REPO_ROOT), + str(WORKSPACE_ROOT / "diffusers" / "src"), + ] + existing_pythonpath = env.get("PYTHONPATH") + if existing_pythonpath: + pythonpath_parts.append(existing_pythonpath) + env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts) + env.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + return env + + +def _resolve_image_source(configured: str | None) -> str: + if configured: + candidate = Path(configured) + if candidate.exists(): + return str(candidate.resolve()) + return configured + return IMAGE_URL + + +def _artifact_paths(image_source: str) -> tuple[Path, Path]: + artifact_dir = video_artifact_dir(RESULT_ROOT, image_source) + artifact_dir.mkdir(parents=True, exist_ok=True) + return artifact_dir / "online.mp4", artifact_dir / "offline.mp4" + + +def _offline_image_source(image_source: str) -> str: + artifact_dir = video_artifact_dir(RESULT_ROOT, image_source) + return materialize_image_source(image_source, artifact_dir) + + +def _build_offline_command(*, image_source: str, output_path: Path) -> list[str]: + return [ + sys.executable, + str(RUNNER_PATH), + "--model", + MODEL_NAME, + "--image", + image_source, + "--prompt", + PROMPT, + "--height", + str(HEIGHT), + "--width", + str(WIDTH), + "--num-frames", + str(NUM_FRAMES), + "--num-inference-steps", + str(NUM_INFERENCE_STEPS), + "--guidance-scale", + str(GUIDANCE_SCALE), + "--flow-shift", + str(FLOW_SHIFT), + "--fps", + str(FPS), + "--seed", + str(SEED), + "--enforce-eager", + "--output", + str(output_path), + ] + + +def _generate_offline_video(*, image_source: str) -> Path: + _, offline_path = _artifact_paths(image_source) + offline_image_source = _offline_image_source(image_source) + subprocess.run( + _build_offline_command(image_source=offline_image_source, output_path=offline_path), + cwd=REPO_ROOT, + env=_runner_env(), + check=True, + timeout=VIDEO_TIMEOUT_SECONDS, + ) + return offline_path + + +def _generate_online_video( + *, + omni_server, + online_client, + image_source: str, + timeout_seconds: int, +) -> Path: + online_path, _ = _artifact_paths(image_source) + online_image_source = _offline_image_source(image_source) + request_config = { + "model": omni_server.model, + "form_data": { + "prompt": PROMPT, + "height": HEIGHT, + "width": WIDTH, + "fps": FPS, + "num_frames": NUM_FRAMES, + "guidance_scale": GUIDANCE_SCALE, + "flow_shift": FLOW_SHIFT, + "num_inference_steps": NUM_INFERENCE_STEPS, + "seed": SEED, + }, + "image_reference": build_online_image_reference(online_image_source), + } + online_video_bytes = send_video_request_with_timeout( + online_client, + request_config, + timeout_seconds=timeout_seconds, + ) + online_path.write_bytes(online_video_bytes) + return online_path + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) +def test_hunyuanvideo15_i2v_diffusers_offline_generates_video( + hunyuanvideo15_i2v_image_source: str | None, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 I2V offline accuracy test requires CUDA.") + + probe_binary("ffprobe") + if not RUNNER_PATH.exists(): + raise AssertionError(f"Offline runner does not exist: {RUNNER_PATH}") + + image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source) + validate_image_source(image_source) + offline_path = _generate_offline_video(image_source=image_source) + assert offline_path.exists(), f"Expected offline video artifact at {offline_path}" + offline_metadata = probe_video(offline_path) + assert_video_metadata(offline_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) +@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True) +def test_hunyuanvideo15_i2v_online_serving_generates_video( + omni_server, + online_client, + hunyuanvideo15_i2v_image_source: str | None, + hunyuanvideo15_online_timeout_seconds: int, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 I2V online accuracy test requires CUDA.") + + probe_binary("ffprobe") + image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source) + validate_image_source(image_source) + online_path = _generate_online_video( + omni_server=omni_server, + online_client=online_client, + image_source=image_source, + timeout_seconds=hunyuanvideo15_online_timeout_seconds, + ) + assert online_path.exists(), f"Expected online video artifact at {online_path}" + online_metadata = probe_video(online_path) + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) +def test_hunyuanvideo15_i2v_serving_matches_offline_video_similarity( + hunyuanvideo15_i2v_image_source: str | None, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 I2V video similarity test requires CUDA.") + + probe_binary("ffmpeg") + probe_binary("ffprobe") + image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source) + # The input image was already consumed by the prerequisite generation + # tests. Avoid re-fetching a remote source when only comparing artifacts. + online_path, offline_path = _artifact_paths(image_source) + if not online_path.exists(): + pytest.skip(f"Missing online artifact from prerequisite test: {online_path}") + if not offline_path.exists(): + pytest.skip(f"Missing offline artifact from prerequisite test: {offline_path}") + + online_metadata = probe_video(online_path) + offline_metadata = probe_video(offline_path) + assert online_metadata == offline_metadata, ( + f"Video metadata mismatch:\n" + f"online={online_metadata}\n" + f"offline={offline_metadata}\n" + f"online_path={online_path}\n" + f"offline_path={offline_path}" + ) + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + assert_video_similarity_metrics( + label="hunyuanvideo15_i2v", + online_path=online_path, + offline_path=offline_path, + ssim_threshold=SSIM_THRESHOLD, + psnr_threshold=PSNR_THRESHOLD, + ) diff --git a/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py b/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py new file mode 100644 index 000000000..f16813f12 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py @@ -0,0 +1,214 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import torch + +from tests.e2e.accuracy.helpers import ( + assert_video_metadata, + assert_video_similarity_metrics, + probe_binary, + probe_video, + send_video_request_with_timeout, +) +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +MODEL_NAME = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" +PROMPT = "A cat walks on the grass, realistic style." +WIDTH = 832 +HEIGHT = 480 +FPS = 24 +NUM_FRAMES = 33 +NUM_INFERENCE_STEPS = 50 +GUIDANCE_SCALE = 6.0 +FLOW_SHIFT = 5.0 +SEED = 42 +SSIM_THRESHOLD = 0.94 +PSNR_THRESHOLD = 28.0 + +REPO_ROOT = Path(__file__).resolve().parents[4] +WORKSPACE_ROOT = REPO_ROOT.parent +RUNNER_PATH = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py" +RESULT_ROOT = Path(__file__).parent / "result" +ARTIFACT_DIR = RESULT_ROOT / "cat_grass" +VIDEO_TIMEOUT_SECONDS = 60 * 60 +SERVER_CASES = [ + pytest.param( + OmniServerParams( + model=MODEL_NAME, + server_args=[ + "--flow-shift", + str(FLOW_SHIFT), + ], + env_dict={"VLLM_OMNI_STORAGE_PATH": str(RESULT_ROOT / "storage")}, + use_omni=True, + ), + id="hunyuanvideo15_t2v_default", + ) +] + + +def _runner_env() -> dict[str, str]: + env = os.environ.copy() + pythonpath_parts = [ + str(REPO_ROOT), + str(WORKSPACE_ROOT / "diffusers" / "src"), + ] + existing_pythonpath = env.get("PYTHONPATH") + if existing_pythonpath: + pythonpath_parts.append(existing_pythonpath) + env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts) + env.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + return env + + +def _artifact_paths() -> tuple[Path, Path]: + ARTIFACT_DIR.mkdir(parents=True, exist_ok=True) + return ARTIFACT_DIR / "online.mp4", ARTIFACT_DIR / "offline.mp4" + + +def _build_offline_command(*, output_path: Path) -> list[str]: + return [ + sys.executable, + str(RUNNER_PATH), + "--model", + MODEL_NAME, + "--prompt", + PROMPT, + "--height", + str(HEIGHT), + "--width", + str(WIDTH), + "--num-frames", + str(NUM_FRAMES), + "--num-inference-steps", + str(NUM_INFERENCE_STEPS), + "--guidance-scale", + str(GUIDANCE_SCALE), + "--flow-shift", + str(FLOW_SHIFT), + "--fps", + str(FPS), + "--seed", + str(SEED), + "--output", + str(output_path), + ] + + +def _generate_offline_video() -> Path: + _, offline_path = _artifact_paths() + subprocess.run( + _build_offline_command(output_path=offline_path), + cwd=REPO_ROOT, + env=_runner_env(), + check=True, + timeout=VIDEO_TIMEOUT_SECONDS, + ) + return offline_path + + +def _generate_online_video(*, omni_server, online_client, timeout_seconds: int) -> Path: + online_path, _ = _artifact_paths() + request_config = { + "model": omni_server.model, + "form_data": { + "prompt": PROMPT, + "height": HEIGHT, + "width": WIDTH, + "fps": FPS, + "num_frames": NUM_FRAMES, + "guidance_scale": GUIDANCE_SCALE, + "flow_shift": FLOW_SHIFT, + "num_inference_steps": NUM_INFERENCE_STEPS, + "seed": SEED, + }, + } + online_video_bytes = send_video_request_with_timeout( + online_client, + request_config, + timeout_seconds=timeout_seconds, + ) + online_path.write_bytes(online_video_bytes) + return online_path + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) +def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 T2V offline accuracy test requires CUDA.") + + probe_binary("ffprobe") + if not RUNNER_PATH.exists(): + raise AssertionError(f"Offline runner does not exist: {RUNNER_PATH}") + + offline_path = _generate_offline_video() + assert offline_path.exists(), f"Expected offline video artifact at {offline_path}" + offline_metadata = probe_video(offline_path) + assert_video_metadata(offline_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) +@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True) +def test_hunyuanvideo15_t2v_online_serving_generates_video( + omni_server, + online_client, + hunyuanvideo15_online_timeout_seconds: int, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 T2V online accuracy test requires CUDA.") + + probe_binary("ffprobe") + online_path = _generate_online_video( + omni_server=omni_server, + online_client=online_client, + timeout_seconds=hunyuanvideo15_online_timeout_seconds, + ) + assert online_path.exists(), f"Expected online video artifact at {online_path}" + online_metadata = probe_video(online_path) + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) +def test_hunyuanvideo15_t2v_serving_matches_offline_video_similarity() -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 T2V video similarity test requires CUDA.") + + probe_binary("ffmpeg") + probe_binary("ffprobe") + online_path, offline_path = _artifact_paths() + if not online_path.exists(): + pytest.skip(f"Missing online artifact from prerequisite test: {online_path}") + if not offline_path.exists(): + pytest.skip(f"Missing offline artifact from prerequisite test: {offline_path}") + + online_metadata = probe_video(online_path) + offline_metadata = probe_video(offline_path) + assert online_metadata == offline_metadata, ( + f"Video metadata mismatch:\n" + f"online={online_metadata}\n" + f"offline={offline_metadata}\n" + f"online_path={online_path}\n" + f"offline_path={offline_path}" + ) + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + assert_video_similarity_metrics( + label="hunyuanvideo15_t2v", + online_path=online_path, + offline_path=offline_path, + ssim_threshold=SSIM_THRESHOLD, + psnr_threshold=PSNR_THRESHOLD, + ) diff --git a/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py b/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py new file mode 100644 index 000000000..41a020bd0 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS accuracy regression coverage. + +Daily-Omni settings follow the MiniCPM interleaved AV recipe that reaches +~77% overall accuracy on Daily-Omni (``minicpm-interleave``, ``temperature=0``, +text modalities, and server ``--interleave-mm-strings`` + 1fps / 128-frame +media-io kwargs, also pinned in ``minicpmo_4_5.yaml``). + +Seed-TTS WER is gated over ``/v1/chat/completions`` (whole-utterance TTS) on +the duplex deploy config, once with the stock Code2Wav and once with its +bfloat16 attention cache. The native per-unit full-duplex path is not WER +gated here: the checkpoint's own per-unit boundaries split words across +Talker conditions, so that path scores ~0.075 on this benchmark (the HF +``MiniCPMODuplex`` reference scores ~0.14) and cannot meet the 0.05 gate. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path + +import pytest + +from tests.e2e.accuracy.qwen3_omni import run_qwen_omni_acc_benchmark as _acc_bench +from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import ( + build_acc_benchmark_cli_argv, + find_vllm_cli, +) +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams +from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config + +_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5") +_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml") +# CUDA only: the NPU Code2Wav rejects ``code2wav_bfloat16_attention_cache``. +_BF16_ATTENTION_CACHE_DEPLOY_CONFIG = modify_stage_config( + _DEPLOY_CONFIG, + updates={ + "base_config": _DEPLOY_CONFIG, + "connectors.connector_of_shared_memory.extra.code2wav_bfloat16_attention_cache": True, + }, +) +_RESULT_DIR = Path( + os.environ.get( + "ACC_BENCH_RESULT_DIR", + str(Path(__file__).resolve().parent / "results"), + ) +) + +# The 0.78 gate (934/1197) sits 1 item above the observed NPU mean +# (~933/1197) and inside the observed 932~940 spread, so it flips nightly - +# 3 failures in the latest 7 dated NPU nightlies (#6887). 0.77 (922/1197) +# sits 10 items below the observed minimum (932) while still failing a +# >=1pp real accuracy regression. +_MIN_DAILY_OMNI_ACCURACY = 0.77 +_MAX_SEED_TTS_MEAN_WER = 0.05 +# Match the validated Daily-Omni client body from daily_omni_bench.sh. +_DAILY_EXTRA_BODY = { + "modalities": ["text"], + "chat_template_kwargs": {"enable_thinking": False}, +} +_SEED_EXTRA_BODY = { + "modalities": ["text", "audio"], + "chat_template_kwargs": { + "enable_thinking": False, + "use_tts_template": True, + }, +} +# Server flags required for MiniCPM interleaved image/audio packs (Daily-Omni only). +# Do not reuse these for Seed-TTS: ``--interleave-mm-strings`` + TTS ref_audio can trip +# msgspec ValidationError and kill the orchestrator mid-suite. +# Also pinned in ``minicpmo_4_5.yaml`` stage 0 so they survive config-path routing. +_DAILY_OMNI_SERVER_ARGS = [ + "--trust-remote-code", + "--interleave-mm-strings", + "--media-io-kwargs", + '{"video":{"fps":1,"num_frames":128}}', +] +_SEED_TTS_SERVER_ARGS = [ + "--trust-remote-code", +] + +pytestmark = [pytest.mark.full_model, pytest.mark.omni] + +# Evaluated at setup time, before the ``omni_server`` fixture boots a server. An +# ``importorskip`` inside the test body would skip only after paying several minutes +# of startup, and the resulting skip is easy to miss in a nightly log. +_missing_daily_omni_deps = [name for name in ("datasets", "huggingface_hub") if importlib.util.find_spec(name) is None] +requires_daily_omni_deps = pytest.mark.skipif( + bool(_missing_daily_omni_deps), + reason=f"Daily-Omni accuracy bench needs {', '.join(_missing_daily_omni_deps)}", +) + +daily_test_params = [ + OmniServerParams( + model=_MODEL, + stage_config_path=_DEPLOY_CONFIG, + server_args=list(_DAILY_OMNI_SERVER_ARGS), + ) +] + +seed_test_params = [ + OmniServerParams( + model=_MODEL, + stage_config_path=_DEPLOY_CONFIG, + server_args=list(_SEED_TTS_SERVER_ARGS), + ) +] + +seed_bf16_attention_cache_test_params = [ + pytest.param( + OmniServerParams( + model=_MODEL, + stage_config_path=_BF16_ATTENTION_CACHE_DEPLOY_CONFIG, + server_args=list(_SEED_TTS_SERVER_ARGS), + ), + id="bf16-attention-cache", + ), +] + + +def _require_vllm_cli() -> None: + try: + find_vllm_cli() + except FileNotFoundError as exc: + pytest.skip(str(exc)) + + +@pytest.fixture(autouse=True) +def _inline_daily_omni_media(monkeypatch: pytest.MonkeyPatch) -> None: + """Inline cached Hub/local videos because the test server has no local-media allowlist.""" + original = _acc_bench.daily_omni_bench_argv + + def _wrapped() -> list[str]: + argv = list(original()) + if "--daily-omni-inline-local-video" not in argv: + argv.append("--daily-omni-inline-local-video") + return argv + + monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrapped) + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + + +@requires_daily_omni_deps +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1) +@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True) +def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None: + _require_vllm_cli() + + argv = build_acc_benchmark_cli_argv( + omni_server, + skip_seed=True, + skip_daily=False, + ) + argv.extend( + [ + "--result-dir", + str(_RESULT_DIR), + "--min-daily-omni-accuracy", + str(_MIN_DAILY_OMNI_ACCURACY), + "--temperature", + "0", + "--output-len", + "512", + "--daily-omni-input-mode", + "all", + "--daily-omni-pack-mode", + "minicpm-interleave", + "--daily-extra-body-json", + json.dumps(_DAILY_EXTRA_BODY, separators=(",", ":")), + "--trust-remote-code", + ] + ) + + assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0 + + +def _run_seed_tts_wer_bench(omni_server) -> None: + _require_vllm_cli() + pytest.importorskip("huggingface_hub") + + argv = build_acc_benchmark_cli_argv( + omni_server, + skip_seed=False, + skip_daily=True, + ) + argv.extend( + [ + "--result-dir", + str(_RESULT_DIR), + "--max-seed-tts-mean-wer", + str(_MAX_SEED_TTS_MEAN_WER), + "--seed-extra-body-json", + json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")), + "--trust-remote-code", + ] + ) + + assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0 + + +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A3"}, num_cards=1) +@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True) +def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None: + _run_seed_tts_wer_bench(omni_server) + + +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=1) +@pytest.mark.parametrize("omni_server", seed_bf16_attention_cache_test_params, indirect=True) +def test_minicpmo_4_5_seed_tts_wer_bench_bf16_attention_cache(omni_server) -> None: + """Same Seed-TTS gate with the Code2Wav bfloat16 attention cache enabled.""" + _run_seed_tts_wer_bench(omni_server) diff --git a/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py b/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py new file mode 100644 index 000000000..4ae45e4e1 --- /dev/null +++ b/test/fixtures/remote_ci_owner/repo/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py @@ -0,0 +1,387 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +from pathlib import Path + +import pytest +import requests +import torch +from huggingface_hub import snapshot_download + +from tests.e2e.accuracy.helpers import ( + assert_video_metadata, + assert_video_similarity_metrics, + probe_binary, + probe_video, + reset_artifact_dir, +) +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServer + +pytestmark = [pytest.mark.benchmark, pytest.mark.diffusion, pytest.mark.full_model] + +MODEL_REPO_ID = "MiniMaxAI/MiniMax-H3" +# The previously pinned revision was removed from the Hugging Face repository. +MODEL_REVISION = "main" +MODEL_ENV_VAR = "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL" +REF2VA_MODEL_ENV_VAR = "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL" +I2VA_REFERENCE_VIDEO_URL = "https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/assets/i2va.mp4" +I2VA_REFERENCE_VIDEO_SHA256 = "5a5d6e0fcdda7e6a5d98cb92d9e5cb4b6b0b44d6e1dfd5a370d16dfaa79b9ae6" +I2VA_IMAGE_URL = ( + "https://cdn.hailuoai.com/prod/hailuo_demo/testsets/H3_AA_I2VA/gallery/" + "sr_v17_variants_seed42_43_20260724/inputs/4a3a90bf9100_KDmcbkhzYo5sjjxr9FqcVmWVnzb.png" +) +I2VA_IMAGE_SHA256 = "352e959e08b886406d3766cee25ebe7bbf679e6d53eeffbd1c607b8dc93e2068" +REF2VA_REFERENCE_VIDEO_URL = "https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/assets/r2va.mp4" +REF2VA_REFERENCE_VIDEO_SHA256 = "5c9d33dfc517438c59e93da7b3d0b2a62e3cf69bc2adb7d1b2d33d7d8df7161f" +REF2VA_INPUT_VIDEO_URL = ( + "https://cdn.hailuoai.com/prod/hailuo_demo/testsets/h3_promo_eval_ref2va/" + "gallery/sr_v2p26_trio_seed42_20260724/inputs/" + "297573323635_00_%E8%A7%86%E9%A2%911_YnyRbxEwio_video_20260525_163755_1927e9d3.mp4" +) +REF2VA_INPUT_VIDEO_SHA256 = "d6edc20ac82aa980c9cbd0b05b371f60d12dc8d018492e571b42de0086819c41" +REF2VA_INPUT_AUDIO_URL = ( + "https://cdn.hailuoai.com/prod/hailuo_demo/testsets/h3_promo_eval_ref2va/" + "gallery/sr_v2p26_trio_seed42_20260724/inputs/" + "f463d523c5ce_01_%E9%9F%B3%E9%A2%911_RSLcbpzJPo_6%E6%9C%885%E6%97%A5(1).mp3" +) +REF2VA_INPUT_AUDIO_SHA256 = "2a48d447b343b38a2c504d5281a0c97c688db0ba87931ed9282c598bd6752454" + +I2VA_PROMPT = """For the target video, at 0.00 seconds into the target video, (from [Shot 1]) is fully referenced. + +integrated_multimodal_description: [Shot 1] This is a live-action, cinematic shot with a shallow depth of field. The camera holds a perfectly static shot throughout the entire eight-second duration, capturing a cozy family gathering in a traditional Japanese dining room. The scene opens with a large, intricately patterned blue and white ceramic bowl of ramen in the immediate foreground, rendered in crisp, sharp focus. The bowl sits on a smooth, polished long wooden table. Inside the bowl, a rich, oily golden-brown broth surrounds yellow wavy noodles, topped with two thick, round slices of chashu pork featuring visible fat marbling and a distinct spiral meat pattern. A generous mound of freshly chopped, bright green scallions rests in the center, and a crisp, dark green rectangular sheet of nori seaweed is tucked into the right edge. To the left of the bowl, a pair of light brown wooden chopsticks rests horizontally on a small, dark rectangular chopstick rest, near a small cylindrical ceramic teacup with blue painted patterns. On the right side of the table, a spherical paper lantern with a ribbed bamboo frame sits on a black wooden base. In the background, a large family of seven is gathered around the table, initially appearing as a soft, blurred presence. Behind them, traditional Japanese sliding shoji screens with wooden lattice frames are open, revealing a bright outdoor scene with lush green trees. Early in the clip, the thick, white steam rising from the hot ramen broth immediately intensifies, billowing upwards in thick, swirling clouds that dance continuously above the bowl. As the clip progresses into the middle seconds, the camera maintains its static position while the focus begins a deliberate, smooth shift deeper into the room. The foreground ramen bowl, its vibrant ingredients, and the rising steam gradually soften into a hazy, out-of-focus blur. Simultaneously, the family members in the background come into sharp, detailed clarity. The heavy steam continues to rise from the foreground, creating a dynamic, translucent veil between the camera and the family. With the focus now firmly locked on the background, the vibrant family dinner comes alive. The man in the dark navy blue long-sleeved shirt on the left leans forward, his mouth moving animatedly in a silent exchange. The young girl in the crisp white short-sleeved t-shirt beside him smiles brightly, looking toward the center of the table. The woman on the far left, wearing a soft light blue long-sleeved blouse, turns her head slightly, smiling gently. Across the table, the woman in the light grey button-down shirt smiles broadly, her eyes crinkling, as she rests her hands near her plate. The woman in the dark grey top further back uses her wooden chopsticks to pick up a small piece of food from a central ceramic dish filled with bright red pickled vegetables. The woman in the center back in the light grey sweater smiles gently, her hands clasped softly in front of her, observing the interaction. Throughout the remainder of the clip, the family continues their lively physical interaction, their mouths moving in continuous, silent cadences of conversation, while the thick, white steam from the blurred ramen bowl in the foreground never stops rising, adding a comforting atmosphere to the warm gathering. + +overall_soundscape: The soundscape begins with a quiet room tone mixed with the faint, airy rustle of the thick steam billowing from the hot ramen bowl in the foreground, accompanied by the subtle, continuous hissing and bubbling of the rich broth. As the visual focus shifts deeper into the room, the physical sounds of the bustling family dinner become dominant in the foreground. The clear, sharp clinking of ceramic bowls and wooden chopsticks touching plates is clearly heard as the family members reach for food. This is followed by the faint, muffled thud of a cup being set down on the smooth wooden table, and the subtle, rhythmic rustle of cotton and wool clothing as the family members lean forward and gesture, perfectly capturing the lively, physical atmosphere of the shared meal. + +non_diegetic_music: A gentle, heartwarming acoustic guitar melody plays softly in the background, accompanied by the subtle, resonant notes of a traditional Japanese koto. The music maintains a slow, comforting tempo that enhances the cozy, nostalgic, and joyful atmosphere of the family gathering.""" + +REF2VA_PROMPT = """subject_definitions: + is the young man with short wavy blonde hair, wearing a bright pink suit jacket, matching pink trousers, an unbuttoned white shirt, and silver rings, holding a small black lamb in his arms in