From d8b3cd79f42fed5ae138ef959a4ec6f4fc18857d Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Fri, 28 Aug 2026 17:50:45 +0200 Subject: [PATCH 1/6] Evaluate saved pruned checkpoints Signed-off-by: Johannes Rausch --- examples/puzzletron/README.md | 2 +- .../qwen3_5/qwen3p5_0p8b/runs/full_smoke.yaml | 34 +++++- .../qwen3p5_0p8b/runs/full_vlm_smoke.yaml | 33 +++++- .../qwen3p5_0p8b/execution.full_smoke.yaml | 2 + .../execution.full_vlm_smoke.yaml | 2 + .../puzzletron/docs/checkpoint_evaluation.md | 2 +- examples/puzzletron/docs/post_mip_pipeline.md | 27 ++++- .../puzzletron/docs/qwen3p5_0p8b_smoke.md | 33 ++++-- .../puzzletron/docs/qwen3p5_0p8b_vlm_smoke.md | 54 +++++++-- examples/puzzletron/evaluation/vlm/model.py | 11 +- .../puzzletron/evaluation/vlm/post_mip.py | 101 ++++++++++++++++ examples/puzzletron/evaluation/vlm/run.py | 110 +++++++++-------- examples/puzzletron/evaluation/vlm/suites.py | 3 +- examples/puzzletron/run_post_mip_node.py | 28 ++++- modelopt/torch/puzzletron/post_mip/runner.py | 32 ++++- .../torch/puzzletron/checkpoint_evaluation.py | 112 ++++++++++++++++++ .../puzzletron/test_qwen3p5_0p8b_smoke.py | 34 ++++-- .../puzzletron/test_qwen3p5_0p8b_vlm_smoke.py | 64 ++++++++-- .../puzzletron/evaluation/vlm/test_run.py | 83 ++++++++++++- .../torch/puzzletron/test_post_mip_runner.py | 70 +++++++++++ .../test_qwen3p5_0p8b_full_smoke_plan.py | 13 ++ .../test_qwen3p5_0p8b_full_vlm_smoke_plan.py | 14 ++- 22 files changed, 755 insertions(+), 109 deletions(-) create mode 100644 examples/puzzletron/evaluation/vlm/post_mip.py create mode 100644 tests/_test_utils/torch/puzzletron/checkpoint_evaluation.py diff --git a/examples/puzzletron/README.md b/examples/puzzletron/README.md index 3c724f2e1e0..bb379857dc1 100644 --- a/examples/puzzletron/README.md +++ b/examples/puzzletron/README.md @@ -145,7 +145,7 @@ dependency-ordered campaign and whole-campaign resume. Candidate evaluation can be part of a post-MIP campaign flow, where metrics, selection, materialization, and report lineage remain connected. Configure that route with the -[post-MIP pipeline guide](docs/post_mip_pipeline.md#downstream-evaluation). +[post-MIP pipeline guide](docs/post_mip_pipeline.md#evaluate-saved-checkpoints). For standalone evaluation in the Puzzletron worker environment, choose the route that matches the checkpoint and task: diff --git a/examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_smoke.yaml b/examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_smoke.yaml index f41b763be66..66fd9b641cd 100644 --- a/examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_smoke.yaml +++ b/examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_smoke.yaml @@ -44,16 +44,39 @@ post_mip: materialized: type: materialize input: best_lm + checkpoint_eval: + type: downstream_evaluation + input: materialized + failure_policy: strict + config: &ifeval_smoke_config + tasks: [ifeval] + limit: 2 + batch_size: 1 + timeout_seconds: 600 + reasoning_parser: qwen3 + dtype: bfloat16 + gpu_memory_utilization: 0.5 + max_model_len: 4096 + topology: + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + data_parallel_size: 1 + prefill_context_parallel_size: 1 + decode_context_parallel_size: 1 + enable_expert_parallel: false + distributed_executor_backend: mp + gpu_group_size: 1 serving: type: aiperf - input: materialized + input: checkpoint_eval config: input_tokens: 128 output_tokens: 32 concurrency: [1] request_count: 4 use_server_token_count: true - benchmark_timeout: 900 + readiness_timeout: 300 + benchmark_timeout: 300 allow_aiperf_v011_online_tokenizer_resolution: true topology: tensor_parallel_size: 1 @@ -84,9 +107,14 @@ post_mip: global_batch_size: 1 local_batch_size: 1 checkpoint_every_steps: 2 + post_kd_checkpoint_eval: + type: downstream_evaluation + input: short_kd + failure_policy: strict + config: *ifeval_smoke_config final_eval: type: evaluation - input: short_kd + input: post_kd_checkpoint_eval config: eval_samples: 2 block_size: 512 diff --git a/examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_vlm_smoke.yaml b/examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_vlm_smoke.yaml index d5245e66551..3169351ccdb 100644 --- a/examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_vlm_smoke.yaml +++ b/examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_vlm_smoke.yaml @@ -51,9 +51,30 @@ post_mip: materialized: type: materialize input: best_vlm_loss + checkpoint_eval: + type: downstream_evaluation + input: materialized + failure_policy: strict + config: &realworldqa_smoke_config + profile: qwen35_vlm_realworldqa + batch_size: 1 + timeout_seconds: 600 + dtype: bfloat16 + gpu_memory_utilization: 0.5 + max_model_len: 4096 + limit_mm_per_prompt: {image: 1} + topology: + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + data_parallel_size: 1 + prefill_context_parallel_size: 1 + decode_context_parallel_size: 1 + enable_expert_parallel: false + distributed_executor_backend: mp + gpu_group_size: 1 vlm_serving: type: aiperf - input: materialized + input: checkpoint_eval config: endpoint_type: chat input_tokens: 100 @@ -65,7 +86,8 @@ post_mip: request_count: 1 extra_inputs: {min_tokens: 80} use_server_token_count: true - benchmark_timeout: 900 + readiness_timeout: 300 + benchmark_timeout: 300 topology: tensor_parallel_size: 1 pipeline_parallel_size: 1 @@ -100,9 +122,14 @@ post_mip: global_batch_size: 1 local_batch_size: 1 checkpoint_every_steps: 2 + post_kd_checkpoint_eval: + type: downstream_evaluation + input: short_vlm_kd + failure_policy: strict + config: *realworldqa_smoke_config final_image_eval: type: evaluation - input: short_vlm_kd + input: post_kd_checkpoint_eval config: eval_samples: 2 block_size: 512 diff --git a/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_smoke.yaml b/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_smoke.yaml index 1933178f004..98090ca8776 100644 --- a/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_smoke.yaml +++ b/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_smoke.yaml @@ -23,8 +23,10 @@ execution: post.params-90.online_eval: {strategy: sharded, instances: 1} post.params-90.best_lm: {strategy: single, instances: 1} post.params-90.materialized: {strategy: sharded, instances: 1} + post.params-90.checkpoint_eval: {strategy: sharded, instances: 1} post.params-90.serving: {strategy: sharded, instances: 1} post.params-90.fastest: {strategy: single, instances: 1} post.params-90.short_kd: {strategy: sharded, instances: 1} + post.params-90.post_kd_checkpoint_eval: {strategy: sharded, instances: 1} post.params-90.final_eval: {strategy: sharded, instances: 1} post.params-90.best: {strategy: single, instances: 1} diff --git a/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_vlm_smoke.yaml b/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_vlm_smoke.yaml index 1ad76742d27..034554976c2 100644 --- a/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_vlm_smoke.yaml +++ b/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_vlm_smoke.yaml @@ -21,8 +21,10 @@ execution: post.params-90.image_eval: {strategy: sharded, instances: 1} post.params-90.best_vlm_loss: {strategy: single, instances: 1} post.params-90.materialized: {strategy: sharded, instances: 1} + post.params-90.checkpoint_eval: {strategy: sharded, instances: 1} post.params-90.vlm_serving: {strategy: sharded, instances: 1} post.params-90.fastest_vlm: {strategy: single, instances: 1} post.params-90.short_vlm_kd: {strategy: sharded, instances: 1} + post.params-90.post_kd_checkpoint_eval: {strategy: sharded, instances: 1} post.params-90.final_image_eval: {strategy: sharded, instances: 1} post.params-90.best: {strategy: single, instances: 1} diff --git a/examples/puzzletron/docs/checkpoint_evaluation.md b/examples/puzzletron/docs/checkpoint_evaluation.md index b65a8473099..69eb7a0f6bc 100644 --- a/examples/puzzletron/docs/checkpoint_evaluation.md +++ b/examples/puzzletron/docs/checkpoint_evaluation.md @@ -117,4 +117,4 @@ identity when applicable. For `lmms-eval`, retain `command.json`, generated YAML and launcher result artifacts. To evaluate candidates as part of a pruning campaign, use -[downstream evaluation](post_mip_pipeline.md#downstream-evaluation) instead. +[saved-checkpoint evaluation](post_mip_pipeline.md#evaluate-saved-checkpoints) instead. diff --git a/examples/puzzletron/docs/post_mip_pipeline.md b/examples/puzzletron/docs/post_mip_pipeline.md index cea2a8af614..e673c474d9b 100644 --- a/examples/puzzletron/docs/post_mip_pipeline.md +++ b/examples/puzzletron/docs/post_mip_pipeline.md @@ -178,15 +178,34 @@ Selection still follows `input`; `model_source` only chooses the artifact operat on. This supports a long KD run selected using short-KD/PTQ results but restarted from the original candidate. -## Downstream evaluation +## Evaluate saved checkpoints `downstream_evaluation` adapts the generic -[checkpoint evaluator](checkpoint_evaluation.md) to materialized campaign -candidates and publishes their task metrics. Add it after a `materialize` node; -the linked example config shows the complete flow. Use the standalone +[checkpoint evaluator](checkpoint_evaluation.md) to campaign checkpoints and +publishes their task metrics. Add it after any checkpoint-producing node, such +as `materialize` or `global_kd`. Use the standalone [checkpoint evaluation](checkpoint_evaluation.md) command when campaign lineage, filtering, and reports are not needed. +Materialization writes a reloadable Hugging Face checkpoint directory. Its +configuration records the realized per-layer block sizes, and its safetensors +contain the physically sliced weights. Evaluation passes that saved directory +unchanged to a fresh `lmms-eval` process backed by vLLM. It does not convert the +AnyModel instance back into an AutoModel instance. Modality-specific profiles +may prepare pinned task adapters and offline dataset snapshots first, but they +delegate checkpoint execution and completion validation to the same evaluator. + +Global KD publishes a consolidated Hugging Face checkpoint and preserves the +realized pruning configuration and required tokenizer or processor assets. A +downstream evaluation node after KD therefore uses the same checkpoint contract +as one after materialization. The Qwen 3.5 text and VLM smoke flows evaluate the +selected checkpoint both before and after their short KD stage. + +The post-MIP graph does not treat the teacher as a candidate revision. To compare +a distilled student with its teacher, evaluate the teacher separately with the +same task, evaluator version, dataset revision, prompt settings, and sample +limit, then compare those metrics with the post-KD node's metrics. + ## Filters `top_k` accepts one integer or separate homogeneous/heterogeneous quotas. diff --git a/examples/puzzletron/docs/qwen3p5_0p8b_smoke.md b/examples/puzzletron/docs/qwen3p5_0p8b_smoke.md index 0e9098ff567..85907ea91e2 100644 --- a/examples/puzzletron/docs/qwen3p5_0p8b_smoke.md +++ b/examples/puzzletron/docs/qwen3p5_0p8b_smoke.md @@ -3,11 +3,15 @@ The checked-in `full_smoke` recipe runs a small end-to-end test of text-only pruning for Qwen 3.5 0.8B. It searches the FFN intermediate sizes `[3072, 2048]`, evaluates the candidates, saves the two strongest candidates as -physical checkpoints, and measures their serving performance with AIPerf. It -then distills the candidate with higher measured output-token throughput for -two steps, evaluates it again, and selects the final checkpoint. The recipe -pins the public checkpoint revision so repeated runs use the same starting -model. +physical checkpoints, and reloads each saved directory through vLLM for two +IFEval samples. It then measures both checkpoints with AIPerf, distills the +candidate with higher measured output-token throughput for two steps, evaluates +the resulting checkpoint with another two IFEval samples and the internal +two-sample LM-loss check, and selects the final checkpoint. The recipe pins the +public checkpoint revision so repeated runs use the same starting model. See +[evaluate saved checkpoints](post_mip_pipeline.md#evaluate-saved-checkpoints) +for how both Hugging Face directories are loaded without an AnyModel-to-AutoModel +conversion. These small budgets check that the complete workflow runs and resumes correctly. They do not establish model quality or production throughput. @@ -61,9 +65,16 @@ python examples/puzzletron/orchestrate.py \ --stage full ``` -The checked-in flow deliberately uses two evaluation samples per candidate, -four AIPerf requests per serving candidate, and two distillation steps. These -budgets validate lifecycle correctness, comparative serving selection, and -resumability; they are not quality or throughput claims. Final acceptance must -reload the selected checkpoint, verify the cumulative report, and confirm that -the resume submits no work for completed stages. +The checked-in flow deliberately uses two candidate-evaluation samples, two +IFEval samples, four AIPerf requests per serving candidate, and two +distillation steps. These budgets validate lifecycle correctness, comparative +serving selection, and resumability; they are not quality or throughput claims. +The worker environment must provide the [pinned evaluator +installation](checkpoint_evaluation.md#quick-start). IFEval task data must be +fetchable from each worker or already present in its Hugging Face cache. + +After completion, inspect the `checkpoint_eval` and `post_kd_checkpoint_eval` +nodes under `artifacts/post_mip/nodes`. Their summaries must name the corresponding +pre-KD and post-KD checkpoints, report two effective IFEval samples, and contain +finite metrics. Also verify the cumulative report and confirm that resuming +submits no work for completed stages. diff --git a/examples/puzzletron/docs/qwen3p5_0p8b_vlm_smoke.md b/examples/puzzletron/docs/qwen3p5_0p8b_vlm_smoke.md index 0a7974a30e9..0cd9763d884 100644 --- a/examples/puzzletron/docs/qwen3p5_0p8b_vlm_smoke.md +++ b/examples/puzzletron/docs/qwen3p5_0p8b_vlm_smoke.md @@ -4,11 +4,19 @@ The checked-in `full_vlm_smoke` recipe runs a small end-to-end test of vision-language pruning for the public `Qwen/Qwen3.5-0.8B` checkpoint. It uses real image-conversation examples to search the FFN intermediate sizes `[3072, 2048]`, evaluate the candidates, and save the two strongest candidates -as physical checkpoints. It measures both checkpoints with 1-, 6-, and -12-image AIPerf requests, distills the candidate with the highest measured -12-image throughput for two steps, and runs a final image-and-text evaluation. -The immutable revision in `model.yaml` ensures that repeated runs use the same -starting model. +as physical checkpoints. It reloads each saved directory through vLLM for two +RealWorldQA image samples using the pinned Qwen 3.5 VLM evaluation profile. The +profile verifies the evaluator revision and immutable offline dataset snapshot, +strips inherited Hub credentials, and records preflight provenance before +delegating execution to the shared checkpoint evaluator. The workflow then +measures both checkpoints with 1-, 6-, and 12-image AIPerf requests, distills +the candidate with the highest measured 12-image throughput for two steps, and +runs the pinned RealWorldQA benchmark and internal image-and-text evaluation on +the resulting checkpoint. The immutable revision in `model.yaml` ensures that +repeated runs use the same starting model. See +[evaluate saved checkpoints](post_mip_pipeline.md#evaluate-saved-checkpoints) +for the direct pre-KD and post-KD reload paths; no AnyModel-to-AutoModel +conversion occurs. These small budgets check that the complete workflow runs and resumes correctly. They do not establish model quality or production throughput. @@ -18,7 +26,11 @@ correctly. They do not establish model quality or production throughput. Prepare the setup and worker environments described in [environment setup](environment_setup.md). The worker environment must provide ModelOpt, NeMo AutoModel's Qwen 3.5 VLM support, the Puzzletron requirements, -and the reviewed AIPerf/vLLM runtime selected by your runner contract. +the pinned `lmms-eval` dependency, and the reviewed AIPerf/vLLM runtime selected +by your runner contract. The worker-visible Hugging Face cache must already +contain the pinned RealWorldQA snapshot. Populate it as described in +[cache benchmark data](vlm_checkpoint_evaluation.md#cache-benchmark-data); the +profile verifies the local snapshot and evaluates it offline. The worker-visible Hugging Face cache must contain, or be allowed to fetch, `Qwen/Qwen3.5-0.8B` at the pinned revision in @@ -128,11 +140,11 @@ python examples/puzzletron/orchestrate.py \ Confirm that all enabled model stages use one GPU, image-backed stages resolve `data.modality=multimodal`, and no text tokenization stage is present. Confirm -that two quality candidates reach `post.params-90.vlm_serving`, which declares -a `chat` workload with 1, 6, and 12 1280x720 images per request rather than a -text-only serving proxy. The `fastest_vlm` filter selects one candidate from -the 12-image throughput metric before KD. Then launch by omitting only -`--dry-run`: +that two quality candidates reach `post.params-90.checkpoint_eval` for the +bounded RealWorldQA run and then `post.params-90.vlm_serving`, which declares a +`chat` workload with 1, 6, and 12 1280x720 images per request rather than a +text-only serving proxy. The `fastest_vlm` filter selects one candidate from the +12-image throughput metric before KD. Then launch by omitting only `--dry-run`: ```bash python examples/puzzletron/orchestrate.py \ @@ -157,7 +169,14 @@ canonical stage summaries: - width scoring and VLM KD processed real image tensors and report a nonzero vision-forward count; - sorting and physical slicing equivalence passed at the configured tolerance; -- the selected checkpoint reloads after materialization and after VLM KD; +- each materialized pre-KD checkpoint has a successful `checkpoint_eval` + summary whose `checkpoint` field names that saved artifact, whose + RealWorldQA sample count equals two, and whose metrics are finite; the + evaluation root's `profile.json` records the pinned dataset revision and + offline preflight; +- the selected post-KD checkpoint has a successful `post_kd_checkpoint_eval` + summary for two RealWorldQA samples with finite metrics, and the internal + final image evaluation reloads the same checkpoint; - the two-step KD summary contains finite main CE/KD and MTP CE/KD metrics plus nonzero trainable-group gradient evidence; - AIPerf completes every 1-, 6-, and 12-image chat workload for both retained @@ -195,3 +214,14 @@ vision path, and comparative selection while keeping the example bounded; it does not isolate vision-encoder latency or establish production performance. Increase request count and concurrency in a separate reviewed performance run before making throughput claims. + +The real-checkpoint lifecycle test is opt-in and is not part of default pytest +or routine CI smoke execution. Point it at the populated cache: + +```bash +PUZZLETRON_VLM_BENCHMARK_HF_HOME=/path/to/hf-home \ + pytest --run-manual tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_vlm_smoke.py +``` + +The checkpoint contract is documented in +[evaluate saved checkpoints](post_mip_pipeline.md#evaluate-saved-checkpoints). diff --git a/examples/puzzletron/evaluation/vlm/model.py b/examples/puzzletron/evaluation/vlm/model.py index 11154b52844..a36b1805f9b 100644 --- a/examples/puzzletron/evaluation/vlm/model.py +++ b/examples/puzzletron/evaluation/vlm/model.py @@ -43,8 +43,15 @@ def verify_checkpoint(checkpoint: Path, *, profile: str) -> None: if config.get("model_type") != _MODEL_TYPE: raise ValueError(f"{profile} checkpoint model_type must be qwen3_5") architectures = config.get("architectures") - if not isinstance(architectures, list) or _ARCHITECTURE not in architectures: - raise ValueError(f"{profile} checkpoint architectures must contain {_ARCHITECTURE}") + native_checkpoint = architectures == [_ARCHITECTURE] + realized_checkpoint = ( + architectures == ["AnyModel"] and config.get("base_architecture") == _ARCHITECTURE + ) + if not native_checkpoint and not realized_checkpoint: + raise ValueError( + f"{profile} checkpoint must identify {_ARCHITECTURE} directly or as the " + "AnyModel base_architecture" + ) text_config = config.get("text_config") if not isinstance(text_config, dict) or text_config.get("model_type") != "qwen3_5_text": raise ValueError(f"{profile} checkpoint text_config.model_type must be qwen3_5_text") diff --git a/examples/puzzletron/evaluation/vlm/post_mip.py b/examples/puzzletron/evaluation/vlm/post_mip.py new file mode 100644 index 00000000000..da89485d3cc --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/post_mip.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Adapt the pinned Qwen 3.5 VLM profile to a post-MIP checkpoint node.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Mapping + from typing import Any + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import run + +__all__ = ["evaluate_realworldqa_checkpoint", "register_profiles"] + +_RUNNER_OVERRIDES = frozenset( + { + "dtype", + "gpu_memory_utilization", + "limit_mm_per_prompt", + "max_model_len", + "topology", + } +) + + +def register_profiles() -> None: + """Install the example-owned profile into the generic post-MIP runner.""" + + from modelopt.torch.puzzletron.post_mip.runner import register_downstream_evaluation_profile + + register_downstream_evaluation_profile( + "qwen35_vlm_realworldqa", + evaluate_realworldqa_checkpoint, + ) + + +def evaluate_realworldqa_checkpoint( + checkpoint_path: str | Path, + *, + output_root: str | Path, + settings: Mapping[str, Any], +) -> dict[str, Any]: + """Run the two-sample pinned RealWorldQA profile for one saved checkpoint.""" + + settings = dict(settings) + unexpected = set(settings) - _RUNNER_OVERRIDES - {"batch_size", "timeout_seconds"} + if unexpected: + raise ValueError(f"unsupported Qwen 3.5 VLM profile settings: {sorted(unexpected)}") + output_dir = Path(output_root).expanduser().absolute() + output_dir.mkdir(parents=True, exist_ok=True) + args = argparse.Namespace( + checkpoint=Path(checkpoint_path).expanduser().absolute(), + output_dir=output_dir, + suite="realworldqa-smoke", + batch_size=int(settings.pop("batch_size", 1)), + seed=42, + timeout_seconds=settings.pop("timeout_seconds", None), + hf_home=Path(os.environ["HF_HOME"]) if os.environ.get("HF_HOME") else None, + quick_manifest=None, + mmvu_judge_api_type=None, + mmvu_judge_model=None, + allow_judge_calls=False, + preflight_only=False, + ) + profile_path = output_dir / "profile.json" + + def write_preflight(report: dict[str, object]) -> None: + checkpoint.write_generated( + profile_path, + json.dumps(report, indent=2, sort_keys=True) + "\n", + ) + + result = run.evaluate( + args, + settings_overrides=settings, + preflight_callback=write_preflight, + ) + runs = result["runs"] + if not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict): + raise RuntimeError("pinned RealWorldQA profile returned an invalid run count") + return {**runs[0], "profile_path": str(profile_path)} diff --git a/examples/puzzletron/evaluation/vlm/run.py b/examples/puzzletron/evaluation/vlm/run.py index f6385896374..a77d30400e6 100644 --- a/examples/puzzletron/evaluation/vlm/run.py +++ b/examples/puzzletron/evaluation/vlm/run.py @@ -21,6 +21,10 @@ import json import sys from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping REPOSITORY_ROOT = Path(__file__).absolute().parents[4] if str(REPOSITORY_ROOT) not in sys.path: @@ -117,57 +121,69 @@ def _build_parser() -> argparse.ArgumentParser: return parser +def evaluate( + args: argparse.Namespace, + *, + settings_overrides: Mapping[str, object] | None = None, + preflight_callback: Callable[[dict[str, object]], None] | None = None, +) -> dict[str, object]: + """Prepare and run one pinned VLM profile invocation.""" + + prepared = preflight.prepare(args) + task_root, configured_tasks = tasks.prepare( + args.output_dir, + suite=prepared.suite, + dataset_snapshots=prepared.dataset_snapshots, + quick_manifest=prepared.quick_manifest, + ) + offline_task_preflight = tasks.verify_offline( + task_root, + configured_tasks, + hf_home=prepared.hf_home, + timeout_seconds=checkpoint.DEFAULT_PREFLIGHT_TIMEOUT_SECONDS, + ) + report = dict(prepared.report) + report.update( + { + "configured_tasks": list(configured_tasks), + "offline_task_preflight": offline_task_preflight, + "task_config_root": str(task_root), + } + ) + if preflight_callback is not None: + preflight_callback(report) + if args.preflight_only: + return {"preflight": report, "runs": []} + settings = preflight.settings( + args, + tasks_root=task_root, + configured_tasks=configured_tasks, + prepared=prepared, + ) + settings.update(settings_overrides or {}) + repetitions = prepared.execution_policy["repetitions"] + runs = [] + with checkpoint.without_huggingface_credentials(): + for repetition in range(1, repetitions + 1): + output_root = args.output_dir + if repetitions > 1: + output_root = output_root / f"short-repetition-{repetition}" + runs.append( + checkpoint.run_lmms_eval_checkpoint( + args.checkpoint, + output_root=output_root, + settings=settings, + ) + ) + return {"preflight": report, "runs": runs} + + def main(argv: list[str] | None = None) -> int: args = _build_parser().parse_args(argv) try: - prepared = preflight.prepare(args) - task_root, configured_tasks = tasks.prepare( - args.output_dir, - suite=prepared.suite, - dataset_snapshots=prepared.dataset_snapshots, - quick_manifest=prepared.quick_manifest, - ) - offline_task_preflight = tasks.verify_offline( - task_root, - configured_tasks, - hf_home=prepared.hf_home, - timeout_seconds=checkpoint.DEFAULT_PREFLIGHT_TIMEOUT_SECONDS, - ) - report = dict(prepared.report) - report.update( - { - "configured_tasks": list(configured_tasks), - "offline_task_preflight": offline_task_preflight, - "task_config_root": str(task_root), - } - ) + result = evaluate(args) if args.preflight_only: - print(json.dumps({"preflight": report}, indent=2, sort_keys=True)) - return 0 - settings = preflight.settings( - args, - tasks_root=task_root, - configured_tasks=configured_tasks, - prepared=prepared, - ) - repetitions = prepared.execution_policy["repetitions"] - runs = [] - with checkpoint.without_huggingface_credentials(): - for repetition in range(1, repetitions + 1): - output_root = args.output_dir - if repetitions > 1: - output_root = output_root / f"short-repetition-{repetition}" - runs.append( - checkpoint.run_lmms_eval_checkpoint( - args.checkpoint, - output_root=output_root, - settings=settings, - ) - ) - result = { - "preflight": report, - "runs": runs, - } + result = {"preflight": result["preflight"]} except Exception as error: payload = { "error": type(error).__name__, diff --git a/examples/puzzletron/evaluation/vlm/suites.py b/examples/puzzletron/evaluation/vlm/suites.py index 6c0264c986a..ce499909b38 100644 --- a/examples/puzzletron/evaluation/vlm/suites.py +++ b/examples/puzzletron/evaluation/vlm/suites.py @@ -59,6 +59,7 @@ QUICK_TASKS = ("realworldqa", "mmmu_val", "mvbench") ADAPTER_SMOKE_TASKS = ("videomme", "perceptiontest_val_mc") SINGLE_TASK_SMOKE_SUITES = { + "realworldqa-smoke": "realworldqa", "video-mmmu-smoke": "video_mmmu", "mmvu-smoke": "mmvu_val", "longvideobench-smoke": "longvideobench_val_v", @@ -155,7 +156,7 @@ def execution_policy(suite: str, *, timeout_seconds: float | None) -> ExecutionP return { "frame": {"reader": "decord", "fps": 2, "max_frames": 32}, "generation": {"temperature": 0, "do_sample": False}, - "limit": 8 if is_smoke else None, + "limit": 2 if suite == "realworldqa-smoke" else 8 if is_smoke else None, "repetitions": 2 if suite == "short" else 1, "timeout_seconds": ( timeout_seconds if timeout_seconds is not None else default_timeout_seconds diff --git a/examples/puzzletron/run_post_mip_node.py b/examples/puzzletron/run_post_mip_node.py index c143ca4e351..4871d4f7efb 100644 --- a/examples/puzzletron/run_post_mip_node.py +++ b/examples/puzzletron/run_post_mip_node.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Run or aggregate one campaign-configured post-MIP node.""" @@ -15,7 +27,20 @@ if str(REPOSITORY_ROOT) not in sys.path: sys.path.insert(0, str(REPOSITORY_ROOT)) -from puzzletron_orchestrator.config import load_experiment_config +from puzzletron_orchestrator.config import load_experiment_config # noqa: E402 + + +def _register_evaluation_profiles(config: dict) -> None: + profiles = { + node.get("config", {}).get("profile") + for flow in (config.get("post_mip", {}).get("flows") or {}).values() + for node in (flow.get("nodes") or {}).values() + if node.get("type") == "downstream_evaluation" + } + if "qwen35_vlm_realworldqa" in profiles: + from examples.puzzletron.evaluation.vlm.post_mip import register_profiles + + register_profiles() def main() -> None: @@ -44,6 +69,7 @@ def main() -> None: from modelopt.torch.puzzletron.post_mip import run_post_mip_node_shard config = pipeline_config_from_path(args.config, overrides=args.override) + _register_evaluation_profiles(config) payload = { "result_path": str( run_post_mip_node_shard( diff --git a/modelopt/torch/puzzletron/post_mip/runner.py b/modelopt/torch/puzzletron/post_mip/runner.py index 838078d92d1..4e18c29b4f4 100644 --- a/modelopt/torch/puzzletron/post_mip/runner.py +++ b/modelopt/torch/puzzletron/post_mip/runner.py @@ -27,7 +27,7 @@ from contextlib import contextmanager from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Iterator, Mapping, Sequence +from typing import Any, Callable, Iterator, Mapping, Sequence from ..evaluation import DEFAULT_LMMS_EVAL_TIMEOUT_SECONDS, run_lmms_eval_checkpoint from ..identity import canonicalize, stable_hash @@ -44,9 +44,26 @@ __all__ = [ "aggregate_post_mip_node", "expected_post_mip_execution_identity", + "register_downstream_evaluation_profile", "run_post_mip_node_shard", ] +_DOWNSTREAM_EVALUATION_PROFILES: dict[str, Callable[..., dict[str, Any]]] = {} + + +def register_downstream_evaluation_profile( + name: str, + evaluator: Callable[..., dict[str, Any]], +) -> None: + """Register an examples-layer evaluator without coupling ModelOpt to that example.""" + + if not name: + raise ValueError("downstream evaluation profile name must not be empty") + existing = _DOWNSTREAM_EVALUATION_PROFILES.get(name) + if existing is not None and existing is not evaluator: + raise ValueError(f"downstream evaluation profile is already registered: {name}") + _DOWNSTREAM_EVALUATION_PROFILES[name] = evaluator + def _puzzle_dir(config: Mapping[str, Any]) -> Path: return Path(config.get("puzzle_dir") or (config.get("experiment") or {})["dir"]) @@ -508,10 +525,21 @@ def _downstream_evaluation( / source.architecture_id / "lmms_eval" ) + settings = dict(node.config.get("config") or {}) + profile = settings.pop("profile", None) + if profile is not None: + evaluator = _DOWNSTREAM_EVALUATION_PROFILES.get(str(profile)) + if evaluator is None: + raise ValueError(f"unsupported downstream evaluation profile: {profile}") + return evaluator( + source.artifact["checkpoint"], + output_root=output_root, + settings=settings, + ) return run_lmms_eval_checkpoint( source.artifact["checkpoint"], output_root=output_root, - settings=node.config.get("config") or {}, + settings=settings, ) diff --git a/tests/_test_utils/torch/puzzletron/checkpoint_evaluation.py b/tests/_test_utils/torch/puzzletron/checkpoint_evaluation.py new file mode 100644 index 00000000000..8f7c077a79c --- /dev/null +++ b/tests/_test_utils/torch/puzzletron/checkpoint_evaluation.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Assertions shared by real-checkpoint post-materialization evaluation tests.""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +from modelopt.torch.puzzletron.post_mip.records import CandidateLedger + +__all__ = ["assert_pruned_checkpoints_completed_benchmark"] + + +def _intermediate_sizes(value: Any) -> list[int]: + if isinstance(value, dict): + sizes = [] + for key, child in value.items(): + if ( + key == "intermediate_size" + and isinstance(child, int) + and not isinstance(child, bool) + ): + sizes.append(child) + else: + sizes.extend(_intermediate_sizes(child)) + return sizes + if isinstance(value, list): + return [size for child in value for size in _intermediate_sizes(child)] + return [] + + +def assert_pruned_checkpoints_completed_benchmark( + run_root: Path, + *, + checkpoint_node: str, + evaluation_node: str, + task: str, + limit: int, +) -> None: + """Verify every checkpoint revision from one node completed the benchmark.""" + + ledger = CandidateLedger(run_root / "artifacts/post_mip") + candidate_set = ledger.load_candidate_set(checkpoint_node) + checkpoints = { + Path(ledger.revisions[revision_id].artifact["checkpoint"]).resolve() + for revision_id in candidate_set.revision_ids + } + assert checkpoints + for checkpoint in checkpoints: + config_path = checkpoint / "config.json" + assert config_path.is_file() + assert any(checkpoint.glob("*.safetensors")) + config = json.loads(config_path.read_text(encoding="utf-8")) + language_config = config.get("text_config", config) + teacher_size = language_config["intermediate_size"] + block_configs = language_config.get("block_configs", config.get("block_configs")) + assert isinstance(block_configs, list) and block_configs + assert any(size < teacher_size for size in _intermediate_sizes(block_configs)) + + summaries = sorted( + run_root.glob( + f"artifacts/post_mip/nodes/{evaluation_node}/executions/*/raw/*/" + "lmms_eval/attempt_*/summary.json" + ) + ) + assert summaries + evaluated = set() + for summary_path in summaries: + summary = json.loads(summary_path.read_text(encoding="utf-8")) + checkpoint = Path(summary["checkpoint"]).resolve() + evaluated.add(checkpoint) + assert checkpoint in checkpoints + task_counts = { + name: count + for name, count in summary["sample_counts"].items() + if name == task or name.startswith(f"{task}_") + } + assert task_counts + assert all(count == limit for count in task_counts.values()) + task_metrics = { + name: value + for name, value in summary["metrics"].items() + if name == task or name.startswith(f"{task}.") + } + assert task_metrics + assert all( + isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + for value in task_metrics.values() + ) + + command = json.loads((summary_path.parent / "command.json").read_text(encoding="utf-8")) + argv = command["argv"] + model_args = argv[argv.index("--model_args") + 1] + assert f"model={checkpoint}" in model_args.split(",") + + assert evaluated == checkpoints diff --git a/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py b/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py index 4dafb13cf34..69aae32ff46 100644 --- a/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py +++ b/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py @@ -32,6 +32,10 @@ import yaml from datasets import Dataset, DatasetDict +from tests._test_utils.torch.puzzletron.checkpoint_evaluation import ( + assert_pruned_checkpoints_completed_benchmark, +) + if TYPE_CHECKING: from pathlib import Path @@ -54,13 +58,18 @@ def _save_messages_dataset(path: Path) -> None: @pytest.mark.integration -@pytest.mark.manual(reason="downloads and prunes the real Qwen 3.5 0.8B checkpoint") -@pytest.mark.timeout(2400) -def test_qwen3p5_0p8b_orchestrated_mip_smoke_completes( +@pytest.mark.manual( + reason=( + "downloads and prunes the real checkpoint, runs IFEval on saved pre-KD and " + "post-KD checkpoints through lmms-eval/vLLM, and benchmarks it with AIPerf" + ) +) +@pytest.mark.timeout(7300) +def test_qwen3p5_0p8b_orchestrated_full_smoke_completes( project_root_path: Path, tmp_path: Path, ) -> None: - """Run the enabled DAG through MIP; requires one H100 80GB GPU.""" + """Run the bounded lifecycle on one H100 with network or populated benchmark caches.""" dataset = tmp_path / "dataset" results = tmp_path / "results" @@ -105,14 +114,14 @@ def test_qwen3p5_0p8b_orchestrated_mip_smoke_completes( "--experiment", str( project_root_path / "examples/puzzletron/configs/families/qwen3_5/" - "qwen3p5_0p8b/runs/mip_smoke.yaml" + "qwen3p5_0p8b/runs/full_smoke.yaml" ), "--runner", str(runner), "--execution", str( project_root_path / "examples/puzzletron/configs/orchestration/qwen3p5_0p8b/" - "execution.smoke.yaml" + "execution.full_smoke.yaml" ), "--stage", "full", @@ -124,7 +133,7 @@ def test_qwen3p5_0p8b_orchestrated_mip_smoke_completes( env=environment, capture_output=True, text=True, - timeout=2300, + timeout=7200, check=False, ) if completed.returncode: @@ -207,3 +216,14 @@ def test_qwen3p5_0p8b_orchestrated_mip_smoke_completes( assert active_profiles["profile_ids"] == ["params-90"] assert [profile["id"] for profile in active_profiles["profiles"]] == ["params-90"] assert active_profiles["profiles"][0]["feasible_count"] >= 1 + for checkpoint_node, evaluation_node in ( + ("materialized", "checkpoint_eval"), + ("short_kd", "post_kd_checkpoint_eval"), + ): + assert_pruned_checkpoints_completed_benchmark( + results, + checkpoint_node=checkpoint_node, + evaluation_node=evaluation_node, + task="ifeval", + limit=2, + ) diff --git a/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_vlm_smoke.py b/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_vlm_smoke.py index 4e94f374a84..2e8922b33c0 100644 --- a/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_vlm_smoke.py +++ b/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_vlm_smoke.py @@ -27,10 +27,17 @@ import yaml from PIL import Image +from examples.puzzletron.evaluation.vlm import profile as evaluation_profile +from examples.puzzletron.evaluation.vlm import suites as evaluation_suites from modelopt.torch.puzzletron.dataset.multimodal import materialize_normalized_conversation_samples +from tests._test_utils.torch.puzzletron.checkpoint_evaluation import ( + assert_pruned_checkpoints_completed_benchmark, +) -RUN_PATH = "examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/mip_vlm_smoke.yaml" -EXECUTION_PATH = "examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.vlm_smoke.yaml" +RUN_PATH = "examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_vlm_smoke.yaml" +EXECUTION_PATH = ( + "examples/puzzletron/configs/orchestration/qwen3p5_0p8b/execution.full_vlm_smoke.yaml" +) def _materialize_image_conversations(path: Path) -> None: @@ -100,18 +107,39 @@ def _write_local_runner(path: Path, project_root: Path) -> None: @pytest.mark.integration -@pytest.mark.manual(reason="downloads and prunes the real Qwen 3.5 0.8B VLM checkpoint") -@pytest.mark.timeout(2400) -def test_qwen3p5_0p8b_orchestrated_vlm_mip_smoke_completes( +@pytest.mark.manual( + reason=( + "downloads and prunes the real checkpoint, evaluates the cached pinned RealWorldQA " + "snapshot on saved pre-KD and post-KD checkpoints through lmms-eval/vLLM, and " + "benchmarks it with AIPerf" + ) +) +@pytest.mark.timeout(8700) +def test_qwen3p5_0p8b_orchestrated_vlm_full_smoke_completes( project_root_path: Path, tmp_path: Path, monkeypatch, ) -> None: - """Run through MIP and prove that image inputs reach the vision tower.""" + """Run the bounded VLM lifecycle on one H100 with populated offline benchmark caches.""" dataset = tmp_path / "dataset" results = tmp_path / "results" - cache = tmp_path / "cache" + benchmark_hf_home = os.environ.get("PUZZLETRON_VLM_BENCHMARK_HF_HOME") + if not benchmark_hf_home: + pytest.fail("requires PUZZLETRON_VLM_BENCHMARK_HF_HOME with the pinned RealWorldQA cache") + benchmark_hf_home_path = Path(benchmark_hf_home).expanduser().absolute() + if not benchmark_hf_home_path.is_dir(): + pytest.fail(f"benchmark cache is not a directory: {benchmark_hf_home_path}") + benchmark_hub_cache = benchmark_hf_home_path / "hub" + monkeypatch.setenv("HF_HUB_CACHE", str(benchmark_hub_cache)) + try: + evaluation_suites.offline_dataset_snapshot( + benchmark_hf_home_path, + "realworldqa", + evaluation_profile.VLM_BENCHMARK_DATASETS["realworldqa"].revision, + ) + except ValueError as error: + pytest.fail(str(error)) runner = tmp_path / "runner.yaml" _materialize_image_conversations(dataset) _write_local_runner(runner, project_root_path) @@ -122,10 +150,11 @@ def test_qwen3p5_0p8b_orchestrated_vlm_mip_smoke_completes( environment = os.environ.copy() environment.update( { - "HF_HOME": str(cache / "huggingface"), - "HF_DATASETS_CACHE": str(cache / "datasets"), - "TORCH_HOME": str(cache / "torch"), - "XDG_CACHE_HOME": str(cache / "xdg"), + "HF_HOME": str(benchmark_hf_home_path), + "HF_HUB_CACHE": str(benchmark_hub_cache), + "HF_DATASETS_CACHE": str(benchmark_hf_home_path / "datasets"), + "TORCH_HOME": str(tmp_path / "cache/torch"), + "XDG_CACHE_HOME": str(tmp_path / "cache/xdg"), } ) completed = subprocess.run( @@ -148,7 +177,7 @@ def test_qwen3p5_0p8b_orchestrated_vlm_mip_smoke_completes( env=environment, capture_output=True, text=True, - timeout=2300, + timeout=8600, check=False, ) if completed.returncode: @@ -197,3 +226,14 @@ def test_qwen3p5_0p8b_orchestrated_vlm_mip_smoke_completes( active_profiles = json.loads((results / "mip/active_profiles.json").read_text()) assert active_profiles["status"] == "success" assert active_profiles["profile_ids"] == ["params-90"] + for checkpoint_node, evaluation_node in ( + ("materialized", "checkpoint_eval"), + ("short_vlm_kd", "post_kd_checkpoint_eval"), + ): + assert_pruned_checkpoints_completed_benchmark( + results, + checkpoint_node=checkpoint_node, + evaluation_node=evaluation_node, + task=evaluation_suites.task_name("realworldqa"), + limit=2, + ) diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py index 85cdd90f375..eede50536c9 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py @@ -25,7 +25,8 @@ import pytest from examples.puzzletron.evaluation import checkpoint -from examples.puzzletron.evaluation.vlm import preflight, profile, suites, tasks +from examples.puzzletron.evaluation.vlm import model as vlm_model +from examples.puzzletron.evaluation.vlm import post_mip, preflight, profile, suites, tasks from examples.puzzletron.evaluation.vlm import run as evaluation _QWEN_CONFIG = { @@ -51,6 +52,32 @@ def _write_checkpoint(root: Path) -> Path: return model +def test_checkpoint_contract_accepts_only_matching_realized_anymodel(tmp_path): + checkpoint_path = _write_checkpoint(tmp_path) + config_path = checkpoint_path / "config.json" + config = json.loads(config_path.read_text()) + config.update( + architectures=["AnyModel"], + base_architecture="Qwen3_5ForConditionalGeneration", + ) + config_path.write_text(json.dumps(config) + "\n") + + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config["base_architecture"] = "OtherForConditionalGeneration" + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="AnyModel base_architecture"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + config.update( + architectures=["AnyModel", "Qwen3_5ForConditionalGeneration"], + base_architecture="Qwen3_5ForConditionalGeneration", + ) + config_path.write_text(json.dumps(config) + "\n") + with pytest.raises(ValueError, match="AnyModel base_architecture"): + vlm_model.verify_checkpoint(checkpoint_path, profile="VLM benchmark") + + def _write_lmms_tasks(root: Path, tasks: tuple[str, ...]) -> Path: lmms_root = root / "lmms_eval" for task in tasks: @@ -236,6 +263,60 @@ def fake_runner(checkpoint_path, *, output_root, settings): assert report["generation_policy"] == settings["gen_kwargs"] +def test_realworldqa_smoke_policy_is_single_run_with_two_samples(): + assert suites.source_tasks("realworldqa-smoke") == ("realworldqa",) + policy = suites.execution_policy("realworldqa-smoke", timeout_seconds=900) + assert policy["limit"] == 2 + assert policy["repetitions"] == 1 + assert policy["timeout_seconds"] == 900 + + +def test_post_mip_realworldqa_adapter_routes_runtime_overrides(monkeypatch, tmp_path): + model = tmp_path / "model" + model.mkdir() + output = tmp_path / "output" + captured = {} + + def fake_evaluate(args, *, settings_overrides, preflight_callback): + captured.update(args=args, settings_overrides=settings_overrides) + preflight = { + "profile": suites.EVALUATION_PROFILE, + "dataset_revisions": { + "realworldqa": profile.VLM_BENCHMARK_DATASETS["realworldqa"].revision + }, + "sample_limit": 2, + } + preflight_callback(preflight) + return { + "preflight": preflight, + "runs": [{"metrics": {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5}}], + } + + monkeypatch.setattr(post_mip.run, "evaluate", fake_evaluate) + result = post_mip.evaluate_realworldqa_checkpoint( + model, + output_root=output, + settings={ + "batch_size": 1, + "timeout_seconds": 900, + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + }, + ) + + assert captured["args"].suite == "realworldqa-smoke" + assert captured["args"].checkpoint == model + assert captured["args"].output_dir == output + assert captured["args"].timeout_seconds == 900 + assert captured["settings_overrides"] == { + "dtype": "bfloat16", + "topology": {"tensor_parallel_size": 1}, + } + assert json.loads((output / "profile.json").read_text())["sample_limit"] == 2 + assert result["metrics"] == {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5} + assert result["profile_path"] == str(output / "profile.json") + + def test_mmvu_guard_is_limited_to_full_suite(monkeypatch, tmp_path): snapshot = tmp_path / "snapshot" snapshot.mkdir() diff --git a/tests/unit/torch/puzzletron/test_post_mip_runner.py b/tests/unit/torch/puzzletron/test_post_mip_runner.py index da2b5e85ea1..e695f1a7bf3 100644 --- a/tests/unit/torch/puzzletron/test_post_mip_runner.py +++ b/tests/unit/torch/puzzletron/test_post_mip_runner.py @@ -22,6 +22,7 @@ from omegaconf import OmegaConf import modelopt.torch.puzzletron.stages.future as future_stages +from examples.puzzletron import run_post_mip_node as post_mip_entrypoint from modelopt.torch.puzzletron.post_mip import runner from modelopt.torch.puzzletron.post_mip.records import ArtifactKind from modelopt.torch.puzzletron.post_mip.runner import ( @@ -79,6 +80,33 @@ def test_post_mip_kd_always_requests_a_consolidated_output(): assert settings["max_steps"] == 8 +def test_worker_entrypoint_registers_configured_vlm_evaluation_profile(monkeypatch): + # Keep the examples-layer VLM dependencies out of core test collection. + from examples.puzzletron.evaluation.vlm import post_mip as vlm_post_mip + + calls = [] + monkeypatch.setattr(vlm_post_mip, "register_profiles", lambda: calls.append(True)) + + post_mip_entrypoint._register_evaluation_profiles( + { + "post_mip": { + "flows": { + "params": { + "nodes": { + "checkpoint_eval": { + "type": "downstream_evaluation", + "config": {"profile": "qwen35_vlm_realworldqa"}, + } + } + } + } + } + } + ) + + assert calls == [True] + + def test_online_eval_settings_deep_merge_automodel_overrides(): scoring = OmegaConf.create( { @@ -293,3 +321,45 @@ def fake_evaluate(checkpoint_path, *, output_root, settings): ), "settings": {"tasks": ["ifeval"], "limit": 4}, } + + +def test_downstream_evaluation_routes_the_pinned_vlm_profile(monkeypatch, tmp_path): + # Keep the examples-layer VLM dependencies out of core test collection. + from examples.puzzletron.evaluation.vlm import post_mip + + checkpoint = tmp_path / "checkpoint" + checkpoint.mkdir() + captured = {} + + def fake_evaluate(checkpoint_path, *, output_root, settings): + captured.update( + checkpoint=checkpoint_path, + output_root=output_root, + settings=settings, + ) + return {"metrics": {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5}} + + monkeypatch.setattr(runner, "_DOWNSTREAM_EVALUATION_PROFILES", {}) + monkeypatch.setattr(post_mip, "evaluate_realworldqa_checkpoint", fake_evaluate) + post_mip.register_profiles() + node = SimpleNamespace( + node_id="checkpoint_eval", + config={ + "config": { + "profile": "qwen35_vlm_realworldqa", + "batch_size": 1, + "timeout_seconds": 600, + } + }, + ) + source = SimpleNamespace( + architecture_id="architecture", + artifact_kind=ArtifactKind.CHECKPOINT, + artifact={"checkpoint": str(checkpoint)}, + ) + + result = runner._downstream_evaluation({"puzzle_dir": str(tmp_path)}, node, source, "execution") + + assert result["metrics"] == {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5} + assert captured["checkpoint"] == str(checkpoint) + assert captured["settings"] == {"batch_size": 1, "timeout_seconds": 600} diff --git a/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_full_smoke_plan.py b/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_full_smoke_plan.py index b69bdea78ce..d667fdd4072 100644 --- a/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_full_smoke_plan.py +++ b/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_full_smoke_plan.py @@ -99,9 +99,11 @@ def test_qwen3p5_0p8b_full_smoke_compiles_the_complete_one_gpu_route( "post.params-90.online_eval", "post.params-90.best_lm", "post.params-90.materialized", + "post.params-90.checkpoint_eval", "post.params-90.serving", "post.params-90.fastest", "post.params-90.short_kd", + "post.params-90.post_kd_checkpoint_eval", "post.params-90.final_eval", "post.params-90.best", ) @@ -124,6 +126,14 @@ def test_qwen3p5_0p8b_full_smoke_keeps_runtime_budgets_bounded( assert config["mip"]["runs"]["params-90"]["solver"]["num_solutions"] == 1 assert config["mip"]["runs"]["params-90"]["homogeneous"]["keep"] == 5 assert nodes["best_lm"]["top_k"] == 2 + assert nodes["checkpoint_eval"]["type"] == "downstream_evaluation" + assert nodes["checkpoint_eval"]["failure_policy"] == "strict" + assert nodes["checkpoint_eval"]["config"]["tasks"] == ["ifeval"] + assert nodes["checkpoint_eval"]["config"]["limit"] == 2 + assert nodes["checkpoint_eval"]["config"]["batch_size"] == 1 + assert nodes["checkpoint_eval"]["config"]["timeout_seconds"] == 600 + assert nodes["serving"]["config"]["readiness_timeout"] == 300 + assert nodes["serving"]["config"]["benchmark_timeout"] == 300 assert nodes["serving"]["config"]["request_count"] == 4 assert nodes["serving"]["config"]["concurrency"] == [1] assert nodes["fastest"]["top_k"] == 1 @@ -133,5 +143,8 @@ def test_qwen3p5_0p8b_full_smoke_keeps_runtime_budgets_bounded( "local_batch_size": 1, "checkpoint_every_steps": 2, } + assert nodes["post_kd_checkpoint_eval"]["type"] == "downstream_evaluation" + assert nodes["post_kd_checkpoint_eval"]["failure_policy"] == "strict" + assert nodes["post_kd_checkpoint_eval"]["config"] == nodes["checkpoint_eval"]["config"] assert nodes["final_eval"]["config"] == {"eval_samples": 2, "block_size": 512} assert nodes["best"]["top_k"] == 1 diff --git a/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_full_vlm_smoke_plan.py b/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_full_vlm_smoke_plan.py index 23eb9e80020..70cfe31f6f6 100644 --- a/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_full_vlm_smoke_plan.py +++ b/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_full_vlm_smoke_plan.py @@ -75,9 +75,11 @@ def test_qwen3p5_0p8b_full_vlm_smoke_compiles_the_one_gpu_lifecycle( "post.params-90.image_eval", "post.params-90.best_vlm_loss", "post.params-90.materialized", + "post.params-90.checkpoint_eval", "post.params-90.vlm_serving", "post.params-90.fastest_vlm", "post.params-90.short_vlm_kd", + "post.params-90.post_kd_checkpoint_eval", "post.params-90.final_image_eval", "post.params-90.best", ) @@ -100,7 +102,14 @@ def test_qwen3p5_0p8b_full_vlm_smoke_bounds_work_and_declares_vlm_kd( assert config["mip"]["runs"]["params-90"]["solver"]["num_solutions"] == 1 assert config["mip"]["runs"]["params-90"]["homogeneous"]["keep"] == 5 assert nodes["best_vlm_loss"]["top_k"] == 2 - assert nodes["vlm_serving"]["input"] == "materialized" + assert nodes["checkpoint_eval"]["type"] == "downstream_evaluation" + assert nodes["checkpoint_eval"]["failure_policy"] == "strict" + assert nodes["checkpoint_eval"]["config"]["profile"] == "qwen35_vlm_realworldqa" + assert nodes["checkpoint_eval"]["config"]["batch_size"] == 1 + assert nodes["checkpoint_eval"]["config"]["timeout_seconds"] == 600 + assert nodes["checkpoint_eval"]["config"]["limit_mm_per_prompt"] == {"image": 1} + assert nodes["vlm_serving"]["config"]["readiness_timeout"] == 300 + assert nodes["vlm_serving"]["config"]["benchmark_timeout"] == 300 assert nodes["fastest_vlm"] == { "type": "filter", "input": "vlm_serving", @@ -116,6 +125,9 @@ def test_qwen3p5_0p8b_full_vlm_smoke_bounds_work_and_declares_vlm_kd( "local_batch_size": 1, "checkpoint_every_steps": 2, } + assert nodes["post_kd_checkpoint_eval"]["type"] == "downstream_evaluation" + assert nodes["post_kd_checkpoint_eval"]["failure_policy"] == "strict" + assert nodes["post_kd_checkpoint_eval"]["config"] == nodes["checkpoint_eval"]["config"] assert nodes["final_image_eval"]["config"] == { "eval_samples": 2, "block_size": 512, From 48e9b8dbd1d55442244e072848051fb4358172ea Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Fri, 28 Aug 2026 22:30:52 +0200 Subject: [PATCH 2/6] Fix checkpoint evaluation review findings Signed-off-by: Johannes Rausch --- examples/puzzletron/evaluation/vlm/run.py | 2 ++ tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/puzzletron/evaluation/vlm/run.py b/examples/puzzletron/evaluation/vlm/run.py index a77d30400e6..81958cde306 100644 --- a/examples/puzzletron/evaluation/vlm/run.py +++ b/examples/puzzletron/evaluation/vlm/run.py @@ -26,6 +26,8 @@ if TYPE_CHECKING: from collections.abc import Callable, Mapping +__all__ = ["evaluate"] + REPOSITORY_ROOT = Path(__file__).absolute().parents[4] if str(REPOSITORY_ROOT) not in sys.path: sys.path.insert(0, str(REPOSITORY_ROOT)) diff --git a/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py b/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py index 69aae32ff46..8769893fafc 100644 --- a/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py +++ b/tests/gpu/torch/puzzletron/test_qwen3p5_0p8b_smoke.py @@ -174,7 +174,7 @@ def test_qwen3p5_0p8b_orchestrated_full_smoke_completes( run_config = yaml.safe_load( ( project_root_path - / "examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/mip_smoke.yaml" + / "examples/puzzletron/configs/families/qwen3_5/qwen3p5_0p8b/runs/full_smoke.yaml" ).read_text(encoding="utf-8") ) width_sanity_config = run_config["width_sanity"] From e86d805b9a8309614d2cdd36ddf96f185de5bc18 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Sat, 29 Aug 2026 02:35:08 +0200 Subject: [PATCH 3/6] Export the VLM checkpoint evaluator Signed-off-by: Johannes Rausch --- .../puzzletron/evaluation/vlm/__init__.py | 35 +++++++++++++++++++ .../puzzletron/evaluation/vlm/post_mip.py | 4 +-- 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 examples/puzzletron/evaluation/vlm/__init__.py diff --git a/examples/puzzletron/evaluation/vlm/__init__.py b/examples/puzzletron/evaluation/vlm/__init__.py new file mode 100644 index 00000000000..6a87cbe0093 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/__init__.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public interfaces for Puzzletron VLM evaluation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .run import evaluate as evaluate + +__all__ = ["evaluate"] + + +def __getattr__(name: str) -> Any: + """Load the CLI-backed evaluator without preloading ``python -m ...run``.""" + + if name == "evaluate": + from .run import evaluate + + return evaluate + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/examples/puzzletron/evaluation/vlm/post_mip.py b/examples/puzzletron/evaluation/vlm/post_mip.py index da89485d3cc..86c45c44293 100644 --- a/examples/puzzletron/evaluation/vlm/post_mip.py +++ b/examples/puzzletron/evaluation/vlm/post_mip.py @@ -28,7 +28,7 @@ from typing import Any from examples.puzzletron.evaluation import checkpoint -from examples.puzzletron.evaluation.vlm import run +from examples.puzzletron.evaluation.vlm import evaluate __all__ = ["evaluate_realworldqa_checkpoint", "register_profiles"] @@ -90,7 +90,7 @@ def write_preflight(report: dict[str, object]) -> None: json.dumps(report, indent=2, sort_keys=True) + "\n", ) - result = run.evaluate( + result = evaluate( args, settings_overrides=settings, preflight_callback=write_preflight, From f89d16fedae4e03541fcfff9497d4b8b0a536d75 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Sat, 29 Aug 2026 02:55:51 +0200 Subject: [PATCH 4/6] Harden VLM evaluation integration Expose the reusable evaluator from a non-CLI module so package imports follow the public API convention without preloading the documented module entry point. Normalize optional post-MIP mappings before profile registration. Signed-off-by: Johannes Rausch --- .../puzzletron/evaluation/vlm/__init__.py | 19 +--- .../puzzletron/evaluation/vlm/evaluator.py | 86 +++++++++++++++++++ examples/puzzletron/evaluation/vlm/run.py | 64 +------------- examples/puzzletron/run_post_mip_node.py | 4 +- .../puzzletron/evaluation/vlm/test_run.py | 58 ++++++------- .../torch/puzzletron/test_post_mip_runner.py | 7 +- 6 files changed, 122 insertions(+), 116 deletions(-) create mode 100644 examples/puzzletron/evaluation/vlm/evaluator.py diff --git a/examples/puzzletron/evaluation/vlm/__init__.py b/examples/puzzletron/evaluation/vlm/__init__.py index 6a87cbe0093..abc8474d28b 100644 --- a/examples/puzzletron/evaluation/vlm/__init__.py +++ b/examples/puzzletron/evaluation/vlm/__init__.py @@ -15,21 +15,4 @@ """Public interfaces for Puzzletron VLM evaluation.""" -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from .run import evaluate as evaluate - -__all__ = ["evaluate"] - - -def __getattr__(name: str) -> Any: - """Load the CLI-backed evaluator without preloading ``python -m ...run``.""" - - if name == "evaluate": - from .run import evaluate - - return evaluate - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +from .evaluator import * diff --git a/examples/puzzletron/evaluation/vlm/evaluator.py b/examples/puzzletron/evaluation/vlm/evaluator.py new file mode 100644 index 00000000000..f67f728b573 --- /dev/null +++ b/examples/puzzletron/evaluation/vlm/evaluator.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reusable VLM checkpoint evaluation interface.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import argparse + from collections.abc import Callable, Mapping + +from examples.puzzletron.evaluation import checkpoint +from examples.puzzletron.evaluation.vlm import preflight, tasks + +__all__ = ["evaluate"] + + +def evaluate( + args: argparse.Namespace, + *, + settings_overrides: Mapping[str, object] | None = None, + preflight_callback: Callable[[dict[str, object]], None] | None = None, +) -> dict[str, object]: + """Prepare and run one pinned VLM profile invocation.""" + + prepared = preflight.prepare(args) + task_root, configured_tasks = tasks.prepare( + args.output_dir, + suite=prepared.suite, + dataset_snapshots=prepared.dataset_snapshots, + quick_manifest=prepared.quick_manifest, + ) + offline_task_preflight = tasks.verify_offline( + task_root, + configured_tasks, + hf_home=prepared.hf_home, + timeout_seconds=checkpoint.DEFAULT_PREFLIGHT_TIMEOUT_SECONDS, + ) + report = dict(prepared.report) + report.update( + { + "configured_tasks": list(configured_tasks), + "offline_task_preflight": offline_task_preflight, + "task_config_root": str(task_root), + } + ) + if preflight_callback is not None: + preflight_callback(report) + if args.preflight_only: + return {"preflight": report, "runs": []} + settings = preflight.settings( + args, + tasks_root=task_root, + configured_tasks=configured_tasks, + prepared=prepared, + ) + settings.update(settings_overrides or {}) + repetitions = prepared.execution_policy["repetitions"] + runs = [] + with checkpoint.without_huggingface_credentials(): + for repetition in range(1, repetitions + 1): + output_root = args.output_dir + if repetitions > 1: + output_root = output_root / f"short-repetition-{repetition}" + runs.append( + checkpoint.run_lmms_eval_checkpoint( + args.checkpoint, + output_root=output_root, + settings=settings, + ) + ) + return {"preflight": report, "runs": runs} diff --git a/examples/puzzletron/evaluation/vlm/run.py b/examples/puzzletron/evaluation/vlm/run.py index 81958cde306..a095f545686 100644 --- a/examples/puzzletron/evaluation/vlm/run.py +++ b/examples/puzzletron/evaluation/vlm/run.py @@ -21,10 +21,6 @@ import json import sys from pathlib import Path -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Callable, Mapping __all__ = ["evaluate"] @@ -33,7 +29,8 @@ sys.path.insert(0, str(REPOSITORY_ROOT)) from examples.puzzletron.evaluation import checkpoint # noqa: E402 -from examples.puzzletron.evaluation.vlm import preflight, suites, tasks # noqa: E402 +from examples.puzzletron.evaluation.vlm import suites # noqa: E402 +from examples.puzzletron.evaluation.vlm.evaluator import evaluate # noqa: E402 def _checkpoint_directory(value: str) -> Path: @@ -123,63 +120,6 @@ def _build_parser() -> argparse.ArgumentParser: return parser -def evaluate( - args: argparse.Namespace, - *, - settings_overrides: Mapping[str, object] | None = None, - preflight_callback: Callable[[dict[str, object]], None] | None = None, -) -> dict[str, object]: - """Prepare and run one pinned VLM profile invocation.""" - - prepared = preflight.prepare(args) - task_root, configured_tasks = tasks.prepare( - args.output_dir, - suite=prepared.suite, - dataset_snapshots=prepared.dataset_snapshots, - quick_manifest=prepared.quick_manifest, - ) - offline_task_preflight = tasks.verify_offline( - task_root, - configured_tasks, - hf_home=prepared.hf_home, - timeout_seconds=checkpoint.DEFAULT_PREFLIGHT_TIMEOUT_SECONDS, - ) - report = dict(prepared.report) - report.update( - { - "configured_tasks": list(configured_tasks), - "offline_task_preflight": offline_task_preflight, - "task_config_root": str(task_root), - } - ) - if preflight_callback is not None: - preflight_callback(report) - if args.preflight_only: - return {"preflight": report, "runs": []} - settings = preflight.settings( - args, - tasks_root=task_root, - configured_tasks=configured_tasks, - prepared=prepared, - ) - settings.update(settings_overrides or {}) - repetitions = prepared.execution_policy["repetitions"] - runs = [] - with checkpoint.without_huggingface_credentials(): - for repetition in range(1, repetitions + 1): - output_root = args.output_dir - if repetitions > 1: - output_root = output_root / f"short-repetition-{repetition}" - runs.append( - checkpoint.run_lmms_eval_checkpoint( - args.checkpoint, - output_root=output_root, - settings=settings, - ) - ) - return {"preflight": report, "runs": runs} - - def main(argv: list[str] | None = None) -> int: args = _build_parser().parse_args(argv) try: diff --git a/examples/puzzletron/run_post_mip_node.py b/examples/puzzletron/run_post_mip_node.py index 4871d4f7efb..45905f47e74 100644 --- a/examples/puzzletron/run_post_mip_node.py +++ b/examples/puzzletron/run_post_mip_node.py @@ -32,8 +32,8 @@ def _register_evaluation_profiles(config: dict) -> None: profiles = { - node.get("config", {}).get("profile") - for flow in (config.get("post_mip", {}).get("flows") or {}).values() + (node.get("config") or {}).get("profile") + for flow in ((config.get("post_mip") or {}).get("flows") or {}).values() for node in (flow.get("nodes") or {}).values() if node.get("type") == "downstream_evaluation" } diff --git a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py index eede50536c9..f988e37b716 100644 --- a/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py +++ b/tests/unit/torch/puzzletron/evaluation/vlm/test_run.py @@ -263,36 +263,30 @@ def fake_runner(checkpoint_path, *, output_root, settings): assert report["generation_policy"] == settings["gen_kwargs"] -def test_realworldqa_smoke_policy_is_single_run_with_two_samples(): - assert suites.source_tasks("realworldqa-smoke") == ("realworldqa",) - policy = suites.execution_policy("realworldqa-smoke", timeout_seconds=900) - assert policy["limit"] == 2 - assert policy["repetitions"] == 1 - assert policy["timeout_seconds"] == 900 - - -def test_post_mip_realworldqa_adapter_routes_runtime_overrides(monkeypatch, tmp_path): - model = tmp_path / "model" - model.mkdir() +def test_post_mip_realworldqa_adapter_runs_pinned_profile(monkeypatch, tmp_path): + model = _write_checkpoint(tmp_path) + lmms_root = _write_lmms_tasks(tmp_path, ("realworldqa",)) + _use_offline_fakes(monkeypatch, lmms_root) + hf_home = tmp_path / "hf-home" + hf_home.mkdir() + monkeypatch.setenv("HF_HOME", str(hf_home)) output = tmp_path / "output" captured = {} - def fake_evaluate(args, *, settings_overrides, preflight_callback): - captured.update(args=args, settings_overrides=settings_overrides) - preflight = { - "profile": suites.EVALUATION_PROFILE, - "dataset_revisions": { - "realworldqa": profile.VLM_BENCHMARK_DATASETS["realworldqa"].revision - }, - "sample_limit": 2, - } - preflight_callback(preflight) + def fake_runner(checkpoint_path, *, output_root, settings): + report = json.loads((output / "profile.json").read_text()) + assert report["configured_tasks"] == ["modelopt_vlm_benchmark_realworldqa"] + assert report["sample_limit"] == 2 + captured.update( + checkpoint=checkpoint_path, + output_root=output_root, + settings=settings, + ) return { - "preflight": preflight, - "runs": [{"metrics": {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5}}], + "metrics": {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5}, } - monkeypatch.setattr(post_mip.run, "evaluate", fake_evaluate) + monkeypatch.setattr(checkpoint, "run_lmms_eval_checkpoint", fake_runner) result = post_mip.evaluate_realworldqa_checkpoint( model, output_root=output, @@ -304,15 +298,13 @@ def fake_evaluate(args, *, settings_overrides, preflight_callback): }, ) - assert captured["args"].suite == "realworldqa-smoke" - assert captured["args"].checkpoint == model - assert captured["args"].output_dir == output - assert captured["args"].timeout_seconds == 900 - assert captured["settings_overrides"] == { - "dtype": "bfloat16", - "topology": {"tensor_parallel_size": 1}, - } - assert json.loads((output / "profile.json").read_text())["sample_limit"] == 2 + assert captured["checkpoint"] == model + assert captured["output_root"] == output + assert captured["settings"]["tasks"] == "modelopt_vlm_benchmark_realworldqa" + assert captured["settings"]["limit"] == 2 + assert captured["settings"]["timeout_seconds"] == 900 + assert captured["settings"]["dtype"] == "bfloat16" + assert captured["settings"]["topology"] == {"tensor_parallel_size": 1} assert result["metrics"] == {"modelopt_vlm_benchmark_realworldqa.accuracy": 0.5} assert result["profile_path"] == str(output / "profile.json") diff --git a/tests/unit/torch/puzzletron/test_post_mip_runner.py b/tests/unit/torch/puzzletron/test_post_mip_runner.py index e695f1a7bf3..5d51beff2db 100644 --- a/tests/unit/torch/puzzletron/test_post_mip_runner.py +++ b/tests/unit/torch/puzzletron/test_post_mip_runner.py @@ -87,16 +87,21 @@ def test_worker_entrypoint_registers_configured_vlm_evaluation_profile(monkeypat calls = [] monkeypatch.setattr(vlm_post_mip, "register_profiles", lambda: calls.append(True)) + post_mip_entrypoint._register_evaluation_profiles({"post_mip": None}) post_mip_entrypoint._register_evaluation_profiles( { "post_mip": { "flows": { "params": { "nodes": { + "generic_eval": { + "type": "downstream_evaluation", + "config": None, + }, "checkpoint_eval": { "type": "downstream_evaluation", "config": {"profile": "qwen35_vlm_realworldqa"}, - } + }, } } } From a1655ab979d44b37e3048e81bf71627b665bebe9 Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Sat, 29 Aug 2026 03:20:02 +0200 Subject: [PATCH 5/6] Handle malformed post-MIP profile entries Signed-off-by: Johannes Rausch --- examples/puzzletron/run_post_mip_node.py | 18 ++++++++++++------ .../torch/puzzletron/test_post_mip_runner.py | 10 +++++++++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/examples/puzzletron/run_post_mip_node.py b/examples/puzzletron/run_post_mip_node.py index 45905f47e74..37ee69306bf 100644 --- a/examples/puzzletron/run_post_mip_node.py +++ b/examples/puzzletron/run_post_mip_node.py @@ -21,6 +21,7 @@ import json import os import sys +from collections.abc import Mapping from pathlib import Path REPOSITORY_ROOT = Path(__file__).resolve().parents[2] @@ -31,12 +32,17 @@ def _register_evaluation_profiles(config: dict) -> None: - profiles = { - (node.get("config") or {}).get("profile") - for flow in ((config.get("post_mip") or {}).get("flows") or {}).values() - for node in (flow.get("nodes") or {}).values() - if node.get("type") == "downstream_evaluation" - } + post_mip = config.get("post_mip") + flows = post_mip.get("flows") if isinstance(post_mip, Mapping) else None + profiles = set() + for flow in flows.values() if isinstance(flows, Mapping) else (): + nodes = flow.get("nodes") if isinstance(flow, Mapping) else None + for node in nodes.values() if isinstance(nodes, Mapping) else (): + if not isinstance(node, Mapping) or node.get("type") != "downstream_evaluation": + continue + node_config = node.get("config") + if isinstance(node_config, Mapping): + profiles.add(node_config.get("profile")) if "qwen35_vlm_realworldqa" in profiles: from examples.puzzletron.evaluation.vlm.post_mip import register_profiles diff --git a/tests/unit/torch/puzzletron/test_post_mip_runner.py b/tests/unit/torch/puzzletron/test_post_mip_runner.py index 5d51beff2db..88a3951e555 100644 --- a/tests/unit/torch/puzzletron/test_post_mip_runner.py +++ b/tests/unit/torch/puzzletron/test_post_mip_runner.py @@ -92,18 +92,26 @@ def test_worker_entrypoint_registers_configured_vlm_evaluation_profile(monkeypat { "post_mip": { "flows": { + "null_flow": None, + "invalid_flow": [], "params": { "nodes": { + "null_node": None, + "invalid_node": [], "generic_eval": { "type": "downstream_evaluation", "config": None, }, + "invalid_config": { + "type": "downstream_evaluation", + "config": [], + }, "checkpoint_eval": { "type": "downstream_evaluation", "config": {"profile": "qwen35_vlm_realworldqa"}, }, } - } + }, } } } From f29bac5d9721af61864cac2594df90d1328a736b Mon Sep 17 00:00:00 2001 From: Johannes Rausch Date: Sat, 29 Aug 2026 03:29:15 +0200 Subject: [PATCH 6/6] Ignore invalid post-MIP profile values Signed-off-by: Johannes Rausch --- examples/puzzletron/run_post_mip_node.py | 4 +++- tests/unit/torch/puzzletron/test_post_mip_runner.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/examples/puzzletron/run_post_mip_node.py b/examples/puzzletron/run_post_mip_node.py index 37ee69306bf..eaeab0dd7a3 100644 --- a/examples/puzzletron/run_post_mip_node.py +++ b/examples/puzzletron/run_post_mip_node.py @@ -42,7 +42,9 @@ def _register_evaluation_profiles(config: dict) -> None: continue node_config = node.get("config") if isinstance(node_config, Mapping): - profiles.add(node_config.get("profile")) + profile = node_config.get("profile") + if isinstance(profile, str): + profiles.add(profile) if "qwen35_vlm_realworldqa" in profiles: from examples.puzzletron.evaluation.vlm.post_mip import register_profiles diff --git a/tests/unit/torch/puzzletron/test_post_mip_runner.py b/tests/unit/torch/puzzletron/test_post_mip_runner.py index 88a3951e555..00db7fc7e66 100644 --- a/tests/unit/torch/puzzletron/test_post_mip_runner.py +++ b/tests/unit/torch/puzzletron/test_post_mip_runner.py @@ -106,6 +106,14 @@ def test_worker_entrypoint_registers_configured_vlm_evaluation_profile(monkeypat "type": "downstream_evaluation", "config": [], }, + "invalid_list_profile": { + "type": "downstream_evaluation", + "config": {"profile": []}, + }, + "invalid_mapping_profile": { + "type": "downstream_evaluation", + "config": {"profile": {}}, + }, "checkpoint_eval": { "type": "downstream_evaluation", "config": {"profile": "qwen35_vlm_realworldqa"},