diff --git a/.dockerignore b/.dockerignore index d5f22e63894..bab52c031e7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ docker examples/**/.git examples/**/saved_models* +examples/puzzletron/reports **/experimental ##### Copied from .gitignore ##### diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index e4dd7560776..2c7811dfba2 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,6 +7,7 @@ on: branches: [main, release/*, feature/*] paths: - ".github/workflows/unit_tests.yml" + - "examples/__init__.py" - "examples/puzzletron/**" - "modelopt/**" - "noxfile.py" @@ -83,6 +84,8 @@ jobs: with: files: | .github/workflows/unit_tests.yml + examples/__init__.py + examples/puzzletron/Dockerfile examples/puzzletron/**/*.html examples/puzzletron/**/*.json examples/puzzletron/**/*.md diff --git a/examples/puzzletron/Dockerfile b/examples/puzzletron/Dockerfile new file mode 100644 index 00000000000..89b0e975dbe --- /dev/null +++ b/examples/puzzletron/Dockerfile @@ -0,0 +1,151 @@ +FROM nvidia/cuda:12.9.2-cudnn-devel-ubuntu24.04@sha256:b4db213759eb86d55a7271909bdad891fab300ec5700fb4f4656463b2f51980f + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG DEBIAN_FRONTEND=noninteractive +ARG TARGETPLATFORM + +ENV VIRTUAL_ENV=/venv +ENV PATH=/venv/bin/:$PATH +ENV PIP_NO_CACHE_DIR=1 +ENV NLTK_DATA=/opt/puzzletron/nltk_data +ENV PUZZLETRON_CI_ENVIRONMENT=/opt/puzzletron/ci_environment.json +ENV PUZZLETRON_REQUIREMENTS=/opt/puzzletron/requirements.txt +ENV PUZZLETRON_ROOT=/opt/puzzletron/src/modelopt +ENV PUZZLETRON_VENV=/venv +ENV PUZZLETRON_VLLM_ANYMODEL=1 +ENV PYTHONPATH=/opt/puzzletron/src/modelopt +ENV PYTHONUNBUFFERED=1 + +COPY examples/puzzletron/ci_environment.json /opt/puzzletron/ci_environment.json +COPY examples/puzzletron/requirements.txt /opt/puzzletron/requirements.txt +COPY examples/__init__.py /opt/puzzletron/src/modelopt/examples/__init__.py +COPY examples/puzzletron/patches /opt/puzzletron/patches +COPY pyproject.toml LICENSE_HEADER /opt/modelopt-dependencies/ + +# Install base build tools before compiling CUDA extensions. +RUN test "${TARGETPLATFORM}" = "linux/amd64" || \ + (echo "Puzzletron requires linux/amd64 because its CUDA stack and video decoder are not validated for Linux ARM" >&2; exit 1) && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential ca-certificates cmake curl git jq ninja-build \ + python3 python3-dev python3-pip python3-venv unzip && \ + rm -rf /var/lib/apt/lists/* && \ + python3 -m venv "${VIRTUAL_ENV}" && \ + python -m pip install --upgrade \ + pip "setuptools>=80,<81" "setuptools-scm>=8,<10" setuptools-rust wheel \ + "packaging>=24.2" "cmake>=3.26.1" ninja jinja2 + +# `pin` keeps ci_environment.json as the single source of dependency versions +# while leaving the install commands readable. +RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ + python -m pip install \ + "torch==$(pin torch)" \ + "torchvision==$(pin torchvision)" \ + "torchaudio==$(pin torch)" \ + --index-url https://download.pytorch.org/whl/cu129 + +# Evaluation and worker dependencies. Install Transformers last so the pinned +# worker version wins over transitive requirements. +RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ + python -m pip install \ + -r "${PUZZLETRON_REQUIREMENTS}" \ + "nemo-automodel @ git+$(pin nemo_automodel.repository)@$(pin nemo_automodel.commit)" \ + "aiperf==$(pin gpu_image.aiperf)" \ + "langdetect==$(pin gpu_image.langdetect)" \ + "nltk==$(pin gpu_image.nltk)" \ + "nox==$(pin gpu_image.nox)" && \ + mkdir -p "${NLTK_DATA}/tokenizers" && \ + while read -r nltk_resource nltk_resource_sha256; do \ + nltk_archive="/tmp/${nltk_resource}.zip" && \ + curl --fail --location --silent --show-error \ + --output "${nltk_archive}" \ + "https://raw.githubusercontent.com/nltk/nltk_data/$(pin gpu_image.nltk_data_commit)/packages/tokenizers/${nltk_resource}.zip" && \ + echo "${nltk_resource_sha256} ${nltk_archive}" | sha256sum --check --strict && \ + unzip -q "${nltk_archive}" -d "${NLTK_DATA}/tokenizers" && \ + rm "${nltk_archive}"; \ + done < <(jq -r \ + '.gpu_image.nltk_resources[] as $name | "\($name) \(.gpu_image.nltk_resource_sha256[$name])"' \ + "${PUZZLETRON_CI_ENVIRONMENT}") && \ + python -m pip install "transformers==$(pin transformers)" + +# Compile the pinned AnyModel vLLM fork for every supported worker GPU. +RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ + export FORCE_CUDA=1 && \ + export TORCH_CUDA_ARCH_LIST="$(pin runtime_image.torch_cuda_arch_list)" && \ + python -m pip install --no-build-isolation \ + "vllm @ git+$(pin vllm.repository)@$(pin vllm.commit)" + +# Compile the remaining CUDA extensions. Mamba is built from its pinned source +# because the compatibility patch is not present in the upstream release. +RUN pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ + mamba_patch="$(pin runtime_image.mamba_ssm.compatibility_patch)" && \ + export FORCE_CUDA=1 && \ + export TORCH_CUDA_ARCH_LIST="$(pin runtime_image.torch_cuda_arch_list)" && \ + python -m pip install --no-build-isolation \ + "causal-conv1d==$(pin runtime_image.causal_conv1d)" && \ + echo "$(pin runtime_image.mamba_ssm.compatibility_patch_sha256) /opt/puzzletron/patches/${mamba_patch}" | \ + sha256sum --check --strict && \ + git clone --filter=blob:none --no-checkout \ + "$(pin runtime_image.mamba_ssm.repository)" /tmp/mamba-ssm && \ + git -C /tmp/mamba-ssm checkout --detach "$(pin runtime_image.mamba_ssm.commit)" && \ + test "$(git -C /tmp/mamba-ssm rev-parse HEAD)" = \ + "$(pin runtime_image.mamba_ssm.commit)" && \ + git -C /tmp/mamba-ssm apply "/opt/puzzletron/patches/${mamba_patch}" && \ + MAMBA_FORCE_BUILD=TRUE python -m pip install --no-build-isolation \ + /tmp/mamba-ssm && \ + rm -rf /tmp/mamba-ssm && \ + python -m pip install \ + "flash-linear-attention[cuda]==$(pin runtime_image.flash_linear_attention)" && \ + export TORCH_CUDA_ARCH_LIST="$(pin runtime_image.grouped_gemm_cuda_arch_list)" && \ + python -m pip install --no-build-isolation \ + "$(pin runtime_image.grouped_gemm.distribution) @ git+$(pin runtime_image.grouped_gemm.repository)@$(pin runtime_image.grouped_gemm.commit)" + +# Cache ModelOpt's ordinary Python dependencies separately from its source. +RUN mkdir -p /opt/modelopt-dependencies/modelopt && \ + touch /opt/modelopt-dependencies/modelopt/__init__.py && \ + python -m pip install "/opt/modelopt-dependencies[hf,puzzletron,dev-test]" && \ + python -m pip uninstall -y nvidia-modelopt + +# Keep the immutable source revision below dependency compilation so source-only +# rebuilds reuse the pinned CUDA dependency layers. +ARG MODELOPT_REVISION +RUN [[ "${MODELOPT_REVISION}" =~ ^[0-9a-f]{40}$ ]] && \ + printf '%s\n' "${MODELOPT_REVISION}" > /opt/puzzletron/modelopt_revision + +COPY pyproject.toml LICENSE_HEADER README.md /opt/puzzletron/src/modelopt/ +COPY modelopt /opt/puzzletron/src/modelopt/modelopt +COPY modelopt_recipes /opt/puzzletron/src/modelopt/modelopt_recipes +COPY puzzletron_orchestrator /opt/puzzletron/src/modelopt/puzzletron_orchestrator +COPY puzzletron_setup /opt/puzzletron/src/modelopt/puzzletron_setup +COPY examples/puzzletron /opt/puzzletron/src/modelopt/examples/puzzletron + +# Smoke-test the installed environment. Pins and source revisions are checked +# by the repository tests and by the install commands above. +RUN python -m pip install --no-build-isolation --no-deps -e \ + "/opt/puzzletron/src/modelopt[hf,puzzletron,dev-test]" && \ + pin() { jq -er ".${1}" "${PUZZLETRON_CI_ENVIRONMENT}"; } && \ + for module in \ + aiperf causal_conv1d decord fla grouped_gemm langdetect lmms_eval \ + mamba_ssm modelopt nemo_automodel nltk puzzletron_orchestrator \ + puzzletron_setup tilelang torch transformers vllm; do \ + python -c "import ${module}"; \ + done && \ + test "$(python -c 'import torch; print(torch.version.cuda)')" = \ + "$(pin gpu_image.torch_cuda)" && \ + while read -r nltk_resource; do \ + python -c "import nltk; nltk.data.find('tokenizers/${nltk_resource}')"; \ + done < <(jq -r '.gpu_image.nltk_resources[]' "${PUZZLETRON_CI_ENVIRONMENT}") && \ + lmms_root="$(python -c 'import lmms_eval; print(next(iter(lmms_eval.__path__)))')" && \ + while read -r task_config; do \ + test -f "${lmms_root}/${task_config}"; \ + done < <(jq -r '.lmms_eval.task_configs[]' "${PUZZLETRON_CI_ENVIRONMENT}") + +LABEL org.opencontainers.image.source="https://github.com/NVIDIA/Model-Optimizer" \ + org.opencontainers.image.revision="${MODELOPT_REVISION}" \ + com.nvidia.modelopt.puzzletron.environment="examples/puzzletron/ci_environment.json" \ + com.nvidia.modelopt.puzzletron.environment-recipe="examples/puzzletron/Dockerfile" + +WORKDIR /opt/puzzletron/src/modelopt + +CMD ["bash"] diff --git a/examples/puzzletron/README.md b/examples/puzzletron/README.md index b80bdb4baf0..4474e3d0730 100644 --- a/examples/puzzletron/README.md +++ b/examples/puzzletron/README.md @@ -36,6 +36,12 @@ conversion, training, evaluation, and benchmarking run in the worker environment or container selected during setup. Prepare the [worker environment](docs/environment_setup.md) before launching a campaign. +### Worker image + +The repository includes a Dockerfile and build command for Puzzletron workers. +Follow the [image guide](docs/worker_image.md) to build, check, or export an +Enroot/Pyxis SquashFS image. + ### 2. Generate a campaign Start the guided setup with the repository defaults: diff --git a/examples/puzzletron/build_worker_image.py b/examples/puzzletron/build_worker_image.py new file mode 100644 index 00000000000..5a13fed0832 --- /dev/null +++ b/examples/puzzletron/build_worker_image.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build and optionally export the Linux amd64 Puzzletron worker image.""" + +from __future__ import annotations + +import argparse +import os +import platform +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +__all__ = [ + "build_parser", + "main", + "sqsh_name", +] + +_PLATFORM = "linux/amd64" +_REVISION_PATTERN = re.compile(r"[0-9a-f]{40}") + + +def sqsh_name(revision: str) -> str: + """Return the SquashFS filename for an image built from ``revision``.""" + + if not _REVISION_PATTERN.fullmatch(revision): + raise ValueError("Puzzletron image revision must be a full lowercase Git commit") + return f"modelopt-puzzletron-linux-amd64-git-{revision[:12]}.sqsh" + + +def _run(command: list[str], **kwargs) -> subprocess.CompletedProcess: + return subprocess.run(command, check=True, **kwargs) + + +def _source_revision(repository_root: Path) -> str: + status = subprocess.check_output( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=repository_root, + text=True, + ) + if status: + raise RuntimeError("Build the Puzzletron worker image from a clean Git checkout") + revision = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=repository_root, text=True + ).strip() + if not _REVISION_PATTERN.fullmatch(revision): + raise RuntimeError(f"Git returned an invalid Puzzletron image revision: {revision!r}") + return revision + + +def _require_tools(*tools: str) -> None: + missing = [tool for tool in tools if shutil.which(tool) is None] + if missing: + raise RuntimeError(f"Missing required command(s): {', '.join(missing)}") + + +def _require_linux_amd64() -> None: + if platform.system() != "Linux" or platform.machine() not in {"amd64", "x86_64"}: + raise RuntimeError("Build the Puzzletron worker image on a Linux amd64 host") + + +def _output_path(output_dir: Path, name: str) -> Path: + path = output_dir / name + if path.exists(): + raise FileExistsError(f"Refusing to overwrite existing artifact: {path}") + return path + + +def _export_sqsh(image: str, output: Path) -> None: + partial = output.with_name(f".{output.stem}.partial.sqsh") + if partial.exists(): + raise FileExistsError(f"Refusing to overwrite incomplete artifact: {partial}") + + with tempfile.TemporaryDirectory(prefix="puzzletron-enroot-") as enroot_root: + environment = { + **os.environ, + "ENROOT_CACHE_PATH": f"{enroot_root}/cache", + "ENROOT_DATA_PATH": f"{enroot_root}/data", + "ENROOT_RUNTIME_PATH": f"{enroot_root}/runtime", + } + try: + _run( + ["enroot", "import", "--output", str(partial), f"dockerd://{image}"], + env=environment, + ) + except (OSError, subprocess.CalledProcessError): + partial.unlink(missing_ok=True) + raise + partial.replace(output) + + +def build_parser() -> argparse.ArgumentParser: + """Build the worker-image command-line parser.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + help="directory for optional exported artifacts", + ) + parser.add_argument( + "--sqsh", + action="store_true", + help="export an Enroot/Pyxis SquashFS image", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Build, verify, and optionally export the Puzzletron worker image.""" + + parser = build_parser() + args = parser.parse_args(argv) + if args.sqsh and args.output_dir is None: + parser.error("--output-dir is required with --sqsh") + if args.output_dir is not None and not args.sqsh: + parser.error("--output-dir requires --sqsh") + + repository_root = Path(__file__).resolve().parents[2] + revision = _source_revision(repository_root) + _require_linux_amd64() + image = f"modelopt-puzzletron:linux-amd64-git-{revision[:12]}" + + sqsh = None + if args.output_dir is not None: + args.output_dir = args.output_dir.expanduser().resolve() + if args.output_dir.is_relative_to(repository_root): + parser.error("--output-dir must be outside the repository") + sqsh = _output_path(args.output_dir, sqsh_name(revision)) + + required_tools = ["docker"] + if args.sqsh: + required_tools.append("enroot") + _require_tools(*required_tools) + if args.output_dir is not None: + args.output_dir.mkdir(parents=True, exist_ok=True) + + _run( + [ + "docker", + "build", + "--platform", + _PLATFORM, + "--file", + "examples/puzzletron/Dockerfile", + "--build-arg", + f"MODELOPT_REVISION={revision}", + "--tag", + image, + ".", + ], + cwd=repository_root, + ) + recorded_revision = subprocess.check_output( + [ + "docker", + "image", + "inspect", + "--format", + '{{ index .Config.Labels "org.opencontainers.image.revision" }}', + image, + ], + text=True, + ).strip() + if recorded_revision != revision: + raise RuntimeError( + f"Puzzletron image revision mismatch: expected {revision}, found {recorded_revision}" + ) + if sqsh is not None: + _export_sqsh(image, sqsh) + + print(f"Docker image: {image}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/puzzletron/ci_environment.json b/examples/puzzletron/ci_environment.json index 9e39fd5d6b8..d57d7966212 100644 --- a/examples/puzzletron/ci_environment.json +++ b/examples/puzzletron/ci_environment.json @@ -1,18 +1,56 @@ { "schema_version": 1, - "scope": "puzzletron_v2_ci", + "scope": "puzzletron_v2_worker_ci", "python": "3.12", "torch": "2.11.0", "torchvision": "0.26.0", "transformers": "5.8.1", + "gpu_image": { + "base_image": "nvidia/cuda:12.9.2-cudnn-devel-ubuntu24.04@sha256:b4db213759eb86d55a7271909bdad891fab300ec5700fb4f4656463b2f51980f", + "torch_cuda": "12.9", + "aiperf": "0.12.0", + "langdetect": "1.0.9", + "nltk": "3.10.3", + "nltk_data_commit": "550b6625bcef1f2abff2ff770a5a0d272c9c6b2a", + "nltk_resource_sha256": { + "punkt": "51c3078994aeaf650bfc8e028be4fb42b4a0d177d41c012b6a983979653660ec", + "punkt_tab": "e57f64187974277726a3417ca6f181ec5403676c717672eef6a748a7b20e0106" + }, + "nltk_resources": ["punkt", "punkt_tab"], + "nox": "2026.8.17" + }, "lmms_eval": { "base_version": "0.7.0", "repository": "https://github.com/EvolvingLMMs-Lab/lmms-eval.git", - "commit": "15c32bfec165df13c269ddd3cda03b2ed9137825" + "commit": "15c32bfec165df13c269ddd3cda03b2ed9137825", + "task_configs": ["tasks/gsm8k/gsm8k.yaml", "tasks/ifeval/ifeval.yaml"] }, "nemo_automodel": { "base_version": "0.5.0", "repository": "https://github.com/Separius/Automodel.git", "commit": "b22cd029d806197e249f2cc4a42c5de91713b772" + }, + "vllm": { + "repository": "https://github.com/Separius/vllm.git", + "commit": "a056958c78226dcc5476ad5083a26155dd8863c5" + }, + "runtime_image": { + "causal_conv1d": "1.7.0", + "flash_linear_attention": "0.5.1", + "grouped_gemm": { + "base_version": "1.1.4.post8", + "distribution": "nv-grouped-gemm", + "repository": "https://github.com/fanshiqing/grouped_gemm.git", + "commit": "efe8c40eaf4c8ef57191e0ea9aa4117aa5b1a8f2" + }, + "grouped_gemm_cuda_arch_list": "8.0;8.6;9.0", + "mamba_ssm": { + "base_version": "2.3.2.post1", + "repository": "https://github.com/state-spaces/mamba.git", + "commit": "a14b1dff0454a3bc27d9eb31355dc01e4b2490ec", + "compatibility_patch": "mamba_ssm_tilelang_0_1_9.patch", + "compatibility_patch_sha256": "5ac3654a620e44db347b30231bafdceaf328058c16fca061d5dadb25ebff7291" + }, + "torch_cuda_arch_list": "8.0;8.6;9.0;10.0;12.0" } } diff --git a/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/runner.slurm.yaml b/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/runner.slurm.yaml index e67c1e54fc9..89d03a316f9 100644 --- a/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/runner.slurm.yaml +++ b/examples/puzzletron/configs/orchestration/qwen3p5_0p8b/runner.slurm.yaml @@ -11,9 +11,9 @@ runner: max_nodes: 1 time_limit: "1:00:00" execution_contract: - repository: REPLACE_WITH_WORKER_VISIBLE_MODELOPT_CHECKOUT - venv: REPLACE_WITH_WORKER_VISIBLE_MODELOPT_VENV - container: REPLACE_WITH_REVIEWED_PUZZLETRON_IMAGE + repository: /opt/puzzletron/src/modelopt + venv: /venv + container: REPLACE_WITH_PUZZLETRON_IMAGE container_mounts: REPLACE_WITH_REQUIRED_CONTAINER_MOUNTS prerun_commands: [] postrun_commands: [] diff --git a/examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml b/examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml index b3b2489c682..7363d863c92 100644 --- a/examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml +++ b/examples/puzzletron/configs/orchestration/qwen_moe/runner.slurm.yaml @@ -11,18 +11,14 @@ runner: max_nodes: 20 time_limit: "4:00:00" execution_contract: - # Replace with the ModelOpt checkout path visible on every worker and in the container. - repository: REPLACE_WITH_WORKER_VISIBLE_MODELOPT_CHECKOUT - # Replace with the virtual environment path to source on every worker. - venv: REPLACE_WITH_WORKER_VISIBLE_MODELOPT_VENV - # Replace with an image or path accepted by the site's srun container plugin. - container: REPLACE_WITH_SLURM_CONTAINER_IMAGE + # These paths are provided by the Puzzletron worker image. + repository: /opt/puzzletron/src/modelopt + venv: /venv + # Replace with a registry reference or cluster-readable copy of that image. + container: REPLACE_WITH_PUZZLETRON_IMAGE # Replace with the host and container paths required by the campaign. container_mounts: "REPLACE_WITH_HOST_PATH:REPLACE_WITH_CONTAINER_PATH" - # Replace these values with the site's setup script and source checkout paths. - prerun_commands: - - source REPLACE_WITH_SITE_SETUP_SCRIPT - - export VLLM_ROOT=REPLACE_WITH_WORKER_VISIBLE_VLLM_CHECKOUT - - export AUTOMODEL_ROOT=REPLACE_WITH_WORKER_VISIBLE_AUTOMODEL_CHECKOUT + # Optional site setup, for example cache or authentication variables. + prerun_commands: [] # Optional shell commands run when the stage payload exits. postrun_commands: [] diff --git a/examples/puzzletron/configs/orchestration/runner.slurm.example.yaml b/examples/puzzletron/configs/orchestration/runner.slurm.example.yaml index cce5898216b..2ba0126009b 100644 --- a/examples/puzzletron/configs/orchestration/runner.slurm.example.yaml +++ b/examples/puzzletron/configs/orchestration/runner.slurm.example.yaml @@ -16,13 +16,13 @@ runner: time_limit: "4:00:00" log_dir: logs execution_contract: - # Required. Use the checkout path visible on every worker and in the container, if used. - repository: REPLACE_WITH_WORKER_VISIBLE_MODELOPT_CHECKOUT - # Required. Sourced as /bin/activate on every worker. - venv: REPLACE_WITH_WORKER_VISIBLE_MODELOPT_VENV - # Optional. Set to an image/path accepted by the cluster's srun container plugin. - # Leave null to execute directly in the worker environment. - container: + # The Puzzletron worker image provides these paths. Change them only + # when using a custom worker environment. + repository: /opt/puzzletron/src/modelopt + venv: /venv + # Required for the worker image. Replace with a registry + # reference or image path accepted by the site's container plugin. + container: REPLACE_WITH_PUZZLETRON_IMAGE # Optional and used only with a container. Use comma-separated # /host/path:/container/path entries, for example /data:/data,/models:/models. container_mounts: diff --git a/examples/puzzletron/docs/checkpoint_evaluation.md b/examples/puzzletron/docs/checkpoint_evaluation.md index b65a8473099..7df9c8e8ba2 100644 --- a/examples/puzzletron/docs/checkpoint_evaluation.md +++ b/examples/puzzletron/docs/checkpoint_evaluation.md @@ -22,13 +22,9 @@ separate [VLM checkpoint evaluator](vlm_checkpoint_evaluation.md). ## Quick start -Install the Puzzletron worker requirements: - -```bash -python -m pip install -r examples/puzzletron/requirements.txt -``` - -Then run the default smoke: +Run the command in the Puzzletron worker image described in the +[environment setup guide](environment_setup.md#worker-environment). Mount the +checkpoint and output paths into the container, then run the default smoke: ```bash python -m examples.puzzletron.evaluation.text \ diff --git a/examples/puzzletron/docs/environment_setup.md b/examples/puzzletron/docs/environment_setup.md index 396daebd47f..6a091f13f61 100644 --- a/examples/puzzletron/docs/environment_setup.md +++ b/examples/puzzletron/docs/environment_setup.md @@ -33,208 +33,36 @@ ModelOpt, CUDA, the worker container, or the worker virtual environment. ## Worker environment -Use one Python environment for ModelOpt, patched vLLM, AutoModel, and official -AIPerf. Install PyTorch first and build every CUDA extension against that -installation. Mixing PyTorch or CUDA builds can cause import failures or -incorrect GPU execution. +The repository [`Dockerfile`](../Dockerfile) builds the worker image. It +installs ModelOpt, the pinned vLLM and AutoModel sources, AIPerf, LMMS-Eval, +the required CUDA extensions, and the teacher-evaluation resources. Do not +maintain a second set of worker installation commands outside the Dockerfile. -### Choose a container or host environment +Build the Linux amd64 image from the repository root by following the +[image build and validation guide](worker_image.md). That guide provides the +build command and the revision-specific image tag. -This CUDA image provides a reproducible bootstrap: +The amd64 platform is required because the current CUDA extension set and +Linux `eva-decord 0.6.1` dependency do not have a validated ARM build path. -```text -nvcr.io/nvidia/cuda:12.9.2-cudnn-devel-ubuntu24.04 -``` - -The image is an example, not a required runner image. Slurm campaigns can set -`runner.execution_contract.container` to an image or path accepted by the -site, or omit it to execute directly in the worker environment. Bare-metal -runners use the host environment selected by `runner.execution_contract.venv`. - -The commands below assume a container. For bare metal, skip the Docker, -`/workspace`, and `apt-get` steps. Install equivalent Python and build tools -through the host-environment tooling and adapt the paths. +Using the `image` variable from that guide, run the image locally with GPU +access: ```bash -export PUZZLETRON_WORKSPACE=/absolute/path/to/workspace docker run --gpus all --ipc=host --rm -it \ - -v "${PUZZLETRON_WORKSPACE}:/workspace" \ - -w /workspace \ - nvcr.io/nvidia/cuda:12.9.2-cudnn-devel-ubuntu24.04 bash -``` - -Inside the container, install Python and the build tools used by editable -packages and optional CUDA extensions: - -```bash -apt-get update -DEBIAN_FRONTEND=noninteractive apt-get install -y \ - build-essential cmake git ninja-build \ - python3 python3-dev python3-pip python3-venv -``` - -### Clone the tracked forks - -Keep ModelOpt and the two Puzzletron forks as siblings. The machine-readable -[CI environment](../ci_environment.json) records the shared compatibility pins. - -```bash -export MODEL_OPT_ROOT=/workspace/modelopt -export VLLM_ROOT=/workspace/vllm -export AUTOMODEL_ROOT=/workspace/Automodel -export PUZZLETRON_CI_ENVIRONMENT="${MODEL_OPT_ROOT}/examples/puzzletron/ci_environment.json" -export AUTOMODEL_REF="$(python3 -c \ - 'import json, sys; print(json.load(open(sys.argv[1]))["nemo_automodel"]["commit"])' \ - "${PUZZLETRON_CI_ENVIRONMENT}")" - -git clone --branch feature/add_anymodel_to_vllm --single-branch \ - https://github.com/Separius/vllm.git "${VLLM_ROOT}" -git clone --branch puzzletron --single-branch \ - https://github.com/Separius/Automodel.git "${AUTOMODEL_ROOT}" -git -C "${AUTOMODEL_ROOT}" checkout --detach "${AUTOMODEL_REF}" + "${image}" ``` -```text -/workspace/ -├── modelopt/ -├── vllm/ -└── Automodel/ -``` - -### Install runtime packages +Inside the image, the runner contract is: -The patched vLLM branch uses the PyTorch version recorded in the CI environment -with CUDA 12.9. Install that combination before compiling CUDA code: +- `repository: /opt/puzzletron/src/modelopt` +- `venv: /venv` +- `container: ` -```bash -python3 -m venv /workspace/.venv -source /workspace/.venv/bin/activate - -export PUZZLETRON_TORCH_VERSION="$(python -c \ - 'import json, sys; print(json.load(open(sys.argv[1]))["torch"])' \ - "${PUZZLETRON_CI_ENVIRONMENT}")" -export PUZZLETRON_TORCHVISION_VERSION="$(python -c \ - 'import json, sys; print(json.load(open(sys.argv[1]))["torchvision"])' \ - "${PUZZLETRON_CI_ENVIRONMENT}")" -export PUZZLETRON_TRANSFORMERS_VERSION="$(python -c \ - 'import json, sys; print(json.load(open(sys.argv[1]))["transformers"])' \ - "${PUZZLETRON_CI_ENVIRONMENT}")" - -python -m pip install --upgrade \ - pip "setuptools>=80,<81" "setuptools-scm>=8" setuptools-rust \ - wheel "packaging>=24.2" "cmake>=3.26.1" ninja jinja2 - -python -m pip install \ - "torch==${PUZZLETRON_TORCH_VERSION}" \ - "torchvision==${PUZZLETRON_TORCHVISION_VERSION}" \ - "torchaudio==${PUZZLETRON_TORCH_VERSION}" \ - --index-url https://download.pytorch.org/whl/cu129 - -VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_VARIANT=cu129 \ - python -m pip install --no-build-isolation -e "${VLLM_ROOT}" - -python -m pip install -e "${AUTOMODEL_ROOT}" -python -m pip install aiperf -python -m pip install -e "${MODEL_OPT_ROOT}[hf,puzzletron]" -python -m pip install "transformers==${PUZZLETRON_TRANSFORMERS_VERSION}" -python -m pip install -r "${MODEL_OPT_ROOT}/examples/puzzletron/requirements.txt" -``` - -Do not add `--no-deps`; these packages need their declared Python dependencies. -`--no-build-isolation` makes compiled extensions use the active PyTorch -installation. It does not disable dependency installation. - -Install only the kernels required by the target architecture: - -```bash -# Mixture of experts -python -m pip install --no-build-isolation \ - "git+https://github.com/fanshiqing/grouped_gemm@v1.1.4" - -# Mamba -python -m pip install "mamba-ssm[causal-conv1d]" --no-build-isolation - -# Linear attention -python -m pip install "flash-linear-attention[cuda]" -``` - -## Verify the worker environment - -Run these checks inside the same container and virtual environment used by -Puzzletron jobs: - -```bash -test "$(git -C "${VLLM_ROOT}" remote get-url origin)" = \ - "https://github.com/Separius/vllm.git" -test "$(git -C "${VLLM_ROOT}" branch --show-current)" = \ - "feature/add_anymodel_to_vllm" -test "$(git -C "${AUTOMODEL_ROOT}" remote get-url origin)" = \ - "https://github.com/Separius/Automodel.git" -test "$(git -C "${AUTOMODEL_ROOT}" rev-parse HEAD)" = "${AUTOMODEL_REF}" - -git -C "${MODEL_OPT_ROOT}" rev-parse HEAD -git -C "${VLLM_ROOT}" rev-parse HEAD -git -C "${AUTOMODEL_ROOT}" rev-parse HEAD -``` - -```bash -PYTHONPATH="${MODEL_OPT_ROOT}" python - <<'PY' -import importlib.metadata as metadata -import json -import os - -from packaging.version import Version - -from examples.puzzletron.ci_environment import verify_installed_vcs_source - -import aiperf -import lmms_eval -import modelopt -import nemo_automodel -import torch -import transformers -import vllm - -with open(os.environ["PUZZLETRON_CI_ENVIRONMENT"], encoding="utf-8") as stream: - ci_environment = json.load(stream) - -for package in ( - "torch", - "vllm", - "nemo-automodel", - "aiperf", - "lmms-eval", - "nvidia-modelopt", -): - print(package, metadata.version(package)) - -print("torch CUDA", torch.version.cuda) -print("CUDA available", torch.cuda.is_available()) -print("modelopt", modelopt.__file__) -print("vllm", vllm.__file__) - -assert Version(torch.__version__).release == Version(ci_environment["torch"]).release -assert Version(metadata.version("torchvision")).release == Version( - ci_environment["torchvision"] -).release -assert transformers.__version__ == ci_environment["transformers"] -assert Version(metadata.version("lmms-eval")).base_version == ( - ci_environment["lmms_eval"]["base_version"] -) -assert Version(metadata.version("nemo-automodel")).base_version == ( - ci_environment["nemo_automodel"]["base_version"] -) -for package, source in ( - ("lmms-eval", ci_environment["lmms_eval"]), - ("nemo-automodel", ci_environment["nemo_automodel"]), -): - verify_installed_vcs_source(package, source) -assert torch.version.cuda == "12.9" -assert torch.cuda.is_available() -PY - -python -m pip check -``` +Add site-specific data, model, cache, and result mounts through +`container_mounts`. A registry upload or conversion to a cluster container +format changes how the image is delivered, not how its Python environment is +created. -Record the three source revisions and verification output with the campaign. -Repeat verification after pulling either fork or rebuilding a CUDA extension. +CI jobs that need the Puzzletron worker stack should use this image and its +`/venv`; they should not reinstall a separate environment. diff --git a/examples/puzzletron/docs/worker_image.md b/examples/puzzletron/docs/worker_image.md new file mode 100644 index 00000000000..34d4cdf1a39 --- /dev/null +++ b/examples/puzzletron/docs/worker_image.md @@ -0,0 +1,72 @@ +# Build the Puzzletron worker image + +The [`Dockerfile`](../Dockerfile) contains the worker installation steps. +[`ci_environment.json`](../ci_environment.json) stores the versions, source +revisions, CUDA targets, and downloaded-file checksums used by those steps. +The Dockerfile reads the same file for installation and its build-time checks. + +## Build + +Run one repository command from a clean checkout on a Linux amd64 system with +Docker: + +```bash +python examples/puzzletron/build_worker_image.py +``` + +The command always uses `examples/puzzletron/Dockerfile`, the repository root as +the build context, Linux amd64 as the platform, and the current full Git +revision. It checks the installed modules, CUDA version, and required evaluation +data. The full source revision is recorded in the image label and at +`/opt/puzzletron/modelopt_revision`. The Dockerfile remains directly usable by +standard Docker tools, but users do not need to assemble these arguments. + +## Export for another runtime + +The same command can build and export a revision-named Enroot/Pyxis image in one +step. No separate Docker build is required: + +```bash +python examples/puzzletron/build_worker_image.py \ + --output-dir /path/to/output \ + --sqsh +``` + +The helper runs the same controlled Docker build before exporting. Docker reuses +its build cache when the image is already current. Creating a SquashFS image +requires Enroot and Docker on the same Linux amd64 host. Use node-local or other +large storage for the output directory. The local Docker image remains +available after export and can be removed with normal Docker image-management +commands when it is no longer needed. + +The exported file is named after the source revision: + +```text +modelopt-puzzletron-linux-amd64-git-<12-character-commit>.sqsh +``` + +## GPU check + +The build does not require a GPU. On a host with an NVIDIA GPU, check CUDA +access using the local Docker name printed by the build command: + +```bash +image="modelopt-puzzletron:linux-amd64-git-$(git rev-parse --short=12 HEAD)" +docker run --gpus all --ipc=host --rm "${image}" \ + python -c 'import torch; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))' +``` + +## Use + +The image contains the worker environment at `/venv` and the ModelOpt checkout +at `/opt/puzzletron/src/modelopt`. Use the image directly with Docker, publish +it to a registry, or export it for Enroot, Pyxis, or the target Slurm container +plugin. + +The current image supports Linux amd64 only. Its CUDA extensions and +`eva-decord 0.6.1` dependency have not been validated on Linux ARM. + +The artifact filename identifies the recipe revision without relying on a local +Docker tag. Rebuilding that revision may still resolve newer transitive Python +dependencies. Record the registry digest if the image is later published. This +repository does not publish the image automatically. diff --git a/examples/puzzletron/patches/mamba_ssm_tilelang_0_1_9.patch b/examples/puzzletron/patches/mamba_ssm_tilelang_0_1_9.patch new file mode 100644 index 00000000000..7dc23c353f0 --- /dev/null +++ b/examples/puzzletron/patches/mamba_ssm_tilelang_0_1_9.patch @@ -0,0 +1,26 @@ +diff --git a/pyproject.toml b/pyproject.toml +index a3d4d1a..bd274c9 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -17,7 +17,7 @@ classifiers = [ + ] + dependencies = [ + "torch", +- "tilelang==0.1.8", ++ "tilelang==0.1.9", + "apache-tvm-ffi<=0.1.9", + "quack-kernels>=0.3.4", + "triton>=3.5.0", +diff --git a/setup.py b/setup.py +index f9c3320..a3136f9 100755 +--- a/setup.py ++++ b/setup.py +@@ -399,7 +399,7 @@ setup( + "einops", + "triton>=3.5.0", + "transformers", +- "tilelang==0.1.8", ++ "tilelang==0.1.9", + "apache-tvm-ffi<=0.1.9", + "quack-kernels>=0.3.4", + # "causal_conv1d>=1.4.0", diff --git a/modelopt/torch/puzzletron/orchestration/task_launcher.py b/modelopt/torch/puzzletron/orchestration/task_launcher.py index 6245435e902..7a80d4bef81 100644 --- a/modelopt/torch/puzzletron/orchestration/task_launcher.py +++ b/modelopt/torch/puzzletron/orchestration/task_launcher.py @@ -39,7 +39,18 @@ "resolve_task_binding", ] -TASK_IDENTITY_ENV_KEYS = frozenset( +TORCH_DISTRIBUTED_ENV_KEYS = frozenset( + { + "LOCAL_RANK", + "LOCAL_WORLD_SIZE", + "MASTER_ADDR", + "MASTER_PORT", + "RANK", + "WORLD_SIZE", + } +) + +TASK_IDENTITY_ENV_KEYS = TORCH_DISTRIBUTED_ENV_KEYS | frozenset( { "CUDA_VISIBLE_DEVICES", "SLURM_LOCALID", @@ -257,6 +268,17 @@ def main(argv: Sequence[str] | None = None) -> int: PUZZLETRON_RENDEZVOUS_ENDPOINT=rendezvous_endpoint(binding), PUZZLETRON_RENDEZVOUS_ID=binding.rendezvous_id, ) + for key in TORCH_DISTRIBUTED_ENV_KEYS: + env.pop(key, None) + if args.launcher == TaskLauncher.DIRECT.value and binding.group_size == 1: + env.update( + LOCAL_RANK="0", + LOCAL_WORLD_SIZE="1", + MASTER_ADDR=binding.master_addr, + MASTER_PORT=str(binding.master_port), + RANK="0", + WORLD_SIZE="1", + ) print( "puzzletron binding " f"host={binding.hostname} task={binding.task_index} " diff --git a/tests/unit/torch/puzzletron/test_build_worker_image.py b/tests/unit/torch/puzzletron/test_build_worker_image.py new file mode 100644 index 00000000000..4772c4f3be7 --- /dev/null +++ b/tests/unit/torch/puzzletron/test_build_worker_image.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the Puzzletron worker-image build command.""" + +import pytest + +from examples.puzzletron.build_worker_image import sqsh_name + + +def test_sqsh_name_uses_the_git_revision(): + revision = "ba737f1f2301d0526c7d4674e1d21bf3d8c1ff14" + + assert sqsh_name(revision) == "modelopt-puzzletron-linux-amd64-git-ba737f1f2301.sqsh" + + +def test_sqsh_name_requires_a_full_git_revision(): + with pytest.raises(ValueError, match="full lowercase Git commit"): + sqsh_name("ba737f1f2301") diff --git a/tests/unit/torch/puzzletron/test_ci_image_contract.py b/tests/unit/torch/puzzletron/test_ci_image_contract.py new file mode 100644 index 00000000000..40e8e45e8a1 --- /dev/null +++ b/tests/unit/torch/puzzletron/test_ci_image_contract.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the repository-owned Puzzletron worker image.""" + +import hashlib +import json +import re + +import yaml + + +def test_image_recipe_records_pinned_environment(project_root_path): + puzzletron_root = project_root_path / "examples/puzzletron" + environment = json.loads((puzzletron_root / "ci_environment.json").read_text()) + dockerfile = (puzzletron_root / "Dockerfile").read_text() + + base_image = environment["gpu_image"]["base_image"] + assert re.fullmatch(r"nvidia/cuda:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}", base_image) + assert f"FROM {base_image}" in dockerfile + assert "ARG TARGETPLATFORM" in dockerfile + assert 'test "${TARGETPLATFORM}" = "linux/amd64"' in dockerfile + assert "> /opt/puzzletron/modelopt_revision" in dockerfile + assert ( + "COPY examples/puzzletron/ci_environment.json /opt/puzzletron/ci_environment.json" + in dockerfile + ) + assert "for module in" in dockerfile + assert "nltk.data.find" in dockerfile + assert ".lmms_eval.task_configs[]" in dockerfile + + vcs_sources = [ + environment["lmms_eval"], + environment["nemo_automodel"], + environment["vllm"], + environment["runtime_image"]["grouped_gemm"], + environment["runtime_image"]["mamba_ssm"], + ] + assert all(re.fullmatch(r"[0-9a-f]{40}", source["commit"]) for source in vcs_sources) + + nltk_resources = environment["gpu_image"]["nltk_resources"] + nltk_checksums = environment["gpu_image"]["nltk_resource_sha256"] + assert set(nltk_checksums) == set(nltk_resources) + assert all(re.fullmatch(r"[0-9a-f]{64}", checksum) for checksum in nltk_checksums.values()) + + assert "nltk_data/$(pin gpu_image.nltk_data_commit)/packages/tokenizers" in dockerfile + assert ( + 'echo "${nltk_resource_sha256} ${nltk_archive}" | sha256sum --check --strict' in dockerfile + ) + + +def test_mamba_compatibility_patch_is_limited_to_the_tilelang_pin(project_root_path): + puzzletron_root = project_root_path / "examples/puzzletron" + environment = json.loads((puzzletron_root / "ci_environment.json").read_text()) + dockerfile = (puzzletron_root / "Dockerfile").read_text() + + mamba_source = environment["runtime_image"]["mamba_ssm"] + patch_bytes = (puzzletron_root / "patches" / mamba_source["compatibility_patch"]).read_bytes() + assert hashlib.sha256(patch_bytes).hexdigest() == mamba_source["compatibility_patch_sha256"] + changed_lines = [ + line + for line in patch_bytes.decode().splitlines() + if line.startswith(("+", "-")) and not line.startswith(("+++ ", "--- ")) + ] + assert changed_lines == [ + '- "tilelang==0.1.8",', + '+ "tilelang==0.1.9",', + '- "tilelang==0.1.8",', + '+ "tilelang==0.1.9",', + ] + assert '"$(pin runtime_image.mamba_ssm.repository)" /tmp/mamba-ssm' in dockerfile + assert 'git -C /tmp/mamba-ssm checkout --detach "$(pin runtime_image.mamba_ssm.commit)"' in ( + dockerfile + ) + assert 'test "$(git -C /tmp/mamba-ssm rev-parse HEAD)" = \\' in dockerfile + + +def test_cpu_contract_lane_watches_image_recipe_inputs(project_root_path): + workflow = yaml.safe_load((project_root_path / ".github/workflows/unit_tests.yml").read_text()) + + # PyYAML applies YAML 1.1 boolean resolution to GitHub's unquoted `on` key. + push_paths = workflow[True]["push"]["paths"] + changed_files_step = next( + step + for step in workflow["jobs"]["check-file-changes"]["steps"] + if step.get("id") == "puzzletron_changed" + ) + pull_request_paths = changed_files_step["with"]["files"].splitlines() + + assert "examples/__init__.py" in push_paths + assert "examples/__init__.py" in pull_request_paths + assert "examples/puzzletron/**" in push_paths + assert "examples/puzzletron/Dockerfile" in pull_request_paths diff --git a/tests/unit/torch/puzzletron/test_orchestration_task_topology.py b/tests/unit/torch/puzzletron/test_orchestration_task_topology.py index 9b1fadf4cf1..0ea97f8bb0e 100644 --- a/tests/unit/torch/puzzletron/test_orchestration_task_topology.py +++ b/tests/unit/torch/puzzletron/test_orchestration_task_topology.py @@ -166,11 +166,32 @@ def fake_posix_spawnp(executable, command, env) -> int: assert captured["env"]["CUDA_VISIBLE_DEVICES"] == expected_gpus assert captured["env"]["PUZZLETRON_TASK_LAUNCHER"] == "direct" assert captured["env"]["PUZZLETRON_RENDEZVOUS_ENDPOINT"] == "localhost:0" + assert { + key: captured["env"][key] + for key in ("LOCAL_RANK", "LOCAL_WORLD_SIZE", "RANK", "WORLD_SIZE") + } == { + "LOCAL_RANK": "0", + "LOCAL_WORLD_SIZE": "1", + "RANK": "0", + "WORLD_SIZE": "1", + } + assert captured["env"]["MASTER_ADDR"] == "node-a" + assert int(captured["env"]["MASTER_PORT"]) > 0 def test_task_launcher_exports_shared_multi_node_rendezvous(monkeypatch) -> None: captured: dict[str, object] = {} monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1,2,3,4,5,6,7") + distributed_keys = ( + "LOCAL_RANK", + "LOCAL_WORLD_SIZE", + "MASTER_ADDR", + "MASTER_PORT", + "RANK", + "WORLD_SIZE", + ) + for key in distributed_keys: + monkeypatch.setenv(key, "99") monkeypatch.setenv("PUZZLETRON_TASK_INDEX", "1") monkeypatch.setenv("PUZZLETRON_LOCAL_TASK_INDEX", "0") monkeypatch.setenv("PUZZLETRON_TASK_HOSTS", "node-a,node-b") @@ -213,6 +234,7 @@ def fake_posix_spawnp(executable, command, env) -> int: assert env["PUZZLETRON_GROUP_RANK"] == "1" assert env["PUZZLETRON_RENDEZVOUS_ENDPOINT"].startswith("node-a:") assert env["PUZZLETRON_RENDEZVOUS_ID"] == "attempt-a-group-0" + assert all(key not in env for key in distributed_keys) def test_run_worker_consumes_multi_node_task_launcher_identity(tmp_path: Path) -> None: diff --git a/tests/unit/torch/puzzletron/test_portable_configs.py b/tests/unit/torch/puzzletron/test_portable_configs.py index d00fd56282f..64c307a6b4e 100644 --- a/tests/unit/torch/puzzletron/test_portable_configs.py +++ b/tests/unit/torch/puzzletron/test_portable_configs.py @@ -45,9 +45,9 @@ def test_slurm_runner_example_is_portable() -> None: path = REPOSITORY_ROOT / "examples/puzzletron/configs/orchestration/runner.slurm.example.yaml" slurm = load_runner_config(path) - assert slurm.contract.repository == WORKER_REPOSITORY_PLACEHOLDER - assert slurm.contract.venv == WORKER_VENV_PLACEHOLDER - assert slurm.contract.container is None + assert slurm.contract.repository == "/opt/puzzletron/src/modelopt" + assert slurm.contract.venv == "/venv" + assert slurm.contract.container.startswith("REPLACE_WITH_") assert slurm.contract.container_mounts is None assert not slurm.contract.prerun_commands assert slurm.slurm is not None @@ -82,13 +82,9 @@ def test_qwen_slurm_runner_preserves_portable_environment_contract() -> None: runner.contract.container, runner.contract.container_mounts, ) - assert contract_values[:2] == ( - WORKER_REPOSITORY_PLACEHOLDER, - WORKER_VENV_PLACEHOLDER, - ) + assert contract_values[:2] == ("/opt/puzzletron/src/modelopt", "/venv") assert all(value and value.startswith("REPLACE_WITH_") for value in contract_values[2:]) - assert runner.contract.prerun_commands - assert all("REPLACE_WITH_" in command for command in runner.contract.prerun_commands) + assert not runner.contract.prerun_commands assert runner.slurm is not None assert runner.slurm.account.startswith("REPLACE_WITH_") assert runner.slurm.partition.startswith("REPLACE_WITH_") @@ -105,7 +101,9 @@ def test_checked_in_slurm_runners_only_emit_generic_partition( @pytest.mark.parametrize("relative_path", NAMED_SLURM_RUNNER_CONFIGS) -def test_named_slurm_runners_keep_logs_below_the_campaign_root(relative_path: str) -> None: +def test_named_slurm_runners_keep_logs_below_the_campaign_root( + relative_path: str, +) -> None: payload = yaml.safe_load((REPOSITORY_ROOT / relative_path).read_text()) assert "log_dir" not in payload["runner"]["slurm"] diff --git a/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_smoke_plan.py b/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_smoke_plan.py index fb3b6c1ce4e..9035d9d1239 100644 --- a/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_smoke_plan.py +++ b/tests/unit/torch/puzzletron/test_qwen3p5_0p8b_smoke_plan.py @@ -94,8 +94,8 @@ def test_qwen3p5_0p8b_runner_requires_an_explicit_site_contract() -> None: assert runner.slurm.time_limit == "1:00:00" assert runner.slurm.account.startswith("REPLACE_WITH_") assert runner.slurm.partition.startswith("REPLACE_WITH_") - assert runner.contract.repository.startswith("REPLACE_WITH_") - assert runner.contract.venv.startswith("REPLACE_WITH_") + assert runner.contract.repository == "/opt/puzzletron/src/modelopt" + assert runner.contract.venv == "/venv" assert runner.contract.container is not None assert runner.contract.container.startswith("REPLACE_WITH_") assert runner.contract.container_mounts is not None