From 20f91949bfa1cdd0efa184fd2eb25cf4400f0bfd Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Wed, 7 Oct 2026 22:36:48 +0800 Subject: [PATCH 1/6] MediaTek export: run on Modal, sized per window, and read the first LFM2 checkpoints The Namespace profile (15.7 GiB, 4 vCPUs) carried no LFM2 window by the estimate, and a 32k streaming calibration needs cores more than memory. mtk-matrix now gives each window a Sandbox: memory from calibration_bytes (16 GiB floor), cores and minutes from export_mtk.MODAL_SHAPES, and one attention layer per chunk for a hybrid at 16k and up (finding 39). The image carries MediaTek's public Python 3.10 tools; scripts/mtk-setup.sh downloads, checks and deletes the NeuroPilot SDK inside each run, so none of it is stored. LiquidAI/LFM2-350M, -700M and -1.2B describe their layers as full_attn_idxs and block_ff_dim rather than LFM2.5's layer_types and intermediate_size; hub.normalize_config reads both, so sizing, chunking and lfm2.py see the same config. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/export-mtk.yml | 112 +++++++------------------ config/pipeline.yaml | 18 ++-- pipeline/__main__.py | 21 ++++- pipeline/export_mtk.py | 24 +++++- pipeline/families.py | 22 +++++ pipeline/hub.py | 28 ++++++- pipeline/remote.py | 56 +++++++++++-- scripts/mtk-setup.sh | 45 ++++++++++ tests/fixtures/lfm2-1.2b.config.json | 46 ++++++++++ tests/fixtures/lfm2-350m.config.json | 46 ++++++++++ tests/fixtures/lfm2.5-350m.config.json | 60 +++++++++++++ tests/test_families.py | 30 +++++++ tests/test_namespace_runner.py | 21 ++--- tests/test_remote.py | 29 ++++++- 14 files changed, 444 insertions(+), 114 deletions(-) create mode 100755 scripts/mtk-setup.sh create mode 100644 tests/fixtures/lfm2-1.2b.config.json create mode 100644 tests/fixtures/lfm2-350m.config.json create mode 100644 tests/fixtures/lfm2.5-350m.config.json diff --git a/.github/workflows/export-mtk.yml b/.github/workflows/export-mtk.yml index 91dfe8a..7e60256 100644 --- a/.github/workflows/export-mtk.yml +++ b/.github/workflows/export-mtk.yml @@ -75,11 +75,10 @@ jobs: CONTEXTS: ${{ inputs.contexts }} MODEL_ID: ${{ inputs.model_id }} REVISION: ${{ inputs.revision }} - # One entry per window, each carrying the smallest runner that can build it. The - # sizing is pipeline.export_mtk.pick_runner, the same calibration_bytes and - # calibration_disk_bytes the export gates on, so the matrix and the job cannot - # disagree about whether a window fits. Memory is one prompt's calibration steps; - # disk is every prompt's, since writer_batch_size=1 put the rest in the Arrow cache. + # One entry per window, each carrying the Modal Sandbox that can build it: memory from + # pipeline.export_mtk.calibration_bytes (the same estimate the export gates on), + # cores and minutes from export_mtk.MODAL_SHAPES, and the chunk count for a hybrid at + # 16k and up. run: | entries=$(python3 -m pipeline mtk-matrix "$MODEL_ID" --revision "$REVISION" --contexts "$CONTEXTS") echo "$entries" @@ -94,106 +93,59 @@ jobs: matrix: soc: ${{ fromJSON(needs.targets.outputs.socs) }} entry: ${{ fromJSON(needs.targets.outputs.entries) }} - # Stays on x86: mtk_converter 8.13.0 is a cp310 x86 build and the NeuroPilot SDK - # ships x86_64 libraries (finding 14). - # - # Sized per window by the targets job above, because the cost is the window and not the - # model: a 2k export fits the smallest tier and a 32k one does not fit the largest - # without swap. Picking one size for all of them would either waste three quarters of - # the fleet on the small windows or fail the big ones. - runs-on: ${{ matrix.entry.runner }} - timeout-minutes: 350 + # The export runs on Modal, in a Sandbox the targets job sized to this window: memory + # from export_mtk.calibration_bytes plus modal.memory_margin, and cores and minutes from + # export_mtk.MODAL_SHAPES, because streaming calibration's time grows with the window. + # This job only sends it, waits, and publishes what comes back. Modal's hosts are x86, + # which mtk_converter 8.13.0 (a cp310 x86 build) and the SDK's libraries need (finding 14). + runs-on: namespace-profile-execupack + # Past the longest Sandbox (export_mtk.MODAL_SHAPES), so the Sandbox times out first. + timeout-minutes: 630 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} HF_HUB_DISABLE_PROGRESS_BARS: "1" MODEL_ID: ${{ inputs.model_id }} REVISION: ${{ inputs.revision }} SOC: ${{ matrix.soc }} CONTEXT: ${{ matrix.entry.context }} - MTK_MAX_CHUNKS: ${{ inputs.max_chunks }} + MEMORY_MIB: ${{ matrix.entry.memory_mib }} + CORES: ${{ matrix.entry.cpu }} + SANDBOX_MINUTES: ${{ matrix.entry.timeout_minutes }} + # The input wins; otherwise the targets job's one-attention-layer-per-chunk count for a + # hybrid at 16k and up (finding 39), otherwise mtk.max_chunks. + MTK_MAX_CHUNKS: ${{ inputs.max_chunks || matrix.entry.max_chunks }} MTK_PRECISION: ${{ inputs.precision }} MTK_PROMPT_TOKENS: ${{ inputs.prompt_tokens }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The pipeline itself (fetch, reports, publishing) on the repo's usual Python. - - uses: ./.github/actions/setup-export - with: - # Sized with the runner: enough to absorb an estimate that is low, without taking - # so much of the filesystem that the Arrow cache has nowhere to go. Only the 2.6B - # at 32k is expected to spend it, and pick_runner prefers a tier that does not. - swap-gib: "${{ matrix.entry.swap_gib }}" - requirements: requirements/export-mtk.txt + - uses: ./.github/actions/setup-modal - name: Name things run: | - grep -E '^(NEUROPILOT_SDK_[A-Z0-9_]+|MTK_[A-Z_]+_VERSION|EXECUTORCH_COMMIT)=' config/versions.env >>"$GITHUB_ENV" soc_lower=$(echo "$SOC" | tr '[:upper:]' '[:lower:]') echo "TARGET=$soc_lower" >>"$GITHUB_ENV" echo "ARTIFACT=mtk-${soc_lower}-${MODEL_ID//\//--}-${CONTEXT}" >>"$GITHUB_ENV" - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - id: mtk-python - with: - python-version: ${{ env.MTK_PYTHON_VERSION }} - update-environment: false - - # MediaTek's tools in their own Python 3.10 virtualenv (requirements/mtk-tools.txt says - # why). The NeuroPilot Express SDK is downloaded from MediaTek on every run, checked - # against the pinned digest, and deleted after its two wheels are installed: it is - # licensed by MediaTek Inc. under its own terms (THIRD_PARTY_NOTICES.md), which every - # run accepts, and nothing from it is cached, uploaded or published. - - name: MediaTek tools - run: | - python_bin="${{ steps.mtk-python.outputs.python-path }}" - "$python_bin" -m venv "$WORK_DIR/mtk-venv" - tool_python="$WORK_DIR/mtk-venv/bin/python" - "$tool_python" -m pip install --quiet --upgrade pip - "$tool_python" -m pip install --quiet "$(grep -E '^torch==' requirements/mtk-tools.txt)" \ - --index-url https://download.pytorch.org/whl/cpu - - sdk="$RUNNER_TEMP/neuropilot" - mkdir -p "$sdk" - curl -fsSL --retry 3 --retry-all-errors -o "$sdk.tar.gz" "$NEUROPILOT_SDK_URL" - echo "$NEUROPILOT_SDK_SHA256 $sdk.tar.gz" | sha256sum -c - - tar xzf "$sdk.tar.gz" -C "$sdk" --strip-components=1 --wildcards \ - "*/mtk_neuron-${MTK_NEURON_VERSION}-*.whl" "*/mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" - # The 8.0.8 archive keeps mtk_converter only under mtk_converter-*_public_packages/. - neuron_whl=$(find "$sdk" -name "mtk_neuron-${MTK_NEURON_VERSION}-*.whl" | head -1) - converter_whl=$(find "$sdk" -name "mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" | head -1) - test -n "$neuron_whl" && test -n "$converter_whl" - "$tool_python" -m pip install --quiet -r requirements/mtk-tools.txt "$neuron_whl" "$converter_whl" - rm -rf "$sdk" "$sdk.tar.gz" - "$tool_python" -m pip list | grep -iE '^(torch|torchao|executorch|transformers|mtk.converter|mtk.neuron) ' - echo "MTK_PYTHON=$tool_python" >>"$GITHUB_ENV" - - # The LLM export scripts are in the ExecuTorch source, not the wheel (BSD, fetched at - # the pinned commit), with this repo's patches from third_party/executorch/patches. - - name: ExecuTorch examples/mediatek - run: | - src="$WORK_DIR/executorch" - git init -q "$src" - git -C "$src" remote add origin https://github.com/pytorch/executorch.git - git -C "$src" sparse-checkout set examples/mediatek - git -C "$src" fetch -q --depth 1 origin "$EXECUTORCH_COMMIT" - git -C "$src" checkout -q FETCH_HEAD - for patch in third_party/executorch/patches/mediatek-*.patch; do - git -C "$src" apply -v "$GITHUB_WORKSPACE/$patch" - done - echo "MTK_EXAMPLES=$src/examples/mediatek" >>"$GITHUB_ENV" - - # Exit 4 means this runner cannot build this window: recorded as a skip, so the other - # jobs of the matrix stand on their own. - - name: Export + # scripts/mtk-setup.sh downloads the NeuroPilot SDK inside the Sandbox, checks it, installs + # its two wheels and deletes it, then fetches and patches ExecuTorch's examples/mediatek: + # nothing of the SDK is in the image, cached or brought back. Exit 4 means this window + # cannot be built (the export's own gate): recorded as a skip, so the other jobs of the + # matrix stand on their own. + - name: Export on Modal run: | set +e - python -m pipeline export-mtk "$MODEL_ID" --soc "$SOC" --revision "$REVISION" \ - --out "$WORK_DIR/out" --work "$WORK_DIR/work" --context "$CONTEXT" + python -m pipeline modal-run --memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES" \ + --requirements requirements/export-mtk.txt --mtk-tools requirements/mtk-tools.txt \ + --get "$WORK_DIR/out" \ + --script 'bash scripts/mtk-setup.sh && python -m pipeline export-mtk "$MODEL_ID" --soc "$SOC" --revision "$REVISION" --out "$OUT" --work "$WORK/export" --context "$CONTEXT" --tool-python /opt/mtk-venv/bin/python --examples "$WORK/executorch/examples/mediatek"' rc=$? set -e if [ "$rc" -eq 4 ]; then echo "SKIPPED=1" >>"$GITHUB_ENV" - echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens on $SOC cannot be exported on this runner" + echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens on $SOC cannot be exported" exit 0 fi exit "$rc" diff --git a/config/pipeline.yaml b/config/pipeline.yaml index 1cf2b60..505a779 100644 --- a/config/pipeline.yaml +++ b/config/pipeline.yaml @@ -164,17 +164,17 @@ export: # largest window a tier is given, so a calibration lands on cores that finish it inside the # job limit. # - # One tier since 2026-10-07, the Namespace profile (vulkan.runner_tiers says what was - # measured): 15.7 GiB, no swap, 4 vCPUs. calibration_bytes puts a 1.2B LFM2.5 build at - # about 17.4 GiB (15 bytes a parameter, measured on 4-chunk builds), so none is placed; the - # 8-chunk 16k build peaked at 13.4 GiB, so a refit for more chunks may bring it back. - # max_window 2048 because 4 vCPUs are half the 8 that were given at most 4096. + # On Modal since 2026-10-07, like XNNPACK and Vulkan (see vulkan.runner_tiers): one tier at + # Modal's ceiling, and mtk-matrix sizes each window's Sandbox from calibration_bytes (memory, + # with export_mtk.MODAL_FLOOR_BYTES under it) and export_mtk.MODAL_SHAPES (cores and minutes, + # which is what grows with the window when calibration streams). x86 like every Modal host. + # Before: the Namespace profile (15.7 GiB, no swap, 4 vCPUs; finding 40), which carried no + # LFM2 window at all by this estimate. runner_tiers: - - label: namespace-profile-execupack - ram_bytes: 16833269760 - disk_bytes: 101427666944 + - label: modal + ram_bytes: 360777252864 + disk_bytes: 549755813888 swap_gib: 0 - max_window: 2048 # Modal: where the exports run since 2026-10-07 (pipeline/remote.py). The GitHub jobs stay on # the Namespace profile and drive a Sandbox per window, sized to that window, with the repo's diff --git a/pipeline/__main__.py b/pipeline/__main__.py index 94e9872..ce0b72f 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -131,7 +131,7 @@ def _mtk_matrix(args) -> int: """ import dataclasses - from pipeline import export_mtk, families, hub, settings + from pipeline import export_mtk, families, hub, remote, settings cfg = settings.load() source = hub.fetch(args.model, args.revision) @@ -162,7 +162,22 @@ def _mtk_matrix(args) -> int: # that refuses to start teaches less than nine jobs that finish. print(f"no runner tier can build a {window}-token window", file=sys.stderr) continue - entries.append({"context": window, "runner": tier.label, "swap_gib": tier.swap_gib}) + entry = {"context": window, "runner": tier.label, "swap_gib": tier.swap_gib, "max_chunks": ""} + if window >= export_mtk.ONE_ATTENTION_PER_CHUNK_FROM: + chunks = families.mtk_chunks_one_attention_each(source.config) + if chunks is not None and chunks > cfg.mtk.max_chunks: + entry["max_chunks"] = str(chunks) + if tier.label == settings.MODAL_TIER: + need = export_mtk.calibration_bytes( + source.config, recipe, args.prompts, source.total_params, streaming=streaming + ) + entry["memory_mib"] = remote.memory_request(max(need, export_mtk.MODAL_FLOOR_BYTES), cfg.modal) + if entry["memory_mib"] is None: + print(f"{window} needs more than Modal allows ({need:,} B)", file=sys.stderr) + continue + entry["cpu"], entry["timeout_minutes"] = export_mtk.modal_shape(window) + print(f"{window}: needs {need / 2**30:.1f} GiB, {entry}", file=sys.stderr) + entries.append(entry) print(json.dumps(entries)) return 0 @@ -484,6 +499,8 @@ def main(argv: list[str] | None = None) -> int: modal_run.add_argument("--get", help="local directory that receives $OUT") modal_run.add_argument("--env", action="append", help="NAME=VALUE set in the Sandbox (repeatable)") modal_run.add_argument("--keep", action="store_true", help="leave the run's files on the Volume") + modal_run.add_argument("--timeout-minutes", type=int, default=None, help="default modal.timeout_minutes") + modal_run.add_argument("--mtk-tools", help="requirements for MediaTek's Python 3.10 virtualenv in the image") modal_run.set_defaults(func=_modal_run) watch = commands.add_parser("watch", help="check the watched orgs, dispatch exports, update state") diff --git a/pipeline/export_mtk.py b/pipeline/export_mtk.py index e6df747..049c598 100644 --- a/pipeline/export_mtk.py +++ b/pipeline/export_mtk.py @@ -129,6 +129,16 @@ def state_layout(script: str) -> str: # 14.4 bytes per parameter, and 17.8 GB for the 1.2B at 4k, so it does not grow with the window. LOWERING_BYTES_PER_PARAM = 16 REPO_ROOT = Path(__file__).resolve().parents[1] +# On Modal (mtk.runner_tiers "modal"), the cores and Sandbox minutes a window gets: (largest +# window, physical cores, minutes). Streaming calibration runs a corpus that fills the window +# through the model twice, so the time and not the memory grows with the window: the 16k +# 8-chunk LFM2.5-1.2B build took 3,235 s on 8 physical cores (16 vCPUs, finding 39), and a +# 32k calibration runs twice the tokens over twice the cache. +MODAL_SHAPES = ((4096, 8, 240), (16384, 16, 330), (32768, 32, 600)) +# The smallest Sandbox: lowering and MediaTek's compiler service on top of the estimate. +MODAL_FLOOR_BYTES = 16 * 2**30 +# From this window up, a hybrid is cut so no chunk holds two attention layers (finding 39). +ONE_ATTENTION_PER_CHUNK_FROM = 16384 TOOL_PACKAGES = ("executorch", "torch", "torchao", "transformers", "mtk_converter", "mtk-neuron") _LOAD_PROGRAM = """ @@ -295,6 +305,14 @@ def carries(tier: settings.RunnerTier, *, with_swap: bool) -> bool: return None +def modal_shape(window: int) -> tuple[int, int]: + """Physical cores and Sandbox minutes for a window on Modal (MODAL_SHAPES).""" + for largest, cores, minutes in MODAL_SHAPES: + if window <= largest: + return cores, minutes + raise ValueError(f"no Modal shape for a {window}-token window") + + def exp_name(weight_dir: Path, precision: str, chunks: int) -> str: """The name the scripts give the output folder and files (utils.get_exp_name).""" return f"{weight_dir.name}_{precision}_{chunks}_chunks" @@ -499,10 +517,12 @@ def run( started = time.time() hub.download(source, weight_dir) tokenizer, licenses = copy_side_files(source, weight_dir, out_dir) + # The MediaTek script reads the weights' own config.json: give it the keys it expects + # (hub.normalize_config: the first LFM2 checkpoints spell them the older way). + model_config = hub.normalize_config(json.loads((weight_dir / "config.json").read_text(encoding="utf-8"))) if plan.tokenizer: - model_config = json.loads((weight_dir / "config.json").read_text(encoding="utf-8")) model_config["tokenizer"] = plan.tokenizer - (weight_dir / "config.json").write_text(json.dumps(model_config, indent=2), encoding="utf-8") + (weight_dir / "config.json").write_text(json.dumps(model_config, indent=2), encoding="utf-8") bos, eos = hub.special_token_ids(source) corpus = None diff --git a/pipeline/families.py b/pipeline/families.py index be1a877..a29058b 100644 --- a/pipeline/families.py +++ b/pipeline/families.py @@ -152,6 +152,28 @@ def mtk_chunks(n_layers: int, max_chunks: int) -> int: return next(n for n in range(min(max_chunks, n_layers), 0, -1) if n_layers % n == 0) +def mtk_chunks_one_attention_each(config: dict) -> int | None: + """Fewest even chunks that leave no chunk more than one attention layer, for a hybrid. + + What the 16k LFM2.5 builds needed (docs/research, finding 39): 8 for the 1.2B (attention + at 2, 5, 8, 10, 12, 14 of 16), where 4 chunks put two in a chunk and lost the compiler. + None for a model whose every layer attends, where the rule would mean a chunk per layer. + """ + c = text_config(config) + layer_types = c.get("layer_types") + if not layer_types or attending_layers(c) is None: + return None + n_layers = len(layer_types) + for chunks in range(1, n_layers + 1): + if n_layers % chunks: + continue + size = n_layers // chunks + groups = [layer_types[i : i + size] for i in range(0, n_layers, size)] + if all(sum(1 for t in group if t != "conv") <= 1 for group in groups): + return chunks + return None + + def mtk_plan(family: Family, config: dict, max_chunks: int) -> MtkPlan: if family.key not in _MTK_SCRIPTS: raise UnsupportedModel(f"no MediaTek export script mapped for {family.key}") diff --git a/pipeline/hub.py b/pipeline/hub.py index 65d10ee..f5b74ab 100644 --- a/pipeline/hub.py +++ b/pipeline/hub.py @@ -62,6 +62,32 @@ def _json_file(hf, repo_id: str, revision: str, filename: str, files: list[str]) return json.loads(Path(path).read_text(encoding="utf-8")) +def normalize_config(config: dict) -> dict: + """The config with the older LFM2 keys spelled the way LFM2.5's are. + + LiquidAI's first LFM2 checkpoints (LFM2-350M, -700M, -1.2B) name their attention layers + `full_attn_idxs` and their feed-forward width `block_ff_dim`, where LFM2.5 writes + `layer_types` and `intermediate_size`; transformers' Lfm2Config accepts both. The pipeline + and the patched MediaTek script read the newer keys, so the older ones are translated here: + every layer is a convolution except the ones `full_attn_idxs` lists, and intermediate_size + takes block_ff_dim's unadjusted value, which families.lfm2_hidden_dim and the MediaTek + script both put through block_auto_adjust_ff_dim's rule as they do for LFM2.5. A config + that already has the newer keys is returned as it is. + """ + if config.get("model_type") != "lfm2": + return config + config = dict(config) + idxs = config.get("full_attn_idxs") + if not config.get("layer_types") and idxs is not None and config.get("num_hidden_layers"): + attending = set(int(i) for i in idxs) + config["layer_types"] = [ + "full_attention" if i in attending else "conv" for i in range(int(config["num_hidden_layers"])) + ] + if config.get("intermediate_size") is None and config.get("block_ff_dim") is not None: + config["intermediate_size"] = int(config["block_ff_dim"]) + return config + + def fetch(repo_id: str, revision: str = "main", token: str | None = None) -> SourceModel: hf = api(token) info = hf.model_info( @@ -87,7 +113,7 @@ def fetch(repo_id: str, revision: str = "main", token: str | None = None) -> Sou files=files, ) try: - source.config = _json_file(hf, repo_id, sha, "config.json", files) + source.config = normalize_config(_json_file(hf, repo_id, sha, "config.json", files)) source.generation_config = _json_file(hf, repo_id, sha, "generation_config.json", files) source.tokenizer_config = _json_file(hf, repo_id, sha, "tokenizer_config.json", files) except GatedRepoError: diff --git a/pipeline/remote.py b/pipeline/remote.py index dc5fac4..14655e4 100644 --- a/pipeline/remote.py +++ b/pipeline/remote.py @@ -46,7 +46,20 @@ VOLUME_MOUNT = "/vol" CPU_INDEX = "https://download.pytorch.org/whl/cpu" # Passed into the Sandbox when set here, besides --env. -PASSTHROUGH = ("HF_HUB_DISABLE_PROGRESS_BARS", "MODEL_ID", "REVISION", "CONTEXT", "QMODE", "GPTQ") +PASSTHROUGH = ( + "HF_HUB_DISABLE_PROGRESS_BARS", + "MODEL_ID", + "REVISION", + "CONTEXT", + "QMODE", + "GPTQ", + "SOC", + "MTK_MAX_CHUNKS", + "MTK_PRECISION", + "MTK_PROMPT_TOKENS", +) +# MediaTek's tools in the image (scripts/mtk-setup.sh adds the NeuroPilot wheels at run time). +MTK_VENV = "/opt/mtk-venv" def python_version() -> str: @@ -68,14 +81,36 @@ def memory_request(need_bytes: int, cfg: settings.ModalSettings) -> int | None: return mib if mib <= cfg.max_memory_mib else None -def image(requirements: Path): +def image(requirements: Path, mtk_tools: Path | None = None): + """The step's image; with `mtk_tools`, also MediaTek's Python 3.10 virtualenv at MTK_VENV. + + The virtualenv holds the public half of requirements/mtk-tools.txt (torch from the CPU + index, executorch, transformers...), which Modal caches like any layer. The NeuroPilot + wheels are not in it: they come from MediaTek's SDK archive, downloaded and deleted on + every run by scripts/mtk-setup.sh, so nothing of the SDK is ever stored in an image. + """ import modal - img = modal.Image.debian_slim(python_version=python_version()).apt_install("git", "build-essential") + img = modal.Image.debian_slim(python_version=python_version()).apt_install("git", "build-essential", "curl") pins = cpu_pins(requirements) if pins: img = img.pip_install(*pins, index_url=CPU_INDEX) img = img.pip_install_from_requirements(str(requirements)) + if mtk_tools is not None: + mtk_python = settings.read_env_file(settings.CONFIG_DIR / "versions.env")["MTK_PYTHON_VERSION"] + tool_python = f"{MTK_VENV}/bin/python" + img = ( + img.pip_install("uv") + .add_local_file(str(mtk_tools), "/tmp/mtk-tools.txt", copy=True) + .run_commands( + f"uv venv --python {mtk_python} {MTK_VENV}", + *[ + f"uv pip install --python {tool_python} '{pin}' --index-url {CPU_INDEX}" + for pin in cpu_pins(mtk_tools) + ], + f"uv pip install --python {tool_python} -r /tmp/mtk-tools.txt", + ) + ) for name in REPO_DIRS: img = img.add_local_dir(str(ROOT / name), f"{REMOTE_ROOT}/{name}", ignore=["**/__pycache__/**"]) return img @@ -114,6 +149,8 @@ def run( env: dict[str, str], cpu: float | None = None, keep: bool = False, + timeout_minutes: int | None = None, + mtk_tools: Path | None = None, ) -> int: import modal @@ -136,6 +173,7 @@ def run( sandbox_env.update(env) sandbox_env.update(IN=f"{base}/in", OUT=f"{base}/out", WORK="/work", HF_HOME="/work/hf-home", PYTHONUNBUFFERED="1") cores = cpu or cfg.cpu + timeout = timeout_minutes or cfg.timeout_minutes sandbox = None code = None @@ -149,7 +187,7 @@ def run( batch.put_file(str(path), f"/{key}/in/{path.name}") print(f"uploaded {len(puts)} input(s) to {cfg.volume}:/{key}/in", flush=True) - print(f"Modal Sandbox: {memory_mib} MiB, {cores:g} cores, timeout {cfg.timeout_minutes} min", flush=True) + print(f"Modal Sandbox: {memory_mib} MiB, {cores:g} cores, timeout {timeout} min", flush=True) started = time.time() with modal.enable_output(): sandbox = modal.Sandbox.create( @@ -157,10 +195,10 @@ def run( "-c", wrap(script), app=app, - image=image(requirements), + image=image(requirements, mtk_tools), memory=memory_mib, cpu=cores, - timeout=cfg.timeout_minutes * 60, + timeout=timeout * 60, volumes={VOLUME_MOUNT: volume}, secrets=secrets, env=sandbox_env, @@ -177,7 +215,7 @@ def run( sandbox.wait(raise_on_termination=False) code = sandbox.returncode except modal.exception.SandboxTimeoutError: - print(f"::error::the Modal Sandbox ran past its {cfg.timeout_minutes}-minute timeout", flush=True) + print(f"::error::the Modal Sandbox ran past its {timeout}-minute timeout", flush=True) for reader in readers: reader.join(timeout=30) print(f"Modal Sandbox exited {code} after {(time.time() - started) / 60:.1f} min", flush=True) @@ -219,6 +257,8 @@ def wrap(script: str) -> str: "}\n" "trap finish EXIT\n" 'mkdir -p "$IN" "$OUT" "$WORK"\n' + # nproc is the cores asked for; left to itself, OpenMP may size its pool to the host. + 'export OMP_NUM_THREADS="${OMP_NUM_THREADS:-$(nproc)}"\n' f"{script}\n" ) @@ -237,4 +277,6 @@ def main(args) -> int: env=env, cpu=args.cpu, keep=args.keep, + timeout_minutes=args.timeout_minutes, + mtk_tools=Path(args.mtk_tools) if args.mtk_tools else None, ) diff --git a/scripts/mtk-setup.sh b/scripts/mtk-setup.sh new file mode 100755 index 0000000..1c9a14c --- /dev/null +++ b/scripts/mtk-setup.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# MediaTek's half of an export, set up inside the Modal Sandbox before export-mtk runs: +# +# bash scripts/mtk-setup.sh && python -m pipeline export-mtk ... \ +# --tool-python /opt/mtk-venv/bin/python --examples "$WORK/executorch/examples/mediatek" +# +# The image already has the Python 3.10 virtualenv with the public half of +# requirements/mtk-tools.txt (pipeline/remote.py, image). This adds what may not be stored: +# the NeuroPilot Express SDK is downloaded from MediaTek on every run, checked against the +# pinned digest, and deleted once its two wheels are installed. It is licensed by MediaTek +# Inc. under its own terms (THIRD_PARTY_NOTICES.md), which every run accepts, and nothing +# from it is cached, uploaded or published. +# +# Then the LLM export scripts, which are in the ExecuTorch source and not the wheel (BSD, +# fetched at the pinned commit), with this repo's patches from third_party/executorch/patches. +set -euo pipefail + +eval "$(grep -E '^(NEUROPILOT_SDK_[A-Z0-9_]+|MTK_[A-Z_]+_VERSION|EXECUTORCH_COMMIT)=' config/versions.env | sed 's/^/export /')" +tool_python=/opt/mtk-venv/bin/python +work=${WORK:?} + +sdk="$work/neuropilot" +mkdir -p "$sdk" +curl -fsSL --retry 3 --retry-all-errors -o "$sdk.tar.gz" "$NEUROPILOT_SDK_URL" +echo "$NEUROPILOT_SDK_SHA256 $sdk.tar.gz" | sha256sum -c - +tar xzf "$sdk.tar.gz" -C "$sdk" --strip-components=1 --wildcards \ + "*/mtk_neuron-${MTK_NEURON_VERSION}-*.whl" "*/mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" +# The 8.0.8 archive keeps mtk_converter only under mtk_converter-*_public_packages/. +neuron_whl=$(find "$sdk" -name "mtk_neuron-${MTK_NEURON_VERSION}-*.whl" | head -1) +converter_whl=$(find "$sdk" -name "mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" | head -1) +test -n "$neuron_whl" && test -n "$converter_whl" +uv pip install --quiet --python "$tool_python" -r requirements/mtk-tools.txt "$neuron_whl" "$converter_whl" +rm -rf "$sdk" "$sdk.tar.gz" +uv pip list --python "$tool_python" | grep -iE '^(torch|torchao|executorch|transformers|mtk.converter|mtk.neuron) ' + +src="$work/executorch" +rm -rf "$src" +git init -q "$src" +git -C "$src" remote add origin https://github.com/pytorch/executorch.git +git -C "$src" sparse-checkout set examples/mediatek +git -C "$src" fetch -q --depth 1 origin "$EXECUTORCH_COMMIT" +git -C "$src" checkout -q FETCH_HEAD +for patch in third_party/executorch/patches/mediatek-*.patch; do + git -C "$src" apply -v "$PWD/$patch" +done diff --git a/tests/fixtures/lfm2-1.2b.config.json b/tests/fixtures/lfm2-1.2b.config.json new file mode 100644 index 0000000..b6d5f20 --- /dev/null +++ b/tests/fixtures/lfm2-1.2b.config.json @@ -0,0 +1,46 @@ +{ + "architectures": [ + "Lfm2ForCausalLM" + ], + "block_auto_adjust_ff_dim": true, + "block_dim": 2048, + "block_ff_dim": 12288, + "block_ffn_dim_multiplier": 1.0, + "block_mlp_init_scale": 1.0, + "block_multiple_of": 256, + "block_norm_eps": 1e-05, + "block_out_init_scale": 1.0, + "block_use_swiglu": true, + "block_use_xavier_init": true, + "bos_token_id": 1, + "conv_L_cache": 3, + "conv_bias": false, + "conv_dim": 2048, + "conv_dim_out": 2048, + "conv_use_xavier_init": true, + "eos_token_id": 7, + "full_attn_idxs": [ + 2, + 5, + 8, + 10, + 12, + 14 + ], + "hidden_size": 2048, + "initializer_range": 0.02, + "max_position_embeddings": 128000, + "model_type": "lfm2", + "norm_eps": 1e-05, + "num_attention_heads": 32, + "num_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 0, + "rope_theta": 1000000.0, + "torch_dtype": "bfloat16", + "transformers_version": "4.54.0.dev0", + "use_cache": true, + "use_pos_enc": true, + "vocab_size": 65536 +} diff --git a/tests/fixtures/lfm2-350m.config.json b/tests/fixtures/lfm2-350m.config.json new file mode 100644 index 0000000..50536b1 --- /dev/null +++ b/tests/fixtures/lfm2-350m.config.json @@ -0,0 +1,46 @@ +{ + "architectures": [ + "Lfm2ForCausalLM" + ], + "block_auto_adjust_ff_dim": true, + "block_dim": 1024, + "block_ff_dim": 6656, + "block_ffn_dim_multiplier": 1.0, + "block_mlp_init_scale": 1.0, + "block_multiple_of": 256, + "block_norm_eps": 1e-05, + "block_out_init_scale": 1.0, + "block_use_swiglu": true, + "block_use_xavier_init": true, + "bos_token_id": 1, + "conv_L_cache": 3, + "conv_bias": false, + "conv_dim": 1024, + "conv_dim_out": 1024, + "conv_use_xavier_init": true, + "eos_token_id": 7, + "full_attn_idxs": [ + 2, + 5, + 8, + 10, + 12, + 14 + ], + "hidden_size": 1024, + "initializer_range": 0.02, + "max_position_embeddings": 128000, + "model_type": "lfm2", + "norm_eps": 1e-05, + "num_attention_heads": 16, + "num_heads": 16, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 0, + "rope_theta": 1000000.0, + "torch_dtype": "bfloat16", + "transformers_version": "4.54.0.dev0", + "use_cache": true, + "use_pos_enc": true, + "vocab_size": 65536 +} diff --git a/tests/fixtures/lfm2.5-350m.config.json b/tests/fixtures/lfm2.5-350m.config.json new file mode 100644 index 0000000..9d07848 --- /dev/null +++ b/tests/fixtures/lfm2.5-350m.config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "Lfm2ForCausalLM" + ], + "block_auto_adjust_ff_dim": true, + "block_dim": 1024, + "block_ff_dim": 6656, + "block_ffn_dim_multiplier": 1.0, + "block_mlp_init_scale": 1.0, + "block_multiple_of": 256, + "block_norm_eps": 1e-05, + "block_out_init_scale": 1.0, + "block_use_swiglu": true, + "block_use_xavier_init": true, + "bos_token_id": 1, + "conv_L_cache": 3, + "conv_bias": false, + "conv_dim": 1024, + "conv_use_xavier_init": true, + "dtype": "bfloat16", + "eos_token_id": 7, + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 6656, + "layer_types": [ + "conv", + "conv", + "full_attention", + "conv", + "conv", + "full_attention", + "conv", + "conv", + "full_attention", + "conv", + "full_attention", + "conv", + "full_attention", + "conv", + "full_attention", + "conv" + ], + "max_position_embeddings": 128000, + "model_type": "lfm2", + "norm_eps": 1e-05, + "num_attention_heads": 16, + "num_heads": 16, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 0, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "tie_embedding": true, + "transformers_version": "5.0.0rc1", + "use_cache": true, + "use_pos_enc": true, + "vocab_size": 65536 +} diff --git a/tests/test_families.py b/tests/test_families.py index 4690e0e..616af91 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -179,3 +179,33 @@ def test_the_memory_model_uses_the_attending_count(): plain = dataclasses.replace(arch, attention_layers=None) assert sizing.causal_mask_bytes(plain, 1024) == 4 * 1024 * 1024 assert sizing.kv_cache_bytes(arch, 1024) < sizing.kv_cache_bytes(plain, 1024) + + +def test_the_first_lfm2_checkpoints_are_read_like_lfm25(): + # LFM2-350M/700M/1.2B name their attention layers full_attn_idxs and their feed-forward + # block_ff_dim; hub.normalize_config spells them as LFM2.5 does. Checked against the + # weights (safetensors headers, 2026-10-07): q_proj only in layers 2, 5, 8, 10, 12, 14, and + # feed-forward widths 4,608 (350M) and 8,192 (1.2B). + from pipeline import hub + + for name, width in (("lfm2-350m", 4608), ("lfm2-1.2b", 8192)): + raw = load_json(f"{name}.config.json") + assert "layer_types" not in raw and "full_attn_idxs" in raw + config = hub.normalize_config(raw) + attending = [i for i, t in enumerate(config["layer_types"]) if t == "full_attention"] + assert attending == [2, 5, 8, 10, 12, 14] + assert len(config["layer_types"]) == 16 + assert families.lfm2_hidden_dim(config) == width + # The XNNPACK plan accepts it, so does sizing, and the original config is untouched. + assert families.xnnpack_plan(families.family_for(config), config).params["layer_types"] == config["layer_types"] + assert families.architecture(config, 1).attention_layers == 6 + assert "layer_types" not in raw + + +def test_a_config_with_the_newer_keys_is_left_alone(): + from pipeline import hub + + config = load_json("lfm2.5-350m.config.json") + assert hub.normalize_config(config) == config + qwen = {"model_type": "qwen3", "num_hidden_layers": 28} + assert hub.normalize_config(qwen) is qwen diff --git a/tests/test_namespace_runner.py b/tests/test_namespace_runner.py index 025c4ce..ec161d3 100644 --- a/tests/test_namespace_runner.py +++ b/tests/test_namespace_runner.py @@ -25,23 +25,20 @@ def arch(model_id): return families.architecture(hf_config(model_id), TOTAL_PARAMS[model_id]) -# What the profile could export if an export_llm backend ran on it, which is why they moved -# to Modal: the profile as a tier, as config/pipeline.yaml's mtk section records it. -PROFILE_TIER = CFG.mtk.runner_tiers[0] +# What the profile could export if an export ran on it, which is why every one moved to Modal +# (XNNPACK and Vulkan, then MediaTek the same day): the profile as the tier it was configured as. +PROFILE_TIER = settings.RunnerTier( + label=PROFILE, ram_bytes=MEM_TOTAL_BYTES, disk_bytes=101427666944, swap_gib=0, max_window=2048 +) def windows(model_id, backend): return [w for w in WINDOWS if export_xnnpack.pick_runner(arch(model_id), w, (PROFILE_TIER,), backend)] -@pytest.mark.parametrize("backend", ["mtk"]) -def test_the_recipes_not_on_modal_run_on_the_namespace_profile_as_measured(backend): - # XNNPACK and Vulkan moved to Modal the same day (test_remote.py); MediaTek has not yet. - (tier,) = getattr(CFG, backend).runner_tiers - assert tier.label == PROFILE - assert tier.ram_bytes == MEM_TOTAL_BYTES - # No swap: the container does not allow it, so nothing may be sized on swap. - assert tier.swap_gib == 0 +def test_every_export_recipe_runs_on_modal(): + for backend in ("xnnpack", "vulkan", "mtk"): + assert [t.label for t in getattr(CFG, backend).runner_tiers] == [settings.MODAL_TIER], backend def test_no_workflow_runs_anywhere_else(): @@ -87,5 +84,5 @@ def test_mediatek_lfm2_does_not_fit_at_any_window(): "hidden_size": 2048, } for window in (512, 2048, 4096): - recipe = dataclasses.replace(CFG.mtk, cache_size=window) + recipe = dataclasses.replace(CFG.mtk, cache_size=window, runner_tiers=(PROFILE_TIER,)) assert export_mtk.pick_runner(lfm, recipe, 23, 1_170_340_608, streaming=True) is None diff --git a/tests/test_remote.py b/tests/test_remote.py index 9f536cd..464ad53 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -11,7 +11,7 @@ import pytest from conftest import TOTAL_PARAMS, hf_config -from pipeline import export_xnnpack, families, remote, settings, sizing +from pipeline import export_mtk, export_xnnpack, families, remote, settings, sizing CFG = settings.load() WINDOWS = (2048, 4096, 8192, 16384, 32768) @@ -80,6 +80,33 @@ def test_the_workflow_sends_each_window_to_modal_with_its_own_memory(workflow): assert "./.github/actions/setup-export" not in text +def test_the_mediatek_workflow_sends_each_window_with_its_memory_cores_and_minutes(): + text = (settings.ROOT / ".github/workflows/export-mtk.yml").read_text(encoding="utf-8") + assert '--memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES"' in text + assert "--mtk-tools requirements/mtk-tools.txt" in text + assert "bash scripts/mtk-setup.sh &&" in text + assert "./.github/actions/setup-export" not in text + # The job outlives the longest Sandbox, so the Sandbox is what times out and says so. + longest = max(minutes for _, _, minutes in export_mtk.MODAL_SHAPES) + assert f"timeout-minutes: {longest + 30}" in text + + +def test_the_neuropilot_sdk_is_fetched_at_run_time_and_never_put_in_the_image(): + setup = (settings.ROOT / "scripts/mtk-setup.sh").read_text(encoding="utf-8") + assert "sha256sum -c -" in setup and 'rm -rf "$sdk" "$sdk.tar.gz"' in setup + image = (settings.ROOT / "pipeline/remote.py").read_text(encoding="utf-8").split("def image(")[1] + assert "NEUROPILOT_SDK_URL" not in image.split("def _stream(")[0] + + +@pytest.mark.parametrize( + ("window", "cores", "minutes"), + [(2048, 8, 240), (4096, 8, 240), (8192, 16, 330), (16384, 16, 330), (32768, 32, 600)], +) +def test_a_mediatek_window_gets_more_cores_and_time_as_it_grows(window, cores, minutes): + assert export_mtk.modal_shape(window) == (cores, minutes) + assert cores <= 64 + + def test_the_matrix_entry_shape_the_workflow_reads(): entry = {"context": 2048, "runner": "modal", "swap_gib": 0, "memory_mib": 6286} assert json.loads(json.dumps(entry))["memory_mib"] == 6286 From 17e6866f4dee089004bcd7c14056801b7342c88b Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Wed, 7 Oct 2026 22:44:14 +0800 Subject: [PATCH 2/6] QNN export: run on Modal, sized per model and window The Namespace profile has 15.7 GiB and no swap, and the one QNN export measured, Qwen3-0.6B, already peaked at 15.6 GB of RSS at 2k and 4k (findings 8 and 10), so nothing larger could build there. qnn-matrix gives each window a Sandbox: 20 bytes a parameter (24 GiB floor), half again by 8k, 16 or 32 cores and up to 23 hours by model size. The request is a minimum Modal lets the run exceed, and each report records the peak to refit it against. The wheel still downloads QAIRT on first import, now inside the Sandbox, so the Actions cache that kept a copy is gone and the SDK is never stored. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/export-qnn.yml | 99 ++++++++++++-------------------- pipeline/__main__.py | 40 +++++++++++++ pipeline/export_qnn.py | 26 +++++++++ tests/test_remote.py | 31 +++++++++- 4 files changed, 134 insertions(+), 62 deletions(-) diff --git a/.github/workflows/export-qnn.yml b/.github/workflows/export-qnn.yml index a6a1f54..f7c58e0 100644 --- a/.github/workflows/export-qnn.yml +++ b/.github/workflows/export-qnn.yml @@ -38,11 +38,14 @@ concurrency: jobs: targets: - # Hub metadata only, no model download. + # Hub metadata only, no model download: a parameter count and arithmetic. runs-on: namespace-profile-execupack + env: + # Llama 3.2 is gated, so even its metadata needs the token. + HF_TOKEN: ${{ secrets.HF_TOKEN }} outputs: socs: ${{ steps.socs.outputs.socs }} - contexts: ${{ steps.contexts.outputs.contexts }} + entries: ${{ steps.contexts.outputs.entries }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/setup-light @@ -57,96 +60,70 @@ jobs: - id: contexts env: CONTEXTS: ${{ inputs.contexts }} - # Accepts a JSON list, one number, or a comma-separated list; empty means every tier. + MODEL_ID: ${{ inputs.model_id }} + REVISION: ${{ inputs.revision }} + # One entry per window with the Modal Sandbox it gets: memory, cores and minutes from + # pipeline.export_qnn.modal_request. Accepts a JSON list, one number, or a + # comma-separated list; empty means every tier. run: | - CONTEXTS=$(python3 - "$CONTEXTS" <<'PY' - import json, sys, yaml - raw = sys.argv[1].strip() - if not raw: - values = yaml.safe_load(open("config/pipeline.yaml"))["export"]["context_tiers"] - else: - try: - values = json.loads(raw) - except ValueError: - values = [v for v in raw.strip("[]").split(",") if v.strip()] - if not isinstance(values, list): - values = [values] - print(json.dumps(sorted({int(v) for v in values}, reverse=True))) - PY - ) - echo "contexts=$CONTEXTS" >>"$GITHUB_OUTPUT" + entries=$(python3 -m pipeline qnn-matrix "$MODEL_ID" --revision "$REVISION" --contexts "$CONTEXTS") + echo "$entries" + echo "entries=$entries" >>"$GITHUB_OUTPUT" export: needs: targets + # An empty matrix is an error to GitHub. + if: ${{ needs.targets.outputs.entries != '[]' }} strategy: fail-fast: false matrix: soc: ${{ fromJSON(needs.targets.outputs.socs) }} - context: ${{ fromJSON(needs.targets.outputs.contexts) }} - # Stays on x86: the Qualcomm backend ships only in the executorch Linux x86_64 - # wheel, and the QAIRT SDK it downloads is x86_64 (this workflow checks for - # libQnnHtp.so under an x86_64 path). + entry: ${{ fromJSON(needs.targets.outputs.entries) }} + # The export runs on Modal, in a Sandbox the targets job sized to this model and window; + # this job only sends it, waits, and publishes what comes back. Modal's hosts are x86, + # which the Qualcomm backend needs: it ships only in the executorch Linux x86_64 wheel, + # and the QAIRT SDK is x86_64. The wheel downloads QAIRT inside the Sandbox on first + # import (export_qnn.qnn_sdk), so the SDK is fetched from Qualcomm on every run and never + # stored in an image, a cache or the Volume. runs-on: namespace-profile-execupack - timeout-minutes: 350 + # Past the longest Sandbox (export_qnn.QNN_MODAL_SHAPES), so the Sandbox times out first. + timeout-minutes: 1410 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} HF_HUB_DISABLE_PROGRESS_BARS: "1" MODEL_ID: ${{ inputs.model_id }} REVISION: ${{ inputs.revision }} SOC: ${{ matrix.soc }} - CONTEXT: ${{ matrix.context }} + CONTEXT: ${{ matrix.entry.context }} + MEMORY_MIB: ${{ matrix.entry.memory_mib }} + CORES: ${{ matrix.entry.cpu }} + SANDBOX_MINUTES: ${{ matrix.entry.timeout_minutes }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/setup-export - with: - swap-gib: "24" - requirements: requirements/export-qnn.txt + - uses: ./.github/actions/setup-modal - name: Name things run: | - grep -E '^QAIRT_VERSION=' config/versions.env >>"$GITHUB_ENV" soc_lower=$(echo "$SOC" | tr '[:upper:]' '[:lower:]') echo "TARGET=$soc_lower" >>"$GITHUB_ENV" echo "ARTIFACT=qnn-${soc_lower}-${MODEL_ID//\//--}-${CONTEXT}" >>"$GITHUB_ENV" - # The executorch wheel downloads the QAIRT SDK and a libc++ into ~/.cache/executorch/qnn - # on first import. Cached per version in this repository only; never re-hosted. Restore - # and save are separate steps so the cache is kept even when the export fails. - - name: Restore QAIRT - id: qairt - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/executorch/qnn - key: qairt-${{ env.QAIRT_VERSION }}-${{ runner.os }}-${{ runner.arch }} - - - name: Install QAIRT - run: | - python -c "import executorch.backends.qualcomm" 2>&1 | grep -v "Downloading\|Extracting" || true - find ~/.cache/executorch/qnn/sdk-"$QAIRT_VERSION" -name libQnnHtp.so -path '*x86_64*' | grep -q . - # The archives are only needed until extraction; keep them out of the cache. - find ~/.cache/executorch/qnn \( -name '*.zip' -o -name '*.tar.xz' \) -delete - du -sh ~/.cache/executorch/qnn - - - name: Save QAIRT - if: steps.qairt.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/executorch/qnn - key: qairt-${{ env.QAIRT_VERSION }}-${{ runner.os }}-${{ runner.arch }} - - # Exit 4 means this runner cannot build this window: recorded as a skip, so the other - # jobs of the matrix stand on their own. - - name: Export + # Exit 4 means this window cannot be built: recorded as a skip, so the other jobs of the + # matrix stand on their own. + - name: Export on Modal run: | set +e - python -m pipeline export-qnn "$MODEL_ID" --soc "$SOC" --revision "$REVISION" \ - --out "$WORK_DIR/out" --work "$WORK_DIR/work" --context "$CONTEXT" + python -m pipeline modal-run --memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES" \ + --requirements requirements/export-qnn.txt --get "$WORK_DIR/out" \ + --script 'python -m pipeline export-qnn "$MODEL_ID" --soc "$SOC" --revision "$REVISION" --out "$OUT" --work "$WORK/export" --context "$CONTEXT"' rc=$? set -e if [ "$rc" -eq 4 ]; then echo "SKIPPED=1" >>"$GITHUB_ENV" - echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens on $SOC cannot be exported on this runner" + echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens on $SOC cannot be exported" exit 0 fi exit "$rc" diff --git a/pipeline/__main__.py b/pipeline/__main__.py index ce0b72f..aa02928 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -182,6 +182,40 @@ def _mtk_matrix(args) -> int: return 0 +def _qnn_matrix(args) -> int: + """One matrix entry per window for the Qualcomm export, with the Modal Sandbox it gets + (export_qnn.modal_request), from Hub metadata alone.""" + from pipeline import export_qnn, hub, remote, settings + + cfg = settings.load() + source = hub.fetch(args.model, args.revision) + if not source.total_params: + print(f"no parameter count for {args.model}", file=sys.stderr) + return 2 + if args.contexts: + try: + wanted = json.loads(args.contexts) + except ValueError: + wanted = [v for v in args.contexts.strip("[]").split(",") if v.strip()] + if not isinstance(wanted, list): + wanted = [wanted] + else: + wanted = list(cfg.context_tiers) + + entries = [] + for window in sorted({int(v) for v in wanted}, reverse=True): + need, cores, minutes = export_qnn.modal_request(source.total_params, window) + memory_mib = remote.memory_request(need, cfg.modal) + if memory_mib is None: + print(f"{window} needs more than Modal allows ({need:,} B)", file=sys.stderr) + continue + entry = {"context": window, "memory_mib": memory_mib, "cpu": cores, "timeout_minutes": minutes} + print(f"{window}: {entry}", file=sys.stderr) + entries.append(entry) + print(json.dumps(entries)) + return 0 + + def _export_matrix(args) -> int: """One matrix entry per window for an export_llm backend: the window and the smallest runner that can build it (export_xnnpack.pick_runner), from the model's config alone. @@ -474,6 +508,12 @@ def main(argv: list[str] | None = None) -> int: ) matrix.set_defaults(func=_mtk_matrix) + qnn_matrix = commands.add_parser("qnn-matrix", help="windows and the Modal Sandbox each gets, as a CI matrix") + qnn_matrix.add_argument("model") + qnn_matrix.add_argument("--revision", default="main") + qnn_matrix.add_argument("--contexts", default="", help="JSON or comma-separated windows; empty for every tier") + qnn_matrix.set_defaults(func=_qnn_matrix) + export_matrix = commands.add_parser( "export-matrix", help="windows and the runner each needs for an export_llm backend, as a CI matrix" ) diff --git a/pipeline/export_qnn.py b/pipeline/export_qnn.py index b56449a..5849dab 100644 --- a/pipeline/export_qnn.py +++ b/pipeline/export_qnn.py @@ -46,6 +46,32 @@ QNN_BACKEND_ID = "QnnBackend" META_FILES = ("original/consolidated.00.pth", "original/params.json", "original/tokenizer.model") SOC_NAMES = {"SM8650": "Snapdragon 8 Gen 3", "SM8750": "Snapdragon 8 Elite"} + +# The Modal Sandbox a window is given (pipeline/remote.py). Only Qwen3-0.6B has been measured: +# 15.6 GB of peak RSS at both 2k and 4k (findings 8 and 10), on 16 GB runners with swap, so +# that figure is a floor on its need rather than the need. Modal treats a plain memory request +# as a minimum, not a limit (the Sandbox may use more) and bills the higher of request and +# use, so this asks for a modest share and lets the peak burst past it; each report records +# the peak, which is what to refit this against. +QNN_BYTES_PER_PARAM = 20 +QNN_FLOOR_BYTES = 24 * 2**30 +# The window enters through calibration and the two graphs' attention, at most half again by 8k. +QNN_WINDOW_SCALE = 16384 +# Time grows with the model and faster than the window: Qwen3-0.6B took 2,434 s at 2k and +# 5,987 s at 4k on a 4 vCPU runner (findings 8 and 10). (largest parameter count, physical +# cores, Sandbox minutes); the minutes cost nothing unless a run hangs. +QNN_MODAL_SHAPES = ((1_300_000_000, 16, 720), (4_500_000_000, 32, 1380)) + + +def modal_request(params: int, window: int) -> tuple[int, int, int]: + """Memory in bytes, physical cores and minutes for a QNN export on Modal.""" + need = max(QNN_FLOOR_BYTES, params * QNN_BYTES_PER_PARAM) * (1 + window / QNN_WINDOW_SCALE) + for largest, cores, minutes in QNN_MODAL_SHAPES: + if params <= largest: + return int(need), cores, minutes + raise ValueError(f"no Modal shape for a {params:,}-parameter QNN export") + + EXECUTORCH_SOURCE_FILES = settings.ROOT / "third_party" / "executorch" diff --git a/tests/test_remote.py b/tests/test_remote.py index 464ad53..5c77b3e 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -11,7 +11,7 @@ import pytest from conftest import TOTAL_PARAMS, hf_config -from pipeline import export_mtk, export_xnnpack, families, remote, settings, sizing +from pipeline import export_mtk, export_qnn, export_xnnpack, families, remote, settings, sizing CFG = settings.load() WINDOWS = (2048, 4096, 8192, 16384, 32768) @@ -107,6 +107,35 @@ def test_a_mediatek_window_gets_more_cores_and_time_as_it_grows(window, cores, m assert cores <= 64 +def test_the_qualcomm_workflow_sends_each_window_with_its_memory_cores_and_minutes(): + text = (settings.ROOT / ".github/workflows/export-qnn.yml").read_text(encoding="utf-8") + assert '--memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES"' in text + assert "--requirements requirements/export-qnn.txt" in text + # QAIRT is downloaded inside each Sandbox now; no Actions cache keeps a copy. + assert "actions/cache" not in text + longest = max(minutes for _, _, minutes in export_qnn.QNN_MODAL_SHAPES) + assert f"timeout-minutes: {longest + 30}" in text + + +@pytest.mark.parametrize( + ("model_id", "cores"), + [ + ("HuggingFaceTB/SmolLM2-360M-Instruct", 16), + ("Qwen/Qwen3-0.6B", 16), + ("meta-llama/Llama-3.2-1B-Instruct", 16), + ("Qwen/Qwen3-1.7B", 32), + ("Qwen/Qwen3-4B", 32), + ], +) +def test_a_qualcomm_export_fits_a_sandbox_at_every_window_it_is_given(model_id, cores): + for window in (2048, 4096, 8192): + need, got_cores, minutes = export_qnn.modal_request(TOTAL_PARAMS[model_id], window) + # Qwen3-0.6B measured 15.6 GB of RSS at 2k and 4k (findings 8 and 10): asked for above that. + assert need >= 15_702_085_632 + assert remote.memory_request(need, CFG.modal) is not None + assert got_cores == cores and minutes < 1440 # Modal's Sandbox limit is 24 hours + + def test_the_matrix_entry_shape_the_workflow_reads(): entry = {"context": 2048, "runner": "modal", "swap_gib": 0, "memory_mib": 6286} assert json.loads(json.dumps(entry))["memory_mib"] == 6286 From e00b74562b77000f4998694164c70c02cb6296d3 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Wed, 7 Oct 2026 22:45:27 +0800 Subject: [PATCH 3/6] MediaTek matrix: size for the prompt batch the export uses, and keep hybrid-only chunking to hybrids From Codex's review of the Modal move: the Sandbox was sized at the default 128-token prompt batch while the export took the prompt_tokens input (73 GiB against 24 GiB for the 1.2B at 32k with 4096); a window past MODAL_SHAPES raised and dropped the whole matrix instead of that one window; and a plain transformer whose config lists its layers as all full_attention was cut one chunk per layer. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/export-mtk.yml | 4 +++- pipeline/__main__.py | 13 +++++++++++-- pipeline/export_mtk.py | 6 +++--- pipeline/families.py | 5 +++-- scripts/mtk-setup.sh | 2 +- tests/test_families.py | 14 +++++++++++--- tests/test_remote.py | 4 ++++ 7 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.github/workflows/export-mtk.yml b/.github/workflows/export-mtk.yml index 7e60256..e2308ed 100644 --- a/.github/workflows/export-mtk.yml +++ b/.github/workflows/export-mtk.yml @@ -75,12 +75,14 @@ jobs: CONTEXTS: ${{ inputs.contexts }} MODEL_ID: ${{ inputs.model_id }} REVISION: ${{ inputs.revision }} + PROMPT_TOKENS: ${{ inputs.prompt_tokens }} # One entry per window, each carrying the Modal Sandbox that can build it: memory from # pipeline.export_mtk.calibration_bytes (the same estimate the export gates on), # cores and minutes from export_mtk.MODAL_SHAPES, and the chunk count for a hybrid at # 16k and up. run: | - entries=$(python3 -m pipeline mtk-matrix "$MODEL_ID" --revision "$REVISION" --contexts "$CONTEXTS") + entries=$(python3 -m pipeline mtk-matrix "$MODEL_ID" --revision "$REVISION" --contexts "$CONTEXTS" \ + ${PROMPT_TOKENS:+--prompt-tokens "$PROMPT_TOKENS"}) echo "$entries" echo "entries=$entries" >>"$GITHUB_OUTPUT" diff --git a/pipeline/__main__.py b/pipeline/__main__.py index aa02928..4c6e33a 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -155,7 +155,11 @@ def _mtk_matrix(args) -> int: entries = [] for window in sorted({int(v) for v in wanted}, reverse=True): - recipe = dataclasses.replace(cfg.mtk, cache_size=window) + # The export's own prompt_tokens (MTK_PROMPT_TOKENS): the prompt graph's attention + # scores grow with it, so sizing at the default would under-ask for a larger batch. + recipe = dataclasses.replace( + cfg.mtk, cache_size=window, prompt_tokens=args.prompt_tokens or cfg.mtk.prompt_tokens + ) tier = export_mtk.pick_runner(source.config, recipe, args.prompts, source.total_params, streaming=streaming) if tier is None: # Left out rather than failed: the other windows are independent and a matrix @@ -175,7 +179,11 @@ def _mtk_matrix(args) -> int: if entry["memory_mib"] is None: print(f"{window} needs more than Modal allows ({need:,} B)", file=sys.stderr) continue - entry["cpu"], entry["timeout_minutes"] = export_mtk.modal_shape(window) + shape = export_mtk.modal_shape(window) + if shape is None: + print(f"no Modal shape for a {window}-token window (export_mtk.MODAL_SHAPES)", file=sys.stderr) + continue + entry["cpu"], entry["timeout_minutes"] = shape print(f"{window}: needs {need / 2**30:.1f} GiB, {entry}", file=sys.stderr) entries.append(entry) print(json.dumps(entries)) @@ -506,6 +514,7 @@ def main(argv: list[str] | None = None) -> int: help="lines in mtk.calibration, which sets the disk the Arrow cache needs; the export " "counts the real file and its disk gate catches a mismatch", ) + matrix.add_argument("--prompt-tokens", type=int, default=None, help="the export's prompt_tokens, if not mtk's") matrix.set_defaults(func=_mtk_matrix) qnn_matrix = commands.add_parser("qnn-matrix", help="windows and the Modal Sandbox each gets, as a CI matrix") diff --git a/pipeline/export_mtk.py b/pipeline/export_mtk.py index 049c598..4029bca 100644 --- a/pipeline/export_mtk.py +++ b/pipeline/export_mtk.py @@ -305,12 +305,12 @@ def carries(tier: settings.RunnerTier, *, with_swap: bool) -> bool: return None -def modal_shape(window: int) -> tuple[int, int]: - """Physical cores and Sandbox minutes for a window on Modal (MODAL_SHAPES).""" +def modal_shape(window: int) -> tuple[int, int] | None: + """Physical cores and Sandbox minutes for a window on Modal (MODAL_SHAPES), or None past them.""" for largest, cores, minutes in MODAL_SHAPES: if window <= largest: return cores, minutes - raise ValueError(f"no Modal shape for a {window}-token window") + return None def exp_name(weight_dir: Path, precision: str, chunks: int) -> str: diff --git a/pipeline/families.py b/pipeline/families.py index a29058b..f07cb51 100644 --- a/pipeline/families.py +++ b/pipeline/families.py @@ -161,9 +161,10 @@ def mtk_chunks_one_attention_each(config: dict) -> int | None: """ c = text_config(config) layer_types = c.get("layer_types") - if not layer_types or attending_layers(c) is None: + attending = attending_layers(c) + n_layers = len(layer_types or ()) + if attending is None or attending >= n_layers: return None - n_layers = len(layer_types) for chunks in range(1, n_layers + 1): if n_layers % chunks: continue diff --git a/scripts/mtk-setup.sh b/scripts/mtk-setup.sh index 1c9a14c..4af2379 100755 --- a/scripts/mtk-setup.sh +++ b/scripts/mtk-setup.sh @@ -29,7 +29,7 @@ tar xzf "$sdk.tar.gz" -C "$sdk" --strip-components=1 --wildcards \ neuron_whl=$(find "$sdk" -name "mtk_neuron-${MTK_NEURON_VERSION}-*.whl" | head -1) converter_whl=$(find "$sdk" -name "mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" | head -1) test -n "$neuron_whl" && test -n "$converter_whl" -uv pip install --quiet --python "$tool_python" -r requirements/mtk-tools.txt "$neuron_whl" "$converter_whl" +uv pip install --quiet --no-cache --python "$tool_python" -r requirements/mtk-tools.txt "$neuron_whl" "$converter_whl" rm -rf "$sdk" "$sdk.tar.gz" uv pip list --python "$tool_python" | grep -iE '^(torch|torchao|executorch|transformers|mtk.converter|mtk.neuron) ' diff --git a/tests/test_families.py b/tests/test_families.py index 616af91..eec040a 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -4,7 +4,7 @@ import pytest from conftest import hf_config, load_json -from pipeline import families +from pipeline import families, hub def plan_for(config): @@ -186,7 +186,6 @@ def test_the_first_lfm2_checkpoints_are_read_like_lfm25(): # block_ff_dim; hub.normalize_config spells them as LFM2.5 does. Checked against the # weights (safetensors headers, 2026-10-07): q_proj only in layers 2, 5, 8, 10, 12, 14, and # feed-forward widths 4,608 (350M) and 8,192 (1.2B). - from pipeline import hub for name, width in (("lfm2-350m", 4608), ("lfm2-1.2b", 8192)): raw = load_json(f"{name}.config.json") @@ -203,9 +202,18 @@ def test_the_first_lfm2_checkpoints_are_read_like_lfm25(): def test_a_config_with_the_newer_keys_is_left_alone(): - from pipeline import hub config = load_json("lfm2.5-350m.config.json") assert hub.normalize_config(config) == config qwen = {"model_type": "qwen3", "num_hidden_layers": 28} assert hub.normalize_config(qwen) is qwen + + +def test_one_attention_layer_per_chunk_is_only_for_hybrids(): + lfm2 = hub.normalize_config(load_json("lfm2-1.2b.config.json")) + assert families.mtk_chunks_one_attention_each(lfm2) == 8 + qwen = dict(hf_config("Qwen/Qwen3-0.6B")) + assert families.mtk_chunks_one_attention_each(qwen) is None + # An explicit all-attention layout is still not a hybrid: no chunk per layer for it. + qwen["layer_types"] = ["full_attention"] * int(qwen["num_hidden_layers"]) + assert families.mtk_chunks_one_attention_each(qwen) is None diff --git a/tests/test_remote.py b/tests/test_remote.py index 5c77b3e..f6814cb 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -107,6 +107,10 @@ def test_a_mediatek_window_gets_more_cores_and_time_as_it_grows(window, cores, m assert cores <= 64 +def test_a_window_past_every_shape_is_left_out_not_fatal(): + assert export_mtk.modal_shape(65536) is None + + def test_the_qualcomm_workflow_sends_each_window_with_its_memory_cores_and_minutes(): text = (settings.ROOT / ".github/workflows/export-qnn.yml").read_text(encoding="utf-8") assert '--memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES"' in text From 7a12b44759c0aa152b1ca3d90710681d259eb9cd Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Wed, 7 Oct 2026 22:48:51 +0800 Subject: [PATCH 4/6] Modal exports: link each report to its run, and record the container's own memory peak From Codex's review of the QNN move, true of every export on Modal: the Sandbox did not get GITHUB_SERVER_URL, GITHUB_REPOSITORY or GITHUB_RUN_ID, so run_info was empty and the model cards lost their build links; and MemorySampler's peak_in_use_bytes comes from /proc/meminfo, which in a Sandbox is the host's. It now also records the cgroup's peak where the container has one, as peak_cgroup_bytes. Co-Authored-By: Claude Opus 5.5 --- pipeline/export_qnn.py | 4 ++-- pipeline/exporting.py | 21 ++++++++++++++++++++- pipeline/remote.py | 4 ++++ tests/test_remote.py | 17 +++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/pipeline/export_qnn.py b/pipeline/export_qnn.py index 5849dab..51569a6 100644 --- a/pipeline/export_qnn.py +++ b/pipeline/export_qnn.py @@ -51,8 +51,8 @@ # 15.6 GB of peak RSS at both 2k and 4k (findings 8 and 10), on 16 GB runners with swap, so # that figure is a floor on its need rather than the need. Modal treats a plain memory request # as a minimum, not a limit (the Sandbox may use more) and bills the higher of request and -# use, so this asks for a modest share and lets the peak burst past it; each report records -# the peak, which is what to refit this against. +# use, so this asks for a modest share and lets the peak burst past it. Refit it against each +# report's peak_rss_export_bytes and peak_cgroup_bytes; on Modal peak_in_use_bytes is the host's. QNN_BYTES_PER_PARAM = 20 QNN_FLOOR_BYTES = 24 * 2**30 # The window enters through calibration and the two graphs' attention, at most half again by 8k. diff --git a/pipeline/exporting.py b/pipeline/exporting.py index 9fba5ed..9afd024 100644 --- a/pipeline/exporting.py +++ b/pipeline/exporting.py @@ -81,6 +81,11 @@ def host_info() -> dict: return info +# The container's own memory in use, cgroup v2 then v1. On a Modal Sandbox /proc/meminfo is the +# host's (pipeline/remote.py), so peak_in_use_bytes there counts other tenants; this does not. +CGROUP_MEMORY = (Path("/sys/fs/cgroup/memory.current"), Path("/sys/fs/cgroup/memory/memory.usage_in_bytes")) + + class MemorySampler: """Samples /proc/meminfo while an export runs. @@ -92,9 +97,16 @@ class MemorySampler: (docs/research, finding 23). """ - def __init__(self, interval: float = 5.0, meminfo: Path = Path("/proc/meminfo")): + def __init__( + self, + interval: float = 5.0, + meminfo: Path = Path("/proc/meminfo"), + cgroup: tuple[Path, ...] = CGROUP_MEMORY, + ): self.interval = interval self.meminfo = meminfo + self.cgroup = next((path for path in cgroup if path.exists()), None) + self.peak_cgroup = None self.peak_swap_used = None self.min_available = None self.peak_in_use = None @@ -103,6 +115,12 @@ def __init__(self, interval: float = 5.0, meminfo: Path = Path("/proc/meminfo")) self._thread = None def sample(self) -> None: + if self.cgroup is not None: + try: + used = int(self.cgroup.read_text().split()[0]) + self.peak_cgroup = max(used, self.peak_cgroup or 0) + except (OSError, ValueError, IndexError): + pass if not self.meminfo.exists(): return fields = {} @@ -142,6 +160,7 @@ def result(self) -> dict: "min_mem_available_bytes": self.min_available, "peak_in_use_bytes": self.peak_in_use, "peak_in_use_at": self.peak_in_use_at, + "peak_cgroup_bytes": self.peak_cgroup, } diff --git a/pipeline/remote.py b/pipeline/remote.py index 14655e4..1762063 100644 --- a/pipeline/remote.py +++ b/pipeline/remote.py @@ -57,6 +57,10 @@ "MTK_MAX_CHUNKS", "MTK_PRECISION", "MTK_PROMPT_TOKENS", + # manifest.run_info: each export report links the GitHub run that built it. + "GITHUB_SERVER_URL", + "GITHUB_REPOSITORY", + "GITHUB_RUN_ID", ) # MediaTek's tools in the image (scripts/mtk-setup.sh adds the NeuroPilot wheels at run time). MTK_VENV = "/opt/mtk-venv" diff --git a/tests/test_remote.py b/tests/test_remote.py index f6814cb..c20e4ef 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -167,3 +167,20 @@ def test_a_failed_sync_fails_a_step_that_had_succeeded(tmp_path): # A step that already failed keeps its own status. skipped = remote.wrap("exit 4").replace(f"sync {remote.VOLUME_MOUNT}", "false") assert subprocess.run(["bash", "-c", skipped], env=env, capture_output=True).returncode == 4 + + +def test_the_run_that_built_a_file_reaches_the_sandbox(): + assert {"GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID"} <= set(remote.PASSTHROUGH) + + +def test_the_container_peak_is_read_from_its_cgroup(tmp_path): + from pipeline.exporting import MemorySampler + + current = tmp_path / "memory.current" + current.write_text("3221225472\n") + sampler = MemorySampler(meminfo=tmp_path / "missing", cgroup=(tmp_path / "absent", current)) + sampler.sample() + current.write_text("1073741824\n") + sampler.sample() + assert sampler.result()["peak_cgroup_bytes"] == 3221225472 + assert MemorySampler(meminfo=tmp_path / "missing", cgroup=()).result()["peak_cgroup_bytes"] is None From f3a9757e1131547c6f4c8a7067681fafe5e530bc Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Wed, 7 Oct 2026 22:58:18 +0800 Subject: [PATCH 5/6] QNN export: ask ExecuTorch 1.5.1 for the QAIRT SDK, which its import no longer fetches 1.5.1 emptied backends/qualcomm/__init__.py and moved the download to utils/qnn_sdk_setup.setup_qnn_sdk, called by the paths that start a backend. The probe still imported the package and read the paths it would use, so the first export since the bump (SmolLM2-135M at 2k on Modal) found no libQnnHtp.so there and failed inside the Qualcomm script. The probe now calls setup_qnn_sdk and checks the libraries exist before the script. Co-Authored-By: Claude Opus 5.5 --- pipeline/export_qnn.py | 14 +++++++++++++- tests/test_qnn.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/pipeline/export_qnn.py b/pipeline/export_qnn.py index 51569a6..84672d0 100644 --- a/pipeline/export_qnn.py +++ b/pipeline/export_qnn.py @@ -171,6 +171,10 @@ def structural_check(pte: Path, model_mode: str) -> dict: _SDK_PROBE = """ import json +# ExecuTorch 1.5.1 moved the download out of the package's __init__, which now has no side +# effects (backends/qualcomm/utils/qnn_sdk_setup.py): it happens here or not at all. +from executorch.backends.qualcomm.utils.qnn_sdk_setup import setup_qnn_sdk +setup_qnn_sdk() from executorch.backends.qualcomm.scripts import download_qnn_sdk as d print(json.dumps({ "version": d.QNN_VERSION, @@ -193,7 +197,15 @@ def qnn_sdk() -> dict: result = subprocess.run([sys.executable, "-c", _SDK_PROBE], capture_output=True, text=True, timeout=1800) if result.returncode != 0: raise ExportError(f"could not set up the QAIRT SDK:\n{result.stderr[-4000:]}") - return json.loads(result.stdout.strip().splitlines()[-1]) + sdk = json.loads(result.stdout.strip().splitlines()[-1]) + # The paths are computed, not found: check the download put the libraries there, so a + # missing SDK fails here and not as a dlopen error deep in the Qualcomm script. + wanted = [Path(sdk["root"]) / "lib" / "x86_64-linux-clang" / "libQnnHtp.so"] + wanted += [Path(sdk["libcxx_dir"]) / name for name in sdk["libcxx_files"]] + missing = [str(path) for path in wanted if not path.exists()] + if missing: + raise ExportError(f"the QAIRT SDK setup left no {', '.join(missing)}:\n{result.stderr[-4000:]}") + return sdk def soname(filename: str) -> str: diff --git a/tests/test_qnn.py b/tests/test_qnn.py index a96eab9..37df1d1 100644 --- a/tests/test_qnn.py +++ b/tests/test_qnn.py @@ -93,20 +93,47 @@ def test_copied_params_describe_the_hf_checkpoint(decoder, hf_fixture): } -def test_the_sdk_is_probed_in_a_child_so_this_process_keeps_its_environment(monkeypatch): +def installed_sdk(root: Path) -> dict: + """SDK with its paths under `root`, and the files qnn_sdk checks for created there.""" + sdk = {**SDK, "root": str(root / "sdk"), "libcxx_dir": str(root / "libcxx")} + htp = root / "sdk" / "lib" / "x86_64-linux-clang" / "libQnnHtp.so" + htp.parent.mkdir(parents=True) + htp.touch() + (root / "libcxx").mkdir() + for name in SDK["libcxx_files"]: + (root / "libcxx" / name).touch() + return sdk + + +def test_the_sdk_is_probed_in_a_child_so_this_process_keeps_its_environment(monkeypatch, tmp_path): calls = [] + sdk = installed_sdk(tmp_path) def fake_run(command, **kwargs): calls.append(command) - return subprocess.CompletedProcess(command, 0, stdout=json.dumps(SDK) + "\n", stderr="Loaded libc++.so.1.0\n") + return subprocess.CompletedProcess(command, 0, stdout=json.dumps(sdk) + "\n", stderr="Loaded libc++.so.1.0\n") monkeypatch.setattr(export_qnn.subprocess, "run", fake_run) monkeypatch.delenv("QNN_SDK_ROOT", raising=False) - assert export_qnn.qnn_sdk() == SDK + assert export_qnn.qnn_sdk() == sdk assert calls and calls[0][1] == "-c" + # ExecuTorch 1.5.1 downloads only when asked: the package import no longer does it. + assert "setup_qnn_sdk()" in calls[0][2] assert "QNN_SDK_ROOT" not in os.environ +def test_a_probe_that_downloaded_nothing_fails_before_the_script_runs(monkeypatch, tmp_path): + # What the first Modal run hit: the paths came back, the files were never fetched. + sdk = {**SDK, "root": str(tmp_path / "sdk"), "libcxx_dir": str(tmp_path / "libcxx")} + monkeypatch.setattr( + export_qnn.subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess(command, 0, stdout=json.dumps(sdk) + "\n", stderr=""), + ) + with pytest.raises(export_qnn.ExportError, match="libQnnHtp.so"): + export_qnn.qnn_sdk() + + def test_sonames(): assert export_qnn.soname("libc++.so.1.0") == "libc++.so.1" assert export_qnn.soname("libc++abi.so.1.0") == "libc++abi.so.1" From d6ec6e783c0c987196a573a889b29f9ec384d074 Mon Sep 17 00:00:00 2001 From: Alpha Romer Coma Date: Wed, 7 Oct 2026 23:23:39 +0800 Subject: [PATCH 6/6] Tests: keep the no-/proc sampler test off the host's cgroup, which a Linux CI runner has Co-Authored-By: Claude Opus 5.5 --- tests/test_exporting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_exporting.py b/tests/test_exporting.py index ac96109..1c064a1 100644 --- a/tests/test_exporting.py +++ b/tests/test_exporting.py @@ -36,7 +36,7 @@ def test_memory_sampler_keeps_the_peak_of_ram_plus_swap_within_one_sample(tmp_pa def test_memory_sampler_is_a_no_op_without_proc(tmp_path): - with exporting.MemorySampler(interval=3600, meminfo=tmp_path / "missing") as sampler: + with exporting.MemorySampler(interval=3600, meminfo=tmp_path / "missing", cgroup=()) as sampler: pass assert set(sampler.result().values()) == {None}