diff --git a/.github/workflows/export-mtk.yml b/.github/workflows/export-mtk.yml index 91dfe8a..e2308ed 100644 --- a/.github/workflows/export-mtk.yml +++ b/.github/workflows/export-mtk.yml @@ -75,13 +75,14 @@ jobs: CONTEXTS: ${{ inputs.contexts }} MODEL_ID: ${{ inputs.model_id }} REVISION: ${{ inputs.revision }} - # One entry per window, each carrying the smallest runner that can build it. The - # sizing is pipeline.export_mtk.pick_runner, the same calibration_bytes and - # calibration_disk_bytes the export gates on, so the matrix and the job cannot - # disagree about whether a window fits. Memory is one prompt's calibration steps; - # disk is every prompt's, since writer_batch_size=1 put the rest in the Arrow cache. + PROMPT_TOKENS: ${{ inputs.prompt_tokens }} + # One entry per window, each carrying the Modal Sandbox that can build it: memory from + # pipeline.export_mtk.calibration_bytes (the same estimate the export gates on), + # cores and minutes from export_mtk.MODAL_SHAPES, and the chunk count for a hybrid at + # 16k and up. run: | - entries=$(python3 -m pipeline mtk-matrix "$MODEL_ID" --revision "$REVISION" --contexts "$CONTEXTS") + entries=$(python3 -m pipeline mtk-matrix "$MODEL_ID" --revision "$REVISION" --contexts "$CONTEXTS" \ + ${PROMPT_TOKENS:+--prompt-tokens "$PROMPT_TOKENS"}) echo "$entries" echo "entries=$entries" >>"$GITHUB_OUTPUT" @@ -94,106 +95,59 @@ jobs: matrix: soc: ${{ fromJSON(needs.targets.outputs.socs) }} entry: ${{ fromJSON(needs.targets.outputs.entries) }} - # Stays on x86: mtk_converter 8.13.0 is a cp310 x86 build and the NeuroPilot SDK - # ships x86_64 libraries (finding 14). - # - # Sized per window by the targets job above, because the cost is the window and not the - # model: a 2k export fits the smallest tier and a 32k one does not fit the largest - # without swap. Picking one size for all of them would either waste three quarters of - # the fleet on the small windows or fail the big ones. - runs-on: ${{ matrix.entry.runner }} - timeout-minutes: 350 + # The export runs on Modal, in a Sandbox the targets job sized to this window: memory + # from export_mtk.calibration_bytes plus modal.memory_margin, and cores and minutes from + # export_mtk.MODAL_SHAPES, because streaming calibration's time grows with the window. + # This job only sends it, waits, and publishes what comes back. Modal's hosts are x86, + # which mtk_converter 8.13.0 (a cp310 x86 build) and the SDK's libraries need (finding 14). + runs-on: namespace-profile-execupack + # Past the longest Sandbox (export_mtk.MODAL_SHAPES), so the Sandbox times out first. + timeout-minutes: 630 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} HF_HUB_DISABLE_PROGRESS_BARS: "1" MODEL_ID: ${{ inputs.model_id }} REVISION: ${{ inputs.revision }} SOC: ${{ matrix.soc }} CONTEXT: ${{ matrix.entry.context }} - MTK_MAX_CHUNKS: ${{ inputs.max_chunks }} + MEMORY_MIB: ${{ matrix.entry.memory_mib }} + CORES: ${{ matrix.entry.cpu }} + SANDBOX_MINUTES: ${{ matrix.entry.timeout_minutes }} + # The input wins; otherwise the targets job's one-attention-layer-per-chunk count for a + # hybrid at 16k and up (finding 39), otherwise mtk.max_chunks. + MTK_MAX_CHUNKS: ${{ inputs.max_chunks || matrix.entry.max_chunks }} MTK_PRECISION: ${{ inputs.precision }} MTK_PROMPT_TOKENS: ${{ inputs.prompt_tokens }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The pipeline itself (fetch, reports, publishing) on the repo's usual Python. - - uses: ./.github/actions/setup-export - with: - # Sized with the runner: enough to absorb an estimate that is low, without taking - # so much of the filesystem that the Arrow cache has nowhere to go. Only the 2.6B - # at 32k is expected to spend it, and pick_runner prefers a tier that does not. - swap-gib: "${{ matrix.entry.swap_gib }}" - requirements: requirements/export-mtk.txt + - uses: ./.github/actions/setup-modal - name: Name things run: | - grep -E '^(NEUROPILOT_SDK_[A-Z0-9_]+|MTK_[A-Z_]+_VERSION|EXECUTORCH_COMMIT)=' config/versions.env >>"$GITHUB_ENV" soc_lower=$(echo "$SOC" | tr '[:upper:]' '[:lower:]') echo "TARGET=$soc_lower" >>"$GITHUB_ENV" echo "ARTIFACT=mtk-${soc_lower}-${MODEL_ID//\//--}-${CONTEXT}" >>"$GITHUB_ENV" - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - id: mtk-python - with: - python-version: ${{ env.MTK_PYTHON_VERSION }} - update-environment: false - - # MediaTek's tools in their own Python 3.10 virtualenv (requirements/mtk-tools.txt says - # why). The NeuroPilot Express SDK is downloaded from MediaTek on every run, checked - # against the pinned digest, and deleted after its two wheels are installed: it is - # licensed by MediaTek Inc. under its own terms (THIRD_PARTY_NOTICES.md), which every - # run accepts, and nothing from it is cached, uploaded or published. - - name: MediaTek tools - run: | - python_bin="${{ steps.mtk-python.outputs.python-path }}" - "$python_bin" -m venv "$WORK_DIR/mtk-venv" - tool_python="$WORK_DIR/mtk-venv/bin/python" - "$tool_python" -m pip install --quiet --upgrade pip - "$tool_python" -m pip install --quiet "$(grep -E '^torch==' requirements/mtk-tools.txt)" \ - --index-url https://download.pytorch.org/whl/cpu - - sdk="$RUNNER_TEMP/neuropilot" - mkdir -p "$sdk" - curl -fsSL --retry 3 --retry-all-errors -o "$sdk.tar.gz" "$NEUROPILOT_SDK_URL" - echo "$NEUROPILOT_SDK_SHA256 $sdk.tar.gz" | sha256sum -c - - tar xzf "$sdk.tar.gz" -C "$sdk" --strip-components=1 --wildcards \ - "*/mtk_neuron-${MTK_NEURON_VERSION}-*.whl" "*/mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" - # The 8.0.8 archive keeps mtk_converter only under mtk_converter-*_public_packages/. - neuron_whl=$(find "$sdk" -name "mtk_neuron-${MTK_NEURON_VERSION}-*.whl" | head -1) - converter_whl=$(find "$sdk" -name "mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" | head -1) - test -n "$neuron_whl" && test -n "$converter_whl" - "$tool_python" -m pip install --quiet -r requirements/mtk-tools.txt "$neuron_whl" "$converter_whl" - rm -rf "$sdk" "$sdk.tar.gz" - "$tool_python" -m pip list | grep -iE '^(torch|torchao|executorch|transformers|mtk.converter|mtk.neuron) ' - echo "MTK_PYTHON=$tool_python" >>"$GITHUB_ENV" - - # The LLM export scripts are in the ExecuTorch source, not the wheel (BSD, fetched at - # the pinned commit), with this repo's patches from third_party/executorch/patches. - - name: ExecuTorch examples/mediatek - run: | - src="$WORK_DIR/executorch" - git init -q "$src" - git -C "$src" remote add origin https://github.com/pytorch/executorch.git - git -C "$src" sparse-checkout set examples/mediatek - git -C "$src" fetch -q --depth 1 origin "$EXECUTORCH_COMMIT" - git -C "$src" checkout -q FETCH_HEAD - for patch in third_party/executorch/patches/mediatek-*.patch; do - git -C "$src" apply -v "$GITHUB_WORKSPACE/$patch" - done - echo "MTK_EXAMPLES=$src/examples/mediatek" >>"$GITHUB_ENV" - - # Exit 4 means this runner cannot build this window: recorded as a skip, so the other - # jobs of the matrix stand on their own. - - name: Export + # scripts/mtk-setup.sh downloads the NeuroPilot SDK inside the Sandbox, checks it, installs + # its two wheels and deletes it, then fetches and patches ExecuTorch's examples/mediatek: + # nothing of the SDK is in the image, cached or brought back. Exit 4 means this window + # cannot be built (the export's own gate): recorded as a skip, so the other jobs of the + # matrix stand on their own. + - name: Export on Modal run: | set +e - python -m pipeline export-mtk "$MODEL_ID" --soc "$SOC" --revision "$REVISION" \ - --out "$WORK_DIR/out" --work "$WORK_DIR/work" --context "$CONTEXT" + python -m pipeline modal-run --memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES" \ + --requirements requirements/export-mtk.txt --mtk-tools requirements/mtk-tools.txt \ + --get "$WORK_DIR/out" \ + --script 'bash scripts/mtk-setup.sh && python -m pipeline export-mtk "$MODEL_ID" --soc "$SOC" --revision "$REVISION" --out "$OUT" --work "$WORK/export" --context "$CONTEXT" --tool-python /opt/mtk-venv/bin/python --examples "$WORK/executorch/examples/mediatek"' rc=$? set -e if [ "$rc" -eq 4 ]; then echo "SKIPPED=1" >>"$GITHUB_ENV" - echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens on $SOC cannot be exported on this runner" + echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens on $SOC cannot be exported" exit 0 fi exit "$rc" diff --git a/.github/workflows/export-qnn.yml b/.github/workflows/export-qnn.yml index a6a1f54..f7c58e0 100644 --- a/.github/workflows/export-qnn.yml +++ b/.github/workflows/export-qnn.yml @@ -38,11 +38,14 @@ concurrency: jobs: targets: - # Hub metadata only, no model download. + # Hub metadata only, no model download: a parameter count and arithmetic. runs-on: namespace-profile-execupack + env: + # Llama 3.2 is gated, so even its metadata needs the token. + HF_TOKEN: ${{ secrets.HF_TOKEN }} outputs: socs: ${{ steps.socs.outputs.socs }} - contexts: ${{ steps.contexts.outputs.contexts }} + entries: ${{ steps.contexts.outputs.entries }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/setup-light @@ -57,96 +60,70 @@ jobs: - id: contexts env: CONTEXTS: ${{ inputs.contexts }} - # Accepts a JSON list, one number, or a comma-separated list; empty means every tier. + MODEL_ID: ${{ inputs.model_id }} + REVISION: ${{ inputs.revision }} + # One entry per window with the Modal Sandbox it gets: memory, cores and minutes from + # pipeline.export_qnn.modal_request. Accepts a JSON list, one number, or a + # comma-separated list; empty means every tier. run: | - CONTEXTS=$(python3 - "$CONTEXTS" <<'PY' - import json, sys, yaml - raw = sys.argv[1].strip() - if not raw: - values = yaml.safe_load(open("config/pipeline.yaml"))["export"]["context_tiers"] - else: - try: - values = json.loads(raw) - except ValueError: - values = [v for v in raw.strip("[]").split(",") if v.strip()] - if not isinstance(values, list): - values = [values] - print(json.dumps(sorted({int(v) for v in values}, reverse=True))) - PY - ) - echo "contexts=$CONTEXTS" >>"$GITHUB_OUTPUT" + entries=$(python3 -m pipeline qnn-matrix "$MODEL_ID" --revision "$REVISION" --contexts "$CONTEXTS") + echo "$entries" + echo "entries=$entries" >>"$GITHUB_OUTPUT" export: needs: targets + # An empty matrix is an error to GitHub. + if: ${{ needs.targets.outputs.entries != '[]' }} strategy: fail-fast: false matrix: soc: ${{ fromJSON(needs.targets.outputs.socs) }} - context: ${{ fromJSON(needs.targets.outputs.contexts) }} - # Stays on x86: the Qualcomm backend ships only in the executorch Linux x86_64 - # wheel, and the QAIRT SDK it downloads is x86_64 (this workflow checks for - # libQnnHtp.so under an x86_64 path). + entry: ${{ fromJSON(needs.targets.outputs.entries) }} + # The export runs on Modal, in a Sandbox the targets job sized to this model and window; + # this job only sends it, waits, and publishes what comes back. Modal's hosts are x86, + # which the Qualcomm backend needs: it ships only in the executorch Linux x86_64 wheel, + # and the QAIRT SDK is x86_64. The wheel downloads QAIRT inside the Sandbox on first + # import (export_qnn.qnn_sdk), so the SDK is fetched from Qualcomm on every run and never + # stored in an image, a cache or the Volume. runs-on: namespace-profile-execupack - timeout-minutes: 350 + # Past the longest Sandbox (export_qnn.QNN_MODAL_SHAPES), so the Sandbox times out first. + timeout-minutes: 1410 env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} HF_HUB_DISABLE_PROGRESS_BARS: "1" MODEL_ID: ${{ inputs.model_id }} REVISION: ${{ inputs.revision }} SOC: ${{ matrix.soc }} - CONTEXT: ${{ matrix.context }} + CONTEXT: ${{ matrix.entry.context }} + MEMORY_MIB: ${{ matrix.entry.memory_mib }} + CORES: ${{ matrix.entry.cpu }} + SANDBOX_MINUTES: ${{ matrix.entry.timeout_minutes }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ./.github/actions/setup-export - with: - swap-gib: "24" - requirements: requirements/export-qnn.txt + - uses: ./.github/actions/setup-modal - name: Name things run: | - grep -E '^QAIRT_VERSION=' config/versions.env >>"$GITHUB_ENV" soc_lower=$(echo "$SOC" | tr '[:upper:]' '[:lower:]') echo "TARGET=$soc_lower" >>"$GITHUB_ENV" echo "ARTIFACT=qnn-${soc_lower}-${MODEL_ID//\//--}-${CONTEXT}" >>"$GITHUB_ENV" - # The executorch wheel downloads the QAIRT SDK and a libc++ into ~/.cache/executorch/qnn - # on first import. Cached per version in this repository only; never re-hosted. Restore - # and save are separate steps so the cache is kept even when the export fails. - - name: Restore QAIRT - id: qairt - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/executorch/qnn - key: qairt-${{ env.QAIRT_VERSION }}-${{ runner.os }}-${{ runner.arch }} - - - name: Install QAIRT - run: | - python -c "import executorch.backends.qualcomm" 2>&1 | grep -v "Downloading\|Extracting" || true - find ~/.cache/executorch/qnn/sdk-"$QAIRT_VERSION" -name libQnnHtp.so -path '*x86_64*' | grep -q . - # The archives are only needed until extraction; keep them out of the cache. - find ~/.cache/executorch/qnn \( -name '*.zip' -o -name '*.tar.xz' \) -delete - du -sh ~/.cache/executorch/qnn - - - name: Save QAIRT - if: steps.qairt.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/executorch/qnn - key: qairt-${{ env.QAIRT_VERSION }}-${{ runner.os }}-${{ runner.arch }} - - # Exit 4 means this runner cannot build this window: recorded as a skip, so the other - # jobs of the matrix stand on their own. - - name: Export + # Exit 4 means this window cannot be built: recorded as a skip, so the other jobs of the + # matrix stand on their own. + - name: Export on Modal run: | set +e - python -m pipeline export-qnn "$MODEL_ID" --soc "$SOC" --revision "$REVISION" \ - --out "$WORK_DIR/out" --work "$WORK_DIR/work" --context "$CONTEXT" + python -m pipeline modal-run --memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES" \ + --requirements requirements/export-qnn.txt --get "$WORK_DIR/out" \ + --script 'python -m pipeline export-qnn "$MODEL_ID" --soc "$SOC" --revision "$REVISION" --out "$OUT" --work "$WORK/export" --context "$CONTEXT"' rc=$? set -e if [ "$rc" -eq 4 ]; then echo "SKIPPED=1" >>"$GITHUB_ENV" - echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens on $SOC cannot be exported on this runner" + echo "::warning title=Window skipped::$MODEL_ID at $CONTEXT tokens on $SOC cannot be exported" exit 0 fi exit "$rc" diff --git a/config/pipeline.yaml b/config/pipeline.yaml index 1cf2b60..505a779 100644 --- a/config/pipeline.yaml +++ b/config/pipeline.yaml @@ -164,17 +164,17 @@ export: # largest window a tier is given, so a calibration lands on cores that finish it inside the # job limit. # - # One tier since 2026-10-07, the Namespace profile (vulkan.runner_tiers says what was - # measured): 15.7 GiB, no swap, 4 vCPUs. calibration_bytes puts a 1.2B LFM2.5 build at - # about 17.4 GiB (15 bytes a parameter, measured on 4-chunk builds), so none is placed; the - # 8-chunk 16k build peaked at 13.4 GiB, so a refit for more chunks may bring it back. - # max_window 2048 because 4 vCPUs are half the 8 that were given at most 4096. + # On Modal since 2026-10-07, like XNNPACK and Vulkan (see vulkan.runner_tiers): one tier at + # Modal's ceiling, and mtk-matrix sizes each window's Sandbox from calibration_bytes (memory, + # with export_mtk.MODAL_FLOOR_BYTES under it) and export_mtk.MODAL_SHAPES (cores and minutes, + # which is what grows with the window when calibration streams). x86 like every Modal host. + # Before: the Namespace profile (15.7 GiB, no swap, 4 vCPUs; finding 40), which carried no + # LFM2 window at all by this estimate. runner_tiers: - - label: namespace-profile-execupack - ram_bytes: 16833269760 - disk_bytes: 101427666944 + - label: modal + ram_bytes: 360777252864 + disk_bytes: 549755813888 swap_gib: 0 - max_window: 2048 # Modal: where the exports run since 2026-10-07 (pipeline/remote.py). The GitHub jobs stay on # the Namespace profile and drive a Sandbox per window, sized to that window, with the repo's diff --git a/pipeline/__main__.py b/pipeline/__main__.py index 94e9872..4c6e33a 100644 --- a/pipeline/__main__.py +++ b/pipeline/__main__.py @@ -131,7 +131,7 @@ def _mtk_matrix(args) -> int: """ import dataclasses - from pipeline import export_mtk, families, hub, settings + from pipeline import export_mtk, families, hub, remote, settings cfg = settings.load() source = hub.fetch(args.model, args.revision) @@ -155,14 +155,71 @@ def _mtk_matrix(args) -> int: entries = [] for window in sorted({int(v) for v in wanted}, reverse=True): - recipe = dataclasses.replace(cfg.mtk, cache_size=window) + # The export's own prompt_tokens (MTK_PROMPT_TOKENS): the prompt graph's attention + # scores grow with it, so sizing at the default would under-ask for a larger batch. + recipe = dataclasses.replace( + cfg.mtk, cache_size=window, prompt_tokens=args.prompt_tokens or cfg.mtk.prompt_tokens + ) tier = export_mtk.pick_runner(source.config, recipe, args.prompts, source.total_params, streaming=streaming) if tier is None: # Left out rather than failed: the other windows are independent and a matrix # that refuses to start teaches less than nine jobs that finish. print(f"no runner tier can build a {window}-token window", file=sys.stderr) continue - entries.append({"context": window, "runner": tier.label, "swap_gib": tier.swap_gib}) + entry = {"context": window, "runner": tier.label, "swap_gib": tier.swap_gib, "max_chunks": ""} + if window >= export_mtk.ONE_ATTENTION_PER_CHUNK_FROM: + chunks = families.mtk_chunks_one_attention_each(source.config) + if chunks is not None and chunks > cfg.mtk.max_chunks: + entry["max_chunks"] = str(chunks) + if tier.label == settings.MODAL_TIER: + need = export_mtk.calibration_bytes( + source.config, recipe, args.prompts, source.total_params, streaming=streaming + ) + entry["memory_mib"] = remote.memory_request(max(need, export_mtk.MODAL_FLOOR_BYTES), cfg.modal) + if entry["memory_mib"] is None: + print(f"{window} needs more than Modal allows ({need:,} B)", file=sys.stderr) + continue + shape = export_mtk.modal_shape(window) + if shape is None: + print(f"no Modal shape for a {window}-token window (export_mtk.MODAL_SHAPES)", file=sys.stderr) + continue + entry["cpu"], entry["timeout_minutes"] = shape + print(f"{window}: needs {need / 2**30:.1f} GiB, {entry}", file=sys.stderr) + entries.append(entry) + print(json.dumps(entries)) + return 0 + + +def _qnn_matrix(args) -> int: + """One matrix entry per window for the Qualcomm export, with the Modal Sandbox it gets + (export_qnn.modal_request), from Hub metadata alone.""" + from pipeline import export_qnn, hub, remote, settings + + cfg = settings.load() + source = hub.fetch(args.model, args.revision) + if not source.total_params: + print(f"no parameter count for {args.model}", file=sys.stderr) + return 2 + if args.contexts: + try: + wanted = json.loads(args.contexts) + except ValueError: + wanted = [v for v in args.contexts.strip("[]").split(",") if v.strip()] + if not isinstance(wanted, list): + wanted = [wanted] + else: + wanted = list(cfg.context_tiers) + + entries = [] + for window in sorted({int(v) for v in wanted}, reverse=True): + need, cores, minutes = export_qnn.modal_request(source.total_params, window) + memory_mib = remote.memory_request(need, cfg.modal) + if memory_mib is None: + print(f"{window} needs more than Modal allows ({need:,} B)", file=sys.stderr) + continue + entry = {"context": window, "memory_mib": memory_mib, "cpu": cores, "timeout_minutes": minutes} + print(f"{window}: {entry}", file=sys.stderr) + entries.append(entry) print(json.dumps(entries)) return 0 @@ -457,8 +514,15 @@ def main(argv: list[str] | None = None) -> int: help="lines in mtk.calibration, which sets the disk the Arrow cache needs; the export " "counts the real file and its disk gate catches a mismatch", ) + matrix.add_argument("--prompt-tokens", type=int, default=None, help="the export's prompt_tokens, if not mtk's") matrix.set_defaults(func=_mtk_matrix) + qnn_matrix = commands.add_parser("qnn-matrix", help="windows and the Modal Sandbox each gets, as a CI matrix") + qnn_matrix.add_argument("model") + qnn_matrix.add_argument("--revision", default="main") + qnn_matrix.add_argument("--contexts", default="", help="JSON or comma-separated windows; empty for every tier") + qnn_matrix.set_defaults(func=_qnn_matrix) + export_matrix = commands.add_parser( "export-matrix", help="windows and the runner each needs for an export_llm backend, as a CI matrix" ) @@ -484,6 +548,8 @@ def main(argv: list[str] | None = None) -> int: modal_run.add_argument("--get", help="local directory that receives $OUT") modal_run.add_argument("--env", action="append", help="NAME=VALUE set in the Sandbox (repeatable)") modal_run.add_argument("--keep", action="store_true", help="leave the run's files on the Volume") + modal_run.add_argument("--timeout-minutes", type=int, default=None, help="default modal.timeout_minutes") + modal_run.add_argument("--mtk-tools", help="requirements for MediaTek's Python 3.10 virtualenv in the image") modal_run.set_defaults(func=_modal_run) watch = commands.add_parser("watch", help="check the watched orgs, dispatch exports, update state") diff --git a/pipeline/export_mtk.py b/pipeline/export_mtk.py index e6df747..4029bca 100644 --- a/pipeline/export_mtk.py +++ b/pipeline/export_mtk.py @@ -129,6 +129,16 @@ def state_layout(script: str) -> str: # 14.4 bytes per parameter, and 17.8 GB for the 1.2B at 4k, so it does not grow with the window. LOWERING_BYTES_PER_PARAM = 16 REPO_ROOT = Path(__file__).resolve().parents[1] +# On Modal (mtk.runner_tiers "modal"), the cores and Sandbox minutes a window gets: (largest +# window, physical cores, minutes). Streaming calibration runs a corpus that fills the window +# through the model twice, so the time and not the memory grows with the window: the 16k +# 8-chunk LFM2.5-1.2B build took 3,235 s on 8 physical cores (16 vCPUs, finding 39), and a +# 32k calibration runs twice the tokens over twice the cache. +MODAL_SHAPES = ((4096, 8, 240), (16384, 16, 330), (32768, 32, 600)) +# The smallest Sandbox: lowering and MediaTek's compiler service on top of the estimate. +MODAL_FLOOR_BYTES = 16 * 2**30 +# From this window up, a hybrid is cut so no chunk holds two attention layers (finding 39). +ONE_ATTENTION_PER_CHUNK_FROM = 16384 TOOL_PACKAGES = ("executorch", "torch", "torchao", "transformers", "mtk_converter", "mtk-neuron") _LOAD_PROGRAM = """ @@ -295,6 +305,14 @@ def carries(tier: settings.RunnerTier, *, with_swap: bool) -> bool: return None +def modal_shape(window: int) -> tuple[int, int] | None: + """Physical cores and Sandbox minutes for a window on Modal (MODAL_SHAPES), or None past them.""" + for largest, cores, minutes in MODAL_SHAPES: + if window <= largest: + return cores, minutes + return None + + def exp_name(weight_dir: Path, precision: str, chunks: int) -> str: """The name the scripts give the output folder and files (utils.get_exp_name).""" return f"{weight_dir.name}_{precision}_{chunks}_chunks" @@ -499,10 +517,12 @@ def run( started = time.time() hub.download(source, weight_dir) tokenizer, licenses = copy_side_files(source, weight_dir, out_dir) + # The MediaTek script reads the weights' own config.json: give it the keys it expects + # (hub.normalize_config: the first LFM2 checkpoints spell them the older way). + model_config = hub.normalize_config(json.loads((weight_dir / "config.json").read_text(encoding="utf-8"))) if plan.tokenizer: - model_config = json.loads((weight_dir / "config.json").read_text(encoding="utf-8")) model_config["tokenizer"] = plan.tokenizer - (weight_dir / "config.json").write_text(json.dumps(model_config, indent=2), encoding="utf-8") + (weight_dir / "config.json").write_text(json.dumps(model_config, indent=2), encoding="utf-8") bos, eos = hub.special_token_ids(source) corpus = None diff --git a/pipeline/export_qnn.py b/pipeline/export_qnn.py index b56449a..84672d0 100644 --- a/pipeline/export_qnn.py +++ b/pipeline/export_qnn.py @@ -46,6 +46,32 @@ QNN_BACKEND_ID = "QnnBackend" META_FILES = ("original/consolidated.00.pth", "original/params.json", "original/tokenizer.model") SOC_NAMES = {"SM8650": "Snapdragon 8 Gen 3", "SM8750": "Snapdragon 8 Elite"} + +# The Modal Sandbox a window is given (pipeline/remote.py). Only Qwen3-0.6B has been measured: +# 15.6 GB of peak RSS at both 2k and 4k (findings 8 and 10), on 16 GB runners with swap, so +# that figure is a floor on its need rather than the need. Modal treats a plain memory request +# as a minimum, not a limit (the Sandbox may use more) and bills the higher of request and +# use, so this asks for a modest share and lets the peak burst past it. Refit it against each +# report's peak_rss_export_bytes and peak_cgroup_bytes; on Modal peak_in_use_bytes is the host's. +QNN_BYTES_PER_PARAM = 20 +QNN_FLOOR_BYTES = 24 * 2**30 +# The window enters through calibration and the two graphs' attention, at most half again by 8k. +QNN_WINDOW_SCALE = 16384 +# Time grows with the model and faster than the window: Qwen3-0.6B took 2,434 s at 2k and +# 5,987 s at 4k on a 4 vCPU runner (findings 8 and 10). (largest parameter count, physical +# cores, Sandbox minutes); the minutes cost nothing unless a run hangs. +QNN_MODAL_SHAPES = ((1_300_000_000, 16, 720), (4_500_000_000, 32, 1380)) + + +def modal_request(params: int, window: int) -> tuple[int, int, int]: + """Memory in bytes, physical cores and minutes for a QNN export on Modal.""" + need = max(QNN_FLOOR_BYTES, params * QNN_BYTES_PER_PARAM) * (1 + window / QNN_WINDOW_SCALE) + for largest, cores, minutes in QNN_MODAL_SHAPES: + if params <= largest: + return int(need), cores, minutes + raise ValueError(f"no Modal shape for a {params:,}-parameter QNN export") + + EXECUTORCH_SOURCE_FILES = settings.ROOT / "third_party" / "executorch" @@ -145,6 +171,10 @@ def structural_check(pte: Path, model_mode: str) -> dict: _SDK_PROBE = """ import json +# ExecuTorch 1.5.1 moved the download out of the package's __init__, which now has no side +# effects (backends/qualcomm/utils/qnn_sdk_setup.py): it happens here or not at all. +from executorch.backends.qualcomm.utils.qnn_sdk_setup import setup_qnn_sdk +setup_qnn_sdk() from executorch.backends.qualcomm.scripts import download_qnn_sdk as d print(json.dumps({ "version": d.QNN_VERSION, @@ -167,7 +197,15 @@ def qnn_sdk() -> dict: result = subprocess.run([sys.executable, "-c", _SDK_PROBE], capture_output=True, text=True, timeout=1800) if result.returncode != 0: raise ExportError(f"could not set up the QAIRT SDK:\n{result.stderr[-4000:]}") - return json.loads(result.stdout.strip().splitlines()[-1]) + sdk = json.loads(result.stdout.strip().splitlines()[-1]) + # The paths are computed, not found: check the download put the libraries there, so a + # missing SDK fails here and not as a dlopen error deep in the Qualcomm script. + wanted = [Path(sdk["root"]) / "lib" / "x86_64-linux-clang" / "libQnnHtp.so"] + wanted += [Path(sdk["libcxx_dir"]) / name for name in sdk["libcxx_files"]] + missing = [str(path) for path in wanted if not path.exists()] + if missing: + raise ExportError(f"the QAIRT SDK setup left no {', '.join(missing)}:\n{result.stderr[-4000:]}") + return sdk def soname(filename: str) -> str: diff --git a/pipeline/exporting.py b/pipeline/exporting.py index 9fba5ed..9afd024 100644 --- a/pipeline/exporting.py +++ b/pipeline/exporting.py @@ -81,6 +81,11 @@ def host_info() -> dict: return info +# The container's own memory in use, cgroup v2 then v1. On a Modal Sandbox /proc/meminfo is the +# host's (pipeline/remote.py), so peak_in_use_bytes there counts other tenants; this does not. +CGROUP_MEMORY = (Path("/sys/fs/cgroup/memory.current"), Path("/sys/fs/cgroup/memory/memory.usage_in_bytes")) + + class MemorySampler: """Samples /proc/meminfo while an export runs. @@ -92,9 +97,16 @@ class MemorySampler: (docs/research, finding 23). """ - def __init__(self, interval: float = 5.0, meminfo: Path = Path("/proc/meminfo")): + def __init__( + self, + interval: float = 5.0, + meminfo: Path = Path("/proc/meminfo"), + cgroup: tuple[Path, ...] = CGROUP_MEMORY, + ): self.interval = interval self.meminfo = meminfo + self.cgroup = next((path for path in cgroup if path.exists()), None) + self.peak_cgroup = None self.peak_swap_used = None self.min_available = None self.peak_in_use = None @@ -103,6 +115,12 @@ def __init__(self, interval: float = 5.0, meminfo: Path = Path("/proc/meminfo")) self._thread = None def sample(self) -> None: + if self.cgroup is not None: + try: + used = int(self.cgroup.read_text().split()[0]) + self.peak_cgroup = max(used, self.peak_cgroup or 0) + except (OSError, ValueError, IndexError): + pass if not self.meminfo.exists(): return fields = {} @@ -142,6 +160,7 @@ def result(self) -> dict: "min_mem_available_bytes": self.min_available, "peak_in_use_bytes": self.peak_in_use, "peak_in_use_at": self.peak_in_use_at, + "peak_cgroup_bytes": self.peak_cgroup, } diff --git a/pipeline/families.py b/pipeline/families.py index be1a877..f07cb51 100644 --- a/pipeline/families.py +++ b/pipeline/families.py @@ -152,6 +152,29 @@ def mtk_chunks(n_layers: int, max_chunks: int) -> int: return next(n for n in range(min(max_chunks, n_layers), 0, -1) if n_layers % n == 0) +def mtk_chunks_one_attention_each(config: dict) -> int | None: + """Fewest even chunks that leave no chunk more than one attention layer, for a hybrid. + + What the 16k LFM2.5 builds needed (docs/research, finding 39): 8 for the 1.2B (attention + at 2, 5, 8, 10, 12, 14 of 16), where 4 chunks put two in a chunk and lost the compiler. + None for a model whose every layer attends, where the rule would mean a chunk per layer. + """ + c = text_config(config) + layer_types = c.get("layer_types") + attending = attending_layers(c) + n_layers = len(layer_types or ()) + if attending is None or attending >= n_layers: + return None + for chunks in range(1, n_layers + 1): + if n_layers % chunks: + continue + size = n_layers // chunks + groups = [layer_types[i : i + size] for i in range(0, n_layers, size)] + if all(sum(1 for t in group if t != "conv") <= 1 for group in groups): + return chunks + return None + + def mtk_plan(family: Family, config: dict, max_chunks: int) -> MtkPlan: if family.key not in _MTK_SCRIPTS: raise UnsupportedModel(f"no MediaTek export script mapped for {family.key}") diff --git a/pipeline/hub.py b/pipeline/hub.py index 65d10ee..f5b74ab 100644 --- a/pipeline/hub.py +++ b/pipeline/hub.py @@ -62,6 +62,32 @@ def _json_file(hf, repo_id: str, revision: str, filename: str, files: list[str]) return json.loads(Path(path).read_text(encoding="utf-8")) +def normalize_config(config: dict) -> dict: + """The config with the older LFM2 keys spelled the way LFM2.5's are. + + LiquidAI's first LFM2 checkpoints (LFM2-350M, -700M, -1.2B) name their attention layers + `full_attn_idxs` and their feed-forward width `block_ff_dim`, where LFM2.5 writes + `layer_types` and `intermediate_size`; transformers' Lfm2Config accepts both. The pipeline + and the patched MediaTek script read the newer keys, so the older ones are translated here: + every layer is a convolution except the ones `full_attn_idxs` lists, and intermediate_size + takes block_ff_dim's unadjusted value, which families.lfm2_hidden_dim and the MediaTek + script both put through block_auto_adjust_ff_dim's rule as they do for LFM2.5. A config + that already has the newer keys is returned as it is. + """ + if config.get("model_type") != "lfm2": + return config + config = dict(config) + idxs = config.get("full_attn_idxs") + if not config.get("layer_types") and idxs is not None and config.get("num_hidden_layers"): + attending = set(int(i) for i in idxs) + config["layer_types"] = [ + "full_attention" if i in attending else "conv" for i in range(int(config["num_hidden_layers"])) + ] + if config.get("intermediate_size") is None and config.get("block_ff_dim") is not None: + config["intermediate_size"] = int(config["block_ff_dim"]) + return config + + def fetch(repo_id: str, revision: str = "main", token: str | None = None) -> SourceModel: hf = api(token) info = hf.model_info( @@ -87,7 +113,7 @@ def fetch(repo_id: str, revision: str = "main", token: str | None = None) -> Sou files=files, ) try: - source.config = _json_file(hf, repo_id, sha, "config.json", files) + source.config = normalize_config(_json_file(hf, repo_id, sha, "config.json", files)) source.generation_config = _json_file(hf, repo_id, sha, "generation_config.json", files) source.tokenizer_config = _json_file(hf, repo_id, sha, "tokenizer_config.json", files) except GatedRepoError: diff --git a/pipeline/remote.py b/pipeline/remote.py index dc5fac4..1762063 100644 --- a/pipeline/remote.py +++ b/pipeline/remote.py @@ -46,7 +46,24 @@ VOLUME_MOUNT = "/vol" CPU_INDEX = "https://download.pytorch.org/whl/cpu" # Passed into the Sandbox when set here, besides --env. -PASSTHROUGH = ("HF_HUB_DISABLE_PROGRESS_BARS", "MODEL_ID", "REVISION", "CONTEXT", "QMODE", "GPTQ") +PASSTHROUGH = ( + "HF_HUB_DISABLE_PROGRESS_BARS", + "MODEL_ID", + "REVISION", + "CONTEXT", + "QMODE", + "GPTQ", + "SOC", + "MTK_MAX_CHUNKS", + "MTK_PRECISION", + "MTK_PROMPT_TOKENS", + # manifest.run_info: each export report links the GitHub run that built it. + "GITHUB_SERVER_URL", + "GITHUB_REPOSITORY", + "GITHUB_RUN_ID", +) +# MediaTek's tools in the image (scripts/mtk-setup.sh adds the NeuroPilot wheels at run time). +MTK_VENV = "/opt/mtk-venv" def python_version() -> str: @@ -68,14 +85,36 @@ def memory_request(need_bytes: int, cfg: settings.ModalSettings) -> int | None: return mib if mib <= cfg.max_memory_mib else None -def image(requirements: Path): +def image(requirements: Path, mtk_tools: Path | None = None): + """The step's image; with `mtk_tools`, also MediaTek's Python 3.10 virtualenv at MTK_VENV. + + The virtualenv holds the public half of requirements/mtk-tools.txt (torch from the CPU + index, executorch, transformers...), which Modal caches like any layer. The NeuroPilot + wheels are not in it: they come from MediaTek's SDK archive, downloaded and deleted on + every run by scripts/mtk-setup.sh, so nothing of the SDK is ever stored in an image. + """ import modal - img = modal.Image.debian_slim(python_version=python_version()).apt_install("git", "build-essential") + img = modal.Image.debian_slim(python_version=python_version()).apt_install("git", "build-essential", "curl") pins = cpu_pins(requirements) if pins: img = img.pip_install(*pins, index_url=CPU_INDEX) img = img.pip_install_from_requirements(str(requirements)) + if mtk_tools is not None: + mtk_python = settings.read_env_file(settings.CONFIG_DIR / "versions.env")["MTK_PYTHON_VERSION"] + tool_python = f"{MTK_VENV}/bin/python" + img = ( + img.pip_install("uv") + .add_local_file(str(mtk_tools), "/tmp/mtk-tools.txt", copy=True) + .run_commands( + f"uv venv --python {mtk_python} {MTK_VENV}", + *[ + f"uv pip install --python {tool_python} '{pin}' --index-url {CPU_INDEX}" + for pin in cpu_pins(mtk_tools) + ], + f"uv pip install --python {tool_python} -r /tmp/mtk-tools.txt", + ) + ) for name in REPO_DIRS: img = img.add_local_dir(str(ROOT / name), f"{REMOTE_ROOT}/{name}", ignore=["**/__pycache__/**"]) return img @@ -114,6 +153,8 @@ def run( env: dict[str, str], cpu: float | None = None, keep: bool = False, + timeout_minutes: int | None = None, + mtk_tools: Path | None = None, ) -> int: import modal @@ -136,6 +177,7 @@ def run( sandbox_env.update(env) sandbox_env.update(IN=f"{base}/in", OUT=f"{base}/out", WORK="/work", HF_HOME="/work/hf-home", PYTHONUNBUFFERED="1") cores = cpu or cfg.cpu + timeout = timeout_minutes or cfg.timeout_minutes sandbox = None code = None @@ -149,7 +191,7 @@ def run( batch.put_file(str(path), f"/{key}/in/{path.name}") print(f"uploaded {len(puts)} input(s) to {cfg.volume}:/{key}/in", flush=True) - print(f"Modal Sandbox: {memory_mib} MiB, {cores:g} cores, timeout {cfg.timeout_minutes} min", flush=True) + print(f"Modal Sandbox: {memory_mib} MiB, {cores:g} cores, timeout {timeout} min", flush=True) started = time.time() with modal.enable_output(): sandbox = modal.Sandbox.create( @@ -157,10 +199,10 @@ def run( "-c", wrap(script), app=app, - image=image(requirements), + image=image(requirements, mtk_tools), memory=memory_mib, cpu=cores, - timeout=cfg.timeout_minutes * 60, + timeout=timeout * 60, volumes={VOLUME_MOUNT: volume}, secrets=secrets, env=sandbox_env, @@ -177,7 +219,7 @@ def run( sandbox.wait(raise_on_termination=False) code = sandbox.returncode except modal.exception.SandboxTimeoutError: - print(f"::error::the Modal Sandbox ran past its {cfg.timeout_minutes}-minute timeout", flush=True) + print(f"::error::the Modal Sandbox ran past its {timeout}-minute timeout", flush=True) for reader in readers: reader.join(timeout=30) print(f"Modal Sandbox exited {code} after {(time.time() - started) / 60:.1f} min", flush=True) @@ -219,6 +261,8 @@ def wrap(script: str) -> str: "}\n" "trap finish EXIT\n" 'mkdir -p "$IN" "$OUT" "$WORK"\n' + # nproc is the cores asked for; left to itself, OpenMP may size its pool to the host. + 'export OMP_NUM_THREADS="${OMP_NUM_THREADS:-$(nproc)}"\n' f"{script}\n" ) @@ -237,4 +281,6 @@ def main(args) -> int: env=env, cpu=args.cpu, keep=args.keep, + timeout_minutes=args.timeout_minutes, + mtk_tools=Path(args.mtk_tools) if args.mtk_tools else None, ) diff --git a/scripts/mtk-setup.sh b/scripts/mtk-setup.sh new file mode 100755 index 0000000..4af2379 --- /dev/null +++ b/scripts/mtk-setup.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# MediaTek's half of an export, set up inside the Modal Sandbox before export-mtk runs: +# +# bash scripts/mtk-setup.sh && python -m pipeline export-mtk ... \ +# --tool-python /opt/mtk-venv/bin/python --examples "$WORK/executorch/examples/mediatek" +# +# The image already has the Python 3.10 virtualenv with the public half of +# requirements/mtk-tools.txt (pipeline/remote.py, image). This adds what may not be stored: +# the NeuroPilot Express SDK is downloaded from MediaTek on every run, checked against the +# pinned digest, and deleted once its two wheels are installed. It is licensed by MediaTek +# Inc. under its own terms (THIRD_PARTY_NOTICES.md), which every run accepts, and nothing +# from it is cached, uploaded or published. +# +# Then the LLM export scripts, which are in the ExecuTorch source and not the wheel (BSD, +# fetched at the pinned commit), with this repo's patches from third_party/executorch/patches. +set -euo pipefail + +eval "$(grep -E '^(NEUROPILOT_SDK_[A-Z0-9_]+|MTK_[A-Z_]+_VERSION|EXECUTORCH_COMMIT)=' config/versions.env | sed 's/^/export /')" +tool_python=/opt/mtk-venv/bin/python +work=${WORK:?} + +sdk="$work/neuropilot" +mkdir -p "$sdk" +curl -fsSL --retry 3 --retry-all-errors -o "$sdk.tar.gz" "$NEUROPILOT_SDK_URL" +echo "$NEUROPILOT_SDK_SHA256 $sdk.tar.gz" | sha256sum -c - +tar xzf "$sdk.tar.gz" -C "$sdk" --strip-components=1 --wildcards \ + "*/mtk_neuron-${MTK_NEURON_VERSION}-*.whl" "*/mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" +# The 8.0.8 archive keeps mtk_converter only under mtk_converter-*_public_packages/. +neuron_whl=$(find "$sdk" -name "mtk_neuron-${MTK_NEURON_VERSION}-*.whl" | head -1) +converter_whl=$(find "$sdk" -name "mtk_converter-${MTK_CONVERTER_VERSION}+public-cp310-*.whl" | head -1) +test -n "$neuron_whl" && test -n "$converter_whl" +uv pip install --quiet --no-cache --python "$tool_python" -r requirements/mtk-tools.txt "$neuron_whl" "$converter_whl" +rm -rf "$sdk" "$sdk.tar.gz" +uv pip list --python "$tool_python" | grep -iE '^(torch|torchao|executorch|transformers|mtk.converter|mtk.neuron) ' + +src="$work/executorch" +rm -rf "$src" +git init -q "$src" +git -C "$src" remote add origin https://github.com/pytorch/executorch.git +git -C "$src" sparse-checkout set examples/mediatek +git -C "$src" fetch -q --depth 1 origin "$EXECUTORCH_COMMIT" +git -C "$src" checkout -q FETCH_HEAD +for patch in third_party/executorch/patches/mediatek-*.patch; do + git -C "$src" apply -v "$PWD/$patch" +done diff --git a/tests/fixtures/lfm2-1.2b.config.json b/tests/fixtures/lfm2-1.2b.config.json new file mode 100644 index 0000000..b6d5f20 --- /dev/null +++ b/tests/fixtures/lfm2-1.2b.config.json @@ -0,0 +1,46 @@ +{ + "architectures": [ + "Lfm2ForCausalLM" + ], + "block_auto_adjust_ff_dim": true, + "block_dim": 2048, + "block_ff_dim": 12288, + "block_ffn_dim_multiplier": 1.0, + "block_mlp_init_scale": 1.0, + "block_multiple_of": 256, + "block_norm_eps": 1e-05, + "block_out_init_scale": 1.0, + "block_use_swiglu": true, + "block_use_xavier_init": true, + "bos_token_id": 1, + "conv_L_cache": 3, + "conv_bias": false, + "conv_dim": 2048, + "conv_dim_out": 2048, + "conv_use_xavier_init": true, + "eos_token_id": 7, + "full_attn_idxs": [ + 2, + 5, + 8, + 10, + 12, + 14 + ], + "hidden_size": 2048, + "initializer_range": 0.02, + "max_position_embeddings": 128000, + "model_type": "lfm2", + "norm_eps": 1e-05, + "num_attention_heads": 32, + "num_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 0, + "rope_theta": 1000000.0, + "torch_dtype": "bfloat16", + "transformers_version": "4.54.0.dev0", + "use_cache": true, + "use_pos_enc": true, + "vocab_size": 65536 +} diff --git a/tests/fixtures/lfm2-350m.config.json b/tests/fixtures/lfm2-350m.config.json new file mode 100644 index 0000000..50536b1 --- /dev/null +++ b/tests/fixtures/lfm2-350m.config.json @@ -0,0 +1,46 @@ +{ + "architectures": [ + "Lfm2ForCausalLM" + ], + "block_auto_adjust_ff_dim": true, + "block_dim": 1024, + "block_ff_dim": 6656, + "block_ffn_dim_multiplier": 1.0, + "block_mlp_init_scale": 1.0, + "block_multiple_of": 256, + "block_norm_eps": 1e-05, + "block_out_init_scale": 1.0, + "block_use_swiglu": true, + "block_use_xavier_init": true, + "bos_token_id": 1, + "conv_L_cache": 3, + "conv_bias": false, + "conv_dim": 1024, + "conv_dim_out": 1024, + "conv_use_xavier_init": true, + "eos_token_id": 7, + "full_attn_idxs": [ + 2, + 5, + 8, + 10, + 12, + 14 + ], + "hidden_size": 1024, + "initializer_range": 0.02, + "max_position_embeddings": 128000, + "model_type": "lfm2", + "norm_eps": 1e-05, + "num_attention_heads": 16, + "num_heads": 16, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 0, + "rope_theta": 1000000.0, + "torch_dtype": "bfloat16", + "transformers_version": "4.54.0.dev0", + "use_cache": true, + "use_pos_enc": true, + "vocab_size": 65536 +} diff --git a/tests/fixtures/lfm2.5-350m.config.json b/tests/fixtures/lfm2.5-350m.config.json new file mode 100644 index 0000000..9d07848 --- /dev/null +++ b/tests/fixtures/lfm2.5-350m.config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "Lfm2ForCausalLM" + ], + "block_auto_adjust_ff_dim": true, + "block_dim": 1024, + "block_ff_dim": 6656, + "block_ffn_dim_multiplier": 1.0, + "block_mlp_init_scale": 1.0, + "block_multiple_of": 256, + "block_norm_eps": 1e-05, + "block_out_init_scale": 1.0, + "block_use_swiglu": true, + "block_use_xavier_init": true, + "bos_token_id": 1, + "conv_L_cache": 3, + "conv_bias": false, + "conv_dim": 1024, + "conv_use_xavier_init": true, + "dtype": "bfloat16", + "eos_token_id": 7, + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 6656, + "layer_types": [ + "conv", + "conv", + "full_attention", + "conv", + "conv", + "full_attention", + "conv", + "conv", + "full_attention", + "conv", + "full_attention", + "conv", + "full_attention", + "conv", + "full_attention", + "conv" + ], + "max_position_embeddings": 128000, + "model_type": "lfm2", + "norm_eps": 1e-05, + "num_attention_heads": 16, + "num_heads": 16, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": 0, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "tie_embedding": true, + "transformers_version": "5.0.0rc1", + "use_cache": true, + "use_pos_enc": true, + "vocab_size": 65536 +} diff --git a/tests/test_exporting.py b/tests/test_exporting.py index ac96109..1c064a1 100644 --- a/tests/test_exporting.py +++ b/tests/test_exporting.py @@ -36,7 +36,7 @@ def test_memory_sampler_keeps_the_peak_of_ram_plus_swap_within_one_sample(tmp_pa def test_memory_sampler_is_a_no_op_without_proc(tmp_path): - with exporting.MemorySampler(interval=3600, meminfo=tmp_path / "missing") as sampler: + with exporting.MemorySampler(interval=3600, meminfo=tmp_path / "missing", cgroup=()) as sampler: pass assert set(sampler.result().values()) == {None} diff --git a/tests/test_families.py b/tests/test_families.py index 4690e0e..eec040a 100644 --- a/tests/test_families.py +++ b/tests/test_families.py @@ -4,7 +4,7 @@ import pytest from conftest import hf_config, load_json -from pipeline import families +from pipeline import families, hub def plan_for(config): @@ -179,3 +179,41 @@ def test_the_memory_model_uses_the_attending_count(): plain = dataclasses.replace(arch, attention_layers=None) assert sizing.causal_mask_bytes(plain, 1024) == 4 * 1024 * 1024 assert sizing.kv_cache_bytes(arch, 1024) < sizing.kv_cache_bytes(plain, 1024) + + +def test_the_first_lfm2_checkpoints_are_read_like_lfm25(): + # LFM2-350M/700M/1.2B name their attention layers full_attn_idxs and their feed-forward + # block_ff_dim; hub.normalize_config spells them as LFM2.5 does. Checked against the + # weights (safetensors headers, 2026-10-07): q_proj only in layers 2, 5, 8, 10, 12, 14, and + # feed-forward widths 4,608 (350M) and 8,192 (1.2B). + + for name, width in (("lfm2-350m", 4608), ("lfm2-1.2b", 8192)): + raw = load_json(f"{name}.config.json") + assert "layer_types" not in raw and "full_attn_idxs" in raw + config = hub.normalize_config(raw) + attending = [i for i, t in enumerate(config["layer_types"]) if t == "full_attention"] + assert attending == [2, 5, 8, 10, 12, 14] + assert len(config["layer_types"]) == 16 + assert families.lfm2_hidden_dim(config) == width + # The XNNPACK plan accepts it, so does sizing, and the original config is untouched. + assert families.xnnpack_plan(families.family_for(config), config).params["layer_types"] == config["layer_types"] + assert families.architecture(config, 1).attention_layers == 6 + assert "layer_types" not in raw + + +def test_a_config_with_the_newer_keys_is_left_alone(): + + config = load_json("lfm2.5-350m.config.json") + assert hub.normalize_config(config) == config + qwen = {"model_type": "qwen3", "num_hidden_layers": 28} + assert hub.normalize_config(qwen) is qwen + + +def test_one_attention_layer_per_chunk_is_only_for_hybrids(): + lfm2 = hub.normalize_config(load_json("lfm2-1.2b.config.json")) + assert families.mtk_chunks_one_attention_each(lfm2) == 8 + qwen = dict(hf_config("Qwen/Qwen3-0.6B")) + assert families.mtk_chunks_one_attention_each(qwen) is None + # An explicit all-attention layout is still not a hybrid: no chunk per layer for it. + qwen["layer_types"] = ["full_attention"] * int(qwen["num_hidden_layers"]) + assert families.mtk_chunks_one_attention_each(qwen) is None diff --git a/tests/test_namespace_runner.py b/tests/test_namespace_runner.py index 025c4ce..ec161d3 100644 --- a/tests/test_namespace_runner.py +++ b/tests/test_namespace_runner.py @@ -25,23 +25,20 @@ def arch(model_id): return families.architecture(hf_config(model_id), TOTAL_PARAMS[model_id]) -# What the profile could export if an export_llm backend ran on it, which is why they moved -# to Modal: the profile as a tier, as config/pipeline.yaml's mtk section records it. -PROFILE_TIER = CFG.mtk.runner_tiers[0] +# What the profile could export if an export ran on it, which is why every one moved to Modal +# (XNNPACK and Vulkan, then MediaTek the same day): the profile as the tier it was configured as. +PROFILE_TIER = settings.RunnerTier( + label=PROFILE, ram_bytes=MEM_TOTAL_BYTES, disk_bytes=101427666944, swap_gib=0, max_window=2048 +) def windows(model_id, backend): return [w for w in WINDOWS if export_xnnpack.pick_runner(arch(model_id), w, (PROFILE_TIER,), backend)] -@pytest.mark.parametrize("backend", ["mtk"]) -def test_the_recipes_not_on_modal_run_on_the_namespace_profile_as_measured(backend): - # XNNPACK and Vulkan moved to Modal the same day (test_remote.py); MediaTek has not yet. - (tier,) = getattr(CFG, backend).runner_tiers - assert tier.label == PROFILE - assert tier.ram_bytes == MEM_TOTAL_BYTES - # No swap: the container does not allow it, so nothing may be sized on swap. - assert tier.swap_gib == 0 +def test_every_export_recipe_runs_on_modal(): + for backend in ("xnnpack", "vulkan", "mtk"): + assert [t.label for t in getattr(CFG, backend).runner_tiers] == [settings.MODAL_TIER], backend def test_no_workflow_runs_anywhere_else(): @@ -87,5 +84,5 @@ def test_mediatek_lfm2_does_not_fit_at_any_window(): "hidden_size": 2048, } for window in (512, 2048, 4096): - recipe = dataclasses.replace(CFG.mtk, cache_size=window) + recipe = dataclasses.replace(CFG.mtk, cache_size=window, runner_tiers=(PROFILE_TIER,)) assert export_mtk.pick_runner(lfm, recipe, 23, 1_170_340_608, streaming=True) is None diff --git a/tests/test_qnn.py b/tests/test_qnn.py index a96eab9..37df1d1 100644 --- a/tests/test_qnn.py +++ b/tests/test_qnn.py @@ -93,20 +93,47 @@ def test_copied_params_describe_the_hf_checkpoint(decoder, hf_fixture): } -def test_the_sdk_is_probed_in_a_child_so_this_process_keeps_its_environment(monkeypatch): +def installed_sdk(root: Path) -> dict: + """SDK with its paths under `root`, and the files qnn_sdk checks for created there.""" + sdk = {**SDK, "root": str(root / "sdk"), "libcxx_dir": str(root / "libcxx")} + htp = root / "sdk" / "lib" / "x86_64-linux-clang" / "libQnnHtp.so" + htp.parent.mkdir(parents=True) + htp.touch() + (root / "libcxx").mkdir() + for name in SDK["libcxx_files"]: + (root / "libcxx" / name).touch() + return sdk + + +def test_the_sdk_is_probed_in_a_child_so_this_process_keeps_its_environment(monkeypatch, tmp_path): calls = [] + sdk = installed_sdk(tmp_path) def fake_run(command, **kwargs): calls.append(command) - return subprocess.CompletedProcess(command, 0, stdout=json.dumps(SDK) + "\n", stderr="Loaded libc++.so.1.0\n") + return subprocess.CompletedProcess(command, 0, stdout=json.dumps(sdk) + "\n", stderr="Loaded libc++.so.1.0\n") monkeypatch.setattr(export_qnn.subprocess, "run", fake_run) monkeypatch.delenv("QNN_SDK_ROOT", raising=False) - assert export_qnn.qnn_sdk() == SDK + assert export_qnn.qnn_sdk() == sdk assert calls and calls[0][1] == "-c" + # ExecuTorch 1.5.1 downloads only when asked: the package import no longer does it. + assert "setup_qnn_sdk()" in calls[0][2] assert "QNN_SDK_ROOT" not in os.environ +def test_a_probe_that_downloaded_nothing_fails_before_the_script_runs(monkeypatch, tmp_path): + # What the first Modal run hit: the paths came back, the files were never fetched. + sdk = {**SDK, "root": str(tmp_path / "sdk"), "libcxx_dir": str(tmp_path / "libcxx")} + monkeypatch.setattr( + export_qnn.subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess(command, 0, stdout=json.dumps(sdk) + "\n", stderr=""), + ) + with pytest.raises(export_qnn.ExportError, match="libQnnHtp.so"): + export_qnn.qnn_sdk() + + def test_sonames(): assert export_qnn.soname("libc++.so.1.0") == "libc++.so.1" assert export_qnn.soname("libc++abi.so.1.0") == "libc++abi.so.1" diff --git a/tests/test_remote.py b/tests/test_remote.py index 9f536cd..c20e4ef 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -11,7 +11,7 @@ import pytest from conftest import TOTAL_PARAMS, hf_config -from pipeline import export_xnnpack, families, remote, settings, sizing +from pipeline import export_mtk, export_qnn, export_xnnpack, families, remote, settings, sizing CFG = settings.load() WINDOWS = (2048, 4096, 8192, 16384, 32768) @@ -80,6 +80,66 @@ def test_the_workflow_sends_each_window_to_modal_with_its_own_memory(workflow): assert "./.github/actions/setup-export" not in text +def test_the_mediatek_workflow_sends_each_window_with_its_memory_cores_and_minutes(): + text = (settings.ROOT / ".github/workflows/export-mtk.yml").read_text(encoding="utf-8") + assert '--memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES"' in text + assert "--mtk-tools requirements/mtk-tools.txt" in text + assert "bash scripts/mtk-setup.sh &&" in text + assert "./.github/actions/setup-export" not in text + # The job outlives the longest Sandbox, so the Sandbox is what times out and says so. + longest = max(minutes for _, _, minutes in export_mtk.MODAL_SHAPES) + assert f"timeout-minutes: {longest + 30}" in text + + +def test_the_neuropilot_sdk_is_fetched_at_run_time_and_never_put_in_the_image(): + setup = (settings.ROOT / "scripts/mtk-setup.sh").read_text(encoding="utf-8") + assert "sha256sum -c -" in setup and 'rm -rf "$sdk" "$sdk.tar.gz"' in setup + image = (settings.ROOT / "pipeline/remote.py").read_text(encoding="utf-8").split("def image(")[1] + assert "NEUROPILOT_SDK_URL" not in image.split("def _stream(")[0] + + +@pytest.mark.parametrize( + ("window", "cores", "minutes"), + [(2048, 8, 240), (4096, 8, 240), (8192, 16, 330), (16384, 16, 330), (32768, 32, 600)], +) +def test_a_mediatek_window_gets_more_cores_and_time_as_it_grows(window, cores, minutes): + assert export_mtk.modal_shape(window) == (cores, minutes) + assert cores <= 64 + + +def test_a_window_past_every_shape_is_left_out_not_fatal(): + assert export_mtk.modal_shape(65536) is None + + +def test_the_qualcomm_workflow_sends_each_window_with_its_memory_cores_and_minutes(): + text = (settings.ROOT / ".github/workflows/export-qnn.yml").read_text(encoding="utf-8") + assert '--memory-mib "$MEMORY_MIB" --cpu "$CORES" --timeout-minutes "$SANDBOX_MINUTES"' in text + assert "--requirements requirements/export-qnn.txt" in text + # QAIRT is downloaded inside each Sandbox now; no Actions cache keeps a copy. + assert "actions/cache" not in text + longest = max(minutes for _, _, minutes in export_qnn.QNN_MODAL_SHAPES) + assert f"timeout-minutes: {longest + 30}" in text + + +@pytest.mark.parametrize( + ("model_id", "cores"), + [ + ("HuggingFaceTB/SmolLM2-360M-Instruct", 16), + ("Qwen/Qwen3-0.6B", 16), + ("meta-llama/Llama-3.2-1B-Instruct", 16), + ("Qwen/Qwen3-1.7B", 32), + ("Qwen/Qwen3-4B", 32), + ], +) +def test_a_qualcomm_export_fits_a_sandbox_at_every_window_it_is_given(model_id, cores): + for window in (2048, 4096, 8192): + need, got_cores, minutes = export_qnn.modal_request(TOTAL_PARAMS[model_id], window) + # Qwen3-0.6B measured 15.6 GB of RSS at 2k and 4k (findings 8 and 10): asked for above that. + assert need >= 15_702_085_632 + assert remote.memory_request(need, CFG.modal) is not None + assert got_cores == cores and minutes < 1440 # Modal's Sandbox limit is 24 hours + + def test_the_matrix_entry_shape_the_workflow_reads(): entry = {"context": 2048, "runner": "modal", "swap_gib": 0, "memory_mib": 6286} assert json.loads(json.dumps(entry))["memory_mib"] == 6286 @@ -107,3 +167,20 @@ def test_a_failed_sync_fails_a_step_that_had_succeeded(tmp_path): # A step that already failed keeps its own status. skipped = remote.wrap("exit 4").replace(f"sync {remote.VOLUME_MOUNT}", "false") assert subprocess.run(["bash", "-c", skipped], env=env, capture_output=True).returncode == 4 + + +def test_the_run_that_built_a_file_reaches_the_sandbox(): + assert {"GITHUB_SERVER_URL", "GITHUB_REPOSITORY", "GITHUB_RUN_ID"} <= set(remote.PASSTHROUGH) + + +def test_the_container_peak_is_read_from_its_cgroup(tmp_path): + from pipeline.exporting import MemorySampler + + current = tmp_path / "memory.current" + current.write_text("3221225472\n") + sampler = MemorySampler(meminfo=tmp_path / "missing", cgroup=(tmp_path / "absent", current)) + sampler.sample() + current.write_text("1073741824\n") + sampler.sample() + assert sampler.result()["peak_cgroup_bytes"] == 3221225472 + assert MemorySampler(meminfo=tmp_path / "missing", cgroup=()).result()["peak_cgroup_bytes"] is None