fix: Salt failure is a hard error

docs/compliance_audit_plan.md

@@ -140,7 +140,7 @@ benchmark from-config
 ### Program flow (output-caching audit / MLPerf TEST04, two phases)
 
 Every decision gate is shown, with the exit code it produces. Exit codes:
-`0` PASS · `1` FAIL · `3` SetupError · `4` ExecutionError · `130` interrupted
+`0` PASS · `1` FAIL · `2` InputValidationError · `3` SetupError · `4` ExecutionError · `130` interrupted
 (during the main run: the audit never starts; during the audit: the perf
 report is already written).
 
@@ -397,13 +397,15 @@ The generic loop never names a specific test:
    or re-scores them) and has `audit=None` to prevent re-entry into `run_audit`. A phase can
    override this per-`AuditRunSpec` via `test_mode` (`ACC`/`BOTH` keeps its accuracy datasets);
    the orchestrator reads `spec.test_mode` rather than hardcoding perf-only. This override is
-   supported but currently unused — every registered audit (TEST04) runs perf-only. If any phase raises
-   (`SetupError` / `ExecutionError`), `run_audit` aborts **without verifying** — a crashed
-   phase must never produce a result. A phase that returns but whose `Report.complete` is
-   `False` (metrics drain timed out, or the run was interrupted → partial stats) is likewise
-   rejected with `ExecutionError` — a result is never certified on partial data. Errors
-   propagate to the standard CLI handler (`main.py`), which maps `SetupError` → exit `3` and
-   `ExecutionError` → exit `4`.
+   supported but currently unused — every registered audit (TEST04) runs perf-only. If any phase
+   raises a typed CLI error (`InputValidationError` / `SetupError` / `ExecutionError`), `run_audit`
+   aborts **without verifying** — a crashed phase must never produce a result. In particular a
+   phase's `setup_benchmark` can raise `InputValidationError` (e.g. `DatasetValidationError` for an
+   unsaltable warmup dataset); it propagates verbatim rather than being recast as `ExecutionError`.
+   A phase that returns but whose `Report.complete` is `False` (metrics drain timed out, or the run
+   was interrupted → partial stats) is likewise rejected with `ExecutionError` — a result is never
+   certified on partial data. Errors propagate to the standard CLI handler (`main.py`), which maps
+   `InputValidationError` → exit `2`, `SetupError` → exit `3`, and `ExecutionError` → exit `4`.
 5. `result = test.verify(runs, cfg)`
 6. Atomically write the result (`tmp → fsync → rename → fsync(parent)`).
 7. Return the typed `AuditResult`. Because `run_benchmark` currently returns `None` and

src/inference_endpoint/commands/audit.py

@@ -20,10 +20,12 @@
 
 run_audit returns an AuditResult; cli.py maps PASS/FAIL and main.run() maps
 exceptions to process exit codes:
-  0  PASS           — result.passed is True
-  1  FAIL           — result.passed is False (cli.py raises CLIError)
-  3  SetupError     — config invalid for the audit (bad sample count/index)
-  4  ExecutionError — a phase run failed or produced partial data
+  0  PASS                 — result.passed is True
+  1  FAIL                 — result.passed is False (cli.py raises CLIError)
+  2  InputValidationError — a phase rejected user input (e.g. an unsaltable
+                            warmup dataset), propagated verbatim from setup
+  3  SetupError           — config invalid for the audit (bad sample count/index)
+  4  ExecutionError       — a phase run failed or produced partial data
 """
 
 from __future__ import annotations
@@ -35,7 +37,7 @@
 from ..compliance import AuditRunArtifacts, get_audit_test
 from ..compliance.result import AuditResult, write_result
 from ..config.schema import BenchmarkConfig, DatasetType
-from ..exceptions import ExecutionError, SetupError
+from ..exceptions import CLIError, ExecutionError, SetupError
 from .benchmark.execute import (
     BenchmarkResult,
     TestMode,
@@ -65,6 +67,9 @@ def run_audit(config: BenchmarkConfig, base_report_dir: Path) -> AuditResult:
         AuditResult — always returned; caller maps passed/failed to exit code.
 
     Raises:
+        InputValidationError: A phase rejected user input (e.g. setup_benchmark
+            raising DatasetValidationError for an unsaltable warmup dataset);
+            propagated verbatim rather than recast as a phase ExecutionError.
         SetupError: Config invalid for audit (missing audit block, bad sample
             count/index for the dataset).
         ExecutionError: A phase benchmark run failed.
@@ -116,7 +121,11 @@ def run_audit(config: BenchmarkConfig, base_report_dir: Path) -> AuditResult:
                 )
             bench = run_benchmark_async(ctx)
             finalize_benchmark(ctx, bench)
-        except (SetupError, ExecutionError):
+        except CLIError:
+            # Typed CLI errors already carry the right exit code — SetupError (3),
+            # ExecutionError (4), and the InputValidationError (2) that
+            # setup_benchmark raises for an unsaltable warmup dataset. Propagate
+            # them; only genuinely unexpected exceptions become a phase failure.
             raise
         except Exception as exc:
             raise ExecutionError(f"Audit phase '{spec.label}' failed: {exc}") from exc

src/inference_endpoint/commands/benchmark/cli.py

@@ -38,7 +38,7 @@
 )
 from inference_endpoint.exceptions import (
     CLIError,
-    DatasetValidationError,
+    DatasetParseError,
     InputValidationError,
 )
 
@@ -67,9 +67,9 @@ def _run(
                 f"{'.'.join(str(p) for p in err['loc'])}: {err['msg']}"
                 for err in e.errors()
             )
-            raise DatasetValidationError(f"Invalid --dataset: {msgs}") from e
+            raise DatasetParseError(f"Invalid --dataset: {msgs}") from e
         except ValueError as e:
-            raise DatasetValidationError(f"Invalid --dataset: {e}") from e
+            raise DatasetParseError(f"Invalid --dataset: {e}") from e
     if config.audit is None:
         run_benchmark(config, mode)
         return

src/inference_endpoint/commands/benchmark/execute.py

@@ -398,6 +398,23 @@ def _load_datasets(
         except Exception as e:
             raise SetupError(f"Failed to load dataset: {e}") from e
 
+        # Fail fast on a warmup dataset that salt cannot bust — at load time,
+        # before any worker/aggregator subprocess is spawned. with_salt() runs
+        # the same check later; this earlier call is deliberate (not redundant),
+        # so an invalid dataset aborts before the subprocess fan-out.
+        warmup = config.settings.warmup
+        if warmup.enabled:
+            if warmup.salt:
+                dataloader.validate_saltable()
+            else:
+                logger.warning(
+                    "Warmup is enabled without salt (--warmup-salt): warmup "
+                    "prompts are issued verbatim, so the server may serve the "
+                    "measured phase from a warm KV/prefix cache and understate "
+                    "latency. Enable --warmup-salt on text-'prompt' datasets to "
+                    "bust the cache."
+                )
+
         if perf_cfg.accuracy_config is not None:
             accuracy_config = perf_cfg.accuracy_config
             scorer_cls, extractor_cls = _resolve_accuracy_components(

src/inference_endpoint/config/schema.py

@@ -764,12 +764,14 @@ class WarmupConfig(BaseModel):
     ] = Field(None, gt=0, description="Warmup request count (None = full dataset once)")
     salt: Annotated[
         bool,
-        cyclopts.Parameter(
-            alias="--warmup-salt",
-            help="Prepend a unique random hex salt to each warmup prompt",
-        ),
+        cyclopts.Parameter(alias="--warmup-salt"),
     ] = Field(
-        True, description="Prepend a unique random hex salt to each warmup prompt"
+        False,
+        description=(
+            "Prepend a unique random hex salt to each warmup prompt. Requires "
+            "text-'prompt' samples; enabling it on a pre-tokenized "
+            "('input_tokens') dataset is a hard error."
+        ),
     )
     drain: Annotated[
         bool,

src/inference_endpoint/config/templates/concurrency_template_full.yaml

@@ -93,7 +93,7 @@ settings:
   warmup:
     enabled: false  # Enable warmup phase before performance run
     n_requests: null  # Warmup request count (None = full dataset once)
-    salt: true  # Prepend a unique random hex salt to each warmup prompt
+    salt: false  # Prepend a unique random hex salt to each warmup prompt. Requires text-'prompt' samples; enabling it on a pre-tokenized ('input_tokens') dataset is a hard error.
     drain: false  # Drain in-flight warmup requests before starting the performance phase
     warmup_random_seed: 42  # RNG seed for warmup scheduling and sample ordering
   profiling:

src/inference_endpoint/config/templates/offline_template_full.yaml

@@ -93,7 +93,7 @@ settings:
   warmup:
     enabled: false  # Enable warmup phase before performance run
     n_requests: null  # Warmup request count (None = full dataset once)
-    salt: true  # Prepend a unique random hex salt to each warmup prompt
+    salt: false  # Prepend a unique random hex salt to each warmup prompt. Requires text-'prompt' samples; enabling it on a pre-tokenized ('input_tokens') dataset is a hard error.
     drain: false  # Drain in-flight warmup requests before starting the performance phase
     warmup_random_seed: 42  # RNG seed for warmup scheduling and sample ordering
   profiling:

src/inference_endpoint/config/templates/online_template_full.yaml

@@ -94,7 +94,7 @@ settings:
   warmup:
     enabled: false  # Enable warmup phase before performance run
     n_requests: null  # Warmup request count (None = full dataset once)
-    salt: true  # Prepend a unique random hex salt to each warmup prompt
+    salt: false  # Prepend a unique random hex salt to each warmup prompt. Requires text-'prompt' samples; enabling it on a pre-tokenized ('input_tokens') dataset is a hard error.
     drain: false  # Drain in-flight warmup requests before starting the performance phase
     warmup_random_seed: 42  # RNG seed for warmup scheduling and sample ordering
   profiling:

src/inference_endpoint/config/utils.py

@@ -119,6 +119,14 @@ def parse_dataset_string(s: str) -> dict[str, object]:
         for seg in segments[:-1]:
             if seg not in target:
                 target[seg] = {}  # type: ignore[index]
+            elif not isinstance(target[seg], dict):
+                # e.g. 'parser=x,parser.prompt=y' — 'parser' is set as both a
+                # scalar and a nested key. Without this, target[seg] would be a
+                # str and the next assignment raises an uncaught TypeError.
+                raise ValueError(
+                    f"Conflicting option '{key}': '{seg}' is set both as a value "
+                    f"and as a nested key. Use one form for '{seg}'."
+                )
             target = target[seg]  # type: ignore[assignment]
         target[segments[-1]] = value  # type: ignore[index]
 

src/inference_endpoint/dataset_manager/dataset.py

@@ -30,6 +30,7 @@
 from datasets import load_dataset, load_from_disk
 
 from ..config.schema import APIType, ModelParams
+from ..exceptions import DatasetValidationError
 from .transforms import (
     ColumnFilter,
     Transform,
@@ -257,6 +258,34 @@ def load_from_huggingface(
     return ds[split].to_pandas()
 
 
+def _check_unsaltable(sample: Any) -> DatasetValidationError.Reason | None:
+    """Return the Reason a sample cannot be salted, or None if it can.
+
+    Salt requires a dict sample with a str 'prompt' and neither 'input_tokens'
+    nor 'messages'. Both are sent to the server ahead of 'prompt' — adapters
+    forward 'input_tokens' verbatim, and the OpenAI chat adapter prefers
+    'messages' over 'prompt' (openai_msgspec_adapter.py) — so a sample carrying
+    either would ship an unsalted payload even after 'prompt' is salted.
+    A list-form 'prompt' (an OpenAI batch / token-ID array, or this project's
+    multimodal content-part convention) is an explicitly unsupported salt path
+    and is rejected with its own reason.
+    """
+    Reason = DatasetValidationError.Reason
+    if not isinstance(sample, dict):
+        return Reason.TYPE_MISMATCH
+    if "input_tokens" in sample:
+        return Reason.INPUT_TOKENS_SHADOWING
+    if "messages" in sample:
+        return Reason.MESSAGES_SHADOWING
+    if "prompt" not in sample:
+        return Reason.PROMPT_MISSING
+    if isinstance(sample["prompt"], list):
+        return Reason.PROMPT_LIST_UNSUPPORTED
+    if not isinstance(sample["prompt"], str):
+        return Reason.PROMPT_TYPE_MISMATCH
+    return None
+
+
 class Dataset:
     """Class for loading and managing benchmark datasets.
 
@@ -440,55 +469,64 @@ def load_sample(self, index: int) -> Any:
             data = self._apply_salt(data)
         return data
 
+    def validate_saltable(self) -> None:
+        """Raise if any loaded sample cannot be salted.
+
+        salt requires a dict sample with a text ('str') 'prompt' and neither
+        'input_tokens' nor 'messages' (adapters send those verbatim / prefer
+        'messages' over 'prompt', so a salted 'prompt' would never reach the
+        server); a list-form 'prompt' (batch / token-IDs / multimodal content
+        parts) is an unsupported salt path and is rejected too. A non-saltable
+        sample is an error, not a silent
+        skip: skipping would leave the KV cache un-busted. Every sample is
+        checked — a single invalid item fails the run, because the seeded warmup
+        subset can draw any index and salt correctness is all-or-nothing. Called
+        before any load is issued — at benchmark setup and again from with_salt().
+
+        Raises:
+            DatasetValidationError: naming the first offending sample. The index
+                is into the loaded, post-transform sample order, not the source
+                file line.
+        """
+        assert self.data is not None, "Dataset not loaded. Call load() first."
+        for i, sample in enumerate(self.data):
+            if reason := _check_unsaltable(sample):
+                raise DatasetValidationError(
+                    reason,
+                    detail=(
+                        f"sample {i} (index into the loaded, post-transform "
+                        f"order); disable salt (--no-warmup-salt / warmup.salt: "
+                        f"false) or use a text-prompt dataset"
+                    ),
+                )
+
     def with_salt(self, rng: random.Random) -> "Dataset":
         """Return a shallow copy of this dataset that salts each load_sample() call.
 
         The returned dataset shares the same loaded data — no re-loading needed.
         Each load_sample() call on the returned dataset prepends a unique hex salt
-        derived from rng to the prompt field, preventing KV-cache reuse.
+        derived from rng to the 'prompt' field, preventing KV-cache reuse.
+
+        Validates every sample first (see validate_saltable): a non-saltable
+        dataset raises here, before any load is issued.
+
+        Raises:
+            DatasetValidationError: if any sample cannot be salted.
         """
+        self.validate_saltable()
         clone = copy.copy(self)
         clone._salt_rng = rng
         return clone
 
-    def _apply_salt(self, data: Any) -> Any:
-        """Prepend a unique salt to the prompt field of a sample dict."""
+    def _apply_salt(self, data: dict[str, Any]) -> dict[str, Any]:
+        """Prepend a unique salt to the 'prompt' field.
+
+        with_salt() has validated every sample, so ``data`` is guaranteed to be a
+        dict with a str 'prompt' and neither 'input_tokens' nor 'messages'.
+        """
         assert self._salt_rng is not None
-        if not isinstance(data, dict):
-            return data
-        if "input_tokens" in data and "prompt" not in data:
-            self.logger.warning(
-                "salt=True: sample has 'input_tokens' but no 'prompt' — "
-                "salt cannot be applied to pre-tokenized input; KV-cache reuse may not be prevented"
-            )
-            return data
-        if "input_tokens" in data and "prompt" in data:
-            self.logger.warning(
-                "salt=True: sample has both 'input_tokens' and 'prompt' — "
-                "salt applied to 'prompt' only; adapters that use 'input_tokens' "
-                "directly will still reuse the KV cache"
-            )
-        if "prompt" not in data:
-            return data
-        prompt = data["prompt"]
         salt = self._salt_rng.randbytes(8).hex()
-        if isinstance(prompt, str):
-            return {**data, "prompt": f"[{salt}] {prompt}"}
-        if isinstance(prompt, list) and prompt:
-            # Find the first text part at any index (image-first prompts place text at index 1+)
-            for i, part in enumerate(prompt):
-                if isinstance(part, dict) and part.get("type") == "text":
-                    salted_parts = [
-                        *prompt[:i],
-                        {**part, "text": f"[{salt}] {part['text']}"},
-                        *prompt[i + 1 :],
-                    ]
-                    return {**data, "prompt": salted_parts}
-            self.logger.warning(
-                "salt=True: multimodal prompt has no text part — "
-                "salt cannot be applied; KV-cache reuse may not be prevented"
-            )
-        return data  # unsupported prompt type — skip salting
+        return {**data, "prompt": f"[{salt}] {data['prompt']}"}
 
     def num_samples(self) -> int:
         assert self.data is not None, "Dataset not loaded. Call load() first."