From abe26126f822b6a98d5d8da1f7eae43be5bdc445 Mon Sep 17 00:00:00 2001 From: Hendar Mawan <176287116+h00w@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:32:28 -0700 Subject: [PATCH 1/2] Harden PromptPulse release behavior --- promptpulse/comparison.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/promptpulse/comparison.py b/promptpulse/comparison.py index efed80b..9663a54 100644 --- a/promptpulse/comparison.py +++ b/promptpulse/comparison.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import asdict, dataclass +from math import isfinite from typing import Any, Mapping from .evaluation import evaluate_response @@ -29,8 +30,8 @@ def compare_runs( *, mean_tolerance: float = 0.02, ) -> ComparisonReport: - if not dataset or mean_tolerance < 0: - raise ValueError("A non-empty dataset and non-negative tolerance are required.") + if not dataset or not isinstance(mean_tolerance, (int, float)) or not isfinite(mean_tolerance) or mean_tolerance < 0: + raise ValueError("A non-empty dataset and finite non-negative tolerance are required.") cases = {row["id"]: row for row in dataset} if len(cases) != len(dataset): raise ValueError("Dataset case IDs must be unique.") From 11b6d0ace478b40d525508064bab501e6357706b Mon Sep 17 00:00:00 2001 From: Hendar Mawan <176287116+h00w@users.noreply.github.com> Date: Tue, 29 Sep 2026 08:32:42 -0700 Subject: [PATCH 2/2] Harden PromptPulse release behavior --- tests/test_comparison.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_comparison.py b/tests/test_comparison.py index fc3a53f..46e036b 100644 --- a/tests/test_comparison.py +++ b/tests/test_comparison.py @@ -1,5 +1,6 @@ from promptpulse.comparison import compare_runs from promptpulse.data import load_dataset +import pytest def _answers(): @@ -30,3 +31,10 @@ def test_policy_regression_holds_despite_other_good_cases(): report = compare_runs(rows, answers, candidate) assert report.decision == "HOLD" assert "refund-policy" in report.regressed_cases + + +@pytest.mark.parametrize("tolerance", [float("nan"), float("inf"), -0.01]) +def test_invalid_tolerance_cannot_turn_regression_into_pass(tolerance): + rows, answers = _answers() + with pytest.raises(ValueError, match="finite non-negative"): + compare_runs(rows, answers, answers, mean_tolerance=tolerance)