From 54eaecc1edd469ea9ceecc36ad54d96eded87ea3 Mon Sep 17 00:00:00 2001
From: Hendar Mawan <176287116+h00w@users.noreply.github.com>
Date: Fri, 2 Oct 2026 07:24:06 -0700
Subject: [PATCH] Validate evaluation pass thresholds before scoring

---
 promptpulse/evaluation.py |  4 ++++
 tests/test_evaluation.py  | 10 ++++++++++
 2 files changed, 14 insertions(+)

diff --git a/promptpulse/evaluation.py b/promptpulse/evaluation.py
index 05c5f8e..e5ce19e 100644
--- a/promptpulse/evaluation.py
+++ b/promptpulse/evaluation.py
@@ -2,6 +2,7 @@
 
 import re
 from dataclasses import asdict, dataclass
+from math import isfinite
 from typing import Iterable
 
 from .config import PULSE_PASS_THRESHOLD
@@ -101,6 +102,9 @@ def evaluate_response(
     forbidden_terms: Iterable[str] = (),
     pass_threshold: float = PULSE_PASS_THRESHOLD,
 ) -> EvaluationResult:
+    if (isinstance(pass_threshold, bool) or not isinstance(pass_threshold, (int, float))
+            or not isfinite(pass_threshold) or not 0 <= pass_threshold <= 1):
+        raise ValueError("pass_threshold must be a finite number within [0, 1]")
     relevance = query_relevance(user_query, response)
     ground = groundedness(reference_context, response)
     coverage = reference_coverage(reference_context, response)
diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py
index 98232ea..a542509 100644
--- a/tests/test_evaluation.py
+++ b/tests/test_evaluation.py
@@ -71,3 +71,13 @@ def test_policy_terms_do_not_match_inside_other_words_or_numbers():
     score, failures = policy_compliance("Refund in 30 days; policy is safe.", required_terms=["30 days"], forbidden_terms=["safe"])
     assert score == 0.5
     assert failures == ["Forbidden term present: safe"]
+
+
+@pytest.mark.parametrize("threshold", [float("nan"), float("inf"), float("-inf"), -0.1, 1.1, True, False, "0.7", None])
+def test_invalid_evaluation_threshold_cannot_change_release_decision(threshold):
+    row = load_dataset()[0]
+    with pytest.raises(ValueError, match="pass_threshold"):
+        evaluate_response(
+            user_query=row["user_query"], reference_context=row["reference_context"],
+            response=row["expected_answer"], pass_threshold=threshold,
+        )