diff --git a/promptpulse/data.py b/promptpulse/data.py index c7f1c05..dd624d0 100644 --- a/promptpulse/data.py +++ b/promptpulse/data.py @@ -25,10 +25,17 @@ def load_dataset(path: str | Path = DEFAULT_DATASET) -> list[dict[str, Any]]: missing = required - row.keys() if missing: raise ValueError(f"Dataset row {index} is missing: {sorted(missing)}") + for field in required: + if not isinstance(row[field], str) or not row[field].strip(): + raise ValueError(f"Dataset row {index} requires a non-empty string for {field}.") if row["id"] in seen: raise ValueError(f"Duplicate dataset id: {row['id']}") seen.add(row["id"]) row.setdefault("required_terms", []) row.setdefault("forbidden_terms", []) + for field in ("required_terms", "forbidden_terms"): + terms = row[field] + if not isinstance(terms, list) or any(not isinstance(term, str) or not term.strip() for term in terms): + raise ValueError(f"Dataset row {index} requires a list of non-empty strings for {field}.") return data diff --git a/tests/test_evaluation.py b/tests/test_evaluation.py index fd9601d..98232ea 100644 --- a/tests/test_evaluation.py +++ b/tests/test_evaluation.py @@ -1,5 +1,7 @@ from promptpulse.data import load_dataset from promptpulse.evaluation import evaluate_response, policy_compliance +import json +import pytest def test_dataset_schema_and_unique_ids(): @@ -8,6 +10,22 @@ def test_dataset_schema_and_unique_ids(): assert len({row["id"] for row in rows}) == len(rows) +def test_malformed_policy_terms_cannot_be_scored_as_character_checks(tmp_path): + row = dict(load_dataset()[0], required_terms="30 days") + path = tmp_path / "invalid.json" + path.write_text(json.dumps([row])) + with pytest.raises(ValueError, match="required_terms"): + load_dataset(path) + + +def test_empty_reference_is_rejected_before_evaluation(tmp_path): + row = dict(load_dataset()[0], reference_context=" ") + path = tmp_path / "invalid.json" + path.write_text(json.dumps([row])) + with pytest.raises(ValueError, match="reference_context"): + load_dataset(path) + + def test_expected_answers_clear_deterministic_gate(): for row in load_dataset(): result = evaluate_response(