From 3b5f5e812c2276d15cfdd906fce1c13318092a8c Mon Sep 17 00:00:00 2001 From: erauner Date: Sat, 26 Sep 2026 11:26:37 -0500 Subject: [PATCH 1/6] Give rubric_based_* metrics a public rubric path (#132) The rubric-based metrics have been reachable only through the internal build_eval_metric(..., rubrics=...) call: the metric builder accepted rubrics, the eval-set format documented them on cases and invocations, and nothing on the config, CLI or API path passed any of them through, so the metrics were listed as not working. Rubrics can now be declared on a builtin evaluator in the eval config, as {id, text} mappings or plain strings with positional ids, with duplicate ids refused. At run time the runner matches the eval case, not only its conversation, and the matched case's rubrics and each expected invocation's rubrics are attached to the actual invocations the way ADK's own eval service attaches them, with the metric's rubric type given to rubrics that have none, since ADK applies invocation rubrics by type. A rubric metric with no rubric from the config, the case or an invocation reports an error before any judge call rather than an empty evaluation. Its result carries per-rubric scores per invocation and overall in details. The API lists the rubric metrics as working. --- README.md | 13 ++ docs/custom-evaluators.md | 15 ++ docs/eval-set-format.md | 14 ++ src/agentevals/adk_bridge.py | 13 ++ src/agentevals/api/routes.py | 2 +- src/agentevals/builtin_metrics.py | 134 ++++++++++++++++- src/agentevals/config.py | 37 +++++ src/agentevals/custom_evaluators.py | 4 +- src/agentevals/mcp_server.py | 2 +- tests/test_rubrics.py | 218 ++++++++++++++++++++++++++++ 10 files changed, 441 insertions(+), 11 deletions(-) create mode 100644 tests/test_rubrics.py diff --git a/README.md b/README.md index 706f22c1..1ccac8da 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,19 @@ evaluators: threshold: 0.7 ``` +Rubric metrics accept named properties in the evaluator config: + +```yaml +evaluators: + - name: rubric_based_final_response_quality_v1 + type: builtin + rubrics: + - id: latest_correction + text: The response uses the user's latest correction. +``` + +Rubrics can also come from a matched eval case or its invocations; see [Rubrics](docs/eval-set-format.md#rubrics). + `type: remote` pulls community evaluators from GitHub. Protocol, SDK helpers and runtimes: [Custom Evaluators](docs/custom-evaluators.md). ## Run it diff --git a/docs/custom-evaluators.md b/docs/custom-evaluators.md index 87cb251e..58987aa1 100644 --- a/docs/custom-evaluators.md +++ b/docs/custom-evaluators.md @@ -89,6 +89,21 @@ agentevals run traces/my_trace.json \ Each evaluator entry in the `evaluators` list uses the following fields. The `type` field determines which other fields are valid. +### `type: builtin` (ADK metrics) + +| Field | Required | Default | Description | +|---|---|---|---| +| `name` | yes | | The metric, for example `tool_trajectory_avg_score` or `rubric_based_final_response_quality_v1` | +| `type` | yes | | `builtin` | +| `threshold` | no | `0.5` | Score at or above this value means PASSED | +| `judge_model` | no | ADK default | Judge model for LLM-backed metrics | +| `trajectory_match_type` | no | `EXACT` | `EXACT`, `IN_ORDER` or `ANY_ORDER`, for `tool_trajectory_avg_score` | +| `rubrics` | for `rubric_based_*` | | A list of `{id, text}` mappings, or plain strings (given ids `rubric_0`, `rubric_1`, ...). Ids must be unique. An entry may also carry `type`, an ADK rubric type; unset, the metric's own type is used | +| `credential_ref` | no | | Name of a credential the judge key is resolved from (API runs) | +| `judge_base_url` | no | | Base URL of an OpenAI-compatible judge endpoint | + +A `rubric_based_*` metric with no rubric from the config, the matched eval case or its invocations reports an error rather than scoring nothing. Its result carries `details.per_invocation[].rubric_scores` and `details.overall_rubric_scores`, one entry per rubric id with its score and the judge's rationale. + ### `type: code` (local scripts) | Field | Required | Default | Description | diff --git a/docs/eval-set-format.md b/docs/eval-set-format.md index b637a765..1b57603d 100644 --- a/docs/eval-set-format.md +++ b/docs/eval-set-format.md @@ -176,6 +176,20 @@ An eval case can have multiple invocations to represent a conversation. Each inv | `rubrics` | list[Rubric] | no | Scoring rubrics for this specific invocation | | `creation_timestamp` | float | no | Unix timestamp | +### Rubrics + +`rubrics` on an eval case apply to every invocation of that case; `rubrics` on an invocation apply to that invocation alone. Each rubric is an ADK `Rubric`: `rubric_id`, `rubric_content.text_property`, and an optional `type`. When a `rubric_based_*` metric runs, the runner adds the matched case's rubrics and each invocation's rubrics to the rubrics the metric's config declares, the way ADK's own eval service does, and a rubric without a `type` is given the metric's (`FINAL_RESPONSE_QUALITY` or `TOOL_USE_QUALITY`), since ADK applies invocation-level rubrics by type. Rubric ids must be distinct across the metric, the case and the invocation; a clash is reported as an error. With no rubric from any of the three, the metric reports an error. + +```json +{ + "eval_id": "resume-correction", + "rubrics": [ + {"rubric_id": "corrected_value_wins", "rubric_content": {"text_property": "The response uses the operator's latest correction."}} + ], + "conversation": [ ... ] +} +``` + ### Content Uses the Google GenAI `Content` format: diff --git a/src/agentevals/adk_bridge.py b/src/agentevals/adk_bridge.py index 1ef84e73..8b872156 100644 --- a/src/agentevals/adk_bridge.py +++ b/src/agentevals/adk_bridge.py @@ -15,6 +15,9 @@ from google.adk.evaluation.eval_case import EvalCase, IntermediateData, Invocation from google.adk.evaluation.eval_set import EvalSet +from google.adk.evaluation.eval_rubrics import Rubric, RubricContent + +from .config import RubricDef from google.genai import types as genai_types from .genai.matching import ExpectedConversation, ExpectedTurn @@ -71,6 +74,16 @@ def expected_invocations(expected: ExpectedConversation | None) -> list[Invocati return list(expected.raw.conversation or []) +def criterion_rubrics(definitions: list[RubricDef] | None) -> list[Rubric]: + return [Rubric(rubric_id=r.id, rubric_content=RubricContent(text_property=r.text)) for r in definitions or []] + + +def expected_rubrics(expected: ExpectedConversation | None) -> list[Rubric]: + if expected is None or not isinstance(expected.raw, EvalCase): + return [] + return list(expected.raw.rubrics or []) + + def multi_turn_cases(eval_set: EvalSet | None) -> bool: return any(len(case.conversation or []) > 1 for case in (eval_set.eval_cases if eval_set else [])) diff --git a/src/agentevals/api/routes.py b/src/agentevals/api/routes.py index 17e670f2..92cef48c 100644 --- a/src/agentevals/api/routes.py +++ b/src/agentevals/api/routes.py @@ -278,7 +278,7 @@ def _needs_rubrics(name: str) -> bool: requires_gcp=m.metric_name in METRICS_NEEDING_GCP, requires_rubrics=_needs_rubrics(m.metric_name), description=m.description or "No description available", - working=not _needs_rubrics(m.metric_name), + working=True, ) ) diff --git a/src/agentevals/builtin_metrics.py b/src/agentevals/builtin_metrics.py index 6f869ff2..52b66463 100644 --- a/src/agentevals/builtin_metrics.py +++ b/src/agentevals/builtin_metrics.py @@ -167,6 +167,18 @@ def _enrich_app_details(invocations: list[Invocation]) -> list[Invocation]: return [inv.model_copy(update={"app_details": app_details}) for inv in invocations] +METRICS_NEEDING_RUBRICS = { + "rubric_based_final_response_quality_v1", + "rubric_based_tool_use_quality_v1", +} + +# ADK filters case and invocation rubrics by type, but not criterion rubrics. +_METRIC_RUBRIC_TYPES = { + "rubric_based_final_response_quality_v1": "FINAL_RESPONSE_QUALITY", + "rubric_based_tool_use_quality_v1": "TOOL_USE_QUALITY", +} + + def rubric_strings_to_objects(rubric_texts: list[str]) -> list[Rubric]: """Convert plain-text rubric strings into ADK Rubric objects.""" return [ @@ -178,14 +190,90 @@ def rubric_strings_to_objects(rubric_texts: list[str]) -> list[Rubric]: ] +def to_rubric_objects(rubrics: list[str | Rubric] | None) -> list[Rubric]: + """Plain strings become positional rubrics; ADK ``Rubric`` objects pass through.""" + return [ + Rubric(rubric_id=f"rubric_{index}", rubric_content=RubricContent(text_property=r)) + if isinstance(r, str) else r + for index, r in enumerate(rubrics or []) + ] + + +def attach_case_rubrics( + metric_name: str, + actual_invocations: list[Invocation], + expected_invocations: list[Invocation] | None, + case_rubrics: list[Rubric] | None, + criterion_rubrics: list[Rubric] | None = None, +) -> tuple[list[Invocation], list[str]]: + """Bind case and invocation rubrics without changing ADK types or turn scope. + + Check effective properties before the first judge call, and report dropped + properties by id and source instead of silently defaulting their type. + """ + metric_type = _METRIC_RUBRIC_TYPES.get(metric_name) + attached = [] + warnings = [] + for index, actual in enumerate(actual_invocations): + expected = expected_invocations[index] if expected_invocations and index < len(expected_invocations) else None + sources = [ + (f"actual invocation {index}", actual.rubrics or []), + ("eval case", case_rubrics or []), + (f"expected invocation {index}", expected.rubrics or [] if expected else []), + ] + combined = [r for _, rubrics in sources for r in rubrics] + for source, rubrics in sources: + for rubric in rubrics: + if rubric.type != metric_type: + warning = (f"{source} rubric '{rubric.rubric_id}' ignored: type {rubric.type!r}; " + f"metric requires {metric_type!r}") + if warning not in warnings: + warnings.append(warning) + effective = [*(criterion_rubrics or []), *(r for r in combined if r.type == metric_type)] + ids = [r.rubric_id for r in effective] + if len(ids) != len(set(ids)): + raise ValueError(f"Rubric ids must be distinct across metric, case and invocation {index}: {ids}") + attached.append(actual.model_copy(update={"rubrics": combined})) + return attached, warnings + + +def extract_rubric_details(eval_result: EvaluationResult) -> dict[str, Any]: + """Per-rubric scores, per invocation and overall, from a rubric metric.""" + + def _score(rubric_score: Any) -> dict[str, Any]: + return { + "rubric_id": rubric_score.rubric_id, + "score": rubric_score.score, + "rationale": rubric_score.rationale, + } + + per_invocation = [] + for per_inv_result in eval_result.per_invocation_results: + actual_inv = per_inv_result.actual_invocation + per_invocation.append( + { + "invocation_id": actual_inv.invocation_id if actual_inv else None, + "rubric_scores": [_score(r) for r in per_inv_result.rubric_scores or []], + } + ) + return { + "per_invocation": per_invocation, + "overall_rubric_scores": [_score(r) for r in eval_result.overall_rubric_scores or []], + } + + def build_eval_metric( metric_name: str, judge_model: str | None, threshold: float | None, - rubrics: list[str] | None = None, + rubrics: list[str | Rubric] | None = None, match_type: str | None = None, ) -> EvalMetric: - """Construct an ADK ``EvalMetric`` with the appropriate criterion.""" + """Construct an ADK ``EvalMetric`` with the appropriate criterion. + + ``rubrics`` feed the ``rubric_based_*`` metrics' criterion: plain strings + get positional ids, ADK ``Rubric`` objects are used as given. + """ effective_threshold = threshold if threshold is not None else 0.5 criterion: BaseCriterion | None = None @@ -218,7 +306,7 @@ def build_eval_metric( judge_opts = JudgeModelOptions() if judge_model: judge_opts.judge_model = judge_model - rubric_objects = rubric_strings_to_objects(rubrics) if rubrics else [] + rubric_objects = to_rubric_objects(rubrics) criterion = RubricsBasedCriterion( threshold=effective_threshold, judge_model_options=judge_opts, @@ -378,11 +466,12 @@ async def evaluate_builtin_metric( credential_ref: str | None = None, judge_base_url: str | None = None, expected_reason: str | None = None, + rubrics: list[str | Rubric] | None = None, + case_rubrics: list[Rubric] | None = None, ): - """Evaluate a single built-in ADK metric and return a ``MetricResult``. - - A metric that compares against a golden conversation is ``NOT_EVALUATED`` when none was - selected, with the reason in ``details``; it is never scored against an arbitrary case. + """Evaluate a built-in metric. Case rubrics apply to every invocation; + invocation rubrics apply by position, with their ADK types unchanged. + Unmatched golden comparisons remain NOT_EVALUATED. """ from .runner import MetricResult @@ -393,8 +482,33 @@ async def evaluate_builtin_metric( details={"reason": expected_reason or "no eval set provided"}, ) + rubric_warnings: list[str] = [] + if metric_name in METRICS_NEEDING_RUBRICS: + rubric_objects = to_rubric_objects(rubrics) + try: + actual_invocations, rubric_warnings = attach_case_rubrics( + metric_name, actual_invocations, expected_invocations, case_rubrics, rubric_objects + ) + except ValueError as exc: + return MetricResult(metric_name=metric_name, error=str(exc), error_type="ValueError") + missing = [ + index for index, inv in enumerate(actual_invocations) + if not rubric_objects and not any(r.type == _METRIC_RUBRIC_TYPES[metric_name] for r in inv.rubrics or []) + ] + if missing or not actual_invocations: + return MetricResult( + metric_name=metric_name, + error=( + f"Metric '{metric_name}' requires rubrics applicable to invocations {missing}: " + "set 'rubrics' on the evaluator in the eval config, or correctly typed 'rubrics' " + "on the matched eval case or its invocations." + ), + error_type="ValueError", + details={"warnings": rubric_warnings} if rubric_warnings else None, + ) + try: - eval_metric = build_eval_metric(metric_name, judge_model, threshold, match_type=match_type) + eval_metric = build_eval_metric(metric_name, judge_model, threshold, rubrics=rubrics, match_type=match_type) evaluator: Evaluator = get_evaluator(eval_metric) if credential_ref: @@ -431,6 +545,10 @@ async def evaluate_builtin_metric( details = None if metric_name == "tool_trajectory_avg_score": details = extract_trajectory_details(eval_result) + elif metric_name in METRICS_NEEDING_RUBRICS: + details = extract_rubric_details(eval_result) + if rubric_warnings: + details["warnings"] = rubric_warnings return MetricResult( metric_name=metric_name, diff --git a/src/agentevals/config.py b/src/agentevals/config.py index 0c773a5c..de167815 100644 --- a/src/agentevals/config.py +++ b/src/agentevals/config.py @@ -25,6 +25,22 @@ def _normalize_trajectory_match_type(v: str | None) -> str | None: GroupBy = Literal["auto", "trace", "conversation"] +class RubricDef(BaseModel): + """A named property for a rubric metric, without ADK-specific configuration.""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True, extra="forbid") + + id: str = Field(min_length=1) + text: str = Field(min_length=1) + + @field_validator("id", "text") + @classmethod + def _not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("Rubric id and text must not be blank") + return value + + class BuiltinMetricDef(BaseModel): """A built-in ADK metric, optionally with threshold/judge overrides.""" @@ -35,6 +51,9 @@ class BuiltinMetricDef(BaseModel): threshold: float | None = Field(default=None, ge=0, le=1) judge_model: str | None = None trajectory_match_type: str | None = None + rubrics: list[RubricDef] | None = Field( + default=None, description="Named {id, text} properties, or strings with positional ids, for rubric metrics." + ) credential_ref: str | None = Field( default=None, description="Logical name of a RunSpec.credential_refs entry whose resolved value is the judge API key.", @@ -56,6 +75,24 @@ def _validate_name(cls, v: str) -> str: raise ValueError(f"Built-in metric '{v}' is not supported: {UNSUPPORTED_BUILTIN_METRICS[v]}") return v + @field_validator("rubrics", mode="before") + @classmethod + def _normalize_rubrics(cls, value: Any) -> Any: + if value is None: + return None + if not isinstance(value, list): + raise ValueError("'rubrics' must be a list of strings or {id, text} mappings") + return [{"id": f"rubric_{index}", "text": entry} if isinstance(entry, str) else entry + for index, entry in enumerate(value)] + + @field_validator("rubrics") + @classmethod + def _unique_rubric_ids(cls, value: list[RubricDef] | None) -> list[RubricDef] | None: + duplicates = sorted(key for key, count in Counter(r.id for r in value or []).items() if count > 1) + if duplicates: + raise ValueError("Rubric ids must be unique within a metric. Duplicate ids: " + ", ".join(duplicates)) + return value + class BaseEvaluatorDef(BaseModel): """Shared fields for all executable evaluator definitions.""" diff --git a/src/agentevals/custom_evaluators.py b/src/agentevals/custom_evaluators.py index 058a11af..398f946f 100644 --- a/src/agentevals/custom_evaluators.py +++ b/src/agentevals/custom_evaluators.py @@ -344,7 +344,7 @@ async def evaluate_custom_evaluator( from .runner import MetricResult if isinstance(evaluator_def, BuiltinMetricDef): - from .adk_bridge import expected_invocations, to_adk_invocations + from .adk_bridge import criterion_rubrics, expected_invocations, expected_rubrics, to_adk_invocations from .builtin_metrics import evaluate_builtin_metric actual_invocations = to_adk_invocations(actual.turns) @@ -365,6 +365,8 @@ async def evaluate_custom_evaluator( credential_ref=evaluator_def.credential_ref, judge_base_url=evaluator_def.judge_base_url, expected_reason=expected_reason, + rubrics=criterion_rubrics(evaluator_def.rubrics), + case_rubrics=expected_rubrics(expected), ) if alignment: result.details = {**(result.details or {}), **alignment} diff --git a/src/agentevals/mcp_server.py b/src/agentevals/mcp_server.py index 62ba67c2..4477d329 100644 --- a/src/agentevals/mcp_server.py +++ b/src/agentevals/mcp_server.py @@ -465,7 +465,7 @@ async def evaluate_sessions( "golden_session_id": golden_session_id, "eval_set_id": eval_set_id or f"eval-{golden_session_id[:12]}", "evaluators": [ - e.model_dump(mode="json", by_alias=True) + e.model_dump(mode="json", by_alias=True, exclude={"rubrics"} if e.rubrics is None else set()) for e in make_builtin_evaluator_entries( metrics, judge_model=judge_model, diff --git a/tests/test_rubrics.py b/tests/test_rubrics.py new file mode 100644 index 00000000..c48dde0d --- /dev/null +++ b/tests/test_rubrics.py @@ -0,0 +1,218 @@ +"""The public path for rubric_based_* metrics: rubrics in the eval config, +on the matched eval case and on its invocations, resolved the way ADK's own +eval service resolves them, with per-rubric scores in the result.""" + +from __future__ import annotations + +import asyncio +import textwrap + +import pytest +from google.adk.evaluation.eval_case import EvalCase, Invocation +from google.adk.evaluation.eval_rubrics import Rubric, RubricContent, RubricScore +from google.adk.evaluation.eval_set import EvalSet +from google.adk.evaluation.evaluator import EvalStatus, EvaluationResult, PerInvocationResult +from google.genai import types as genai_types + +from agentevals import builtin_metrics +from agentevals.builtin_metrics import ( + attach_case_rubrics, + build_eval_metric, + evaluate_builtin_metric, + extract_rubric_details, + to_rubric_objects, +) +from agentevals.config import BuiltinMetricDef, RubricDef, apply_builtin_overrides +from agentevals.custom_evaluators import evaluate_custom_evaluator +from agentevals.eval_config_loader import load_eval_config +from agentevals.runner import _find_eval_case, _find_expected_invocations + +METRIC = "rubric_based_final_response_quality_v1" + + +def _rubric(rubric_id: str, text: str = "a statement", rubric_type: str | None = None) -> Rubric: + return Rubric(rubric_id=rubric_id, rubric_content=RubricContent(text_property=text), type=rubric_type) + + +def _invocation(text: str = "hi", rubrics: list[Rubric] | None = None) -> Invocation: + return Invocation( + invocation_id=text, + user_content=genai_types.Content(role="user", parts=[genai_types.Part(text=text)]), + final_response=genai_types.Content(role="model", parts=[genai_types.Part(text="ok")]), + rubrics=rubrics, + ) + + +class TestConfig: + def test_rubrics_accept_mappings_and_strings(self): + metric = BuiltinMetricDef.model_validate( + { + "name": METRIC, + "type": "builtin", + "rubrics": [{"id": "grounded", "text": "No invented cause."}, "Uses the latest correction."], + } + ) + assert [(r.id, r.text) for r in metric.rubrics] == [ + ("grounded", "No invented cause."), + ("rubric_1", "Uses the latest correction."), + ] + assert metric.rubrics[0].to_adk("FINAL_RESPONSE_QUALITY").type == "FINAL_RESPONSE_QUALITY" + assert RubricDef(id="x", text="t", type="TOOL_USE_QUALITY").to_adk("FINAL_RESPONSE_QUALITY").type == ( + "TOOL_USE_QUALITY" + ) + + @pytest.mark.parametrize( + ("rubrics", "message"), + [ + ([{"id": "a", "text": "one"}, {"id": "a", "text": "two"}], "unique"), + ([" "], "empty"), + ("not a list", "must be a list"), + ([{"id": "a"}], "text"), + ([{"id": "a", "text": "one", "weight": 2}], "weight"), + ], + ) + def test_malformed_rubrics_are_refused(self, rubrics, message): + with pytest.raises(ValueError, match=message): + BuiltinMetricDef.model_validate({"name": METRIC, "type": "builtin", "rubrics": rubrics}) + + def test_rubrics_survive_run_level_overrides(self): + metric = BuiltinMetricDef(name=METRIC, rubrics=[RubricDef(id="a", text="t")]) + (updated,) = apply_builtin_overrides([metric], judge_model="gemini-2.5-flash", threshold=0.9) + assert updated.judge_model == "gemini-2.5-flash" + assert [r.id for r in updated.rubrics] == ["a"] + + def test_loader_reads_rubrics_from_yaml(self, tmp_path): + config_file = tmp_path / "eval_config.yaml" + config_file.write_text( + textwrap.dedent( + f""" + evaluators: + - name: {METRIC} + type: builtin + judge_model: gemini-2.5-flash + rubrics: + - id: corrected_value_wins + text: The response uses the operator's latest correction. + - The response states no root cause the conversation did not establish. + """ + ), + encoding="utf-8", + ) + (metric,) = load_eval_config(config_file).evaluators + assert isinstance(metric, BuiltinMetricDef) + assert [r.id for r in metric.rubrics] == ["corrected_value_wins", "rubric_1"] + + +class TestCriterion: + def test_strings_and_objects_both_reach_the_criterion(self): + metric = build_eval_metric(METRIC, "gemini-2.5-flash", 0.5, rubrics=["plain", _rubric("named", "typed")]) + rubrics = metric.criterion.rubrics + assert [(r.rubric_id, r.rubric_content.text_property) for r in rubrics] == [ + ("rubric_0", "plain"), + ("named", "typed"), + ] + assert to_rubric_objects(None) == [] + + +class TestCaseRubrics: + def test_case_rubrics_reach_every_invocation_and_get_the_metrics_type(self): + actual = [_invocation("a"), _invocation("b")] + attached = attach_case_rubrics(METRIC, actual, None, [_rubric("case")]) + for inv in attached: + (rubric,) = inv.rubrics + assert (rubric.rubric_id, rubric.type) == ("case", "FINAL_RESPONSE_QUALITY") + # The originals are untouched. + assert all(inv.rubrics is None for inv in actual) + + def test_invocation_rubrics_apply_at_their_position_and_keep_their_type(self): + actual = [_invocation("a"), _invocation("b")] + expected = [_invocation("a", [_rubric("first", rubric_type="CUSTOM")]), _invocation("b")] + attached = attach_case_rubrics(METRIC, actual, expected, None) + assert [(r.rubric_id, r.type) for r in attached[0].rubrics] == [("first", "CUSTOM")] + assert attached[1].rubrics is None + + def test_a_duplicate_id_between_case_and_invocation_is_an_error(self): + actual = [_invocation("a")] + expected = [_invocation("a", [_rubric("dup")])] + with pytest.raises(ValueError, match="defined more than once"): + attach_case_rubrics(METRIC, actual, expected, [_rubric("dup")]) + + def test_a_rubric_metric_without_any_rubric_is_an_error_before_any_judge_call(self): + result = asyncio.run( + evaluate_builtin_metric( + METRIC, + [_invocation("a")], + None, + judge_model="gemini-2.5-flash", + threshold=0.5, + ) + ) + assert result.score is None + assert "requires rubrics" in result.error + + def test_the_dispatch_passes_config_and_case_rubrics(self, monkeypatch): + seen = {} + + async def fake(**kwargs): + seen.update(kwargs) + from agentevals.runner import MetricResult + + return MetricResult(metric_name=kwargs["metric_name"], score=1.0, eval_status="PASSED") + + monkeypatch.setattr(builtin_metrics, "evaluate_builtin_metric", fake) + case = EvalCase(eval_id="c", conversation=[_invocation("a")], rubrics=[_rubric("case")]) + metric = BuiltinMetricDef(name=METRIC, rubrics=[RubricDef(id="cfg", text="t")]) + asyncio.run(evaluate_custom_evaluator(metric, [_invocation("a")], case.conversation, eval_case=case)) + assert [r.rubric_id for r in seen["rubrics"]] == ["cfg"] + assert seen["rubrics"][0].type == "FINAL_RESPONSE_QUALITY" + assert [r.rubric_id for r in seen["case_rubrics"]] == ["case"] + + +class TestMatching: + def test_the_matched_case_carries_its_rubrics(self): + eval_set = EvalSet( + eval_set_id="s", + eval_cases=[ + EvalCase(eval_id="one", conversation=[_invocation("first question")]), + EvalCase(eval_id="two", conversation=[_invocation("second question")], rubrics=[_rubric("r")]), + ], + ) + case = _find_eval_case([_invocation("second question")], eval_set) + assert case.eval_id == "two" and [r.rubric_id for r in case.rubrics] == ["r"] + assert _find_expected_invocations([_invocation("second question")], eval_set) is case.conversation + assert _find_eval_case([], EvalSet(eval_set_id="e", eval_cases=[])) is None + + +class TestDetails: + def test_per_rubric_scores_are_reported_per_invocation_and_overall(self): + actual = _invocation("a") + result = EvaluationResult( + overall_score=0.5, + overall_eval_status=EvalStatus.FAILED, + per_invocation_results=[ + PerInvocationResult( + actual_invocation=actual, + score=0.5, + eval_status=EvalStatus.FAILED, + rubric_scores=[ + RubricScore(rubric_id="grounded", score=1.0, rationale="no cause was invented"), + RubricScore(rubric_id="corrected", score=0.0, rationale="the stale value is current"), + ], + ) + ], + overall_rubric_scores=[ + RubricScore(rubric_id="grounded", score=1.0), + RubricScore(rubric_id="corrected", score=0.0), + ], + ) + details = extract_rubric_details(result) + assert details["per_invocation"] == [ + { + "invocation_id": "a", + "rubric_scores": [ + {"rubric_id": "grounded", "score": 1.0, "rationale": "no cause was invented"}, + {"rubric_id": "corrected", "score": 0.0, "rationale": "the stale value is current"}, + ], + } + ] + assert [r["rubric_id"] for r in details["overall_rubric_scores"]] == ["grounded", "corrected"] From 247ad18d4168e5cfc183b26609d6688140b7e4d4 Mon Sep 17 00:00:00 2001 From: erauner Date: Sat, 26 Sep 2026 12:41:35 -0500 Subject: [PATCH 2/6] docs: a case-level rubric is judged once per invocation --- docs/eval-set-format.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/eval-set-format.md b/docs/eval-set-format.md index 1b57603d..74a7c78b 100644 --- a/docs/eval-set-format.md +++ b/docs/eval-set-format.md @@ -180,6 +180,8 @@ An eval case can have multiple invocations to represent a conversation. Each inv `rubrics` on an eval case apply to every invocation of that case; `rubrics` on an invocation apply to that invocation alone. Each rubric is an ADK `Rubric`: `rubric_id`, `rubric_content.text_property`, and an optional `type`. When a `rubric_based_*` metric runs, the runner adds the matched case's rubrics and each invocation's rubrics to the rubrics the metric's config declares, the way ADK's own eval service does, and a rubric without a `type` is given the metric's (`FINAL_RESPONSE_QUALITY` or `TOOL_USE_QUALITY`), since ADK applies invocation-level rubrics by type. Rubric ids must be distinct across the metric, the case and the invocation; a clash is reported as an error. With no rubric from any of the three, the metric reports an error. +The `rubric_based_*` metrics grade each invocation on its own, with that invocation's prompt, steps and final response, so a case-level rubric is judged once per turn. In a multi-turn case a rubric that describes the end state ("produces the completed draft") is therefore judged against the intermediate turns as well, where it fails, and lowers the case's mean. Write such a rubric on the final invocation instead, and keep case-level rubrics for properties every turn must have. + ```json { "eval_id": "resume-correction", From ff6e97063ee7cbdacfcbc0f45749afe1c9794331 Mon Sep 17 00:00:00 2001 From: erauner Date: Mon, 12 Oct 2026 12:24:36 -0500 Subject: [PATCH 3/6] fix(rubrics): retain ADK parity on the public evaluation path --- docs/custom-evaluators.md | 4 +- docs/eval-set-format.md | 6 +- src/agentevals/adk_bridge.py | 5 +- src/agentevals/api/routes.py | 4 +- src/agentevals/builtin_metrics.py | 14 ++- src/agentevals/config.py | 6 +- tests/test_builtin_metrics.py | 9 +- tests/test_rubrics.py | 175 +++++++++++++++++++++++++----- ui/src/lib/types.ts | 4 +- 9 files changed, 178 insertions(+), 49 deletions(-) diff --git a/docs/custom-evaluators.md b/docs/custom-evaluators.md index 58987aa1..b8a1af88 100644 --- a/docs/custom-evaluators.md +++ b/docs/custom-evaluators.md @@ -98,11 +98,11 @@ Each evaluator entry in the `evaluators` list uses the following fields. The `ty | `threshold` | no | `0.5` | Score at or above this value means PASSED | | `judge_model` | no | ADK default | Judge model for LLM-backed metrics | | `trajectory_match_type` | no | `EXACT` | `EXACT`, `IN_ORDER` or `ANY_ORDER`, for `tool_trajectory_avg_score` | -| `rubrics` | for `rubric_based_*` | | A list of `{id, text}` mappings, or plain strings (given ids `rubric_0`, `rubric_1`, ...). Ids must be unique. An entry may also carry `type`, an ADK rubric type; unset, the metric's own type is used | +| `rubrics` | for `rubric_based_*` | | A list of `{id, text}` mappings, or plain strings (given positional ids `rubric_0`, `rubric_1`, ...). Ids must be unique. Config rubrics have no `type` field | | `credential_ref` | no | | Name of a credential the judge key is resolved from (API runs) | | `judge_base_url` | no | | Base URL of an OpenAI-compatible judge endpoint | -A `rubric_based_*` metric with no rubric from the config, the matched eval case or its invocations reports an error rather than scoring nothing. Its result carries `details.per_invocation[].rubric_scores` and `details.overall_rubric_scores`, one entry per rubric id with its score and the judge's rationale. +A `rubric_based_*` metric requires an applicable rubric for each evaluated invocation, from the config, the matched eval case or that invocation. Its result carries `details.per_invocation[].rubric_scores` and `details.overall_rubric_scores`, one entry per rubric id with its score and the judge's rationale. Untyped case/invocation rubrics are listed in `details.warnings`; other types are ignored as in ADK, so response and tool-use properties can share one eval set. Config rubrics are not filtered by type. Invalid or empty effective rubric sets report an error before any judge call. Scores, aggregation and explanations follow ADK. ### `type: code` (local scripts) diff --git a/docs/eval-set-format.md b/docs/eval-set-format.md index 74a7c78b..e2af8d5a 100644 --- a/docs/eval-set-format.md +++ b/docs/eval-set-format.md @@ -178,15 +178,15 @@ An eval case can have multiple invocations to represent a conversation. Each inv ### Rubrics -`rubrics` on an eval case apply to every invocation of that case; `rubrics` on an invocation apply to that invocation alone. Each rubric is an ADK `Rubric`: `rubric_id`, `rubric_content.text_property`, and an optional `type`. When a `rubric_based_*` metric runs, the runner adds the matched case's rubrics and each invocation's rubrics to the rubrics the metric's config declares, the way ADK's own eval service does, and a rubric without a `type` is given the metric's (`FINAL_RESPONSE_QUALITY` or `TOOL_USE_QUALITY`), since ADK applies invocation-level rubrics by type. Rubric ids must be distinct across the metric, the case and the invocation; a clash is reported as an error. With no rubric from any of the three, the metric reports an error. +`rubrics` on an eval case apply to every invocation of that case; `rubrics` on an invocation apply to that invocation alone. Each rubric is an ADK `Rubric`: `rubric_id`, `rubric_content.text_property`, and `type`. Response-quality rubrics use `FINAL_RESPONSE_QUALITY`; tool-use rubrics use `TOOL_USE_QUALITY`. As in ADK, each metric ignores properties of another type. Untyped case/invocation properties are ignored with their ids and locations listed in `details.warnings`. The runner preserves authored types. Config-level `{id, text}` rubrics are not filtered by type. Effective rubric ids must be distinct across the metric, case and invocation. A clash or an invocation with no applicable rubrics reports an error before any judge call. -The `rubric_based_*` metrics grade each invocation on its own, with that invocation's prompt, steps and final response, so a case-level rubric is judged once per turn. In a multi-turn case a rubric that describes the end state ("produces the completed draft") is therefore judged against the intermediate turns as well, where it fails, and lowers the case's mean. Write such a rubric on the final invocation instead, and keep case-level rubrics for properties every turn must have. +The `rubric_based_*` metrics grade each invocation on its own, with that invocation's prompt, steps and final response, so a case-level rubric is judged once per turn. In a multi-turn case a rubric that describes the end state ("produces the completed draft") is therefore judged against the intermediate turns as well, where it fails, and lowers the case's mean. Write such a rubric on the final invocation instead, and keep config/case-level rubrics for properties every turn must have. Every evaluated turn still needs an applicable property. A terminal response may be a clarification request when that is the expected outcome of the case. Earlier conversational context and a dedicated final-turn expectations format are separate from this per-invocation behavior. ```json { "eval_id": "resume-correction", "rubrics": [ - {"rubric_id": "corrected_value_wins", "rubric_content": {"text_property": "The response uses the operator's latest correction."}} + {"rubric_id": "corrected_value_wins", "type": "FINAL_RESPONSE_QUALITY", "rubric_content": {"text_property": "The response uses the operator's latest correction."}} ], "conversation": [ ... ] } diff --git a/src/agentevals/adk_bridge.py b/src/agentevals/adk_bridge.py index 8b872156..dc74d29e 100644 --- a/src/agentevals/adk_bridge.py +++ b/src/agentevals/adk_bridge.py @@ -14,12 +14,11 @@ from typing import Any from google.adk.evaluation.eval_case import EvalCase, IntermediateData, Invocation -from google.adk.evaluation.eval_set import EvalSet from google.adk.evaluation.eval_rubrics import Rubric, RubricContent - -from .config import RubricDef +from google.adk.evaluation.eval_set import EvalSet from google.genai import types as genai_types +from .config import RubricDef from .genai.matching import ExpectedConversation, ExpectedTurn from .genai.messages import text_of from .genai.model import Turn diff --git a/src/agentevals/api/routes.py b/src/agentevals/api/routes.py index 92cef48c..568ceb40 100644 --- a/src/agentevals/api/routes.py +++ b/src/agentevals/api/routes.py @@ -353,7 +353,7 @@ def _needs_rubrics(name: str) -> bool: requires_llm=True, requires_gcp=False, requires_rubrics=True, - working=False, + working=True, description="Rubric-based quality assessment of responses (requires rubrics config)", ), MetricInfo( @@ -363,7 +363,7 @@ def _needs_rubrics(name: str) -> bool: requires_llm=True, requires_gcp=False, requires_rubrics=True, - working=False, + working=True, description="Rubric-based assessment of tool usage quality (requires rubrics config)", ), MetricInfo( diff --git a/src/agentevals/builtin_metrics.py b/src/agentevals/builtin_metrics.py index 52b66463..e6fc03c2 100644 --- a/src/agentevals/builtin_metrics.py +++ b/src/agentevals/builtin_metrics.py @@ -193,8 +193,7 @@ def rubric_strings_to_objects(rubric_texts: list[str]) -> list[Rubric]: def to_rubric_objects(rubrics: list[str | Rubric] | None) -> list[Rubric]: """Plain strings become positional rubrics; ADK ``Rubric`` objects pass through.""" return [ - Rubric(rubric_id=f"rubric_{index}", rubric_content=RubricContent(text_property=r)) - if isinstance(r, str) else r + Rubric(rubric_id=f"rubric_{index}", rubric_content=RubricContent(text_property=r)) if isinstance(r, str) else r for index, r in enumerate(rubrics or []) ] @@ -224,9 +223,11 @@ def attach_case_rubrics( combined = [r for _, rubrics in sources for r in rubrics] for source, rubrics in sources: for rubric in rubrics: - if rubric.type != metric_type: - warning = (f"{source} rubric '{rubric.rubric_id}' ignored: type {rubric.type!r}; " - f"metric requires {metric_type!r}") + if rubric.type is None: + warning = ( + f"{source} rubric '{rubric.rubric_id}' ignored: type {rubric.type!r}; " + f"metric requires {metric_type!r}" + ) if warning not in warnings: warnings.append(warning) effective = [*(criterion_rubrics or []), *(r for r in combined if r.type == metric_type)] @@ -492,7 +493,8 @@ async def evaluate_builtin_metric( except ValueError as exc: return MetricResult(metric_name=metric_name, error=str(exc), error_type="ValueError") missing = [ - index for index, inv in enumerate(actual_invocations) + index + for index, inv in enumerate(actual_invocations) if not rubric_objects and not any(r.type == _METRIC_RUBRIC_TYPES[metric_name] for r in inv.rubrics or []) ] if missing or not actual_invocations: diff --git a/src/agentevals/config.py b/src/agentevals/config.py index de167815..1a77f41f 100644 --- a/src/agentevals/config.py +++ b/src/agentevals/config.py @@ -82,8 +82,10 @@ def _normalize_rubrics(cls, value: Any) -> Any: return None if not isinstance(value, list): raise ValueError("'rubrics' must be a list of strings or {id, text} mappings") - return [{"id": f"rubric_{index}", "text": entry} if isinstance(entry, str) else entry - for index, entry in enumerate(value)] + return [ + {"id": f"rubric_{index}", "text": entry} if isinstance(entry, str) else entry + for index, entry in enumerate(value) + ] @field_validator("rubrics") @classmethod diff --git a/tests/test_builtin_metrics.py b/tests/test_builtin_metrics.py index df37c275..e5f87f08 100644 --- a/tests/test_builtin_metrics.py +++ b/tests/test_builtin_metrics.py @@ -287,7 +287,14 @@ async def test_every_prebuilt_metric_receives_invocation_events(monkeypatch, met actual = _invocation([("a", "1")], [("a", "1", "A")]) expected = _invocation([("a", "1")], []) - result = await evaluate_builtin_metric(metric_name, [actual], [expected], None, 0.5) + result = await evaluate_builtin_metric( + metric_name, + [actual], + [expected], + None, + 0.5, + rubrics=["Uses captured evidence."] if metric_name in builtin_metrics.METRICS_NEEDING_RUBRICS else None, + ) assert result.error is None for inv in [*recorder.actual, *recorder.expected]: diff --git a/tests/test_rubrics.py b/tests/test_rubrics.py index c48dde0d..686566d1 100644 --- a/tests/test_rubrics.py +++ b/tests/test_rubrics.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio +import json import textwrap import pytest @@ -15,6 +16,7 @@ from google.genai import types as genai_types from agentevals import builtin_metrics +from agentevals.adk_bridge import criterion_rubrics, expected_from_case from agentevals.builtin_metrics import ( attach_case_rubrics, build_eval_metric, @@ -25,7 +27,9 @@ from agentevals.config import BuiltinMetricDef, RubricDef, apply_builtin_overrides from agentevals.custom_evaluators import evaluate_custom_evaluator from agentevals.eval_config_loader import load_eval_config -from agentevals.runner import _find_eval_case, _find_expected_invocations +from agentevals.genai.extract import extract_conversation +from agentevals.runner import EvalRunConfig, run_evaluation +from test_runner import _make_eval_set_json, _make_tool_trace METRIC = "rubric_based_final_response_quality_v1" @@ -56,19 +60,18 @@ def test_rubrics_accept_mappings_and_strings(self): ("grounded", "No invented cause."), ("rubric_1", "Uses the latest correction."), ] - assert metric.rubrics[0].to_adk("FINAL_RESPONSE_QUALITY").type == "FINAL_RESPONSE_QUALITY" - assert RubricDef(id="x", text="t", type="TOOL_USE_QUALITY").to_adk("FINAL_RESPONSE_QUALITY").type == ( - "TOOL_USE_QUALITY" - ) + assert criterion_rubrics(metric.rubrics)[0].type is None @pytest.mark.parametrize( ("rubrics", "message"), [ ([{"id": "a", "text": "one"}, {"id": "a", "text": "two"}], "unique"), - ([" "], "empty"), + ([" "], "blank"), ("not a list", "must be a list"), ([{"id": "a"}], "text"), ([{"id": "a", "text": "one", "weight": 2}], "weight"), + ([{"id": "a", "text": "one", "type": "FINAL_RESPONSE_QUALITY"}], "type"), + ([{"id": " ", "text": "one"}], "blank"), ], ) def test_malformed_rubrics_are_refused(self, rubrics, message): @@ -115,27 +118,75 @@ def test_strings_and_objects_both_reach_the_criterion(self): class TestCaseRubrics: - def test_case_rubrics_reach_every_invocation_and_get_the_metrics_type(self): + def test_case_rubrics_reach_every_invocation_without_defaulting_type(self): actual = [_invocation("a"), _invocation("b")] - attached = attach_case_rubrics(METRIC, actual, None, [_rubric("case")]) + attached, warnings = attach_case_rubrics(METRIC, actual, None, [_rubric("case")]) for inv in attached: (rubric,) = inv.rubrics - assert (rubric.rubric_id, rubric.type) == ("case", "FINAL_RESPONSE_QUALITY") + assert (rubric.rubric_id, rubric.type) == ("case", None) + assert warnings == ["eval case rubric 'case' ignored: type None; metric requires 'FINAL_RESPONSE_QUALITY'"] # The originals are untouched. assert all(inv.rubrics is None for inv in actual) def test_invocation_rubrics_apply_at_their_position_and_keep_their_type(self): actual = [_invocation("a"), _invocation("b")] expected = [_invocation("a", [_rubric("first", rubric_type="CUSTOM")]), _invocation("b")] - attached = attach_case_rubrics(METRIC, actual, expected, None) + attached, warnings = attach_case_rubrics(METRIC, actual, expected, None) assert [(r.rubric_id, r.type) for r in attached[0].rubrics] == [("first", "CUSTOM")] - assert attached[1].rubrics is None + assert attached[1].rubrics == [] + assert warnings == [] def test_a_duplicate_id_between_case_and_invocation_is_an_error(self): actual = [_invocation("a")] - expected = [_invocation("a", [_rubric("dup")])] - with pytest.raises(ValueError, match="defined more than once"): - attach_case_rubrics(METRIC, actual, expected, [_rubric("dup")]) + expected = [_invocation("a", [_rubric("dup", rubric_type="FINAL_RESPONSE_QUALITY")])] + with pytest.raises(ValueError, match="distinct"): + attach_case_rubrics(METRIC, actual, expected, [_rubric("dup", rubric_type="FINAL_RESPONSE_QUALITY")]) + + @pytest.mark.parametrize("rubric_type", [None, "TOOL_USE_QUALITY"]) + def test_dropped_rubrics_are_reported_before_judge_construction(self, monkeypatch, rubric_type): + def unexpected(*args): + pytest.fail("Judge must not be constructed without an applicable rubric") + + monkeypatch.setattr(builtin_metrics, "get_evaluator", unexpected) + result = asyncio.run( + evaluate_builtin_metric( + METRIC, + [_invocation("a")], + None, + "gemini-2.5-flash", + 0.5, + case_rubrics=[_rubric("dropped", rubric_type=rubric_type)], + ) + ) + assert result.score is None and result.eval_status == "NOT_EVALUATED" + assert "requires rubrics" in result.error + if rubric_type is None: + assert "eval case rubric 'dropped' ignored" in result.details["warnings"][0] + else: + assert result.details is None + + def test_missing_later_turn_rubrics_prevents_partial_judging(self, monkeypatch): + monkeypatch.setattr(builtin_metrics, "get_evaluator", lambda *args: pytest.fail("Partial judge call")) + result = asyncio.run( + evaluate_builtin_metric( + METRIC, + [_invocation("a"), _invocation("b")], + [_invocation("a", [_rubric("first", rubric_type="FINAL_RESPONSE_QUALITY")]), _invocation("b")], + "gemini-2.5-flash", + 0.5, + ) + ) + assert result.score is None and "invocations [1]" in result.error + + def test_duplicate_between_criterion_and_case_is_rejected(self): + with pytest.raises(ValueError, match="distinct"): + attach_case_rubrics( + METRIC, + [_invocation()], + None, + [_rubric("same", rubric_type="FINAL_RESPONSE_QUALITY")], + [_rubric("same")], + ) def test_a_rubric_metric_without_any_rubric_is_an_error_before_any_judge_call(self): result = asyncio.run( @@ -162,25 +213,93 @@ async def fake(**kwargs): monkeypatch.setattr(builtin_metrics, "evaluate_builtin_metric", fake) case = EvalCase(eval_id="c", conversation=[_invocation("a")], rubrics=[_rubric("case")]) metric = BuiltinMetricDef(name=METRIC, rubrics=[RubricDef(id="cfg", text="t")]) - asyncio.run(evaluate_custom_evaluator(metric, [_invocation("a")], case.conversation, eval_case=case)) + actual = extract_conversation([_make_tool_trace([])]) + asyncio.run(evaluate_custom_evaluator(metric, actual, expected_from_case(case))) assert [r.rubric_id for r in seen["rubrics"]] == ["cfg"] - assert seen["rubrics"][0].type == "FINAL_RESPONSE_QUALITY" + assert seen["rubrics"][0].type is None assert [r.rubric_id for r in seen["case_rubrics"]] == ["case"] -class TestMatching: - def test_the_matched_case_carries_its_rubrics(self): - eval_set = EvalSet( - eval_set_id="s", - eval_cases=[ - EvalCase(eval_id="one", conversation=[_invocation("first question")]), - EvalCase(eval_id="two", conversation=[_invocation("second question")], rubrics=[_rubric("r")]), - ], +class TestPublicPath: + @pytest.mark.parametrize( + ("metric_name", "rubric_type"), + [(METRIC, "FINAL_RESPONSE_QUALITY"), ("rubric_based_tool_use_quality_v1", "TOOL_USE_QUALITY")], + ) + @pytest.mark.parametrize("score", [0.0, 1.0, None]) + def test_authored_properties_survive_matching_and_serialization( + self, tmp_path, monkeypatch, metric_name, rubric_type, score + ): + class Evaluator: + async def evaluate_invocations(self, actual_invocations, expected_invocations): + actual = actual_invocations[0] + assert actual.app_details.agent_details + assert [(r.rubric_id, r.type) for r in actual.rubrics] == [ + ("case", rubric_type), + ("ignored", None), + ("turn", rubric_type), + ] + status = ( + EvalStatus.NOT_EVALUATED if score is None else EvalStatus.PASSED if score else EvalStatus.FAILED + ) + scores = [ + RubricScore(rubric_id=key, score=score, rationale="observed synthetic evidence") + for key in ("cfg", "case", "turn") + ] + return EvaluationResult( + overall_score=score, + overall_eval_status=status, + per_invocation_results=[ + PerInvocationResult( + actual_invocation=actual, score=score, eval_status=status, rubric_scores=scores + ) + ], + overall_rubric_scores=scores, + ) + + def factory(metric): + assert [r.rubric_id for r in metric.criterion.rubrics] == ["cfg"] + return Evaluator() + + monkeypatch.setattr(builtin_metrics, "get_evaluator", factory) + from agentevals.otel.encode import encode_traces + + trace_path = tmp_path / "trace.json" + trace_path.write_text(json.dumps(encode_traces([_make_tool_trace(["lookup"])]))) + golden = _make_eval_set_json(["lookup"]) + case = golden["eval_cases"][0] + case["eval_id"] = "matched-case" + case["rubrics"] = [_rubric("case", rubric_type=rubric_type).model_dump(), _rubric("ignored").model_dump()] + case["conversation"][0]["rubrics"] = [_rubric("turn", rubric_type=rubric_type).model_dump()] + unrelated = json.loads(json.dumps(case)) + unrelated["eval_id"] = "unrelated" + unrelated["conversation"][0]["user_content"]["parts"][0]["text"] = "other question" + golden["eval_cases"].insert(0, unrelated) + golden_path = tmp_path / "golden.json" + golden_path.write_text(json.dumps(golden)) + config_path = tmp_path / "config.yaml" + config_path.write_text( + f"evaluators:\n - name: {metric_name}\n type: builtin\n rubrics:\n - id: cfg\n text: Uses captured evidence.\n" + ) + config = load_eval_config(config_path) + result = asyncio.run( + run_evaluation( + EvalRunConfig( + trace_files=[str(trace_path)], eval_set_file=str(golden_path), evaluators=config.evaluators + ) + ) ) - case = _find_eval_case([_invocation("second question")], eval_set) - assert case.eval_id == "two" and [r.rubric_id for r in case.rubrics] == ["r"] - assert _find_expected_invocations([_invocation("second question")], eval_set) is case.conversation - assert _find_eval_case([], EvalSet(eval_set_id="e", eval_cases=[])) is None + payload = json.loads(result.model_dump_json(by_alias=True)) + trace = payload["traceResults"][0] + metric = trace["metricResults"][0] + assert trace["evalCaseId"] == "matched-case" + assert metric["score"] == score + assert metric["evalStatus"] == ("NOT_EVALUATED" if score is None else "PASSED" if score else "FAILED") + details = metric["details"] + assert details["per_invocation"][0]["invocation_id"] == trace["turnRefs"][0]["spanId"] + assert [ + (r["rubric_id"], r["score"], r["rationale"]) for r in details["per_invocation"][0]["rubric_scores"] + ] == [(key, score, "observed synthetic evidence") for key in ("cfg", "case", "turn")] + assert "eval case rubric 'ignored' ignored" in details["warnings"][0] class TestDetails: diff --git a/ui/src/lib/types.ts b/ui/src/lib/types.ts index 843031fe..e8177af3 100644 --- a/ui/src/lib/types.ts +++ b/ui/src/lib/types.ts @@ -409,7 +409,7 @@ export const AVAILABLE_METRICS: MetricMetadata[] = [ requiresLLM: true, requiresGCP: false, requiresRubrics: true, - working: false, + working: true, description: 'Rubric-based quality assessment of responses (requires rubrics config)' }, { @@ -419,7 +419,7 @@ export const AVAILABLE_METRICS: MetricMetadata[] = [ requiresLLM: true, requiresGCP: false, requiresRubrics: true, - working: false, + working: true, description: 'Rubric-based assessment of tool usage quality (requires rubrics config)' }, // Safety metrics From a5e39b9998891511a6c9c686b8c6c0ff3d74ab2b Mon Sep 17 00:00:00 2001 From: erauner Date: Mon, 12 Oct 2026 13:01:02 -0500 Subject: [PATCH 4/6] fix(rubrics): keep conversation metric unavailable until integrated --- src/agentevals/api/routes.py | 7 ++++++- tests/test_api.py | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/agentevals/api/routes.py b/src/agentevals/api/routes.py index 568ceb40..d33fad1a 100644 --- a/src/agentevals/api/routes.py +++ b/src/agentevals/api/routes.py @@ -278,7 +278,12 @@ def _needs_rubrics(name: str) -> bool: requires_gcp=m.metric_name in METRICS_NEEDING_GCP, requires_rubrics=_needs_rubrics(m.metric_name), description=m.description or "No description available", - working=True, + working=not _needs_rubrics(m.metric_name) + or m.metric_name + in { + "rubric_based_final_response_quality_v1", + "rubric_based_tool_use_quality_v1", + }, ) ) diff --git a/tests/test_api.py b/tests/test_api.py index dd712b9e..d099cec6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -458,6 +458,13 @@ def test_metrics_envelope(self): assert isinstance(body["data"], list) assert body["error"] is None + def test_only_supported_rubric_metrics_are_available(self): + body = _assert_envelope(self.client.get("/api/metrics")) + metrics = {metric["name"]: metric for metric in body["data"]} + assert metrics["rubric_based_final_response_quality_v1"]["working"] is True + assert metrics["rubric_based_tool_use_quality_v1"]["working"] is True + assert metrics["rubric_based_multi_turn_trajectory_quality_v1"]["working"] is False + def test_metrics_all_camel(self): body = self.client.get("/api/metrics").json() _assert_all_keys_camel(body) From 07af740eb614b9c81448d1fb7208b11608e2ed04 Mon Sep 17 00:00:00 2001 From: erauner Date: Mon, 12 Oct 2026 13:25:26 -0500 Subject: [PATCH 5/6] test(rubrics): cover clarification and completion through the public path --- docs/eval-set-format.md | 37 ++++++++++++-- tests/test_rubrics.py | 104 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 3 deletions(-) diff --git a/docs/eval-set-format.md b/docs/eval-set-format.md index e2af8d5a..96cd1d18 100644 --- a/docs/eval-set-format.md +++ b/docs/eval-set-format.md @@ -184,14 +184,45 @@ The `rubric_based_*` metrics grade each invocation on its own, with that invocat ```json { - "eval_id": "resume-correction", + "eval_id": "booking-clarification-and-completion", "rubrics": [ - {"rubric_id": "corrected_value_wins", "type": "FINAL_RESPONSE_QUALITY", "rubric_content": {"text_property": "The response uses the operator's latest correction."}} + { + "rubric_id": "clear-outcome", + "type": "FINAL_RESPONSE_QUALITY", + "rubric_content": {"text_property": "Clearly distinguishes a clarification question from a completed draft."} + } ], - "conversation": [ ... ] + "conversation": [ + { + "invocation_id": "clarification", + "user_content": {"role": "user", "parts": [{"text": "Draft a booking for Paris."}]}, + "final_response": {"role": "model", "parts": [{"text": "What date should the booking use?"}]}, + "rubrics": [ + { + "rubric_id": "asks-date", + "type": "FINAL_RESPONSE_QUALITY", + "rubric_content": {"text_property": "Asks for the missing date without claiming a completed booking."} + } + ] + }, + { + "invocation_id": "completion", + "user_content": {"role": "user", "parts": [{"text": "Friday; actually use Lyon."}]}, + "final_response": {"role": "model", "parts": [{"text": "Draft booking for Lyon on Friday."}]}, + "rubrics": [ + { + "rubric_id": "uses-latest-destination", + "type": "FINAL_RESPONSE_QUALITY", + "rubric_content": {"text_property": "Uses Lyon and Friday in the completed draft."} + } + ] + } + ] } ``` +Here `clear-outcome` applies to both invocations, `asks-date` only to the clarification, and `uses-latest-destination` only to the completed draft. A case intended to stop at clarification can contain just the first invocation and its properties; it does not need to require a completed answer. This uses the existing invocation rubric field without adding a new scope option or changing ADK semantics. + ### Content Uses the Google GenAI `Content` format: diff --git a/tests/test_rubrics.py b/tests/test_rubrics.py index 686566d1..c1a5e77c 100644 --- a/tests/test_rubrics.py +++ b/tests/test_rubrics.py @@ -29,6 +29,7 @@ from agentevals.eval_config_loader import load_eval_config from agentevals.genai.extract import extract_conversation from agentevals.runner import EvalRunConfig, run_evaluation +from otel.builders import request, span from test_runner import _make_eval_set_json, _make_tool_trace METRIC = "rubric_based_final_response_quality_v1" @@ -221,6 +222,109 @@ async def fake(**kwargs): class TestPublicPath: + @pytest.mark.parametrize("ends_at_clarification", [False, True]) + def test_clarification_and_completion_properties_keep_their_authored_turn( + self, tmp_path, monkeypatch, ends_at_clarification + ): + turns = [ + ("Draft a booking.", "Which destination?", "asks-destination"), + ("Paris.", "What date?", "asks-date"), + ("Friday; actually Lyon.", "Draft booking for Lyon on Friday.", "uses-latest-destination"), + ] + if ends_at_clarification: + turns = turns[:1] + spans = [ + span( + span_id=f"{index + 1:016x}", + start=(index + 1) * 1000, + end=(index + 1) * 1000 + 100, + attrs={ + "gen_ai.operation.name": "invoke_agent", + "gen_ai.agent.name": "booking-assistant", + "gen_ai.conversation.id": "booking-conversation", + "gen_ai.input.messages": json.dumps( + [{"role": "user", "parts": [{"type": "text", "content": question}]}] + ), + "gen_ai.output.messages": json.dumps( + [{"role": "assistant", "parts": [{"type": "text", "content": answer}]}] + ), + }, + ) + for index, (question, answer, _) in enumerate(turns) + ] + trace_path = tmp_path / "trace.json" + trace_path.write_text(json.dumps(request(spans))) + golden = { + "eval_set_id": "booking", + "eval_cases": [ + { + "eval_id": "booking-case", + "rubrics": [ + _rubric( + "every-turn", + "Keeps a clear distinction between a question and a completed draft.", + "FINAL_RESPONSE_QUALITY", + ).model_dump() + ], + "conversation": [ + { + "invocation_id": f"expected-{index}", + "user_content": {"role": "user", "parts": [{"text": question}]}, + "final_response": {"role": "model", "parts": [{"text": answer}]}, + "rubrics": [_rubric(key, answer, "FINAL_RESPONSE_QUALITY").model_dump()], + } + for index, (question, answer, key) in enumerate(turns) + ], + } + ], + } + golden_path = tmp_path / "golden.json" + golden_path.write_text(json.dumps(golden)) + + class Evaluator: + async def evaluate_invocations(self, actual_invocations, expected_invocations): + assert len(actual_invocations) == len(turns) + results = [] + for index, (actual, expected) in enumerate(zip(actual_invocations, expected_invocations, strict=True)): + key = turns[index][2] + assert [r.rubric_id for r in actual.rubrics] == ["every-turn", key] + assert expected.final_response.parts[0].text == turns[index][1] + results.append( + PerInvocationResult( + actual_invocation=actual, + score=1.0, + eval_status=EvalStatus.PASSED, + rubric_scores=[ + RubricScore(rubric_id=property_id, score=1.0, rationale="Synthetic observed property.") + for property_id in ("every-turn", key) + ], + ) + ) + return EvaluationResult( + overall_score=1.0, overall_eval_status=EvalStatus.PASSED, per_invocation_results=results + ) + + monkeypatch.setattr(builtin_metrics, "get_evaluator", lambda metric: Evaluator()) + result = asyncio.run( + run_evaluation( + EvalRunConfig( + trace_files=[str(trace_path)], + eval_set_file=str(golden_path), + group_by="conversation", + evaluators=[BuiltinMetricDef(name=METRIC)], + ) + ) + ) + payload = json.loads(result.model_dump_json(by_alias=True)) + (trace,) = payload["traceResults"] + assert trace["numInvocations"] == len(turns) + assert trace["evalCaseId"] == "booking-case" + details = trace["metricResults"][0]["details"]["per_invocation"] + assert [item["invocation_id"] for item in details] == [ref["spanId"] for ref in trace["turnRefs"]] + assert [[prop["rubric_id"] for prop in item["rubric_scores"]] for item in details] == [ + ["every-turn", key] for _, _, key in turns + ] + @pytest.mark.parametrize( ("metric_name", "rubric_type"), [(METRIC, "FINAL_RESPONSE_QUALITY"), ("rubric_based_tool_use_quality_v1", "TOOL_USE_QUALITY")], From ba61adbca8ae3183ddd1c8857c001e5d60256d3c Mon Sep 17 00:00:00 2001 From: krisztianfekete Date: Tue, 13 Oct 2026 11:06:45 +0200 Subject: [PATCH 6/6] refactor(rubrics): one rubric metric constant, rubrics on the v1alpha1 contract --- docs/custom-evaluators.md | 4 +- .../agentevals/api/v1alpha1/evaluation.pb.go | 276 +++++++++++------- .../agentevals/api/v1alpha1/evaluation.proto | 10 +- src/agentevals/api/rpc/convert.py | 3 + src/agentevals/api/v1alpha1/evaluation_pb2.py | 74 ++--- .../api/v1alpha1/evaluation_pb2.pyi | 14 +- src/agentevals/builtin_metrics.py | 28 +- src/agentevals/config.py | 16 + tests/test_builtin_metrics.py | 3 +- .../api/v1alpha1/evaluation_pb.d.ts | 32 +- .../agentevals/api/v1alpha1/evaluation_pb.js | 35 ++- 11 files changed, 313 insertions(+), 182 deletions(-) diff --git a/docs/custom-evaluators.md b/docs/custom-evaluators.md index b8a1af88..0af21d06 100644 --- a/docs/custom-evaluators.md +++ b/docs/custom-evaluators.md @@ -98,11 +98,11 @@ Each evaluator entry in the `evaluators` list uses the following fields. The `ty | `threshold` | no | `0.5` | Score at or above this value means PASSED | | `judge_model` | no | ADK default | Judge model for LLM-backed metrics | | `trajectory_match_type` | no | `EXACT` | `EXACT`, `IN_ORDER` or `ANY_ORDER`, for `tool_trajectory_avg_score` | -| `rubrics` | for `rubric_based_*` | | A list of `{id, text}` mappings, or plain strings (given positional ids `rubric_0`, `rubric_1`, ...). Ids must be unique. Config rubrics have no `type` field | +| `rubrics` | for `rubric_based_final_response_quality_v1` and `rubric_based_tool_use_quality_v1` | | A list of `{id, text}` mappings, or plain strings (given positional ids `rubric_0`, `rubric_1`, ...). Ids must be unique. Config rubrics have no `type` field. Other metrics reject `rubrics` | | `credential_ref` | no | | Name of a credential the judge key is resolved from (API runs) | | `judge_base_url` | no | | Base URL of an OpenAI-compatible judge endpoint | -A `rubric_based_*` metric requires an applicable rubric for each evaluated invocation, from the config, the matched eval case or that invocation. Its result carries `details.per_invocation[].rubric_scores` and `details.overall_rubric_scores`, one entry per rubric id with its score and the judge's rationale. Untyped case/invocation rubrics are listed in `details.warnings`; other types are ignored as in ADK, so response and tool-use properties can share one eval set. Config rubrics are not filtered by type. Invalid or empty effective rubric sets report an error before any judge call. Scores, aggregation and explanations follow ADK. +Each of these metrics requires an applicable rubric for each evaluated invocation, from the config, the matched eval case or that invocation. Its result carries `details.per_invocation[].rubric_scores` and `details.overall_rubric_scores`, one entry per rubric id with its score and the judge's rationale. Untyped case/invocation rubrics are listed in `details.warnings`; other types are ignored as in ADK, so response and tool-use properties can share one eval set. Config rubrics are not filtered by type. Invalid or empty effective rubric sets report an error before any judge call. Scores, aggregation and explanations follow ADK. ### `type: code` (local scripts) diff --git a/gen/go/agentevals/api/v1alpha1/evaluation.pb.go b/gen/go/agentevals/api/v1alpha1/evaluation.pb.go index 03757e60..0e020b86 100644 --- a/gen/go/agentevals/api/v1alpha1/evaluation.pb.go +++ b/gen/go/agentevals/api/v1alpha1/evaluation.pb.go @@ -174,7 +174,7 @@ type Metric struct { RequiresLlm bool `protobuf:"varint,5,opt,name=requires_llm,json=requiresLlm,proto3" json:"requires_llm,omitempty"` // Needs Google Cloud credentials. RequiresGcp bool `protobuf:"varint,6,opt,name=requires_gcp,json=requiresGcp,proto3" json:"requires_gcp,omitempty"` - // Needs rubrics in the eval set. + // Needs rubrics, from the evaluator or the eval set. RequiresRubrics bool `protobuf:"varint,7,opt,name=requires_rubrics,json=requiresRubrics,proto3" json:"requires_rubrics,omitempty"` // Usable with recorded traces. Working bool `protobuf:"varint,8,opt,name=working,proto3" json:"working,omitempty"` @@ -323,7 +323,9 @@ type BuiltinEvaluator struct { // Name of a credential_refs entry whose value is the judge API key. CredentialRef string `protobuf:"bytes,5,opt,name=credential_ref,json=credentialRef,proto3" json:"credential_ref,omitempty"` // Base URL of an OpenAI compatible judge endpoint. - JudgeBaseUrl string `protobuf:"bytes,6,opt,name=judge_base_url,json=judgeBaseUrl,proto3" json:"judge_base_url,omitempty"` + JudgeBaseUrl string `protobuf:"bytes,6,opt,name=judge_base_url,json=judgeBaseUrl,proto3" json:"judge_base_url,omitempty"` + // Rubrics for the rubric_based_* metrics that take them. Ids are unique. + Rubrics []*Rubric `protobuf:"bytes,7,rep,name=rubrics,proto3" json:"rubrics,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -400,6 +402,66 @@ func (x *BuiltinEvaluator) GetJudgeBaseUrl() string { return "" } +func (x *BuiltinEvaluator) GetRubrics() []*Rubric { + if x != nil { + return x.Rubrics + } + return nil +} + +// One property a rubric metric's judge checks. +type Rubric struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Text string `protobuf:"bytes,2,opt,name=text,proto3" json:"text,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Rubric) Reset() { + *x = Rubric{} + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Rubric) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Rubric) ProtoMessage() {} + +func (x *Rubric) ProtoReflect() protoreflect.Message { + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Rubric.ProtoReflect.Descriptor instead. +func (*Rubric) Descriptor() ([]byte, []int) { + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{4} +} + +func (x *Rubric) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Rubric) GetText() string { + if x != nil { + return x.Text + } + return "" +} + // An evaluator file on the server, run as a subprocess. type CodeEvaluator struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -419,7 +481,7 @@ type CodeEvaluator struct { func (x *CodeEvaluator) Reset() { *x = CodeEvaluator{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[4] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -431,7 +493,7 @@ func (x *CodeEvaluator) String() string { func (*CodeEvaluator) ProtoMessage() {} func (x *CodeEvaluator) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[4] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -444,7 +506,7 @@ func (x *CodeEvaluator) ProtoReflect() protoreflect.Message { // Deprecated: Use CodeEvaluator.ProtoReflect.Descriptor instead. func (*CodeEvaluator) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{4} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{5} } func (x *CodeEvaluator) GetName() string { @@ -510,7 +572,7 @@ type RemoteEvaluator struct { func (x *RemoteEvaluator) Reset() { *x = RemoteEvaluator{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[5] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -522,7 +584,7 @@ func (x *RemoteEvaluator) String() string { func (*RemoteEvaluator) ProtoMessage() {} func (x *RemoteEvaluator) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[5] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -535,7 +597,7 @@ func (x *RemoteEvaluator) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoteEvaluator.ProtoReflect.Descriptor instead. func (*RemoteEvaluator) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{5} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{6} } func (x *RemoteEvaluator) GetName() string { @@ -601,7 +663,7 @@ type Evaluator struct { func (x *Evaluator) Reset() { *x = Evaluator{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -613,7 +675,7 @@ func (x *Evaluator) String() string { func (*Evaluator) ProtoMessage() {} func (x *Evaluator) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -626,7 +688,7 @@ func (x *Evaluator) ProtoReflect() protoreflect.Message { // Deprecated: Use Evaluator.ProtoReflect.Descriptor instead. func (*Evaluator) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{6} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{7} } func (x *Evaluator) GetKind() isEvaluator_Kind { @@ -701,7 +763,7 @@ type EvalParams struct { func (x *EvalParams) Reset() { *x = EvalParams{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -713,7 +775,7 @@ func (x *EvalParams) String() string { func (*EvalParams) ProtoMessage() {} func (x *EvalParams) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -726,7 +788,7 @@ func (x *EvalParams) ProtoReflect() protoreflect.Message { // Deprecated: Use EvalParams.ProtoReflect.Descriptor instead. func (*EvalParams) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{7} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{8} } func (x *EvalParams) GetEvaluators() []*Evaluator { @@ -769,7 +831,7 @@ type InvocationResult struct { func (x *InvocationResult) Reset() { *x = InvocationResult{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[8] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -781,7 +843,7 @@ func (x *InvocationResult) String() string { func (*InvocationResult) ProtoMessage() {} func (x *InvocationResult) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[8] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -794,7 +856,7 @@ func (x *InvocationResult) ProtoReflect() protoreflect.Message { // Deprecated: Use InvocationResult.ProtoReflect.Descriptor instead. func (*InvocationResult) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{8} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{9} } func (x *InvocationResult) GetScore() float64 { @@ -830,7 +892,7 @@ type MetricResult struct { func (x *MetricResult) Reset() { *x = MetricResult{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[9] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -842,7 +904,7 @@ func (x *MetricResult) String() string { func (*MetricResult) ProtoMessage() {} func (x *MetricResult) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[9] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -855,7 +917,7 @@ func (x *MetricResult) ProtoReflect() protoreflect.Message { // Deprecated: Use MetricResult.ProtoReflect.Descriptor instead. func (*MetricResult) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{9} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{10} } func (x *MetricResult) GetMetricName() string { @@ -924,7 +986,7 @@ type TurnRef struct { func (x *TurnRef) Reset() { *x = TurnRef{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[10] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -936,7 +998,7 @@ func (x *TurnRef) String() string { func (*TurnRef) ProtoMessage() {} func (x *TurnRef) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[10] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -949,7 +1011,7 @@ func (x *TurnRef) ProtoReflect() protoreflect.Message { // Deprecated: Use TurnRef.ProtoReflect.Descriptor instead. func (*TurnRef) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{10} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{11} } func (x *TurnRef) GetTraceId() string { @@ -995,7 +1057,7 @@ type TraceResult struct { func (x *TraceResult) Reset() { *x = TraceResult{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[11] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1007,7 +1069,7 @@ func (x *TraceResult) String() string { func (*TraceResult) ProtoMessage() {} func (x *TraceResult) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[11] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1020,7 +1082,7 @@ func (x *TraceResult) ProtoReflect() protoreflect.Message { // Deprecated: Use TraceResult.ProtoReflect.Descriptor instead. func (*TraceResult) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{11} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{12} } func (x *TraceResult) GetTraceId() string { @@ -1127,7 +1189,7 @@ type RunResult struct { func (x *RunResult) Reset() { *x = RunResult{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[12] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1139,7 +1201,7 @@ func (x *RunResult) String() string { func (*RunResult) ProtoMessage() {} func (x *RunResult) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[12] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1152,7 +1214,7 @@ func (x *RunResult) ProtoReflect() protoreflect.Message { // Deprecated: Use RunResult.ProtoReflect.Descriptor instead. func (*RunResult) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{12} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{13} } func (x *RunResult) GetTraceResults() []*TraceResult { @@ -1203,7 +1265,7 @@ type EvaluateTracesRequest struct { func (x *EvaluateTracesRequest) Reset() { *x = EvaluateTracesRequest{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[13] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1215,7 +1277,7 @@ func (x *EvaluateTracesRequest) String() string { func (*EvaluateTracesRequest) ProtoMessage() {} func (x *EvaluateTracesRequest) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[13] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1228,7 +1290,7 @@ func (x *EvaluateTracesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use EvaluateTracesRequest.ProtoReflect.Descriptor instead. func (*EvaluateTracesRequest) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{13} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{14} } func (x *EvaluateTracesRequest) GetTraces() *TraceInput { @@ -1288,7 +1350,7 @@ type EvaluationProgress struct { func (x *EvaluationProgress) Reset() { *x = EvaluationProgress{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[14] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1300,7 +1362,7 @@ func (x *EvaluationProgress) String() string { func (*EvaluationProgress) ProtoMessage() {} func (x *EvaluationProgress) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[14] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1313,7 +1375,7 @@ func (x *EvaluationProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use EvaluationProgress.ProtoReflect.Descriptor instead. func (*EvaluationProgress) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{14} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{15} } func (x *EvaluationProgress) GetMessage() string { @@ -1336,7 +1398,7 @@ type GroupPerformance struct { func (x *GroupPerformance) Reset() { *x = GroupPerformance{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[15] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1348,7 +1410,7 @@ func (x *GroupPerformance) String() string { func (*GroupPerformance) ProtoMessage() {} func (x *GroupPerformance) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[15] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1361,7 +1423,7 @@ func (x *GroupPerformance) ProtoReflect() protoreflect.Message { // Deprecated: Use GroupPerformance.ProtoReflect.Descriptor instead. func (*GroupPerformance) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{15} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{16} } func (x *GroupPerformance) GetTraceId() string { @@ -1400,7 +1462,7 @@ type EvaluateTracesResponse struct { func (x *EvaluateTracesResponse) Reset() { *x = EvaluateTracesResponse{} - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[16] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1412,7 +1474,7 @@ func (x *EvaluateTracesResponse) String() string { func (*EvaluateTracesResponse) ProtoMessage() {} func (x *EvaluateTracesResponse) ProtoReflect() protoreflect.Message { - mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[16] + mi := &file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1425,7 +1487,7 @@ func (x *EvaluateTracesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use EvaluateTracesResponse.ProtoReflect.Descriptor instead. func (*EvaluateTracesResponse) Descriptor() ([]byte, []int) { - return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{16} + return file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP(), []int{17} } func (x *EvaluateTracesResponse) GetEvent() isEvaluateTracesResponse_Event { @@ -1517,7 +1579,7 @@ const file_agentevals_api_v1alpha1_evaluation_proto_rawDesc = "" + "\x10requires_rubrics\x18\a \x01(\bR\x0frequiresRubrics\x12\x18\n" + "\aworking\x18\b \x01(\bR\aworking\"P\n" + "\x13ListMetricsResponse\x129\n" + - "\ametrics\x18\x01 \x03(\v2\x1f.agentevals.api.v1alpha1.MetricR\ametrics\"\xa7\x02\n" + + "\ametrics\x18\x01 \x03(\v2\x1f.agentevals.api.v1alpha1.MetricR\ametrics\"\xe2\x02\n" + "\x10BuiltinEvaluator\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12!\n" + "\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1f\n" + @@ -1525,9 +1587,13 @@ const file_agentevals_api_v1alpha1_evaluation_proto_rawDesc = "" + "judgeModel\x12`\n" + "\x15trajectory_match_type\x18\x04 \x01(\x0e2,.agentevals.api.v1alpha1.TrajectoryMatchTypeR\x13trajectoryMatchType\x12%\n" + "\x0ecredential_ref\x18\x05 \x01(\tR\rcredentialRef\x12$\n" + - "\x0ejudge_base_url\x18\x06 \x01(\tR\fjudgeBaseUrlB\f\n" + + "\x0ejudge_base_url\x18\x06 \x01(\tR\fjudgeBaseUrl\x129\n" + + "\arubrics\x18\a \x03(\v2\x1f.agentevals.api.v1alpha1.RubricR\arubricsB\f\n" + "\n" + - "_threshold\"\xe0\x01\n" + + "_threshold\",\n" + + "\x06Rubric\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04text\x18\x02 \x01(\tR\x04text\"\xe0\x01\n" + "\rCodeEvaluator\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12!\n" + "\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n" + @@ -1663,7 +1729,7 @@ func file_agentevals_api_v1alpha1_evaluation_proto_rawDescGZIP() []byte { } var file_agentevals_api_v1alpha1_evaluation_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_agentevals_api_v1alpha1_evaluation_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_agentevals_api_v1alpha1_evaluation_proto_msgTypes = make([]protoimpl.MessageInfo, 19) var file_agentevals_api_v1alpha1_evaluation_proto_goTypes = []any{ (TrajectoryMatchType)(0), // 0: agentevals.api.v1alpha1.TrajectoryMatchType (EvalStatus)(0), // 1: agentevals.api.v1alpha1.EvalStatus @@ -1671,63 +1737,65 @@ var file_agentevals_api_v1alpha1_evaluation_proto_goTypes = []any{ (*Metric)(nil), // 3: agentevals.api.v1alpha1.Metric (*ListMetricsResponse)(nil), // 4: agentevals.api.v1alpha1.ListMetricsResponse (*BuiltinEvaluator)(nil), // 5: agentevals.api.v1alpha1.BuiltinEvaluator - (*CodeEvaluator)(nil), // 6: agentevals.api.v1alpha1.CodeEvaluator - (*RemoteEvaluator)(nil), // 7: agentevals.api.v1alpha1.RemoteEvaluator - (*Evaluator)(nil), // 8: agentevals.api.v1alpha1.Evaluator - (*EvalParams)(nil), // 9: agentevals.api.v1alpha1.EvalParams - (*InvocationResult)(nil), // 10: agentevals.api.v1alpha1.InvocationResult - (*MetricResult)(nil), // 11: agentevals.api.v1alpha1.MetricResult - (*TurnRef)(nil), // 12: agentevals.api.v1alpha1.TurnRef - (*TraceResult)(nil), // 13: agentevals.api.v1alpha1.TraceResult - (*RunResult)(nil), // 14: agentevals.api.v1alpha1.RunResult - (*EvaluateTracesRequest)(nil), // 15: agentevals.api.v1alpha1.EvaluateTracesRequest - (*EvaluationProgress)(nil), // 16: agentevals.api.v1alpha1.EvaluationProgress - (*GroupPerformance)(nil), // 17: agentevals.api.v1alpha1.GroupPerformance - (*EvaluateTracesResponse)(nil), // 18: agentevals.api.v1alpha1.EvaluateTracesResponse - nil, // 19: agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry - (*structpb.Struct)(nil), // 20: google.protobuf.Struct - (GroupBy)(0), // 21: agentevals.api.v1alpha1.GroupBy - (*TraceInput)(nil), // 22: agentevals.api.v1alpha1.TraceInput + (*Rubric)(nil), // 6: agentevals.api.v1alpha1.Rubric + (*CodeEvaluator)(nil), // 7: agentevals.api.v1alpha1.CodeEvaluator + (*RemoteEvaluator)(nil), // 8: agentevals.api.v1alpha1.RemoteEvaluator + (*Evaluator)(nil), // 9: agentevals.api.v1alpha1.Evaluator + (*EvalParams)(nil), // 10: agentevals.api.v1alpha1.EvalParams + (*InvocationResult)(nil), // 11: agentevals.api.v1alpha1.InvocationResult + (*MetricResult)(nil), // 12: agentevals.api.v1alpha1.MetricResult + (*TurnRef)(nil), // 13: agentevals.api.v1alpha1.TurnRef + (*TraceResult)(nil), // 14: agentevals.api.v1alpha1.TraceResult + (*RunResult)(nil), // 15: agentevals.api.v1alpha1.RunResult + (*EvaluateTracesRequest)(nil), // 16: agentevals.api.v1alpha1.EvaluateTracesRequest + (*EvaluationProgress)(nil), // 17: agentevals.api.v1alpha1.EvaluationProgress + (*GroupPerformance)(nil), // 18: agentevals.api.v1alpha1.GroupPerformance + (*EvaluateTracesResponse)(nil), // 19: agentevals.api.v1alpha1.EvaluateTracesResponse + nil, // 20: agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry + (*structpb.Struct)(nil), // 21: google.protobuf.Struct + (GroupBy)(0), // 22: agentevals.api.v1alpha1.GroupBy + (*TraceInput)(nil), // 23: agentevals.api.v1alpha1.TraceInput } var file_agentevals_api_v1alpha1_evaluation_proto_depIdxs = []int32{ 3, // 0: agentevals.api.v1alpha1.ListMetricsResponse.metrics:type_name -> agentevals.api.v1alpha1.Metric 0, // 1: agentevals.api.v1alpha1.BuiltinEvaluator.trajectory_match_type:type_name -> agentevals.api.v1alpha1.TrajectoryMatchType - 20, // 2: agentevals.api.v1alpha1.CodeEvaluator.config:type_name -> google.protobuf.Struct - 20, // 3: agentevals.api.v1alpha1.RemoteEvaluator.config:type_name -> google.protobuf.Struct - 5, // 4: agentevals.api.v1alpha1.Evaluator.builtin:type_name -> agentevals.api.v1alpha1.BuiltinEvaluator - 6, // 5: agentevals.api.v1alpha1.Evaluator.code:type_name -> agentevals.api.v1alpha1.CodeEvaluator - 7, // 6: agentevals.api.v1alpha1.Evaluator.remote:type_name -> agentevals.api.v1alpha1.RemoteEvaluator - 8, // 7: agentevals.api.v1alpha1.EvalParams.evaluators:type_name -> agentevals.api.v1alpha1.Evaluator - 21, // 8: agentevals.api.v1alpha1.EvalParams.group_by:type_name -> agentevals.api.v1alpha1.GroupBy - 1, // 9: agentevals.api.v1alpha1.InvocationResult.status:type_name -> agentevals.api.v1alpha1.EvalStatus - 1, // 10: agentevals.api.v1alpha1.MetricResult.eval_status:type_name -> agentevals.api.v1alpha1.EvalStatus - 10, // 11: agentevals.api.v1alpha1.MetricResult.per_invocation:type_name -> agentevals.api.v1alpha1.InvocationResult - 20, // 12: agentevals.api.v1alpha1.MetricResult.details:type_name -> google.protobuf.Struct - 11, // 13: agentevals.api.v1alpha1.TraceResult.metric_results:type_name -> agentevals.api.v1alpha1.MetricResult - 20, // 14: agentevals.api.v1alpha1.TraceResult.performance_metrics:type_name -> google.protobuf.Struct - 12, // 15: agentevals.api.v1alpha1.TraceResult.turn_refs:type_name -> agentevals.api.v1alpha1.TurnRef - 13, // 16: agentevals.api.v1alpha1.RunResult.trace_results:type_name -> agentevals.api.v1alpha1.TraceResult - 20, // 17: agentevals.api.v1alpha1.RunResult.performance_metrics:type_name -> google.protobuf.Struct - 22, // 18: agentevals.api.v1alpha1.EvaluateTracesRequest.traces:type_name -> agentevals.api.v1alpha1.TraceInput - 20, // 19: agentevals.api.v1alpha1.EvaluateTracesRequest.adk_eval_set:type_name -> google.protobuf.Struct - 9, // 20: agentevals.api.v1alpha1.EvaluateTracesRequest.params:type_name -> agentevals.api.v1alpha1.EvalParams - 19, // 21: agentevals.api.v1alpha1.EvaluateTracesRequest.credential_refs:type_name -> agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry - 20, // 22: agentevals.api.v1alpha1.GroupPerformance.performance_metrics:type_name -> google.protobuf.Struct - 20, // 23: agentevals.api.v1alpha1.GroupPerformance.trace_metadata:type_name -> google.protobuf.Struct - 16, // 24: agentevals.api.v1alpha1.EvaluateTracesResponse.progress:type_name -> agentevals.api.v1alpha1.EvaluationProgress - 17, // 25: agentevals.api.v1alpha1.EvaluateTracesResponse.performance:type_name -> agentevals.api.v1alpha1.GroupPerformance - 13, // 26: agentevals.api.v1alpha1.EvaluateTracesResponse.trace_result:type_name -> agentevals.api.v1alpha1.TraceResult - 14, // 27: agentevals.api.v1alpha1.EvaluateTracesResponse.done:type_name -> agentevals.api.v1alpha1.RunResult - 20, // 28: agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry.value:type_name -> google.protobuf.Struct - 2, // 29: agentevals.api.v1alpha1.EvaluationService.ListMetrics:input_type -> agentevals.api.v1alpha1.ListMetricsRequest - 15, // 30: agentevals.api.v1alpha1.EvaluationService.EvaluateTraces:input_type -> agentevals.api.v1alpha1.EvaluateTracesRequest - 4, // 31: agentevals.api.v1alpha1.EvaluationService.ListMetrics:output_type -> agentevals.api.v1alpha1.ListMetricsResponse - 18, // 32: agentevals.api.v1alpha1.EvaluationService.EvaluateTraces:output_type -> agentevals.api.v1alpha1.EvaluateTracesResponse - 31, // [31:33] is the sub-list for method output_type - 29, // [29:31] is the sub-list for method input_type - 29, // [29:29] is the sub-list for extension type_name - 29, // [29:29] is the sub-list for extension extendee - 0, // [0:29] is the sub-list for field type_name + 6, // 2: agentevals.api.v1alpha1.BuiltinEvaluator.rubrics:type_name -> agentevals.api.v1alpha1.Rubric + 21, // 3: agentevals.api.v1alpha1.CodeEvaluator.config:type_name -> google.protobuf.Struct + 21, // 4: agentevals.api.v1alpha1.RemoteEvaluator.config:type_name -> google.protobuf.Struct + 5, // 5: agentevals.api.v1alpha1.Evaluator.builtin:type_name -> agentevals.api.v1alpha1.BuiltinEvaluator + 7, // 6: agentevals.api.v1alpha1.Evaluator.code:type_name -> agentevals.api.v1alpha1.CodeEvaluator + 8, // 7: agentevals.api.v1alpha1.Evaluator.remote:type_name -> agentevals.api.v1alpha1.RemoteEvaluator + 9, // 8: agentevals.api.v1alpha1.EvalParams.evaluators:type_name -> agentevals.api.v1alpha1.Evaluator + 22, // 9: agentevals.api.v1alpha1.EvalParams.group_by:type_name -> agentevals.api.v1alpha1.GroupBy + 1, // 10: agentevals.api.v1alpha1.InvocationResult.status:type_name -> agentevals.api.v1alpha1.EvalStatus + 1, // 11: agentevals.api.v1alpha1.MetricResult.eval_status:type_name -> agentevals.api.v1alpha1.EvalStatus + 11, // 12: agentevals.api.v1alpha1.MetricResult.per_invocation:type_name -> agentevals.api.v1alpha1.InvocationResult + 21, // 13: agentevals.api.v1alpha1.MetricResult.details:type_name -> google.protobuf.Struct + 12, // 14: agentevals.api.v1alpha1.TraceResult.metric_results:type_name -> agentevals.api.v1alpha1.MetricResult + 21, // 15: agentevals.api.v1alpha1.TraceResult.performance_metrics:type_name -> google.protobuf.Struct + 13, // 16: agentevals.api.v1alpha1.TraceResult.turn_refs:type_name -> agentevals.api.v1alpha1.TurnRef + 14, // 17: agentevals.api.v1alpha1.RunResult.trace_results:type_name -> agentevals.api.v1alpha1.TraceResult + 21, // 18: agentevals.api.v1alpha1.RunResult.performance_metrics:type_name -> google.protobuf.Struct + 23, // 19: agentevals.api.v1alpha1.EvaluateTracesRequest.traces:type_name -> agentevals.api.v1alpha1.TraceInput + 21, // 20: agentevals.api.v1alpha1.EvaluateTracesRequest.adk_eval_set:type_name -> google.protobuf.Struct + 10, // 21: agentevals.api.v1alpha1.EvaluateTracesRequest.params:type_name -> agentevals.api.v1alpha1.EvalParams + 20, // 22: agentevals.api.v1alpha1.EvaluateTracesRequest.credential_refs:type_name -> agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry + 21, // 23: agentevals.api.v1alpha1.GroupPerformance.performance_metrics:type_name -> google.protobuf.Struct + 21, // 24: agentevals.api.v1alpha1.GroupPerformance.trace_metadata:type_name -> google.protobuf.Struct + 17, // 25: agentevals.api.v1alpha1.EvaluateTracesResponse.progress:type_name -> agentevals.api.v1alpha1.EvaluationProgress + 18, // 26: agentevals.api.v1alpha1.EvaluateTracesResponse.performance:type_name -> agentevals.api.v1alpha1.GroupPerformance + 14, // 27: agentevals.api.v1alpha1.EvaluateTracesResponse.trace_result:type_name -> agentevals.api.v1alpha1.TraceResult + 15, // 28: agentevals.api.v1alpha1.EvaluateTracesResponse.done:type_name -> agentevals.api.v1alpha1.RunResult + 21, // 29: agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntry.value:type_name -> google.protobuf.Struct + 2, // 30: agentevals.api.v1alpha1.EvaluationService.ListMetrics:input_type -> agentevals.api.v1alpha1.ListMetricsRequest + 16, // 31: agentevals.api.v1alpha1.EvaluationService.EvaluateTraces:input_type -> agentevals.api.v1alpha1.EvaluateTracesRequest + 4, // 32: agentevals.api.v1alpha1.EvaluationService.ListMetrics:output_type -> agentevals.api.v1alpha1.ListMetricsResponse + 19, // 33: agentevals.api.v1alpha1.EvaluationService.EvaluateTraces:output_type -> agentevals.api.v1alpha1.EvaluateTracesResponse + 32, // [32:34] is the sub-list for method output_type + 30, // [30:32] is the sub-list for method input_type + 30, // [30:30] is the sub-list for extension type_name + 30, // [30:30] is the sub-list for extension extendee + 0, // [0:30] is the sub-list for field type_name } func init() { file_agentevals_api_v1alpha1_evaluation_proto_init() } @@ -1737,20 +1805,20 @@ func file_agentevals_api_v1alpha1_evaluation_proto_init() { } file_agentevals_api_v1alpha1_common_proto_init() file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[3].OneofWrappers = []any{} - file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[4].OneofWrappers = []any{} file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[5].OneofWrappers = []any{} - file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6].OneofWrappers = []any{ + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[6].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7].OneofWrappers = []any{ (*Evaluator_Builtin)(nil), (*Evaluator_Code)(nil), (*Evaluator_Remote)(nil), } - file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[7].OneofWrappers = []any{} file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[8].OneofWrappers = []any{} file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[9].OneofWrappers = []any{} - file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[13].OneofWrappers = []any{ + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[10].OneofWrappers = []any{} + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[14].OneofWrappers = []any{ (*EvaluateTracesRequest_AdkEvalSet)(nil), } - file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[16].OneofWrappers = []any{ + file_agentevals_api_v1alpha1_evaluation_proto_msgTypes[17].OneofWrappers = []any{ (*EvaluateTracesResponse_Progress)(nil), (*EvaluateTracesResponse_Performance)(nil), (*EvaluateTracesResponse_TraceResult)(nil), @@ -1762,7 +1830,7 @@ func file_agentevals_api_v1alpha1_evaluation_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_agentevals_api_v1alpha1_evaluation_proto_rawDesc), len(file_agentevals_api_v1alpha1_evaluation_proto_rawDesc)), NumEnums: 2, - NumMessages: 18, + NumMessages: 19, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/agentevals/api/v1alpha1/evaluation.proto b/proto/agentevals/api/v1alpha1/evaluation.proto index ebfb98c5..8c59a652 100644 --- a/proto/agentevals/api/v1alpha1/evaluation.proto +++ b/proto/agentevals/api/v1alpha1/evaluation.proto @@ -27,7 +27,7 @@ message Metric { bool requires_llm = 5; // Needs Google Cloud credentials. bool requires_gcp = 6; - // Needs rubrics in the eval set. + // Needs rubrics, from the evaluator or the eval set. bool requires_rubrics = 7; // Usable with recorded traces. bool working = 8; @@ -56,6 +56,14 @@ message BuiltinEvaluator { string credential_ref = 5; // Base URL of an OpenAI compatible judge endpoint. string judge_base_url = 6; + // Rubrics for the rubric_based_* metrics that take them. Ids are unique. + repeated Rubric rubrics = 7; +} + +// One property a rubric metric's judge checks. +message Rubric { + string id = 1; + string text = 2; } // An evaluator file on the server, run as a subprocess. diff --git a/src/agentevals/api/rpc/convert.py b/src/agentevals/api/rpc/convert.py index e62a1102..777b6c97 100644 --- a/src/agentevals/api/rpc/convert.py +++ b/src/agentevals/api/rpc/convert.py @@ -179,6 +179,8 @@ def _evaluator_dict(evaluator: evaluation_pb2.Evaluator) -> dict[str, Any]: out[field] = getattr(message, field) if message.trajectory_match_type in TRAJECTORY_MATCH_FROM_PB: out["trajectory_match_type"] = TRAJECTORY_MATCH_FROM_PB[message.trajectory_match_type] + if message.rubrics: + out["rubrics"] = [{"id": r.id, "text": r.text} for r in message.rubrics] return out if message.HasField("timeout"): out["timeout"] = message.timeout @@ -223,6 +225,7 @@ def _evaluator_to_pb(evaluator: Any) -> evaluation_pb2.Evaluator: trajectory_match_type=TRAJECTORY_MATCH.get(evaluator.trajectory_match_type or "", 0), credential_ref=evaluator.credential_ref or "", judge_base_url=evaluator.judge_base_url or "", + rubrics=[evaluation_pb2.Rubric(id=r.id, text=r.text) for r in evaluator.rubrics or []], ) if evaluator.threshold is not None: builtin.threshold = evaluator.threshold diff --git a/src/agentevals/api/v1alpha1/evaluation_pb2.py b/src/agentevals/api/v1alpha1/evaluation_pb2.py index 821d2cce..ad0e12a0 100644 --- a/src/agentevals/api/v1alpha1/evaluation_pb2.py +++ b/src/agentevals/api/v1alpha1/evaluation_pb2.py @@ -26,7 +26,7 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n(agentevals/api/v1alpha1/evaluation.proto\x12\x17\x61gentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a\x1cgoogle/protobuf/struct.proto\"\x14\n\x12ListMetricsRequest\"\x91\x02\n\x06Metric\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n\x08\x63\x61tegory\x18\x02 \x01(\tR\x08\x63\x61tegory\x12 \n\x0b\x64\x65scription\x18\x03 \x01(\tR\x0b\x64\x65scription\x12*\n\x11requires_eval_set\x18\x04 \x01(\x08R\x0frequiresEvalSet\x12!\n\x0crequires_llm\x18\x05 \x01(\x08R\x0brequiresLlm\x12!\n\x0crequires_gcp\x18\x06 \x01(\x08R\x0brequiresGcp\x12)\n\x10requires_rubrics\x18\x07 \x01(\x08R\x0frequiresRubrics\x12\x18\n\x07working\x18\x08 \x01(\x08R\x07working\"P\n\x13ListMetricsResponse\x12\x39\n\x07metrics\x18\x01 \x03(\x0b\x32\x1f.agentevals.api.v1alpha1.MetricR\x07metrics\"\xa7\x02\n\x10\x42uiltinEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1f\n\x0bjudge_model\x18\x03 \x01(\tR\njudgeModel\x12`\n\x15trajectory_match_type\x18\x04 \x01(\x0e\x32,.agentevals.api.v1alpha1.TrajectoryMatchTypeR\x13trajectoryMatchType\x12%\n\x0e\x63redential_ref\x18\x05 \x01(\tR\rcredentialRef\x12$\n\x0ejudge_base_url\x18\x06 \x01(\tR\x0cjudgeBaseUrlB\x0c\n\n_threshold\"\xe0\x01\n\rCodeEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n\x07timeout\x18\x03 \x01(\x05H\x01R\x07timeout\x88\x01\x01\x12/\n\x06\x63onfig\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63onfig\x12\x1a\n\x08\x65xecutor\x18\x05 \x01(\tR\x08\x65xecutor\x12\x12\n\x04path\x18\x06 \x01(\tR\x04pathB\x0c\n\n_thresholdB\n\n\x08_timeout\"\xf8\x01\n\x0fRemoteEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n\x07timeout\x18\x03 \x01(\x05H\x01R\x07timeout\x88\x01\x01\x12/\n\x06\x63onfig\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63onfig\x12\x1a\n\x08\x65xecutor\x18\x05 \x01(\tR\x08\x65xecutor\x12\x16\n\x06source\x18\x06 \x01(\tR\x06source\x12\x10\n\x03ref\x18\x07 \x01(\tR\x03refB\x0c\n\n_thresholdB\n\n\x08_timeout\"\xdc\x01\n\tEvaluator\x12\x45\n\x07\x62uiltin\x18\x01 \x01(\x0b\x32).agentevals.api.v1alpha1.BuiltinEvaluatorH\x00R\x07\x62uiltin\x12<\n\x04\x63ode\x18\x02 \x01(\x0b\x32&.agentevals.api.v1alpha1.CodeEvaluatorH\x00R\x04\x63ode\x12\x42\n\x06remote\x18\x03 \x01(\x0b\x32(.agentevals.api.v1alpha1.RemoteEvaluatorH\x00R\x06remoteB\x06\n\x04kind\"\xb0\x02\n\nEvalParams\x12\x42\n\nevaluators\x18\x01 \x03(\x0b\x32\".agentevals.api.v1alpha1.EvaluatorR\nevaluators\x12\x37\n\x15max_concurrent_traces\x18\x02 \x01(\x05H\x00R\x13maxConcurrentTraces\x88\x01\x01\x12\x35\n\x14max_concurrent_evals\x18\x03 \x01(\x05H\x01R\x12maxConcurrentEvals\x88\x01\x01\x12;\n\x08group_by\x18\x04 \x01(\x0e\x32 .agentevals.api.v1alpha1.GroupByR\x07groupByB\x18\n\x16_max_concurrent_tracesB\x17\n\x15_max_concurrent_evals\"t\n\x10InvocationResult\x12\x19\n\x05score\x18\x01 \x01(\x01H\x00R\x05score\x88\x01\x01\x12;\n\x06status\x18\x02 \x01(\x0e\x32#.agentevals.api.v1alpha1.EvalStatusR\x06statusB\x08\n\x06_score\"\xad\x03\n\x0cMetricResult\x12\x1f\n\x0bmetric_name\x18\x01 \x01(\tR\nmetricName\x12\x19\n\x05score\x18\x02 \x01(\x01H\x00R\x05score\x88\x01\x01\x12\x44\n\x0b\x65val_status\x18\x03 \x01(\x0e\x32#.agentevals.api.v1alpha1.EvalStatusR\nevalStatus\x12P\n\x0eper_invocation\x18\x04 \x03(\x0b\x32).agentevals.api.v1alpha1.InvocationResultR\rperInvocation\x12\x19\n\x05\x65rror\x18\x05 \x01(\tH\x01R\x05\x65rror\x88\x01\x01\x12\"\n\nerror_type\x18\x06 \x01(\tH\x02R\terrorType\x88\x01\x01\x12\x31\n\x07\x64\x65tails\x18\x07 \x01(\x0b\x32\x17.google.protobuf.StructR\x07\x64\x65tails\x12$\n\x0b\x64uration_ms\x18\x08 \x01(\x01H\x03R\ndurationMs\x88\x01\x01\x42\x08\n\x06_scoreB\x08\n\x06_errorB\r\n\x0b_error_typeB\x0e\n\x0c_duration_ms\"=\n\x07TurnRef\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12\x17\n\x07span_id\x18\x02 \x01(\tR\x06spanId\"\xc8\x04\n\x0bTraceResult\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12\'\n\x0fnum_invocations\x18\x02 \x01(\x05R\x0enumInvocations\x12L\n\x0emetric_results\x18\x03 \x03(\x0b\x32%.agentevals.api.v1alpha1.MetricResultR\rmetricResults\x12/\n\x13\x63onversion_warnings\x18\x04 \x03(\tR\x12\x63onversionWarnings\x12H\n\x13performance_metrics\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12!\n\x0cservice_name\x18\x06 \x01(\tR\x0bserviceName\x12\x1d\n\nagent_name\x18\x07 \x01(\tR\tagentName\x12\x1b\n\ttrace_ids\x18\x08 \x03(\tR\x08traceIds\x12\'\n\x0f\x63onversation_id\x18\t \x01(\tR\x0e\x63onversationId\x12\x1b\n\tgroup_key\x18\n \x01(\tR\x08groupKey\x12 \n\x0c\x65val_case_id\x18\x0b \x01(\tR\nevalCaseId\x12&\n\x0f\x65val_case_match\x18\x0c \x01(\tR\revalCaseMatch\x12=\n\tturn_refs\x18\r \x03(\x0b\x32 .agentevals.api.v1alpha1.TurnRefR\x08turnRefs\"\xcf\x01\n\tRunResult\x12I\n\rtrace_results\x18\x01 \x03(\x0b\x32$.agentevals.api.v1alpha1.TraceResultR\x0ctraceResults\x12\x16\n\x06\x65rrors\x18\x02 \x03(\tR\x06\x65rrors\x12H\n\x13performance_metrics\x18\x03 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12\x15\n\x06run_id\x18\x04 \x01(\tR\x05runId\"\xa3\x03\n\x15\x45valuateTracesRequest\x12;\n\x06traces\x18\x01 \x01(\x0b\x32#.agentevals.api.v1alpha1.TraceInputR\x06traces\x12;\n\x0c\x61\x64k_eval_set\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructH\x00R\nadkEvalSet\x12;\n\x06params\x18\x03 \x01(\x0b\x32#.agentevals.api.v1alpha1.EvalParamsR\x06params\x12k\n\x0f\x63redential_refs\x18\x04 \x03(\x0b\x32\x42.agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntryR\x0e\x63redentialRefs\x1aZ\n\x13\x43redentialRefsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x05value:\x02\x38\x01\x42\n\n\x08\x65val_set\".\n\x12\x45valuationProgress\x12\x18\n\x07message\x18\x01 \x01(\tR\x07message\"\xb7\x01\n\x10GroupPerformance\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12H\n\x13performance_metrics\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12>\n\x0etrace_metadata\x18\x03 \x01(\x0b\x32\x17.google.protobuf.StructR\rtraceMetadata\"\xc0\x02\n\x16\x45valuateTracesResponse\x12I\n\x08progress\x18\x01 \x01(\x0b\x32+.agentevals.api.v1alpha1.EvaluationProgressH\x00R\x08progress\x12M\n\x0bperformance\x18\x02 \x01(\x0b\x32).agentevals.api.v1alpha1.GroupPerformanceH\x00R\x0bperformance\x12I\n\x0ctrace_result\x18\x03 \x01(\x0b\x32$.agentevals.api.v1alpha1.TraceResultH\x00R\x0btraceResult\x12\x38\n\x04\x64one\x18\x04 \x01(\x0b\x32\".agentevals.api.v1alpha1.RunResultH\x00R\x04\x64oneB\x07\n\x05\x65vent*\xa6\x01\n\x13TrajectoryMatchType\x12%\n!TRAJECTORY_MATCH_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bTRAJECTORY_MATCH_TYPE_EXACT\x10\x01\x12\"\n\x1eTRAJECTORY_MATCH_TYPE_IN_ORDER\x10\x02\x12#\n\x1fTRAJECTORY_MATCH_TYPE_ANY_ORDER\x10\x03*x\n\nEvalStatus\x12\x1b\n\x17\x45VAL_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12\x45VAL_STATUS_PASSED\x10\x01\x12\x16\n\x12\x45VAL_STATUS_FAILED\x10\x02\x12\x1d\n\x19\x45VAL_STATUS_NOT_EVALUATED\x10\x03\x32\xf2\x01\n\x11\x45valuationService\x12h\n\x0bListMetrics\x12+.agentevals.api.v1alpha1.ListMetricsRequest\x1a,.agentevals.api.v1alpha1.ListMetricsResponse\x12s\n\x0e\x45valuateTraces\x12..agentevals.api.v1alpha1.EvaluateTracesRequest\x1a/.agentevals.api.v1alpha1.EvaluateTracesResponse0\x01\x42QZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n(agentevals/api/v1alpha1/evaluation.proto\x12\x17\x61gentevals.api.v1alpha1\x1a$agentevals/api/v1alpha1/common.proto\x1a\x1cgoogle/protobuf/struct.proto\"\x14\n\x12ListMetricsRequest\"\x91\x02\n\x06Metric\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n\x08\x63\x61tegory\x18\x02 \x01(\tR\x08\x63\x61tegory\x12 \n\x0b\x64\x65scription\x18\x03 \x01(\tR\x0b\x64\x65scription\x12*\n\x11requires_eval_set\x18\x04 \x01(\x08R\x0frequiresEvalSet\x12!\n\x0crequires_llm\x18\x05 \x01(\x08R\x0brequiresLlm\x12!\n\x0crequires_gcp\x18\x06 \x01(\x08R\x0brequiresGcp\x12)\n\x10requires_rubrics\x18\x07 \x01(\x08R\x0frequiresRubrics\x12\x18\n\x07working\x18\x08 \x01(\x08R\x07working\"P\n\x13ListMetricsResponse\x12\x39\n\x07metrics\x18\x01 \x03(\x0b\x32\x1f.agentevals.api.v1alpha1.MetricR\x07metrics\"\xe2\x02\n\x10\x42uiltinEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1f\n\x0bjudge_model\x18\x03 \x01(\tR\njudgeModel\x12`\n\x15trajectory_match_type\x18\x04 \x01(\x0e\x32,.agentevals.api.v1alpha1.TrajectoryMatchTypeR\x13trajectoryMatchType\x12%\n\x0e\x63redential_ref\x18\x05 \x01(\tR\rcredentialRef\x12$\n\x0ejudge_base_url\x18\x06 \x01(\tR\x0cjudgeBaseUrl\x12\x39\n\x07rubrics\x18\x07 \x03(\x0b\x32\x1f.agentevals.api.v1alpha1.RubricR\x07rubricsB\x0c\n\n_threshold\",\n\x06Rubric\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n\x04text\x18\x02 \x01(\tR\x04text\"\xe0\x01\n\rCodeEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n\x07timeout\x18\x03 \x01(\x05H\x01R\x07timeout\x88\x01\x01\x12/\n\x06\x63onfig\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63onfig\x12\x1a\n\x08\x65xecutor\x18\x05 \x01(\tR\x08\x65xecutor\x12\x12\n\x04path\x18\x06 \x01(\tR\x04pathB\x0c\n\n_thresholdB\n\n\x08_timeout\"\xf8\x01\n\x0fRemoteEvaluator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12!\n\tthreshold\x18\x02 \x01(\x01H\x00R\tthreshold\x88\x01\x01\x12\x1d\n\x07timeout\x18\x03 \x01(\x05H\x01R\x07timeout\x88\x01\x01\x12/\n\x06\x63onfig\x18\x04 \x01(\x0b\x32\x17.google.protobuf.StructR\x06\x63onfig\x12\x1a\n\x08\x65xecutor\x18\x05 \x01(\tR\x08\x65xecutor\x12\x16\n\x06source\x18\x06 \x01(\tR\x06source\x12\x10\n\x03ref\x18\x07 \x01(\tR\x03refB\x0c\n\n_thresholdB\n\n\x08_timeout\"\xdc\x01\n\tEvaluator\x12\x45\n\x07\x62uiltin\x18\x01 \x01(\x0b\x32).agentevals.api.v1alpha1.BuiltinEvaluatorH\x00R\x07\x62uiltin\x12<\n\x04\x63ode\x18\x02 \x01(\x0b\x32&.agentevals.api.v1alpha1.CodeEvaluatorH\x00R\x04\x63ode\x12\x42\n\x06remote\x18\x03 \x01(\x0b\x32(.agentevals.api.v1alpha1.RemoteEvaluatorH\x00R\x06remoteB\x06\n\x04kind\"\xb0\x02\n\nEvalParams\x12\x42\n\nevaluators\x18\x01 \x03(\x0b\x32\".agentevals.api.v1alpha1.EvaluatorR\nevaluators\x12\x37\n\x15max_concurrent_traces\x18\x02 \x01(\x05H\x00R\x13maxConcurrentTraces\x88\x01\x01\x12\x35\n\x14max_concurrent_evals\x18\x03 \x01(\x05H\x01R\x12maxConcurrentEvals\x88\x01\x01\x12;\n\x08group_by\x18\x04 \x01(\x0e\x32 .agentevals.api.v1alpha1.GroupByR\x07groupByB\x18\n\x16_max_concurrent_tracesB\x17\n\x15_max_concurrent_evals\"t\n\x10InvocationResult\x12\x19\n\x05score\x18\x01 \x01(\x01H\x00R\x05score\x88\x01\x01\x12;\n\x06status\x18\x02 \x01(\x0e\x32#.agentevals.api.v1alpha1.EvalStatusR\x06statusB\x08\n\x06_score\"\xad\x03\n\x0cMetricResult\x12\x1f\n\x0bmetric_name\x18\x01 \x01(\tR\nmetricName\x12\x19\n\x05score\x18\x02 \x01(\x01H\x00R\x05score\x88\x01\x01\x12\x44\n\x0b\x65val_status\x18\x03 \x01(\x0e\x32#.agentevals.api.v1alpha1.EvalStatusR\nevalStatus\x12P\n\x0eper_invocation\x18\x04 \x03(\x0b\x32).agentevals.api.v1alpha1.InvocationResultR\rperInvocation\x12\x19\n\x05\x65rror\x18\x05 \x01(\tH\x01R\x05\x65rror\x88\x01\x01\x12\"\n\nerror_type\x18\x06 \x01(\tH\x02R\terrorType\x88\x01\x01\x12\x31\n\x07\x64\x65tails\x18\x07 \x01(\x0b\x32\x17.google.protobuf.StructR\x07\x64\x65tails\x12$\n\x0b\x64uration_ms\x18\x08 \x01(\x01H\x03R\ndurationMs\x88\x01\x01\x42\x08\n\x06_scoreB\x08\n\x06_errorB\r\n\x0b_error_typeB\x0e\n\x0c_duration_ms\"=\n\x07TurnRef\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12\x17\n\x07span_id\x18\x02 \x01(\tR\x06spanId\"\xc8\x04\n\x0bTraceResult\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12\'\n\x0fnum_invocations\x18\x02 \x01(\x05R\x0enumInvocations\x12L\n\x0emetric_results\x18\x03 \x03(\x0b\x32%.agentevals.api.v1alpha1.MetricResultR\rmetricResults\x12/\n\x13\x63onversion_warnings\x18\x04 \x03(\tR\x12\x63onversionWarnings\x12H\n\x13performance_metrics\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12!\n\x0cservice_name\x18\x06 \x01(\tR\x0bserviceName\x12\x1d\n\nagent_name\x18\x07 \x01(\tR\tagentName\x12\x1b\n\ttrace_ids\x18\x08 \x03(\tR\x08traceIds\x12\'\n\x0f\x63onversation_id\x18\t \x01(\tR\x0e\x63onversationId\x12\x1b\n\tgroup_key\x18\n \x01(\tR\x08groupKey\x12 \n\x0c\x65val_case_id\x18\x0b \x01(\tR\nevalCaseId\x12&\n\x0f\x65val_case_match\x18\x0c \x01(\tR\revalCaseMatch\x12=\n\tturn_refs\x18\r \x03(\x0b\x32 .agentevals.api.v1alpha1.TurnRefR\x08turnRefs\"\xcf\x01\n\tRunResult\x12I\n\rtrace_results\x18\x01 \x03(\x0b\x32$.agentevals.api.v1alpha1.TraceResultR\x0ctraceResults\x12\x16\n\x06\x65rrors\x18\x02 \x03(\tR\x06\x65rrors\x12H\n\x13performance_metrics\x18\x03 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12\x15\n\x06run_id\x18\x04 \x01(\tR\x05runId\"\xa3\x03\n\x15\x45valuateTracesRequest\x12;\n\x06traces\x18\x01 \x01(\x0b\x32#.agentevals.api.v1alpha1.TraceInputR\x06traces\x12;\n\x0c\x61\x64k_eval_set\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructH\x00R\nadkEvalSet\x12;\n\x06params\x18\x03 \x01(\x0b\x32#.agentevals.api.v1alpha1.EvalParamsR\x06params\x12k\n\x0f\x63redential_refs\x18\x04 \x03(\x0b\x32\x42.agentevals.api.v1alpha1.EvaluateTracesRequest.CredentialRefsEntryR\x0e\x63redentialRefs\x1aZ\n\x13\x43redentialRefsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x05value:\x02\x38\x01\x42\n\n\x08\x65val_set\".\n\x12\x45valuationProgress\x12\x18\n\x07message\x18\x01 \x01(\tR\x07message\"\xb7\x01\n\x10GroupPerformance\x12\x19\n\x08trace_id\x18\x01 \x01(\tR\x07traceId\x12H\n\x13performance_metrics\x18\x02 \x01(\x0b\x32\x17.google.protobuf.StructR\x12performanceMetrics\x12>\n\x0etrace_metadata\x18\x03 \x01(\x0b\x32\x17.google.protobuf.StructR\rtraceMetadata\"\xc0\x02\n\x16\x45valuateTracesResponse\x12I\n\x08progress\x18\x01 \x01(\x0b\x32+.agentevals.api.v1alpha1.EvaluationProgressH\x00R\x08progress\x12M\n\x0bperformance\x18\x02 \x01(\x0b\x32).agentevals.api.v1alpha1.GroupPerformanceH\x00R\x0bperformance\x12I\n\x0ctrace_result\x18\x03 \x01(\x0b\x32$.agentevals.api.v1alpha1.TraceResultH\x00R\x0btraceResult\x12\x38\n\x04\x64one\x18\x04 \x01(\x0b\x32\".agentevals.api.v1alpha1.RunResultH\x00R\x04\x64oneB\x07\n\x05\x65vent*\xa6\x01\n\x13TrajectoryMatchType\x12%\n!TRAJECTORY_MATCH_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bTRAJECTORY_MATCH_TYPE_EXACT\x10\x01\x12\"\n\x1eTRAJECTORY_MATCH_TYPE_IN_ORDER\x10\x02\x12#\n\x1fTRAJECTORY_MATCH_TYPE_ANY_ORDER\x10\x03*x\n\nEvalStatus\x12\x1b\n\x17\x45VAL_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12\x45VAL_STATUS_PASSED\x10\x01\x12\x16\n\x12\x45VAL_STATUS_FAILED\x10\x02\x12\x1d\n\x19\x45VAL_STATUS_NOT_EVALUATED\x10\x03\x32\xf2\x01\n\x11\x45valuationService\x12h\n\x0bListMetrics\x12+.agentevals.api.v1alpha1.ListMetricsRequest\x1a,.agentevals.api.v1alpha1.ListMetricsResponse\x12s\n\x0e\x45valuateTraces\x12..agentevals.api.v1alpha1.EvaluateTracesRequest\x1a/.agentevals.api.v1alpha1.EvaluateTracesResponse0\x01\x42QZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -36,10 +36,10 @@ _globals['DESCRIPTOR']._serialized_options = b'ZOgithub.com/agentevals-dev/agentevals/gen/go/agentevals/api/v1alpha1;apiv1alpha1' _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._loaded_options = None _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._serialized_options = b'8\001' - _globals['_TRAJECTORYMATCHTYPE']._serialized_start=4213 - _globals['_TRAJECTORYMATCHTYPE']._serialized_end=4379 - _globals['_EVALSTATUS']._serialized_start=4381 - _globals['_EVALSTATUS']._serialized_end=4501 + _globals['_TRAJECTORYMATCHTYPE']._serialized_start=4318 + _globals['_TRAJECTORYMATCHTYPE']._serialized_end=4484 + _globals['_EVALSTATUS']._serialized_start=4486 + _globals['_EVALSTATUS']._serialized_end=4606 _globals['_LISTMETRICSREQUEST']._serialized_start=137 _globals['_LISTMETRICSREQUEST']._serialized_end=157 _globals['_METRIC']._serialized_start=160 @@ -47,35 +47,37 @@ _globals['_LISTMETRICSRESPONSE']._serialized_start=435 _globals['_LISTMETRICSRESPONSE']._serialized_end=515 _globals['_BUILTINEVALUATOR']._serialized_start=518 - _globals['_BUILTINEVALUATOR']._serialized_end=813 - _globals['_CODEEVALUATOR']._serialized_start=816 - _globals['_CODEEVALUATOR']._serialized_end=1040 - _globals['_REMOTEEVALUATOR']._serialized_start=1043 - _globals['_REMOTEEVALUATOR']._serialized_end=1291 - _globals['_EVALUATOR']._serialized_start=1294 - _globals['_EVALUATOR']._serialized_end=1514 - _globals['_EVALPARAMS']._serialized_start=1517 - _globals['_EVALPARAMS']._serialized_end=1821 - _globals['_INVOCATIONRESULT']._serialized_start=1823 - _globals['_INVOCATIONRESULT']._serialized_end=1939 - _globals['_METRICRESULT']._serialized_start=1942 - _globals['_METRICRESULT']._serialized_end=2371 - _globals['_TURNREF']._serialized_start=2373 - _globals['_TURNREF']._serialized_end=2434 - _globals['_TRACERESULT']._serialized_start=2437 - _globals['_TRACERESULT']._serialized_end=3021 - _globals['_RUNRESULT']._serialized_start=3024 - _globals['_RUNRESULT']._serialized_end=3231 - _globals['_EVALUATETRACESREQUEST']._serialized_start=3234 - _globals['_EVALUATETRACESREQUEST']._serialized_end=3653 - _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._serialized_start=3551 - _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._serialized_end=3641 - _globals['_EVALUATIONPROGRESS']._serialized_start=3655 - _globals['_EVALUATIONPROGRESS']._serialized_end=3701 - _globals['_GROUPPERFORMANCE']._serialized_start=3704 - _globals['_GROUPPERFORMANCE']._serialized_end=3887 - _globals['_EVALUATETRACESRESPONSE']._serialized_start=3890 - _globals['_EVALUATETRACESRESPONSE']._serialized_end=4210 - _globals['_EVALUATIONSERVICE']._serialized_start=4504 - _globals['_EVALUATIONSERVICE']._serialized_end=4746 + _globals['_BUILTINEVALUATOR']._serialized_end=872 + _globals['_RUBRIC']._serialized_start=874 + _globals['_RUBRIC']._serialized_end=918 + _globals['_CODEEVALUATOR']._serialized_start=921 + _globals['_CODEEVALUATOR']._serialized_end=1145 + _globals['_REMOTEEVALUATOR']._serialized_start=1148 + _globals['_REMOTEEVALUATOR']._serialized_end=1396 + _globals['_EVALUATOR']._serialized_start=1399 + _globals['_EVALUATOR']._serialized_end=1619 + _globals['_EVALPARAMS']._serialized_start=1622 + _globals['_EVALPARAMS']._serialized_end=1926 + _globals['_INVOCATIONRESULT']._serialized_start=1928 + _globals['_INVOCATIONRESULT']._serialized_end=2044 + _globals['_METRICRESULT']._serialized_start=2047 + _globals['_METRICRESULT']._serialized_end=2476 + _globals['_TURNREF']._serialized_start=2478 + _globals['_TURNREF']._serialized_end=2539 + _globals['_TRACERESULT']._serialized_start=2542 + _globals['_TRACERESULT']._serialized_end=3126 + _globals['_RUNRESULT']._serialized_start=3129 + _globals['_RUNRESULT']._serialized_end=3336 + _globals['_EVALUATETRACESREQUEST']._serialized_start=3339 + _globals['_EVALUATETRACESREQUEST']._serialized_end=3758 + _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._serialized_start=3656 + _globals['_EVALUATETRACESREQUEST_CREDENTIALREFSENTRY']._serialized_end=3746 + _globals['_EVALUATIONPROGRESS']._serialized_start=3760 + _globals['_EVALUATIONPROGRESS']._serialized_end=3806 + _globals['_GROUPPERFORMANCE']._serialized_start=3809 + _globals['_GROUPPERFORMANCE']._serialized_end=3992 + _globals['_EVALUATETRACESRESPONSE']._serialized_start=3995 + _globals['_EVALUATETRACESRESPONSE']._serialized_end=4315 + _globals['_EVALUATIONSERVICE']._serialized_start=4609 + _globals['_EVALUATIONSERVICE']._serialized_end=4851 # @@protoc_insertion_point(module_scope) diff --git a/src/agentevals/api/v1alpha1/evaluation_pb2.pyi b/src/agentevals/api/v1alpha1/evaluation_pb2.pyi index 03da133c..c558529a 100644 --- a/src/agentevals/api/v1alpha1/evaluation_pb2.pyi +++ b/src/agentevals/api/v1alpha1/evaluation_pb2.pyi @@ -62,20 +62,30 @@ class ListMetricsResponse(_message.Message): def __init__(self, metrics: _Optional[_Iterable[_Union[Metric, _Mapping]]] = ...) -> None: ... class BuiltinEvaluator(_message.Message): - __slots__ = ("name", "threshold", "judge_model", "trajectory_match_type", "credential_ref", "judge_base_url") + __slots__ = ("name", "threshold", "judge_model", "trajectory_match_type", "credential_ref", "judge_base_url", "rubrics") NAME_FIELD_NUMBER: _ClassVar[int] THRESHOLD_FIELD_NUMBER: _ClassVar[int] JUDGE_MODEL_FIELD_NUMBER: _ClassVar[int] TRAJECTORY_MATCH_TYPE_FIELD_NUMBER: _ClassVar[int] CREDENTIAL_REF_FIELD_NUMBER: _ClassVar[int] JUDGE_BASE_URL_FIELD_NUMBER: _ClassVar[int] + RUBRICS_FIELD_NUMBER: _ClassVar[int] name: str threshold: float judge_model: str trajectory_match_type: TrajectoryMatchType credential_ref: str judge_base_url: str - def __init__(self, name: _Optional[str] = ..., threshold: _Optional[float] = ..., judge_model: _Optional[str] = ..., trajectory_match_type: _Optional[_Union[TrajectoryMatchType, str]] = ..., credential_ref: _Optional[str] = ..., judge_base_url: _Optional[str] = ...) -> None: ... + rubrics: _containers.RepeatedCompositeFieldContainer[Rubric] + def __init__(self, name: _Optional[str] = ..., threshold: _Optional[float] = ..., judge_model: _Optional[str] = ..., trajectory_match_type: _Optional[_Union[TrajectoryMatchType, str]] = ..., credential_ref: _Optional[str] = ..., judge_base_url: _Optional[str] = ..., rubrics: _Optional[_Iterable[_Union[Rubric, _Mapping]]] = ...) -> None: ... + +class Rubric(_message.Message): + __slots__ = ("id", "text") + ID_FIELD_NUMBER: _ClassVar[int] + TEXT_FIELD_NUMBER: _ClassVar[int] + id: str + text: str + def __init__(self, id: _Optional[str] = ..., text: _Optional[str] = ...) -> None: ... class CodeEvaluator(_message.Message): __slots__ = ("name", "threshold", "timeout", "config", "executor", "path") diff --git a/src/agentevals/builtin_metrics.py b/src/agentevals/builtin_metrics.py index 275abadf..9e9a6b34 100644 --- a/src/agentevals/builtin_metrics.py +++ b/src/agentevals/builtin_metrics.py @@ -28,6 +28,7 @@ from google.adk.evaluation.eval_rubrics import Rubric, RubricContent from google.adk.evaluation.evaluator import EvaluationResult, Evaluator +from .config import RUBRIC_METRIC_TYPES from .resolvers import get_resolved_credential logger = logging.getLogger(__name__) @@ -106,7 +107,7 @@ def metric_catalog() -> list[MetricDescription]: requires_llm=name in METRICS_NEEDING_LLM, requires_gcp=name in METRICS_NEEDING_GCP, requires_rubrics=rubrics, - working=not rubrics, + working=not rubrics or name in RUBRIC_METRIC_TYPES, ) ) return out @@ -224,18 +225,6 @@ def _enrich_app_details(invocations: list[Invocation]) -> list[Invocation]: return [inv.model_copy(update={"app_details": app_details}) for inv in invocations] -METRICS_NEEDING_RUBRICS = { - "rubric_based_final_response_quality_v1", - "rubric_based_tool_use_quality_v1", -} - -# ADK filters case and invocation rubrics by type, but not criterion rubrics. -_METRIC_RUBRIC_TYPES = { - "rubric_based_final_response_quality_v1": "FINAL_RESPONSE_QUALITY", - "rubric_based_tool_use_quality_v1": "TOOL_USE_QUALITY", -} - - def rubric_strings_to_objects(rubric_texts: list[str]) -> list[Rubric]: """Convert plain-text rubric strings into ADK Rubric objects.""" return [ @@ -267,7 +256,7 @@ def attach_case_rubrics( Check effective properties before the first judge call, and report dropped properties by id and source instead of silently defaulting their type. """ - metric_type = _METRIC_RUBRIC_TYPES.get(metric_name) + metric_type = RUBRIC_METRIC_TYPES.get(metric_name) attached = [] warnings = [] for index, actual in enumerate(actual_invocations): @@ -357,10 +346,7 @@ def build_eval_metric( threshold=effective_threshold, judge_model_options=judge_opts, ) - elif metric_name in ( - "rubric_based_final_response_quality_v1", - "rubric_based_tool_use_quality_v1", - ): + elif metric_name in RUBRIC_METRIC_TYPES: judge_opts = JudgeModelOptions() if judge_model: judge_opts.judge_model = judge_model @@ -541,7 +527,7 @@ async def evaluate_builtin_metric( ) rubric_warnings: list[str] = [] - if metric_name in METRICS_NEEDING_RUBRICS: + if metric_name in RUBRIC_METRIC_TYPES: rubric_objects = to_rubric_objects(rubrics) try: actual_invocations, rubric_warnings = attach_case_rubrics( @@ -552,7 +538,7 @@ async def evaluate_builtin_metric( missing = [ index for index, inv in enumerate(actual_invocations) - if not rubric_objects and not any(r.type == _METRIC_RUBRIC_TYPES[metric_name] for r in inv.rubrics or []) + if not rubric_objects and not any(r.type == RUBRIC_METRIC_TYPES[metric_name] for r in inv.rubrics or []) ] if missing or not actual_invocations: return MetricResult( @@ -604,7 +590,7 @@ async def evaluate_builtin_metric( details = None if metric_name == "tool_trajectory_avg_score": details = extract_trajectory_details(eval_result) - elif metric_name in METRICS_NEEDING_RUBRICS: + elif metric_name in RUBRIC_METRIC_TYPES: details = extract_rubric_details(eval_result) if rubric_warnings: details["warnings"] = rubric_warnings diff --git a/src/agentevals/config.py b/src/agentevals/config.py index 1a77f41f..e77d67ba 100644 --- a/src/agentevals/config.py +++ b/src/agentevals/config.py @@ -22,6 +22,13 @@ def _normalize_trajectory_match_type(v: str | None) -> str | None: "per_turn_user_simulator_quality_v1": "it scores a simulated user, and agentevals evaluates recorded traces", } +# Rubric metrics that take rubrics, with the ADK rubric type each one keeps from eval case and +# invocation rubrics. ADK never filters config rubrics by type. +RUBRIC_METRIC_TYPES = { + "rubric_based_final_response_quality_v1": "FINAL_RESPONSE_QUALITY", + "rubric_based_tool_use_quality_v1": "TOOL_USE_QUALITY", +} + GroupBy = Literal["auto", "trace", "conversation"] @@ -95,6 +102,15 @@ def _unique_rubric_ids(cls, value: list[RubricDef] | None) -> list[RubricDef] | raise ValueError("Rubric ids must be unique within a metric. Duplicate ids: " + ", ".join(duplicates)) return value + @model_validator(mode="after") + def _rubrics_only_on_rubric_metrics(self) -> BuiltinMetricDef: + if self.rubrics and self.name not in RUBRIC_METRIC_TYPES: + raise ValueError( + f"Built-in metric '{self.name}' does not take rubrics. " + f"Metrics that do: {', '.join(sorted(RUBRIC_METRIC_TYPES))}" + ) + return self + class BaseEvaluatorDef(BaseModel): """Shared fields for all executable evaluator definitions.""" diff --git a/tests/test_builtin_metrics.py b/tests/test_builtin_metrics.py index e5f87f08..b2749124 100644 --- a/tests/test_builtin_metrics.py +++ b/tests/test_builtin_metrics.py @@ -29,6 +29,7 @@ from agentevals import builtin_metrics from agentevals.adk_bridge import to_adk_invocations from agentevals.builtin_metrics import _to_invocation_events, evaluate_builtin_metric +from agentevals.config import RUBRIC_METRIC_TYPES from agentevals.genai.extract import extract_conversation from agentevals.loader import load_traces @@ -293,7 +294,7 @@ async def test_every_prebuilt_metric_receives_invocation_events(monkeypatch, met [expected], None, 0.5, - rubrics=["Uses captured evidence."] if metric_name in builtin_metrics.METRICS_NEEDING_RUBRICS else None, + rubrics=["Uses captured evidence."] if metric_name in RUBRIC_METRIC_TYPES else None, ) assert result.error is None diff --git a/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.d.ts b/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.d.ts index b344014e..57e9e841 100644 --- a/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.d.ts +++ b/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.d.ts @@ -64,7 +64,7 @@ export declare type Metric = Message<"agentevals.api.v1alpha1.Metric"> & { requiresGcp: boolean; /** - * Needs rubrics in the eval set. + * Needs rubrics, from the evaluator or the eval set. * * @generated from field: bool requires_rubrics = 7; */ @@ -141,6 +141,13 @@ export declare type BuiltinEvaluator = Message<"agentevals.api.v1alpha1.BuiltinE * @generated from field: string judge_base_url = 6; */ judgeBaseUrl: string; + + /** + * Rubrics for the rubric_based_* metrics that take them. Ids are unique. + * + * @generated from field: repeated agentevals.api.v1alpha1.Rubric rubrics = 7; + */ + rubrics: Rubric[]; }; /** @@ -149,6 +156,29 @@ export declare type BuiltinEvaluator = Message<"agentevals.api.v1alpha1.BuiltinE */ export declare const BuiltinEvaluatorSchema: GenMessage; +/** + * One property a rubric metric's judge checks. + * + * @generated from message agentevals.api.v1alpha1.Rubric + */ +export declare type Rubric = Message<"agentevals.api.v1alpha1.Rubric"> & { + /** + * @generated from field: string id = 1; + */ + id: string; + + /** + * @generated from field: string text = 2; + */ + text: string; +}; + +/** + * Describes the message agentevals.api.v1alpha1.Rubric. + * Use `create(RubricSchema)` to create a new message. + */ +export declare const RubricSchema: GenMessage; + /** * An evaluator file on the server, run as a subprocess. * diff --git a/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.js b/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.js index f3f758b7..abe5dfd4 100644 --- a/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.js +++ b/ui/src/generated/agentevals/api/v1alpha1/evaluation_pb.js @@ -10,7 +10,7 @@ import { file_google_protobuf_struct } from "@bufbuild/protobuf/wkt"; * Describes the file agentevals/api/v1alpha1/evaluation.proto. */ export const file_agentevals_api_v1alpha1_evaluation = /*@__PURE__*/ - fileDesc("CihhZ2VudGV2YWxzL2FwaS92MWFscGhhMS9ldmFsdWF0aW9uLnByb3RvEhdhZ2VudGV2YWxzLmFwaS52MWFscGhhMSIUChJMaXN0TWV0cmljc1JlcXVlc3QirwEKBk1ldHJpYxIMCgRuYW1lGAEgASgJEhAKCGNhdGVnb3J5GAIgASgJEhMKC2Rlc2NyaXB0aW9uGAMgASgJEhkKEXJlcXVpcmVzX2V2YWxfc2V0GAQgASgIEhQKDHJlcXVpcmVzX2xsbRgFIAEoCBIUCgxyZXF1aXJlc19nY3AYBiABKAgSGAoQcmVxdWlyZXNfcnVicmljcxgHIAEoCBIPCgd3b3JraW5nGAggASgIIkcKE0xpc3RNZXRyaWNzUmVzcG9uc2USMAoHbWV0cmljcxgBIAMoCzIfLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLk1ldHJpYyLYAQoQQnVpbHRpbkV2YWx1YXRvchIMCgRuYW1lGAEgASgJEhYKCXRocmVzaG9sZBgCIAEoAUgAiAEBEhMKC2p1ZGdlX21vZGVsGAMgASgJEksKFXRyYWplY3RvcnlfbWF0Y2hfdHlwZRgEIAEoDjIsLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlRyYWplY3RvcnlNYXRjaFR5cGUSFgoOY3JlZGVudGlhbF9yZWYYBSABKAkSFgoOanVkZ2VfYmFzZV91cmwYBiABKAlCDAoKX3RocmVzaG9sZCKuAQoNQ29kZUV2YWx1YXRvchIMCgRuYW1lGAEgASgJEhYKCXRocmVzaG9sZBgCIAEoAUgAiAEBEhQKB3RpbWVvdXQYAyABKAVIAYgBARInCgZjb25maWcYBCABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0EhAKCGV4ZWN1dG9yGAUgASgJEgwKBHBhdGgYBiABKAlCDAoKX3RocmVzaG9sZEIKCghfdGltZW91dCK/AQoPUmVtb3RlRXZhbHVhdG9yEgwKBG5hbWUYASABKAkSFgoJdGhyZXNob2xkGAIgASgBSACIAQESFAoHdGltZW91dBgDIAEoBUgBiAEBEicKBmNvbmZpZxgEIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSEAoIZXhlY3V0b3IYBSABKAkSDgoGc291cmNlGAYgASgJEgsKA3JlZhgHIAEoCUIMCgpfdGhyZXNob2xkQgoKCF90aW1lb3V0IsUBCglFdmFsdWF0b3ISPAoHYnVpbHRpbhgBIAEoCzIpLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkJ1aWx0aW5FdmFsdWF0b3JIABI2CgRjb2RlGAIgASgLMiYuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuQ29kZUV2YWx1YXRvckgAEjoKBnJlbW90ZRgDIAEoCzIoLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJlbW90ZUV2YWx1YXRvckgAQgYKBGtpbmQi8gEKCkV2YWxQYXJhbXMSNgoKZXZhbHVhdG9ycxgBIAMoCzIiLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkV2YWx1YXRvchIiChVtYXhfY29uY3VycmVudF90cmFjZXMYAiABKAVIAIgBARIhChRtYXhfY29uY3VycmVudF9ldmFscxgDIAEoBUgBiAEBEjIKCGdyb3VwX2J5GAQgASgOMiAuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR3JvdXBCeUIYChZfbWF4X2NvbmN1cnJlbnRfdHJhY2VzQhcKFV9tYXhfY29uY3VycmVudF9ldmFscyJlChBJbnZvY2F0aW9uUmVzdWx0EhIKBXNjb3JlGAEgASgBSACIAQESMwoGc3RhdHVzGAIgASgOMiMuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuRXZhbFN0YXR1c0IICgZfc2NvcmUi2AIKDE1ldHJpY1Jlc3VsdBITCgttZXRyaWNfbmFtZRgBIAEoCRISCgVzY29yZRgCIAEoAUgAiAEBEjgKC2V2YWxfc3RhdHVzGAMgASgOMiMuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuRXZhbFN0YXR1cxJBCg5wZXJfaW52b2NhdGlvbhgEIAMoCzIpLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkludm9jYXRpb25SZXN1bHQSEgoFZXJyb3IYBSABKAlIAYgBARIXCgplcnJvcl90eXBlGAYgASgJSAKIAQESKAoHZGV0YWlscxgHIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSGAoLZHVyYXRpb25fbXMYCCABKAFIA4gBAUIICgZfc2NvcmVCCAoGX2Vycm9yQg0KC19lcnJvcl90eXBlQg4KDF9kdXJhdGlvbl9tcyIsCgdUdXJuUmVmEhAKCHRyYWNlX2lkGAEgASgJEg8KB3NwYW5faWQYAiABKAkilwMKC1RyYWNlUmVzdWx0EhAKCHRyYWNlX2lkGAEgASgJEhcKD251bV9pbnZvY2F0aW9ucxgCIAEoBRI9Cg5tZXRyaWNfcmVzdWx0cxgDIAMoCzIlLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLk1ldHJpY1Jlc3VsdBIbChNjb252ZXJzaW9uX3dhcm5pbmdzGAQgAygJEjQKE3BlcmZvcm1hbmNlX21ldHJpY3MYBSABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0EhQKDHNlcnZpY2VfbmFtZRgGIAEoCRISCgphZ2VudF9uYW1lGAcgASgJEhEKCXRyYWNlX2lkcxgIIAMoCRIXCg9jb252ZXJzYXRpb25faWQYCSABKAkSEQoJZ3JvdXBfa2V5GAogASgJEhQKDGV2YWxfY2FzZV9pZBgLIAEoCRIXCg9ldmFsX2Nhc2VfbWF0Y2gYDCABKAkSMwoJdHVybl9yZWZzGA0gAygLMiAuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHVyblJlZiKeAQoJUnVuUmVzdWx0EjsKDXRyYWNlX3Jlc3VsdHMYASADKAsyJC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5UcmFjZVJlc3VsdBIOCgZlcnJvcnMYAiADKAkSNAoTcGVyZm9ybWFuY2VfbWV0cmljcxgDIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSDgoGcnVuX2lkGAQgASgJIusCChVFdmFsdWF0ZVRyYWNlc1JlcXVlc3QSMwoGdHJhY2VzGAEgASgLMiMuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHJhY2VJbnB1dBIvCgxhZGtfZXZhbF9zZXQYAiABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0SAASMwoGcGFyYW1zGAMgASgLMiMuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuRXZhbFBhcmFtcxJbCg9jcmVkZW50aWFsX3JlZnMYBCADKAsyQi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsdWF0ZVRyYWNlc1JlcXVlc3QuQ3JlZGVudGlhbFJlZnNFbnRyeRpOChNDcmVkZW50aWFsUmVmc0VudHJ5EgsKA2tleRgBIAEoCRImCgV2YWx1ZRgCIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3Q6AjgBQgoKCGV2YWxfc2V0IiUKEkV2YWx1YXRpb25Qcm9ncmVzcxIPCgdtZXNzYWdlGAEgASgJIosBChBHcm91cFBlcmZvcm1hbmNlEhAKCHRyYWNlX2lkGAEgASgJEjQKE3BlcmZvcm1hbmNlX21ldHJpY3MYAiABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0Ei8KDnRyYWNlX21ldGFkYXRhGAMgASgLMhcuZ29vZ2xlLnByb3RvYnVmLlN0cnVjdCKWAgoWRXZhbHVhdGVUcmFjZXNSZXNwb25zZRI/Cghwcm9ncmVzcxgBIAEoCzIrLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkV2YWx1YXRpb25Qcm9ncmVzc0gAEkAKC3BlcmZvcm1hbmNlGAIgASgLMikuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuR3JvdXBQZXJmb3JtYW5jZUgAEjwKDHRyYWNlX3Jlc3VsdBgDIAEoCzIkLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlRyYWNlUmVzdWx0SAASMgoEZG9uZRgEIAEoCzIiLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1blJlc3VsdEgAQgcKBWV2ZW50KqYBChNUcmFqZWN0b3J5TWF0Y2hUeXBlEiUKIVRSQUpFQ1RPUllfTUFUQ0hfVFlQRV9VTlNQRUNJRklFRBAAEh8KG1RSQUpFQ1RPUllfTUFUQ0hfVFlQRV9FWEFDVBABEiIKHlRSQUpFQ1RPUllfTUFUQ0hfVFlQRV9JTl9PUkRFUhACEiMKH1RSQUpFQ1RPUllfTUFUQ0hfVFlQRV9BTllfT1JERVIQAyp4CgpFdmFsU3RhdHVzEhsKF0VWQUxfU1RBVFVTX1VOU1BFQ0lGSUVEEAASFgoSRVZBTF9TVEFUVVNfUEFTU0VEEAESFgoSRVZBTF9TVEFUVVNfRkFJTEVEEAISHQoZRVZBTF9TVEFUVVNfTk9UX0VWQUxVQVRFRBADMvIBChFFdmFsdWF0aW9uU2VydmljZRJoCgtMaXN0TWV0cmljcxIrLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkxpc3RNZXRyaWNzUmVxdWVzdBosLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkxpc3RNZXRyaWNzUmVzcG9uc2UScwoORXZhbHVhdGVUcmFjZXMSLi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsdWF0ZVRyYWNlc1JlcXVlc3QaLy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsdWF0ZVRyYWNlc1Jlc3BvbnNlMAFCUVpPZ2l0aHViLmNvbS9hZ2VudGV2YWxzLWRldi9hZ2VudGV2YWxzL2dlbi9nby9hZ2VudGV2YWxzL2FwaS92MWFscGhhMTthcGl2MWFscGhhMWIGcHJvdG8z", [file_agentevals_api_v1alpha1_common, file_google_protobuf_struct]); + fileDesc("CihhZ2VudGV2YWxzL2FwaS92MWFscGhhMS9ldmFsdWF0aW9uLnByb3RvEhdhZ2VudGV2YWxzLmFwaS52MWFscGhhMSIUChJMaXN0TWV0cmljc1JlcXVlc3QirwEKBk1ldHJpYxIMCgRuYW1lGAEgASgJEhAKCGNhdGVnb3J5GAIgASgJEhMKC2Rlc2NyaXB0aW9uGAMgASgJEhkKEXJlcXVpcmVzX2V2YWxfc2V0GAQgASgIEhQKDHJlcXVpcmVzX2xsbRgFIAEoCBIUCgxyZXF1aXJlc19nY3AYBiABKAgSGAoQcmVxdWlyZXNfcnVicmljcxgHIAEoCBIPCgd3b3JraW5nGAggASgIIkcKE0xpc3RNZXRyaWNzUmVzcG9uc2USMAoHbWV0cmljcxgBIAMoCzIfLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLk1ldHJpYyKKAgoQQnVpbHRpbkV2YWx1YXRvchIMCgRuYW1lGAEgASgJEhYKCXRocmVzaG9sZBgCIAEoAUgAiAEBEhMKC2p1ZGdlX21vZGVsGAMgASgJEksKFXRyYWplY3RvcnlfbWF0Y2hfdHlwZRgEIAEoDjIsLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlRyYWplY3RvcnlNYXRjaFR5cGUSFgoOY3JlZGVudGlhbF9yZWYYBSABKAkSFgoOanVkZ2VfYmFzZV91cmwYBiABKAkSMAoHcnVicmljcxgHIAMoCzIfLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlJ1YnJpY0IMCgpfdGhyZXNob2xkIiIKBlJ1YnJpYxIKCgJpZBgBIAEoCRIMCgR0ZXh0GAIgASgJIq4BCg1Db2RlRXZhbHVhdG9yEgwKBG5hbWUYASABKAkSFgoJdGhyZXNob2xkGAIgASgBSACIAQESFAoHdGltZW91dBgDIAEoBUgBiAEBEicKBmNvbmZpZxgEIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSEAoIZXhlY3V0b3IYBSABKAkSDAoEcGF0aBgGIAEoCUIMCgpfdGhyZXNob2xkQgoKCF90aW1lb3V0Ir8BCg9SZW1vdGVFdmFsdWF0b3ISDAoEbmFtZRgBIAEoCRIWCgl0aHJlc2hvbGQYAiABKAFIAIgBARIUCgd0aW1lb3V0GAMgASgFSAGIAQESJwoGY29uZmlnGAQgASgLMhcuZ29vZ2xlLnByb3RvYnVmLlN0cnVjdBIQCghleGVjdXRvchgFIAEoCRIOCgZzb3VyY2UYBiABKAkSCwoDcmVmGAcgASgJQgwKCl90aHJlc2hvbGRCCgoIX3RpbWVvdXQixQEKCUV2YWx1YXRvchI8CgdidWlsdGluGAEgASgLMikuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuQnVpbHRpbkV2YWx1YXRvckgAEjYKBGNvZGUYAiABKAsyJi5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5Db2RlRXZhbHVhdG9ySAASOgoGcmVtb3RlGAMgASgLMiguYWdlbnRldmFscy5hcGkudjFhbHBoYTEuUmVtb3RlRXZhbHVhdG9ySABCBgoEa2luZCLyAQoKRXZhbFBhcmFtcxI2CgpldmFsdWF0b3JzGAEgAygLMiIuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuRXZhbHVhdG9yEiIKFW1heF9jb25jdXJyZW50X3RyYWNlcxgCIAEoBUgAiAEBEiEKFG1heF9jb25jdXJyZW50X2V2YWxzGAMgASgFSAGIAQESMgoIZ3JvdXBfYnkYBCABKA4yIC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5Hcm91cEJ5QhgKFl9tYXhfY29uY3VycmVudF90cmFjZXNCFwoVX21heF9jb25jdXJyZW50X2V2YWxzImUKEEludm9jYXRpb25SZXN1bHQSEgoFc2NvcmUYASABKAFIAIgBARIzCgZzdGF0dXMYAiABKA4yIy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsU3RhdHVzQggKBl9zY29yZSLYAgoMTWV0cmljUmVzdWx0EhMKC21ldHJpY19uYW1lGAEgASgJEhIKBXNjb3JlGAIgASgBSACIAQESOAoLZXZhbF9zdGF0dXMYAyABKA4yIy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsU3RhdHVzEkEKDnBlcl9pbnZvY2F0aW9uGAQgAygLMikuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuSW52b2NhdGlvblJlc3VsdBISCgVlcnJvchgFIAEoCUgBiAEBEhcKCmVycm9yX3R5cGUYBiABKAlIAogBARIoCgdkZXRhaWxzGAcgASgLMhcuZ29vZ2xlLnByb3RvYnVmLlN0cnVjdBIYCgtkdXJhdGlvbl9tcxgIIAEoAUgDiAEBQggKBl9zY29yZUIICgZfZXJyb3JCDQoLX2Vycm9yX3R5cGVCDgoMX2R1cmF0aW9uX21zIiwKB1R1cm5SZWYSEAoIdHJhY2VfaWQYASABKAkSDwoHc3Bhbl9pZBgCIAEoCSKXAwoLVHJhY2VSZXN1bHQSEAoIdHJhY2VfaWQYASABKAkSFwoPbnVtX2ludm9jYXRpb25zGAIgASgFEj0KDm1ldHJpY19yZXN1bHRzGAMgAygLMiUuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuTWV0cmljUmVzdWx0EhsKE2NvbnZlcnNpb25fd2FybmluZ3MYBCADKAkSNAoTcGVyZm9ybWFuY2VfbWV0cmljcxgFIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSFAoMc2VydmljZV9uYW1lGAYgASgJEhIKCmFnZW50X25hbWUYByABKAkSEQoJdHJhY2VfaWRzGAggAygJEhcKD2NvbnZlcnNhdGlvbl9pZBgJIAEoCRIRCglncm91cF9rZXkYCiABKAkSFAoMZXZhbF9jYXNlX2lkGAsgASgJEhcKD2V2YWxfY2FzZV9tYXRjaBgMIAEoCRIzCgl0dXJuX3JlZnMYDSADKAsyIC5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5UdXJuUmVmIp4BCglSdW5SZXN1bHQSOwoNdHJhY2VfcmVzdWx0cxgBIAMoCzIkLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLlRyYWNlUmVzdWx0Eg4KBmVycm9ycxgCIAMoCRI0ChNwZXJmb3JtYW5jZV9tZXRyaWNzGAMgASgLMhcuZ29vZ2xlLnByb3RvYnVmLlN0cnVjdBIOCgZydW5faWQYBCABKAki6wIKFUV2YWx1YXRlVHJhY2VzUmVxdWVzdBIzCgZ0cmFjZXMYASABKAsyIy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5UcmFjZUlucHV0Ei8KDGFka19ldmFsX3NldBgCIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3RIABIzCgZwYXJhbXMYAyABKAsyIy5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5FdmFsUGFyYW1zElsKD2NyZWRlbnRpYWxfcmVmcxgEIAMoCzJCLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkV2YWx1YXRlVHJhY2VzUmVxdWVzdC5DcmVkZW50aWFsUmVmc0VudHJ5Gk4KE0NyZWRlbnRpYWxSZWZzRW50cnkSCwoDa2V5GAEgASgJEiYKBXZhbHVlGAIgASgLMhcuZ29vZ2xlLnByb3RvYnVmLlN0cnVjdDoCOAFCCgoIZXZhbF9zZXQiJQoSRXZhbHVhdGlvblByb2dyZXNzEg8KB21lc3NhZ2UYASABKAkiiwEKEEdyb3VwUGVyZm9ybWFuY2USEAoIdHJhY2VfaWQYASABKAkSNAoTcGVyZm9ybWFuY2VfbWV0cmljcxgCIAEoCzIXLmdvb2dsZS5wcm90b2J1Zi5TdHJ1Y3QSLwoOdHJhY2VfbWV0YWRhdGEYAyABKAsyFy5nb29nbGUucHJvdG9idWYuU3RydWN0IpYCChZFdmFsdWF0ZVRyYWNlc1Jlc3BvbnNlEj8KCHByb2dyZXNzGAEgASgLMisuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuRXZhbHVhdGlvblByb2dyZXNzSAASQAoLcGVyZm9ybWFuY2UYAiABKAsyKS5hZ2VudGV2YWxzLmFwaS52MWFscGhhMS5Hcm91cFBlcmZvcm1hbmNlSAASPAoMdHJhY2VfcmVzdWx0GAMgASgLMiQuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuVHJhY2VSZXN1bHRIABIyCgRkb25lGAQgASgLMiIuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuUnVuUmVzdWx0SABCBwoFZXZlbnQqpgEKE1RyYWplY3RvcnlNYXRjaFR5cGUSJQohVFJBSkVDVE9SWV9NQVRDSF9UWVBFX1VOU1BFQ0lGSUVEEAASHwobVFJBSkVDVE9SWV9NQVRDSF9UWVBFX0VYQUNUEAESIgoeVFJBSkVDVE9SWV9NQVRDSF9UWVBFX0lOX09SREVSEAISIwofVFJBSkVDVE9SWV9NQVRDSF9UWVBFX0FOWV9PUkRFUhADKngKCkV2YWxTdGF0dXMSGwoXRVZBTF9TVEFUVVNfVU5TUEVDSUZJRUQQABIWChJFVkFMX1NUQVRVU19QQVNTRUQQARIWChJFVkFMX1NUQVRVU19GQUlMRUQQAhIdChlFVkFMX1NUQVRVU19OT1RfRVZBTFVBVEVEEAMy8gEKEUV2YWx1YXRpb25TZXJ2aWNlEmgKC0xpc3RNZXRyaWNzEisuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuTGlzdE1ldHJpY3NSZXF1ZXN0GiwuYWdlbnRldmFscy5hcGkudjFhbHBoYTEuTGlzdE1ldHJpY3NSZXNwb25zZRJzCg5FdmFsdWF0ZVRyYWNlcxIuLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkV2YWx1YXRlVHJhY2VzUmVxdWVzdBovLmFnZW50ZXZhbHMuYXBpLnYxYWxwaGExLkV2YWx1YXRlVHJhY2VzUmVzcG9uc2UwAUJRWk9naXRodWIuY29tL2FnZW50ZXZhbHMtZGV2L2FnZW50ZXZhbHMvZ2VuL2dvL2FnZW50ZXZhbHMvYXBpL3YxYWxwaGExO2FwaXYxYWxwaGExYgZwcm90bzM", [file_agentevals_api_v1alpha1_common, file_google_protobuf_struct]); /** * Describes the message agentevals.api.v1alpha1.ListMetricsRequest. @@ -40,96 +40,103 @@ export const ListMetricsResponseSchema = /*@__PURE__*/ export const BuiltinEvaluatorSchema = /*@__PURE__*/ messageDesc(file_agentevals_api_v1alpha1_evaluation, 3); +/** + * Describes the message agentevals.api.v1alpha1.Rubric. + * Use `create(RubricSchema)` to create a new message. + */ +export const RubricSchema = /*@__PURE__*/ + messageDesc(file_agentevals_api_v1alpha1_evaluation, 4); + /** * Describes the message agentevals.api.v1alpha1.CodeEvaluator. * Use `create(CodeEvaluatorSchema)` to create a new message. */ export const CodeEvaluatorSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 4); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 5); /** * Describes the message agentevals.api.v1alpha1.RemoteEvaluator. * Use `create(RemoteEvaluatorSchema)` to create a new message. */ export const RemoteEvaluatorSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 5); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 6); /** * Describes the message agentevals.api.v1alpha1.Evaluator. * Use `create(EvaluatorSchema)` to create a new message. */ export const EvaluatorSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 6); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 7); /** * Describes the message agentevals.api.v1alpha1.EvalParams. * Use `create(EvalParamsSchema)` to create a new message. */ export const EvalParamsSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 7); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 8); /** * Describes the message agentevals.api.v1alpha1.InvocationResult. * Use `create(InvocationResultSchema)` to create a new message. */ export const InvocationResultSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 8); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 9); /** * Describes the message agentevals.api.v1alpha1.MetricResult. * Use `create(MetricResultSchema)` to create a new message. */ export const MetricResultSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 9); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 10); /** * Describes the message agentevals.api.v1alpha1.TurnRef. * Use `create(TurnRefSchema)` to create a new message. */ export const TurnRefSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 10); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 11); /** * Describes the message agentevals.api.v1alpha1.TraceResult. * Use `create(TraceResultSchema)` to create a new message. */ export const TraceResultSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 11); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 12); /** * Describes the message agentevals.api.v1alpha1.RunResult. * Use `create(RunResultSchema)` to create a new message. */ export const RunResultSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 12); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 13); /** * Describes the message agentevals.api.v1alpha1.EvaluateTracesRequest. * Use `create(EvaluateTracesRequestSchema)` to create a new message. */ export const EvaluateTracesRequestSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 13); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 14); /** * Describes the message agentevals.api.v1alpha1.EvaluationProgress. * Use `create(EvaluationProgressSchema)` to create a new message. */ export const EvaluationProgressSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 14); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 15); /** * Describes the message agentevals.api.v1alpha1.GroupPerformance. * Use `create(GroupPerformanceSchema)` to create a new message. */ export const GroupPerformanceSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 15); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 16); /** * Describes the message agentevals.api.v1alpha1.EvaluateTracesResponse. * Use `create(EvaluateTracesResponseSchema)` to create a new message. */ export const EvaluateTracesResponseSchema = /*@__PURE__*/ - messageDesc(file_agentevals_api_v1alpha1_evaluation, 16); + messageDesc(file_agentevals_api_v1alpha1_evaluation, 17); /** * Describes the enum agentevals.api.v1alpha1.TrajectoryMatchType.