feat(gooddata-eval): add KDA-skill agentic evaluator

packages/gooddata-eval/src/gooddata_eval/core/agentic/__init__.py

@@ -30,6 +30,14 @@
     evaluate_agentic_guardrail,
     run_agentic_guardrail,
 )
+from gooddata_eval.core.agentic.kda_skill import (
+    AgenticKdaSummary,
+    KdaEvaluation,
+    KdaRunResult,
+    KdaSkillAssertionError,
+    evaluate_agentic_kda_skill,
+    run_agentic_kda_skill,
+)
 from gooddata_eval.core.agentic.metric_skill import (
     AgenticMetricSummary,
     MetricRunResult,
@@ -56,6 +64,7 @@
     "AgenticAlertSummary",
     "AgenticGeneralQuestionSummary",
     "AgenticGuardrailSummary",
+    "AgenticKdaSummary",
     "AgenticMetricSummary",
     "AgenticSearchSummary",
     "AgenticRunSummary",
@@ -69,6 +78,9 @@
     "GeneralQuestionResult",
     "GuardrailAssertionError",
     "GuardrailResult",
+    "KdaEvaluation",
+    "KdaRunResult",
+    "KdaSkillAssertionError",
     "MetricRunResult",
     "MetricSkillAssertionError",
     "RunResult",
@@ -81,13 +93,15 @@
     "evaluate_agentic_conversation",
     "evaluate_agentic_general_question",
     "evaluate_agentic_guardrail",
+    "evaluate_agentic_kda_skill",
     "evaluate_agentic_metric_skill",
     "evaluate_agentic_search_tool",
     "evaluate_agentic_visualization",
     "run_agentic_alert_skill",
     "run_agentic_conversation",
     "run_agentic_general_question",
     "run_agentic_guardrail",
+    "run_agentic_kda_skill",
     "run_agentic_metric_skill",
     "run_agentic_search_tool",
     "run_agentic_visualization",

packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py

@@ -31,7 +31,12 @@ def __init__(self, raw: dict) -> None:
         self.id: str = raw.get("id", "")
         self.metadata: dict = raw.get("metadata") or {}
         self.session_id: str | None = raw.get("sessionId") or raw.get("session_id")
-        self.latency: float = float(raw.get("latency") or 0.0)
+        # None (missing/null) is preserved, not coerced to 0.0 -- a trace that hasn't
+        # finished ingesting has UNKNOWN latency, not zero latency, and callers (e.g.
+        # log_quality_and_value_scores below, and every skill's own `pt.latency if pt
+        # else None` gating) rely on that distinction to not treat "unknown" as the best
+        # possible outcome.
+        self.latency: float | None = float(raw["latency"]) if raw.get("latency") is not None else None
         self.total_cost: float = float(raw.get("totalCost") or raw.get("total_cost") or 0.0)
 
 
@@ -358,11 +363,24 @@ def log_quality_and_value_scores(
         data_type="NUMERIC",
         comment=f"{passed}/{total} strict checks passed",
     )
-    speed = 0.0 if latency_sec is None else max(0.0, 1.0 - latency_sec / _MAX_LATENCY_SEC)
-    cost_factor = 0.0 if cost_usd is None else max(0.0, 1.0 - cost_usd / _MAX_COST_USD)
-    value = _QUALITY_WEIGHT * quality + _SPEED_WEIGHT * speed + _COST_WEIGHT * cost_factor
+    # An unresolved latency/cost (trace not yet settled, price not available) is UNKNOWN,
+    # not the best (1.0) or worst (0.0) possible outcome -- substituting either would
+    # silently pull value_score toward one extreme. Drop that weighted term instead and
+    # renormalize over whichever components do have a real value, so value_score always
+    # reflects only the signals actually measured for this run.
+    components = [(_QUALITY_WEIGHT, quality)]
+    speed = None if latency_sec is None else max(0.0, 1.0 - latency_sec / _MAX_LATENCY_SEC)
+    if speed is not None:
+        components.append((_SPEED_WEIGHT, speed))
+    cost_factor = None if cost_usd is None else max(0.0, 1.0 - cost_usd / _MAX_COST_USD)
+    if cost_factor is not None:
+        components.append((_COST_WEIGHT, cost_factor))
+    weight_total = sum(w for w, _ in components)
+    value = sum(w * v for w, v in components) / weight_total
     latency_str = "unknown" if latency_sec is None else f"{latency_sec:.2f}s"
     cost_str = "unknown" if cost_usd is None else f"${cost_usd:.4f}"
+    speed_str = "n/a" if speed is None else f"{speed:.2f}"
+    cost_factor_str = "n/a" if cost_factor is None else f"{cost_factor:.2f}"
     score_safe(
         langfuse,
         trace_id,
@@ -371,8 +389,8 @@ def log_quality_and_value_scores(
         data_type="NUMERIC",
         comment=(
             f"{_QUALITY_WEIGHT}*quality({quality:.2f}) + "
-            f"{_SPEED_WEIGHT}*speed({speed:.2f}) + "
-            f"{_COST_WEIGHT}*cost({cost_factor:.2f}); "
+            f"{_SPEED_WEIGHT}*speed({speed_str}) + "
+            f"{_COST_WEIGHT}*cost({cost_factor_str}), renormalized /{weight_total:.1f}; "
             f"latency={latency_str}; cost={cost_str}"
         ),
     )