diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 2e2b5b9b1..35bee3975 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -88,13 +88,16 @@ def generate_simulated_response(agent_message: str, expected_output: dict) -> st prompt = ( f"You are simulating a user in a conversation with a BI assistant that creates metrics. " f"The assistant said: '{agent_message}'. " - f"The user originally asked to create a metric with MAQL: {expected_maql}. " - f"Reply briefly as the user, providing any clarification the assistant needs." + f"The user's ground-truth intended metric is exactly this MAQL: {expected_maql}. " + f"Reply as the user. You MUST ensure every clause of that MAQL (including any WHERE/filter " + f"conditions) is eventually satisfied, and quote field/label identifiers verbatim from it -- " + f"never paraphrase or drop a clause, even if the assistant's question doesn't explicitly ask " + f"about it. If the assistant's offered options omit a required filter, add it yourself." ) response = client.chat.completions.create( model="gpt-4o-mini", messages=[{"role": "user", "content": prompt}], - max_tokens=150, + max_tokens=300, ) return response.choices[0].message.content or "Please proceed." diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 67a163e92..3de90cdf5 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -1,5 +1,7 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import sys +import types from unittest.mock import MagicMock, patch import pytest @@ -8,6 +10,7 @@ MetricRunResult, _delete_metric, _normalize_maql, + generate_simulated_response, run_agentic_metric_skill, ) from gooddata_eval.core.models import ChatResult @@ -21,6 +24,39 @@ def test_normalize_maql_removes_select_wrapper(): assert _normalize_maql("(SELECT {metric/abc})") == "{metric/abc}" +def test_generate_simulated_response_prompt_preserves_maql_fidelity(monkeypatch): + """Regression test for a live-reproduced bug: the old prompt ("reply briefly", + no instruction to cover clauses the assistant didn't ask about) let the + simulating LLM silently drop a MAQL's WHERE clause or paraphrase a label id -- + confirmed via a 5x-repeated A/B test (1/5 vs 5/5 fidelity) that this was the + prompt, not the model (gpt-4o did not fix it under the old prompt either). + """ + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.choices = [MagicMock(message=MagicMock(content="ok"))] + mock_client.chat.completions.create.return_value = mock_response + + # `openai` is an optional [llm-judge] extra, not installed in this test env -- + # inject a fake module rather than patching a real one (mirrors how the source + # itself does `from openai import OpenAI` as a local, guarded import). + fake_openai_module = types.SimpleNamespace(OpenAI=MagicMock(return_value=mock_client)) + monkeypatch.setitem(sys.modules, "openai", fake_openai_module) + + expected_output = {"maql": 'SELECT {metric/spend_amount_-_cutcgco} WHERE {label/ecommerce_indicator_code} = "1"'} + generate_simulated_response("Which base metric should I use?", expected_output) + + call_kwargs = mock_client.chat.completions.create.call_args.kwargs + sent_prompt = call_kwargs["messages"][0]["content"] + + assert expected_output["maql"] in sent_prompt + assert "verbatim" in sent_prompt + assert "every clause" in sent_prompt + assert "WHERE" in sent_prompt or "filter" in sent_prompt.lower() + assert "reply briefly" not in sent_prompt.lower() + assert call_kwargs["max_tokens"] >= 300 + + def test_metric_run_result_fields(): r = MetricRunResult( conversation_id="c1",