diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 1df7c3238..087f7baa1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -29,6 +29,7 @@ ReasoningStepEvent, ToolCallEvent, build_latency_breakdown, + shift_and_index_events, ) try: @@ -526,21 +527,14 @@ def _run_once(conv_id: str) -> AlertRunResult: chat_result = client.send_message(conv_id, current_question) reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id - for tc in chat_result.tool_call_events or []: - if tc.call_ts is not None: - tc.call_ts += turn_offset - if tc.result_ts is not None: - tc.result_ts += turn_offset - if tc.index is not None: - tc.index += tool_index_offset - for rs in chat_result.reasoning_step_events or []: - rs.ts += turn_offset - rs.index += reasoning_index_offset + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + chat_result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) all_tool_call_events.extend(chat_result.tool_call_events or []) all_reasoning_step_events.extend(chat_result.reasoning_step_events or []) - tool_index_offset += len(chat_result.tool_call_events or []) - reasoning_index_offset += len(chat_result.reasoning_step_events or []) - turn_offset += chat_result.turn_wall_clock_sec or 0.0 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or []) if tool_called: alert_id_to_delete = alert_id diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index f7e1ecb42..d298f5b6a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -31,6 +31,7 @@ ReasoningStepEvent, ToolCallEvent, build_latency_breakdown, + shift_and_index_events, ) from gooddata_eval.core.scoring import ( check_filters, @@ -351,22 +352,15 @@ def run_agentic_conversation( for _iter in range(max_clarification_turns + 1): chat_result = client.send_message(conversation_id, current_message) final_result = chat_result - for tc in chat_result.tool_call_events or []: - if tc.call_ts is not None: - tc.call_ts += turn_offset - if tc.result_ts is not None: - tc.result_ts += turn_offset - if tc.index is not None: - tc.index += tool_index_offset - for rs in chat_result.reasoning_step_events or []: - rs.ts += turn_offset - rs.index += reasoning_index_offset + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + chat_result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) all_tool_calls.extend(chat_result.tool_call_events or []) conversation_tool_call_events.extend(chat_result.tool_call_events or []) conversation_reasoning_step_events.extend(chat_result.reasoning_step_events or []) - tool_index_offset += len(chat_result.tool_call_events or []) - reasoning_index_offset += len(chat_result.reasoning_step_events or []) - turn_offset += chat_result.turn_wall_clock_sec or 0.0 reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 8455dc378..73e0de9cf 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -18,7 +18,13 @@ from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run -from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, +) from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings _DEFAULT_K = 1 @@ -71,6 +77,8 @@ class GeneralQuestionResult: # excluded from pass@K and from Langfuse scoring rather than counted as a failure: # scoring it 0 would publish a verdict the judge never gave. judge_error: str | None = None + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) @dataclass @@ -132,6 +140,8 @@ def _run_single_general_question( # agent still answered, and that measurement is the one worth keeping. timings=PhaseTimings(agent_s=agent_elapsed, judge_s=judge_elapsed), judge_error=verdict.error, + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), ) @@ -303,6 +313,7 @@ def _write_scores(ctx: RunTraceContext) -> None: "judge_passed": best.passed, "judge_reasoning": best.reasoning, "actual_output": best.actual_output, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), # Only present when it happened, so the usual JSON shape is unchanged. A # pass@K computed over fewer runs than --runs asked for is a weaker result and # the report has to say so. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index 37c19ba90..0fb17ffde 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -18,7 +18,15 @@ ) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ToolCallEvent +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ChatResult, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, + shift_and_index_events, +) _log = logging.getLogger(__name__) @@ -170,6 +178,8 @@ class KdaRunResult: turn_wall_clock_sec: float | None = None reasoning_steps: list[str] = field(default_factory=list) response_id: str | None = None + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) @dataclass @@ -240,6 +250,22 @@ def _run_once(conv_id: str) -> KdaRunResult: current_question = question reasoning_steps: list[str] = [] response_id: str | None = None + all_tool_call_events: list[ToolCallEvent] = [] + all_reasoning_step_events: list[ReasoningStepEvent] = [] + turn_offset = 0.0 # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time + tool_index_offset = 0 + reasoning_index_offset = 0 + + def _accumulate(result: ChatResult) -> None: + nonlocal turn_offset, tool_index_offset, reasoning_index_offset + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) + all_tool_call_events.extend(result.tool_call_events or []) + all_reasoning_step_events.extend(result.reasoning_step_events or []) for iteration in range(max_iterations): try: @@ -250,6 +276,7 @@ def _run_once(conv_id: str) -> KdaRunResult: if partial is not None: reasoning_steps.extend(partial.reasoning_steps or []) response_id = partial.response_id or response_id + _accumulate(partial) create_args, execute_result = _extract_kda_calls(partial.tool_call_events or []) if create_args is not None: turn_wall_clock_sec = partial.turn_wall_clock_sec @@ -257,6 +284,7 @@ def _run_once(conv_id: str) -> KdaRunResult: break reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id + _accumulate(chat_result) create_args, execute_result = _extract_kda_calls(chat_result.tool_call_events or []) response_text = (chat_result.text_response or "").strip() turn_completed = chat_result.stream_ended and bool(response_text) @@ -295,6 +323,8 @@ def _run_once(conv_id: str) -> KdaRunResult: turn_wall_clock_sec=turn_wall_clock_sec, reasoning_steps=reasoning_steps, response_id=response_id, + tool_call_events=all_tool_call_events, + reasoning_step_events=all_reasoning_step_events, ) try: @@ -451,6 +481,7 @@ def _write_scores(ctx: RunTraceContext) -> None: "disambiguated": ev.disambiguated, "actual_create_args": best.actual_create_args, "actual_execute_result": best.actual_execute_result, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), } if not summary.pass_at_k: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index 4457ea117..0479a9140 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -28,6 +28,7 @@ ReasoningStepEvent, ToolCallEvent, build_latency_breakdown, + shift_and_index_events, ) from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings @@ -321,21 +322,14 @@ def _execute_single_metric_run( timings.agent_s += agent_elapsed reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id - for tc in chat_result.tool_call_events or []: - if tc.call_ts is not None: - tc.call_ts += turn_offset - if tc.result_ts is not None: - tc.result_ts += turn_offset - if tc.index is not None: - tc.index += tool_index_offset - for rs in chat_result.reasoning_step_events or []: - rs.ts += turn_offset - rs.index += reasoning_index_offset + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + chat_result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) all_tool_call_events.extend(chat_result.tool_call_events or []) all_reasoning_step_events.extend(chat_result.reasoning_step_events or []) - tool_index_offset += len(chat_result.tool_call_events or []) - reasoning_index_offset += len(chat_result.reasoning_step_events or []) - turn_offset += chat_result.turn_wall_clock_sec or 0.0 for metric_id in _extract_created_metric_ids(chat_result.tool_call_events or []): if metric_id not in created_metric_ids: created_metric_ids.append(metric_id) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index b42bb4782..c3a70f81c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -16,7 +16,13 @@ ) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ToolCallEvent +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, +) _DEFAULT_K = 1 @@ -59,6 +65,8 @@ class SearchResult: tool_call_names: list[str] reasoning_steps: list[str] = field(default_factory=list) response_id: str | None = None + tool_call_events: list[ToolCallEvent] = field(default_factory=list) + reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) @dataclass @@ -103,6 +111,8 @@ def run_agentic_search_tool( tool_call_names=[tc.function_name for tc in tcs], reasoning_steps=list(chat_result.reasoning_steps or []), response_id=chat_result.response_id, + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), ) ) finally: @@ -124,6 +134,8 @@ def run_agentic_search_tool( tool_call_names=[tc.function_name for tc in tcs], reasoning_steps=list(chat_result.reasoning_steps or []), response_id=chat_result.response_id, + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), ) ) finally: @@ -233,6 +245,7 @@ def _write_scores(ctx: RunTraceContext) -> None: "tool_selected": best.tool_selected, "tool_correct": best.tool_correct, "tool_call_names": best.tool_call_names, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), } if not summary.pass_at_k: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 748c49787..85978d824 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -34,6 +34,7 @@ ReasoningStepEvent, ToolCallEvent, build_latency_breakdown, + shift_and_index_events, ) from gooddata_eval.core.scoring import get_dimension_uri_set, get_metric_uri_set, uri_to_display_name @@ -192,23 +193,16 @@ def _execute_single_run( for iteration in range(max_iterations): total_turns += 1.0 total_steps += float(current_result.reasoning_step_count) - for tc in current_result.tool_call_events: - if tc.call_ts is not None: - tc.call_ts += turn_offset - if tc.result_ts is not None: - tc.result_ts += turn_offset - if tc.index is not None: - tc.index += tool_index_offset - for rs in current_result.reasoning_step_events: - rs.ts += turn_offset - rs.index += reasoning_index_offset + turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events( + current_result, + turn_offset=turn_offset, + tool_index_offset=tool_index_offset, + reasoning_index_offset=reasoning_index_offset, + ) all_tool_call_events.extend(current_result.tool_call_events) all_reasoning_step_events.extend(current_result.reasoning_step_events) - tool_index_offset += len(current_result.tool_call_events) - reasoning_index_offset += len(current_result.reasoning_step_events) reasoning_steps.extend(current_result.reasoning_steps or []) response_id = current_result.response_id or response_id - turn_offset += current_result.turn_wall_clock_sec or 0.0 viz_produced = bool(current_result.created_visualizations and current_result.created_visualizations.objects) if viz_produced: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/alert_skill.py index 4d19641de..0303af7c4 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/alert_skill.py @@ -6,7 +6,7 @@ from gooddata_eval.core.evaluators._deep_subset import deep_subset from gooddata_eval.core.evaluators.base import ItemEvaluation -from gooddata_eval.core.models import ChatResult, DatasetItem +from gooddata_eval.core.models import ChatResult, DatasetItem, build_latency_breakdown _TRIGGER_MAP = {"Every time": "ALWAYS", "One time": "ONCE"} @@ -72,7 +72,13 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation return ItemEvaluation( passed=False, rank_key=(False,) * 7, - detail={"alert_created": False, "automation_id": None}, + detail={ + "alert_created": False, + "automation_id": None, + "latency_breakdown": build_latency_breakdown( + chat_result.tool_call_events, chat_result.reasoning_step_events + ), + }, ) args = tool_event.parsed_arguments() @@ -145,5 +151,8 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation # lets a caller (e.g. a cleanup step) delete the exact object # created instead of diffing the workspace catalog before/after. "automation_id": automation_id, + "latency_breakdown": build_latency_breakdown( + chat_result.tool_call_events, chat_result.reasoning_step_events + ), }, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/metric_skill.py index e43b5ebb5..622399273 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/metric_skill.py @@ -2,7 +2,7 @@ """Evaluator for metric_skill: agent must create the correct metric via create_metric tool call.""" from gooddata_eval.core.evaluators.base import ItemEvaluation -from gooddata_eval.core.models import ChatResult, DatasetItem +from gooddata_eval.core.models import ChatResult, DatasetItem, build_latency_breakdown def _find_create_metric(chat_result: ChatResult): @@ -28,7 +28,15 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation return ItemEvaluation( passed=False, rank_key=(False, False, False), - detail={"metric_created": False, "maql_correct": False, "format_correct": False, "metric_id": None}, + detail={ + "metric_created": False, + "maql_correct": False, + "format_correct": False, + "metric_id": None, + "latency_breakdown": build_latency_breakdown( + chat_result.tool_call_events, chat_result.reasoning_step_events + ), + }, ) result = tool_event.parsed_result() @@ -59,5 +67,8 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation # delete the exact object created instead of diffing the workspace # catalog before/after and guessing by name. "metric_id": payload.get("metric_id"), + "latency_breakdown": build_latency_breakdown( + chat_result.tool_call_events, chat_result.reasoning_step_events + ), }, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 059c4d32d..824e77c3e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -225,6 +225,46 @@ class ChatResult(BaseModel): turn_wall_clock_sec: float | None = None +def shift_and_index_events( + result: ChatResult, + *, + turn_offset: float, + tool_index_offset: int, + reasoning_index_offset: int, +) -> tuple[float, int, int]: + """Rebase one turn's tool-call/reasoning-step timestamps and indices onto a shared, + conversation-wide timeline, mutating the events in place. + + Each turn's own SSE stream times its events from ~0 and indexes them from 0 within + that turn alone -- call_ts/result_ts/ts and index must be shifted by every prior + turn's contribution before events from multiple turns can be merged into one + chronologically ordered ``build_latency_breakdown`` across a whole multi-turn run. + + Returns the updated ``(turn_offset, tool_index_offset, reasoning_index_offset)`` to + pass into the next turn's call -- the caller still owns accumulating the shifted + events themselves (e.g. into an ``all_tool_call_events`` list) since callers differ + in whether/how they also need the raw per-turn events for other purposes. + + Note: ``turn_offset`` advances by ``turn_wall_clock_sec`` only, so time spent + generating a simulated-user reply between turns is not represented -- inter-turn + gaps compress in the reconstructed timeline. + """ + for tc in result.tool_call_events or []: + if tc.call_ts is not None: + tc.call_ts += turn_offset + if tc.result_ts is not None: + tc.result_ts += turn_offset + if tc.index is not None: + tc.index += tool_index_offset + for rs in result.reasoning_step_events or []: + rs.ts += turn_offset + rs.index += reasoning_index_offset + tool_index_offset += len(result.tool_call_events or []) + reasoning_index_offset += len(result.reasoning_step_events or []) + turn_offset += result.turn_wall_clock_sec or 0.0 + return turn_offset, tool_index_offset, reasoning_index_offset + + class AgenticAssertionError(AssertionError): """Base for every agentic kind's failure, carrying what the runner reports about it. diff --git a/packages/gooddata-eval/tests/test_agentic_general_question.py b/packages/gooddata-eval/tests/test_agentic_general_question.py index 830d25019..06609f21d 100644 --- a/packages/gooddata-eval/tests/test_agentic_general_question.py +++ b/packages/gooddata-eval/tests/test_agentic_general_question.py @@ -197,6 +197,45 @@ def test_run_agentic_general_question_captures_reasoning_steps(): assert summary.best.response_id == "resp-1" +def test_run_agentic_general_question_captures_tool_call_events_on_every_run(): + """Every run must carry its own tool_call_events, not just the first. + + All K runs are built in one place now, so a per-run gap can't arise the way it did + when run 0 and the k>1 loop each constructed their own result -- this guards the + field being populated at all, and that a future re-split doesn't silently drop it + for later runs (which would leave them with an empty latency_breakdown). + """ + client, judge = _pass_client_and_judge() + client.create_conversation.side_effect = ["conv-1", "conv-2"] + # A fresh object per run: one shared object would let a double-mutation bug pass, + # since both runs would point at the same already-shifted events. + client.send_message.side_effect = [ + ChatResult.model_validate( + { + "textResponse": "42", + "toolCallEvents": [{"functionName": "search_objects", "functionArguments": "{}"}], + "reasoningSteps": ["recalling the answer"], + "responseId": response_id, + } + ) + for response_id in ("resp-1", "resp-2") + ] + + with _patched(client, judge): + summary = run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=2, + ) + + assert len(summary.run_results) == 2 + for run in summary.run_results: + assert len(run.tool_call_events) == 1 + + def test_evaluate_agentic_general_question_returns_reasoning_steps_on_pass(): client, judge = _pass_client_and_judge(reasoning_steps=["recalling the answer"]) @@ -217,6 +256,7 @@ def test_evaluate_agentic_general_question_returns_reasoning_steps_on_pass(): "judge_passed": True, "judge_reasoning": "Correct answer", "actual_output": "42", + "latency_breakdown": [], } @@ -243,6 +283,7 @@ def test_evaluate_agentic_general_question_attaches_reasoning_steps_to_exception "judge_passed": False, "judge_reasoning": "Wrong answer", "actual_output": "I don't know", + "latency_breakdown": [], } diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index a9578bdda..f6eecf8f9 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -485,6 +485,83 @@ def test_run_agentic_kda_skill_marks_disambiguated_after_a_simulated_reply(): assert summary.best.evaluation.triggered is True +def test_run_agentic_kda_skill_shifts_timestamps_and_indices_across_iterations(): + """Regression: the shift/accumulate logic must rebase each iteration's own-zeroed + call_ts/result_ts/ts and index onto the whole run's timeline, not just carry events + through unshifted. + + Iteration 1 has one reasoning step and no tool call, so after it: tool_index_offset + stays 0 but reasoning_index_offset becomes 1. Iteration 2's create/execute calls must + land at iteration 1's turn_wall_clock_sec (10.0) plus their own within-turn call_ts, + with index starting from 0; its reasoning event's index must start from 1, not 0 -- + exactly where an off-by-one or a double-shift would hide. + """ + mock_client = _client() + iteration_1 = ChatResult.model_validate( + { + "textResponse": "Which measure should I analyze?", + "toolCallEvents": [], + "reasoningStepEvents": [{"summary": "**Considering measure**", "ts": 0.5, "index": 0}], + "reasoningStepCount": 1, + "stream_ended": True, + "turn_wall_clock_sec": 10.0, + } + ) + iteration_2 = ChatResult.model_validate( + { + "textResponse": "Here is the analysis.", + "toolCallEvents": [ + { + "functionName": "create_key_driver_analysis", + "functionArguments": json.dumps({"measure": {"type": "metric", "id": "revenue"}}), + "call_ts": 0.2, + "result_ts": 0.3, + "index": 0, + }, + { + "functionName": "execute_key_driver_analysis", + "functionArguments": "{}", + "result": json.dumps({"success": True, "data": {"summary": {}}}), + "call_ts": 0.3, + "result_ts": 0.4, + "index": 1, + }, + ], + "reasoningStepEvents": [{"summary": "**Running analysis**", "ts": 0.1, "index": 0}], + "reasoningStepCount": 1, + "stream_ended": True, + "turn_wall_clock_sec": 5.0, + } + ) + mock_client.send_message.side_effect = [iteration_1, iteration_2] + + with _patched(mock_client, simulated_reply="Use the revenue metric."): + summary = run_agentic_kda_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What drove revenue change?", + expected_output=_EXPECTED, + k=1, + max_iterations=2, + ) + + best = summary.best + create_tc, execute_tc = best.tool_call_events + assert create_tc.call_ts == pytest.approx(10.2) + assert create_tc.result_ts == pytest.approx(10.3) + assert create_tc.index == 0 + assert execute_tc.call_ts == pytest.approx(10.3) + assert execute_tc.result_ts == pytest.approx(10.4) + assert execute_tc.index == 1 + + assert len(best.reasoning_step_events) == 2 + assert best.reasoning_step_events[0].ts == pytest.approx(0.5) + assert best.reasoning_step_events[0].index == 0 + assert best.reasoning_step_events[1].ts == pytest.approx(10.1) + assert best.reasoning_step_events[1].index == 1 + + def test_run_agentic_kda_skill_disambiguates_on_question_followed_by_option_list(): # Regression (QA-28800): the real captured response ends with a bullet list of # candidate metrics. Before this module dropped text classification in favor of @@ -1055,6 +1132,7 @@ def test_evaluate_agentic_kda_skill_returns_reasoning_steps_on_pass(): "disambiguated": False, "actual_create_args": {"measure": {"type": "metric", "id": "revenue"}}, "actual_execute_result": {"success": True, "data": {"summary": {}}}, + "latency_breakdown": [], } @@ -1087,6 +1165,7 @@ def test_evaluate_agentic_kda_skill_attaches_reasoning_steps_to_exception_on_fai "disambiguated": False, "actual_create_args": None, "actual_execute_result": None, + "latency_breakdown": [], } diff --git a/packages/gooddata-eval/tests/test_agentic_search_tool.py b/packages/gooddata-eval/tests/test_agentic_search_tool.py index 025c4db17..975004cd4 100644 --- a/packages/gooddata-eval/tests/test_agentic_search_tool.py +++ b/packages/gooddata-eval/tests/test_agentic_search_tool.py @@ -138,6 +138,46 @@ def test_run_agentic_search_tool_captures_reasoning_steps(): assert summary.best.response_id == "resp-1" +def test_run_agentic_search_tool_captures_tool_call_events_on_every_run(): + """Regression: a run other than the first must still carry its own tool_call_events -- + this runner builds SearchResult in two places (run 0 and the k>1 loop), and populating + only the first leaves later runs on the dataclass's empty-list default, so selecting one + of them as `best` silently produces an empty latency_breakdown. + """ + + def _result(response_id: str) -> ChatResult: + # A fresh object per run -- reusing one across both mocked calls would let a + # double-mutation bug pass unnoticed, since both runs would point at the same + # already-shifted events. + return ChatResult.model_validate( + { + "textResponse": "Found it", + "toolCallEvents": [{"functionName": "search_objects", "functionArguments": '{"keywords": "revenue"}'}], + "reasoningSteps": ["deciding what to search for"], + "responseId": response_id, + } + ) + + mock_client = MagicMock() + mock_client.create_conversation.side_effect = ["conv-1", "conv-2"] + mock_client.send_message.side_effect = [_result("resp-1"), _result("resp-2")] + + with patch("gooddata_eval.core.agentic.search_tool.ChatClient", return_value=mock_client): + summary = run_agentic_search_tool( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Search for revenue", + expected_tool_call={"keywords": "revenue"}, + k=2, + ) + + assert len(summary.run_results) == 2 + for run in summary.run_results: + assert len(run.tool_call_events) == 1 + assert run.tool_call_events[0].function_name == "search_objects" + + def test_evaluate_agentic_search_tool_returns_reasoning_steps_on_pass(): mock_client = MagicMock() mock_client.create_conversation.return_value = "conv-1" @@ -167,6 +207,7 @@ def test_evaluate_agentic_search_tool_returns_reasoning_steps_on_pass(): "tool_selected": True, "tool_correct": True, "tool_call_names": ["search_objects"], + "latency_breakdown": [], } @@ -202,4 +243,5 @@ def test_evaluate_agentic_search_tool_attaches_reasoning_steps_to_exception_on_f "tool_selected": False, "tool_correct": False, "tool_call_names": [], + "latency_breakdown": [], }