diff --git a/.gitignore b/.gitignore index 0c3af92da..dd71f60c7 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,10 @@ packages/gooddata-sdk/tests/export/exports/default/ AGENTS.md .aiassistant/rules/aida.md .junie/guidelines.md + +# gooddata-eval local run artifacts. Root-anchored on purpose: a bare `datasets/` would +# also shadow packages/gooddata-pandas/tests/.../ldm/datasets/, which is tracked. +/packages/gooddata-eval/EVAL_results +/packages/gooddata-eval/datasets/ +# MCP tool logs, written to a relative path by whatever is started from the repo root +/logs/ diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index e5f321027..be4f55046 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -114,6 +114,7 @@ gd-eval run \ |---|---| | `--dataset PATH` | Flat folder of JSON files — one question per file. | | `--langfuse-dataset NAME` | Pull items by name from a Langfuse dataset. Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. | +| `--kind TEST_KIND` | Fallback `test_kind` for dataset items that do not embed one. Defaults to `visualization`; use e.g. `agentic_metric_skill` for multi-turn agentic evaluation. Items that declare their own `test_kind` ignore this. | #### Model selection @@ -126,21 +127,62 @@ gd-eval run \ | Flag | Default | Description | |---|---|---| | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. | -| `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. | +| `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests — see *Concurrency and workspace safety* below. | +| `--judge-model MODEL` | `gpt-4o` | Model used for LLM-as-judge scoring — `agentic_general_question`, `agentic_guardrail`, `general_question`, `guardrail` and `dashboard_summary`. Also settable via `GD_EVAL_JUDGE_MODEL`. Two things to weigh before changing it: the gpt-5 family rejects `temperature=0`, so verdicts stop being reproducible (the run warns when this happens); and choosing the same model the agent runs means the judge grades its own family's output. | | `--reasoning-effort LEVEL` | server default | `LOW`, `MEDIUM` or `HIGH`, sent as `options.reasoningEffort` on every chat message. Requires the `enableGenAiReasoningEffort` feature flag on the target organization — without it the server ignores the value. Applies to chat items only; `dashboard_summary` items go through the summary endpoint, which has no such option. | +**Concurrency and workspace safety.** Agentic kinds that create workspace objects +(`agentic_metric_skill`, `agentic_alert_skill`, `agentic_conversation`, `agentic_kda_skill`) always run one at a +time whatever `--concurrency` says — a metric or alert created and dropped mid-run would otherwise be visible to +another item reading the same catalog. **That protection is for the agentic kinds only:** the single-turn +`metric_skill` and `alert_skill` kinds are still fanned out and the agent performs the same server-side writes on +that path, so avoid raising `--concurrency` on a dataset of those against a shared workspace. Progress output +interleaves when K > 1, and per-item latencies rise, so they stop being clean single-request measurements. + #### Output | Flag | Description | |---|---| | `--json PATH` | Write a JSON report to this path. Always uses the nested `{models, runs, comparison}` shape even for a single model. | | `--quiet` | Suppress per-item progress. Per-model result tables and the comparison summary are still printed. | +| `--preserve-failed` | Keep failed conversations on the server instead of deleting them, so they can be inspected afterwards. Applies to the single-turn chat path; agentic kinds manage their own conversation lifecycle. | +| `--timers` | Print per-turn `[timer]` diagnostics — GoodData response, judge, and simulated-user seconds as they happen. Off by default: an 18-item `--runs 2` run emits ~72 lines and buries the progress output. The same measurements are always in the JSON report's `latency_breakdown_s`, so this only adds a live view. Also settable via `GD_EVAL_TIMERS=1`. | #### Langfuse sink | Flag | Description | |---|---| -| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`, suffixed `-effort-{level}` when `--reasoning-effort` is set so runs differing only by effort stay separate). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. | +| `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Names each experiment run `{dataset_name}_{timestamp}_{model}`, suffixed `_effort-{level}` when `--reasoning-effort` is set (so runs differing only by effort stay separate) and `_run{N}` per run when `--runs` > 1 — e.g. `general_question_2026-09-02-11-13_gpt-5.2_run0`. Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. | + +Set `TAVERN_E2E_SKIP_TRACE_LINK=1` to skip trace lookup entirely (scores are then orphaned; the run says so). + +**A local `--dataset` cannot be attached to a Langfuse run.** `--langfuse` is refused alongside `--dataset` +because a local folder's item ids are not Langfuse dataset item ids. But trace linking does not depend on that +flag — each `evaluate_agentic_*` builds its own client whenever `LANGFUSE_PUBLIC_KEY`/`LANGFUSE_SECRET_KEY` are +exported — so a local run still finds its traces and writes its scores onto them, and only the per-run grouping +fails, with one `404 from dataset-run-items` reported per run. The run warns about this before it starts. Use +`--langfuse-dataset` when you want runs that are comparable across models, or `TAVERN_E2E_SKIP_TRACE_LINK=1` to +skip linking altogether. + +**When trace linking happens.** Finding a gen-ai trace means polling until Langfuse has ingested it, which is +lag measured in seconds to minutes. That work produces no verdict — the pass/fail is already decided — so it +does not run inline per item. Every item's Langfuse block is queued and the whole batch runs *after* the agent +phase, draining before any report is written. Two consequences worth knowing: + +- **No item's `latency_s` includes trace linking.** Its cost is reported separately as + `latency_breakdown_s.langfuse_s`, and the run prints + `[langfuse] trace linking finished in Xs for N item(s); slowest Ys`. If `slowest` approaches the **120s** + batched retry budget, links are timing out and scores are being orphaned — look for + `[langfuse] WARNING: no trace found for conversation ...`. +- **The budget depends on who is waiting.** 120s is affordable only because the batch blocks nobody. A direct + library caller (`evaluate_agentic_*` without a `submit_trace_link`) polls inline, on its own critical path, and + gets **35s** instead — the same cost as before batching existed, so no inline caller pays for a budget raised + on the CLI's behalf. Either way a trace that is already ingested costs nothing: the loop looks before it sleeps. +Scores are always final before the command exits — the run blocks on the batch. Interrupting with Ctrl-C drops +whatever is still queued rather than making you wait it out: both the queued trace links and, under +`--concurrency`, the items that have not started. The handful of items already in flight still have to finish — +worker threads are joined at exit and an in-progress agent call cannot be cancelled — so expect to wait up to one +`--concurrency`-wide wave, not the rest of the dataset. ### JSON report shape @@ -162,6 +204,70 @@ The JSON report always uses the nested multi-model shape: Winner is selected by **pass rate → quality score → latency** (lower latency wins all-equal ties). +Each item reports **how many of its runs passed**, not only whether one did: + +```json +"runs": 5, "runs_passed": 4, "pass_at_k": true, "pass_power_k": false +``` + +`pass_at_k` is "did any run pass" and is what `passed` counts. `runs_passed` is the fact that separates a +reliable item from a coin-flip — without it a 5/5 item and a 1/5 item are identical in every field, because +`quality_score` is derived from the best run alone. `pass_power_k` is true only when every run passed, and the +run summary carries `passed_all_runs` beside `passed`; a large gap between the two means the model is +inconsistent rather than wrong. The console shows `4/5 runs passed` in `Notes` for a non-unanimous pass and +stays quiet for a unanimous one, and its summary line reads `3/4 passed, 1 on every run`. + +`runs` is what the item actually ran, which is not always the requested `--runs`: `agentic_conversation` takes +no K and drives its fixture exactly once. + +Each item additionally carries a per-phase breakdown: + +```json +"latency_breakdown_s": { + "agent_s": 4.02, // GoodData's own response time — the system under test + "judge_s": 1.31, // LLM-as-judge scoring, post-hoc + "simulated_user_s": 0.0, // our simulated user composing the next turn (multi-turn kinds) + "langfuse_s": 5.70 // trace lookup + score writing, off the critical path +} +``` + +Every item reports `runs_ungraded` beside `runs_passed`: runs the agent answered but the LLM judge returned +nothing readable for. Agentic items also list them as `unscored_runs` / `judge_errors` in their `detail`, and a +`dashboard_summary` item carries `ungraded_criteria`. Such a run — or, for `dashboard_summary`, such a +criterion — is excluded from pass@K and from the quality score rather than counted as a failure: scoring it 0 +would be indistinguishable from the judge genuinely failing the answer, which is the confusion +`JudgeResponseError` exists to end. `pass@K` still holds on the runs that *were* graded, so an item can pass with +`runs_ungraded` set; `pass^K` cannot, because a run nobody graded leaves "all K passed" unverified. For +`dashboard_summary` an ungraded `must_include` / `must_not_include` criterion likewise cannot carry a pass — +"the judge could not tell" is not evidence the fact is present — while an ungraded `rubric` line only narrows +the quality score. When *no* run could be graded (for `dashboard_summary`: no gating criterion on any run) the +item errors instead of reporting failures. A non-zero count means pass@K was computed over fewer runs than +`--runs` asked for, so treat the result as weaker evidence and check the judge (`GD_EVAL_JUDGE_DIAGNOSTICS=1`, +or raise `JUDGE_MAX_COMPLETION_TOKENS` if the cause is `finish_reason=length`). The console says so in `Notes` +(`1 run(s) ungraded`, `2 criterion(s) ungraded`). + +`agent_s` + `judge_s` + `simulated_user_s` are the instrumented parts of the item's `latency_s`; they do not add +up to it exactly, because `latency_s` is wall-clock around the whole item and also covers the conversation +create/delete round trips, SDK construction and any cleanup. `langfuse_s` sits **beside** `latency_s`, never +inside it, because trace linking runs outside every item's critical path (see above) — summing all four would +re-inflate exactly what that design removes. + +A phase that a kind does not have reports `0.0` rather than an invented number, so read the zeroes as "not +applicable here", not "instant". Today: + +| Field | Populated by | +|---|---| +| `agent_s` | `agentic_general_question`, `agentic_metric_skill` | +| `judge_s` | `agentic_general_question` only — `agentic_metric_skill` compares MAQL by string, it has no LLM judge | +| `simulated_user_s` | `agentic_metric_skill` only — `agentic_general_question` is single-turn, it has no simulated user | +| `langfuse_s` | every agentic kind, but only on the `gd-eval` path and only when Langfuse credentials are present | + +The other six agentic kinds report `0.0` for the first three. Trace linking itself happens whenever +`LANGFUSE_PUBLIC_KEY`/`LANGFUSE_SECRET_KEY` are exported, with or without `--langfuse`, because each +`evaluate_agentic_*` falls back to `try_make_langfuse_client()`. But its *duration* is measured by the CLI +runner rather than by `evaluate_agentic_*`, so a direct library caller sees `langfuse_s: 0.0` even though its +linking ran. Pass `TAVERN_E2E_SKIP_TRACE_LINK=1` to opt out of linking altogether. + --- ## `gd-eval models` diff --git a/packages/gooddata-eval/pyproject.toml b/packages/gooddata-eval/pyproject.toml index c02806fa0..54b032fb1 100644 --- a/packages/gooddata-eval/pyproject.toml +++ b/packages/gooddata-eval/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ ] [project.optional-dependencies] -llm-judge = ["openai>=1.40,<2.0"] +llm-judge = ["openai>=1.45,<2.0"] [project.scripts] gd-eval = "gooddata_eval.cli.main:main" diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index f85bef6ca..0bd6f5cf8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -3,10 +3,13 @@ from __future__ import annotations +import sys import time +from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, TypedDict from gooddata_eval.core.agentic._langfuse import make_langfuse_client +from gooddata_eval.core.agentic._trace_linker import BackgroundTraceLinker, SubmitTraceLink, run_trace_link_inline from gooddata_eval.core.agentic.alert_skill import evaluate_agentic_alert_skill from gooddata_eval.core.agentic.conversation import ConversationFixture, evaluate_agentic_conversation from gooddata_eval.core.agentic.general_question import evaluate_agentic_general_question @@ -27,6 +30,7 @@ class _LfKw(TypedDict, total=False): run_timestamp: str model_version_override: str | None reasoning_effort: ReasoningEffort | None + submit_trace_link: SubmitTraceLink AGENTIC_TEST_KINDS = frozenset( @@ -44,6 +48,43 @@ class _LfKw(TypedDict, total=False): ) +# Kinds cleared to run several at a time. An EXPLICIT allowlist, not a subtraction: nothing +# in this package can prove a kind is read-only, because the mutation happens server-side in +# whichever tools the agent decides to call. So each entry here is a reviewed judgement, and +# anything absent -- including a kind added later -- runs serially. Slow is a recoverable +# mistake; two runs sharing a workspace mid-mutation corrupts eval results silently and +# reads like a model regression. +# +# agentic_general_question, agentic_guardrail answer questions only, no tool writes +# agentic_search search_objects, read-only by definition +# vis_agentic, agentic_visualization visualizations come back as AAC proposals +# in the chat response; nothing is persisted +# and neither module has cleanup code +PARALLEL_SAFE_TEST_KINDS = frozenset( + { + "agentic_general_question", + "agentic_guardrail", + "agentic_search", + "vis_agentic", + "agentic_visualization", + } +) + +# Everything else. metric_skill and alert_skill demonstrably create workspace objects (they +# carry delete_entity_metrics / delete_entity_automations cleanup) and metric_skill._delete_metric +# records that a leaked metric gets reused by a later test. agentic_conversation drives the +# metric skill. agentic_kda_skill is here on suspicion rather than proof: it triggers +# create_key_driver_analysis with no cleanup, and while the evaluator only ever reads that +# call's ARGUMENTS -- never a created object id -- whether the platform persists anything is +# unverified. Move it to the allowlist once someone confirms it does not. +WORKSPACE_MUTATING_TEST_KINDS = frozenset(AGENTIC_TEST_KINDS) - PARALLEL_SAFE_TEST_KINDS + + +def runs_in_parallel(test_kind: str) -> bool: + """True only for kinds explicitly cleared for concurrent execution.""" + return test_kind in PARALLEL_SAFE_TEST_KINDS + + def _parse_visualization_expected(expected_output: Any) -> list[CreatedVisualization]: """Parse expected_output into a list of CreatedVisualization candidates. @@ -86,6 +127,7 @@ def _dispatch_agentic( model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, ) -> AgenticEvalOutcome: """Call the appropriate evaluate_agentic_* function for the item's test_kind. @@ -102,6 +144,7 @@ def _dispatch_agentic( "run_timestamp": run_ts, "model_version_override": model_version_override, "reasoning_effort": reasoning_effort, + "submit_trace_link": submit_trace_link, } if kind in ("vis_agentic", "agentic_visualization"): @@ -160,6 +203,7 @@ def _dispatch_agentic( expected_output=eo if isinstance(eo, str) else str(eo), k=k, agent_id=agent_id, + user_context=item.user_context, **lf_kw, ) elif kind == "agentic_guardrail": @@ -198,6 +242,40 @@ def _dispatch_agentic( raise ValueError(f"Unknown agentic test kind: {kind!r}") +def _apply_run_counts(item_report: ItemReport, source: Any) -> None: + """Copy how many runs passed, and how many actually ran, onto the item report. + + Kinds that report neither keep the requested K and a 0 count, which reads as "not + instrumented" rather than "nothing passed" because ``pass_power_k`` is only consulted + for an item that already passed. + """ + runs_passed = getattr(source, "runs_passed", None) + if runs_passed is not None: + item_report.runs_passed = runs_passed + effective = getattr(source, "runs_effective", None) + if effective: + # Only when the kind knows better than K -- agentic_conversation runs once. + item_report.runs_effective = effective + # The agentic kinds record their unscored runs in the detail; the report field is the + # one place every kind's count is read from. + unscored = (getattr(source, "detail", None) or {}).get("unscored_runs") + if isinstance(unscored, int): + item_report.runs_ungraded = unscored + + +def _apply_timings(item_report: ItemReport, timings: Any) -> None: + """Copy an outcome's phase breakdown onto the item report, if the kind recorded one. + + Kinds with no phase instrumentation pass None and keep their 0.0 defaults rather than + reporting invented numbers. + """ + if timings is None: + return + item_report.agent_latency_s = timings.agent_s + item_report.judge_latency_s = timings.judge_s + item_report.simulated_user_latency_s = timings.simulated_user_s + + def run_agentic_items( items: list[DatasetItem], host: str, @@ -212,14 +290,29 @@ def run_agentic_items( on_item_start: Any = None, on_item_done: Any = None, agent_id: str | None = None, + concurrency: int = 1, ) -> EvalReport: - """Run agentic items through evaluate_agentic_* and return an EvalReport.""" + """Run agentic items through evaluate_agentic_* and return an EvalReport. + + ``concurrency`` > 1 runs PARALLEL_SAFE_TEST_KINDS items simultaneously. + WORKSPACE_MUTATING_TEST_KINDS items always run one at a time, and in a separate phase + from the parallel ones -- a metric being created and dropped mid-run would otherwise be + visible to a catalog-reading item running alongside it. + + Results are collected in dataset order regardless of completion order. + """ langfuse = make_langfuse_client() if use_langfuse else None report = EvalReport(model=model_version) total = len(items) + # Trace linking runs here rather than inside each evaluate_agentic_*, so an item's + # Langfuse poll overlaps the NEXT item's agent call instead of extending its own + # latency. Drained below before this function returns, so every score is written + # before the caller renders a report or decides an exit code. + linker = BackgroundTraceLinker() + _t0 = time.perf_counter() - for index, item in enumerate(items, start=1): + def _process_item(index: int, item: DatasetItem) -> ItemReport: if on_item_start is not None: try: on_item_start(index, total, item) @@ -235,7 +328,17 @@ def run_agentic_items( t0 = time.perf_counter() try: outcome = _dispatch_agentic( - item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id + item, + host, + token, + workspace_id, + k, + langfuse, + run_ts, + model_version, + reasoning_effort, + agent_id, + submit_trace_link=linker.submit, ) if isinstance(outcome, AgenticEvalOutcome): reasoning_steps = outcome.reasoning_steps @@ -250,6 +353,8 @@ def run_agentic_items( item_report.conversation_id = conversation_id item_report.response_id = response_id item_report.best_detail = detail or {} + _apply_timings(item_report, getattr(outcome, "timings", None)) + _apply_run_counts(item_report, outcome) except AssertionError as exc: item_report.pass_at_k = False item_report.runs = k @@ -257,10 +362,17 @@ def run_agentic_items( item_report.conversation_id = getattr(exc, "conversation_id", None) item_report.response_id = getattr(exc, "response_id", None) item_report.best_detail = getattr(exc, "detail", None) or {} + _apply_timings(item_report, getattr(exc, "timings", None)) + _apply_run_counts(item_report, exc) print(f"[agentic] {item.id} FAIL: {exc}", flush=True) except Exception as exc: item_report.error = f"{type(exc).__name__}: {exc}" item_report.runs = 0 + # An item that errored still measured whatever it got through, and those are + # the most useful numbers on the report -- an item unevaluable because its + # judge broke should not also report the agent as costing 0s. Kinds that + # attach no timings to the exception keep their 0.0 defaults. + _apply_timings(item_report, getattr(exc, "timings", None)) finally: item_report.latency_s = time.perf_counter() - t0 @@ -270,7 +382,79 @@ def run_agentic_items( except Exception: pass - report.items.append(item_report) + return item_report + + concurrency = max(1, concurrency) + indexed = list(enumerate(items, start=1)) + if concurrency > 1: + parallel = [(i, it) for i, it in indexed if runs_in_parallel(it.test_kind)] + serial = [(i, it) for i, it in indexed if not runs_in_parallel(it.test_kind)] + if not parallel and serial: + blocked = ", ".join(sorted({it.test_kind for _, it in serial})) + print( + f"warning: --concurrency {concurrency} has no effect here; every item is a " + f"workspace-mutating kind ({blocked}) and those always run one at a time.", + file=sys.stderr, + ) + else: + parallel, serial = [], indexed + + results: dict[int, ItemReport] = {} + try: + # Two phases, never interleaved: a mutating item creating and dropping a metric + # mid-run would otherwise be visible to a catalog-reading item beside it. + if parallel: + # NOT a `with` block. ThreadPoolExecutor.__exit__ is shutdown(wait=True) with + # cancel_futures left False, so an interrupt raised in this thread while it + # waits on as_completed runs every QUEUED item to completion before the + # KeyboardInterrupt is honoured. cancel_futures drops whatever has not started; the handful already in flight + # cannot be cancelled (the interpreter joins those worker threads at exit + # regardless), so this bounds the wait at one wave rather than the dataset. + pool = ThreadPoolExecutor(max_workers=concurrency, thread_name_prefix="agentic") + try: + futures = {pool.submit(_process_item, i, it): i for i, it in parallel} + for future in as_completed(futures): + results[futures[future]] = future.result() + except BaseException: + pool.shutdown(wait=False, cancel_futures=True) + raise + else: + pool.shutdown(wait=True) + for i, it in serial: + results[i] = _process_item(i, it) + report.items.extend(results[i] for i in sorted(results)) + except BaseException: + # Ctrl-C, or anything else escaping the loop: drop the queued polls rather than + # make the user sit through them (see BackgroundTraceLinker.abandon). + linker.abandon() + raise + + # Blocks until every deferred trace link has finished: "async" here means the poll + # overlaps other items' work, never that the command finishes before scores are final. + if linker.pending: + # Said before the wait, not after. The batch runs once the last item is done and + # can take tens of seconds waiting on Langfuse ingestion; without this the + # terminal sits silent right after the final item and looks hung. + print( + f"[langfuse] linking traces for {linker.pending} item(s); waiting on Langfuse ingestion...", + flush=True, + ) + _link_t0 = time.perf_counter() + linker.drain() + _link_elapsed = time.perf_counter() - _link_t0 + for finished in report.items: + finished.langfuse_latency_s = linker.durations.get(finished.id, 0.0) + if linker.durations: + # Surfaces whether the retry budget is binding. A slowest close to + # _langfuse._LINK_BUDGET_SEC means links are timing out and scores are being + # orphaned -- read it together with any "no trace found for conversation" warnings. + slowest = max(linker.durations.values()) + print( + f"[langfuse] trace linking finished in {_link_elapsed:.1f}s " + f"for {len(linker.durations)} item(s); slowest {slowest:.1f}s", + flush=True, + ) + report.wall_clock_s = time.perf_counter() - _t0 if langfuse is not None: try: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 1e40efd77..0a12cefa5 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -16,7 +16,7 @@ from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, run_agentic_items from gooddata_eval.core.chat.sse_client import ChatClient -from gooddata_eval.core.config import ReasoningEffort, RunConfig +from gooddata_eval.core.config import DEFAULT_JUDGE_MODEL, JUDGE_MODEL_ENV_VAR, ReasoningEffort, RunConfig from gooddata_eval.core.connection import ConnectionError_, resolve_connection from gooddata_eval.core.dataset.local import load_local_dataset from gooddata_eval.core.langfuse.sink import LangfuseSink @@ -25,6 +25,7 @@ from gooddata_eval.core.reporting.json_report import write_multi_model_report from gooddata_eval.core.runner import ItemReport, run_items from gooddata_eval.core.summary.http_client import SummaryClient +from gooddata_eval.core.timing import TIMERS_ENV_VAR from gooddata_eval.core.workspace import ModelResolutionError, WorkspaceModelController _EXIT_OK = 0 @@ -96,7 +97,26 @@ def _build_parser() -> argparse.ArgumentParser: type=int, default=1, help="Number of items evaluated concurrently (default 1 = sequential). " - "Increase to load-test the agent under simultaneous requests.", + "Increase to load-test the agent under simultaneous requests. Agentic kinds that " + "create workspace objects (metric_skill, alert_skill, conversation, kda_skill) always " + "run one at a time regardless, to avoid cross-test contamination.", + ) + run.add_argument( + "--judge-model", + dest="judge_model", + metavar="MODEL", + help=f"OpenAI model for LLM-as-judge scoring (default: {DEFAULT_JUDGE_MODEL}). " + "Also settable via GD_EVAL_JUDGE_MODEL. Two things to weigh before changing it: a " + "model that rejects temperature=0 (the gpt-5 family does) makes verdicts " + "non-deterministic, and picking the same model the agent runs means the judge " + "grades its own family's output.", + ) + run.add_argument( + "--timers", + action="store_true", + help="Print per-turn [timer] diagnostics (agent response, judge, simulated user). " + "Off by default because a large run emits hundreds of lines; the same measurements " + "are always in the JSON report's latency_breakdown_s. Equivalent to GD_EVAL_TIMERS=1.", ) run.add_argument("--json", dest="json_path", help="Write a JSON report to this path.") run.add_argument("--quiet", action="store_true", help="Suppress per-item progress output.") @@ -143,6 +163,50 @@ def parse_args(argv: list[str]) -> argparse.Namespace: return _build_parser().parse_args(argv) +def _apply_judge_model(model: str | None) -> None: + """Publish --judge-model for the deep LLMJudge call sites; an explicit flag wins.""" + if model: + os.environ[JUDGE_MODEL_ENV_VAR] = model + + +def _apply_timer_flag(enabled: bool) -> None: + """Publish --timers for the deep [timer] call sites. + + An env var rather than a parameter because the emitters sit four layers down, and it is + how this package already gates cross-cutting behaviour (see TAVERN_E2E_SKIP_TRACE_LINK). + Only ever sets it -- never clears a value the caller exported themselves. + """ + if enabled: + os.environ[TIMERS_ENV_VAR] = "1" + + +def _warn_if_local_dataset_cannot_link(config: RunConfig, agentic_items: list) -> None: + """Say up front that dataset-run assembly will fail, rather than after the run. + + --langfuse is refused outright with a local dataset because local item ids cannot be + linked. But every evaluate_agentic_* falls back to try_make_langfuse_client() when the + caller passes none, so with LANGFUSE_* exported the linking runs anyway and each + conversation earns a 404 from dataset-run-items -- arriving in a block at the very end + of the run, long after the flag that would have prevented it could be changed. The + fallback is deliberate (direct library and tavern callers rely on it), so this warns + instead of disabling it. + """ + from gooddata_eval.core.agentic._langfuse import SKIP_ENV_VAR, langfuse_credentials_present # noqa: PLC0415 + from gooddata_eval.core.config import env_flag # noqa: PLC0415 + + if not agentic_items or config.dataset_folder is None: + return + if not langfuse_credentials_present() or env_flag(SKIP_ENV_VAR): + return + print( + f"warning: --dataset is a local folder, so its item ids are not Langfuse dataset item ids. " + f"Traces will be found and scored, but the per-run grouping that makes models comparable " + f"cannot be created and each conversation will report a 404 from dataset-run-items. " + f"Use --langfuse-dataset for comparable runs, or set {SKIP_ENV_VAR}=1 to skip trace linking.", + file=sys.stderr, + ) + + def _truncate(text: str, limit: int = 80) -> str: return text if len(text) <= limit else text[: limit - 1] + "…" @@ -279,6 +343,7 @@ def _run(config: RunConfig) -> int: items = _load_dataset(config) agentic_items = [i for i in items if i.test_kind in AGENTIC_TEST_KINDS] non_agentic_items = [i for i in items if i.test_kind not in AGENTIC_TEST_KINDS] + _warn_if_local_dataset_cannot_link(config, agentic_items) models = config.models or [] run_ts = datetime.now(timezone.utc).strftime("%Y-%m-%d-%H-%M") n_models = len(models) if models else 1 @@ -359,6 +424,7 @@ def on_langfuse_item_done( on_item_start=on_item_start, on_item_done=on_item_done, agent_id=config.agent_id, + concurrency=config.concurrency, ) # --- non-agentic items (single-turn, use Evaluator) --- @@ -440,6 +506,8 @@ def on_langfuse_item_done( def main(argv: list[str] | None = None) -> int: args = parse_args(argv if argv is not None else sys.argv[1:]) + _apply_timer_flag(getattr(args, "timers", False)) + _apply_judge_model(getattr(args, "judge_model", None)) if hasattr(args, "concurrency") and args.concurrency < 1: print("error: --concurrency must be >= 1.", file=sys.stderr) return _EXIT_OPERATIONAL_ERROR diff --git a/packages/gooddata-eval/src/gooddata_eval/core/_output.py b/packages/gooddata-eval/src/gooddata_eval/core/_output.py new file mode 100644 index 000000000..6cd023232 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/_output.py @@ -0,0 +1,18 @@ +# (C) 2026 GoodData Corporation +"""One-write console output. + +``print`` emits the text and the newline as two separate writes. Trace linking and the +item pool both run on worker threads, so a second write arriving between those two halves +splits a line down the middle. Everything that writes progress or warnings goes through +``emit_line`` instead, which cannot interleave. +""" + +from __future__ import annotations + +import sys + + +def emit_line(message: str) -> None: + """Write one line to stdout in a single write.""" + sys.stdout.write(message + "\n") + sys.stdout.flush() diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py index 67630ce2f..5d8e8e813 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py @@ -6,6 +6,7 @@ import base64 import logging import os +import threading import time import uuid from collections.abc import Iterator @@ -15,7 +16,8 @@ import httpx -from gooddata_eval.core.config import ReasoningEffort, normalize_reasoning_effort +from gooddata_eval.core.agentic._trace_linker import link_cancel_event, linking_is_inline, warn_from_worker +from gooddata_eval.core.config import ReasoningEffort, env_flag, normalize_reasoning_effort _log = logging.getLogger(__name__) @@ -44,14 +46,35 @@ class _TraceAPI: def __init__(self, client: httpx.Client) -> None: self._client = client - def list(self, from_timestamp: Any, to_timestamp: Any, limit: int) -> _TraceListResult: + def list( + self, from_timestamp: Any, to_timestamp: Any, limit: int, session_id: str | None = None + ) -> _TraceListResult: + """List traces in a window, optionally narrowed to one session server-side. + + ``session_id`` is what makes ``limit`` a non-issue. Without it the endpoint returns + every trace in the window newest-first, so an eval workspace busy enough to put more + than ``limit`` traces inside one item's window pushes that item's OWN (oldest) trace + off the page -- it then polls its whole retry budget against a page that can never + contain it, and the score orphans with only a generic "no trace found" line to show + for it. Concurrency makes that likelier by overlapping every item's window. Named + ``session_id`` because ``_fetch_traces_for_session`` probes for exactly that + parameter; gen-ai sets sessionId = conversationId. + """ + def _ts(v: Any) -> str: return v.isoformat() if hasattr(v, "isoformat") else str(v) - resp = self._client.get( - "/api/public/traces", - params={"fromTimestamp": _ts(from_timestamp), "toTimestamp": _ts(to_timestamp), "limit": limit}, - ) + params: dict[str, Any] = { + "fromTimestamp": _ts(from_timestamp), + "toTimestamp": _ts(to_timestamp), + "limit": limit, + } + # `is not None`, not truthiness: an empty id is a real filter value that matches + # nothing. Dropped here, the query would return the whole padded window for the + # caller's post-check to throw away, page after page, for the poll's whole budget. + if session_id is not None: + params["sessionId"] = session_id + resp = self._client.get("/api/public/traces", params=params) resp.raise_for_status() return _TraceListResult([_TraceObj(t) for t in resp.json().get("data", [])]) @@ -161,6 +184,15 @@ def make_langfuse_client() -> HttpxLangfuseClient: return HttpxLangfuseClient() +def langfuse_credentials_present() -> bool: + """Whether the environment could build a Langfuse client. + + Separate from ``try_make_langfuse_client`` so a caller can ask the question without + opening an httpx client it does not intend to use. + """ + return bool(os.environ.get("LANGFUSE_PUBLIC_KEY")) and bool(os.environ.get("LANGFUSE_SECRET_KEY")) + + def try_make_langfuse_client() -> HttpxLangfuseClient | None: """Create Langfuse client from env vars; return None if credentials are missing.""" try: @@ -173,6 +205,28 @@ def try_make_langfuse_client() -> HttpxLangfuseClient | None: SKIP_ENV_VAR = "TAVERN_E2E_SKIP_TRACE_LINK" +# Run names whose dataset-run assembly has already been reported as impossible. A 404 from +# dataset-run-items means the dataset item id is not in Langfuse, which is a property of +# the dataset and not of the attempt -- so it recurs identically for every item and every +# run of that dataset, and reporting it per conversation buries the run's real output under +# dozens of copies of the same HTTP error. Guarded by a lock because linking runs on the +# drain pool's worker threads. +_UNLINKABLE_RUNS: set[str] = set() +_UNLINKABLE_RUNS_LOCK = threading.Lock() + + +def _first_report_for_run(run_name: str) -> bool: + """True the first time this run is seen, False afterwards. Thread-safe.""" + # A run is suffixed _run0.._runK per K, one per pass over the same dataset; the cause + # is shared across all of them, so collapse to the base name. + base = run_name.rsplit("_run", 1)[0] + with _UNLINKABLE_RUNS_LOCK: + if base in _UNLINKABLE_RUNS: + return False + _UNLINKABLE_RUNS.add(base) + return True + + _MAX_LATENCY_SEC = 60.0 _MAX_COST_USD = 0.05 _QUALITY_WEIGHT = 0.6 @@ -180,8 +234,22 @@ def try_make_langfuse_client() -> HttpxLangfuseClient | None: _COST_WEIGHT = 0.2 _INITIAL_DELAY = 0.5 -_MAX_ATTEMPTS = 8 _BACKOFF = 1.6 +# Ceiling on any single backoff sleep, so a long budget is spent on many steady retries +# rather than one enormous final wait. +_MAX_DELAY = 15.0 +# Budget for one item's batched poll, which blocks nobody. Sized against Langfuse ingestion +# lag on us.cloud, which runs from tens of seconds to several minutes. Shared across the +# item's conversations, so the batch tail does not grow with --runs. +_LINK_BUDGET_SEC = 120.0 +# Budget for a poll that is NOT batched: a direct library caller (the tavern e2e suite) gets +# run_trace_link_inline, so the wait lands on the test that triggered it, under a step +# timeout. Deliberately tighter than _LINK_BUDGET_SEC: no inline caller should pay for a +# budget sized for the CLI's batched tail. +_INLINE_LINK_BUDGET_SEC = 35.0 +# Sanity bound so the retry loop can never spin: the deadline is wall-clock and only the +# sleeps advance it. Comfortably above the ~12 attempts the budget actually affords. +_MAX_ATTEMPTS = 20 _WINDOW_PADDING_SEC = 2 _FETCH_LIMIT = 100 @@ -208,6 +276,15 @@ def get_model_version( return "" +def _matches_session(trace: Any, session_id: str) -> bool: + """Whether a trace belongs to the conversation, by sessionId or by its metadata.""" + sid = getattr(trace, "session_id", None) + if isinstance(sid, str) and sid == session_id: + return True + metadata = getattr(trace, "metadata", None) + return isinstance(metadata, dict) and metadata.get("conversation_id") == session_id + + def _fetch_traces_for_session( langfuse: Any, session_id: str, @@ -221,7 +298,8 @@ def _fetch_traces_for_session( "to_timestamp": window_end + pad, "limit": _FETCH_LIMIT, } - # Langfuse v4+ supports sessionId as a direct filter; older SDK / httpx path may not. + # Langfuse v4+ SDKs and the httpx client take session_id as a server-side filter; older + # SDKs do not, and then the page is the whole window. try: import inspect # noqa: PLC0415 @@ -231,50 +309,106 @@ def _fetch_traces_for_session( except Exception: pass response = langfuse.api.trace.list(**kwargs) - traces = response.data or [] - # If sessionId filter was not applied server-side, filter locally. - if "session_id" not in kwargs: - traces = [ - t - for t in traces - if (isinstance(getattr(t, "session_id", None), str) and t.session_id == session_id) - or (isinstance(getattr(t, "metadata", None), dict) and t.metadata.get("conversation_id") == session_id) - ] - return traces + # A post-check, not a fallback: the Langfuse API drops a query parameter it does not know + # rather than rejecting it, and an unfiltered page would hand the caller's max-latency + # pick a stranger's trace with no warning. On a server that did filter it is a no-op. + return [t for t in (response.data or []) if _matches_session(t, session_id)] + + +# Longest a running poll may stay asleep after cancellation is signalled. Without a bound, +# a Ctrl-C mid-drain waits out the rest of that poll's backoff -- up to _MAX_DELAY per sleep +# and _LINK_BUDGET_SEC overall -- because the interpreter joins executor workers at exit. +_CANCEL_CHECK_SEC = 0.5 + + +def _wait_between_attempts(delay: float) -> bool: + """Wait ``delay`` before the next poll attempt. False means "stop polling". + + Outside a batched drain nobody can cancel the wait -- an inline poll is charged to the + caller that asked for it -- so it stays a single sleep. Inside a drain the wait is served + in slices, so an interrupt is noticed within ``_CANCEL_CHECK_SEC`` rather than after the + whole backoff. Slicing keeps the total unchanged, so the retry ladder is unaffected. + """ + cancel = link_cancel_event() + if cancel is None: + time.sleep(delay) + return True + remaining = delay + while remaining > 0: + if cancel.is_set(): + return False + step = min(_CANCEL_CHECK_SEC, remaining) + time.sleep(step) + remaining -= step + return not cancel.is_set() def find_traces_per_conversation( langfuse: Any, conversation_ids: list[str], window_start: datetime, + window_end: datetime | None = None, + deadline: float | None = None, ) -> dict[str, Any]: - """Poll Langfuse until traces matching all conversation_ids are found or retries exhaust.""" - if bool(os.environ.get(SKIP_ENV_VAR)): + """Poll Langfuse until traces matching all conversation_ids are found or retries exhaust. + + ``window_end`` bounds the trace query and should be pinned by the caller to the moment + the conversations ended. It matters because this poll is normally deferred onto a + worker thread (see ``agentic/_trace_linker.py``): defaulting it to "now" would stretch + the window by however long the task waited in the queue, and since + ``_fetch_traces_for_session`` pages at ``_FETCH_LIMIT`` and filters by session locally, + a wide enough window can push the wanted trace off the page. Defaults to now only for + direct callers that poll immediately. + """ + if env_flag(SKIP_ENV_VAR): + # Say so. Skipping returns all-None, which downstream renders as observe()'s + # generic "No trace found for dataset run ...; scores will be orphaned" -- the + # same message a real lookup failure produces. Left silent, an eval run looks + # like Langfuse is broken when trace linking was simply switched off. + warn_from_worker( + f"[langfuse] trace linking SKIPPED by {SKIP_ENV_VAR}: " + f"{len(conversation_ids)} conversation(s) will have orphaned scores. " + f"Unset it to link traces." + ) return dict.fromkeys(conversation_ids) by_conv: dict[str, Any] = dict.fromkeys(conversation_ids) - window_end = datetime.now(timezone.utc) + window_end = window_end or datetime.now(timezone.utc) pad = timedelta(seconds=_WINDOW_PADDING_SEC) + # One budget for the whole item. Per conversation it would multiply by --runs, and the + # batch tail is meant to be bounded by the budget however many runs an item has. + budget = _INLINE_LINK_BUDGET_SEC if linking_is_inline() else _LINK_BUDGET_SEC + stop_at = deadline if deadline is not None else time.monotonic() + budget + for cid in conversation_ids: + cancel = link_cancel_event() + if cancel is not None and cancel.is_set(): + # The run is being interrupted; the remaining conversations are not worth a + # round trip, and their scores were never going to be written. + break delay = _INITIAL_DELAY found: list[Any] = [] - for _ in range(_MAX_ATTEMPTS): - time.sleep(delay) + for _attempt in range(_MAX_ATTEMPTS): + # Attempt first, sleep only between attempts: a trace that is already ingested + # when we look must cost nothing, which is the common case once linking is + # batched to the end of the run. try: found = _fetch_traces_for_session(langfuse, cid, window_start, window_end, pad) except Exception as exc: _log.debug("Langfuse trace fetch failed for %s: %s", cid, exc) - if found: + if found or time.monotonic() + delay > stop_at: + break + if not _wait_between_attempts(delay): break - delay *= _BACKOFF + delay = min(delay * _BACKOFF, _MAX_DELAY) if found: by_conv[cid] = max(found, key=lambda t: getattr(t, "latency", None) or 0.0) else: _log.warning( "[langfuse] No trace found for conversation %s in window [%s, %s]", cid, window_start, window_end ) - print(f"[langfuse] WARNING: no trace found for conversation {cid}", flush=True) + warn_from_worker(f"[langfuse] WARNING: no trace found for conversation {cid}") return by_conv @@ -313,11 +447,33 @@ def observe( _log.debug( "[langfuse] Created dataset run item: run=%s trace=%s item=%s", run_name, trace_id, dataset_item_id ) + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + _log.warning("Failed to link trace %s to run %s: %s", trace_id, run_name, exc) + warn_from_worker( + f"[langfuse] WARNING: failed to create dataset run item " + f"run={run_name} trace={trace_id} item={dataset_item_id}: {exc}" + ) + elif _first_report_for_run(run_name): + # Say what it means and what to do, once. A raw 404 names an endpoint, + # which tells the reader nothing about the cause being their --dataset. + _log.warning( + "Dataset item %s is not in Langfuse; run %s cannot be assembled.", dataset_item_id, run_name + ) + warn_from_worker( + f"[langfuse] WARNING: dataset item {dataset_item_id!r} does not exist in Langfuse, " + f"so the run {run_name!r} cannot be assembled (404 from dataset-run-items). " + f"Scores ARE still written to the traces themselves -- only the per-run grouping " + f"used to compare models is missing. This is what happens when --dataset points at " + f"a local folder: its item ids are local, not Langfuse dataset item ids. Use " + f"--langfuse-dataset to get comparable runs, or set {SKIP_ENV_VAR}=1 to skip linking " + f"altogether. Further occurrences for this run are suppressed." + ) except Exception as exc: _log.warning("Failed to link trace %s to run %s: %s", trace_id, run_name, exc) - print( - f"[langfuse] WARNING: failed to create dataset run item run={run_name} trace={trace_id} item={dataset_item_id}: {exc}", - flush=True, + warn_from_worker( + f"[langfuse] WARNING: failed to create dataset run item " + f"run={run_name} trace={trace_id} item={dataset_item_id}: {exc}" ) model_version = (run_metadata or {}).get("model_version") if model_version: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py new file mode 100644 index 000000000..ce4bc1a8f --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_trace_linker.py @@ -0,0 +1,302 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""Runs Langfuse trace lookup and scoring off the evaluation's critical path. + +Polling for a gen-ai trace waits on Langfuse ingestion and produces no pass/fail verdict, +so charging it to the item's latency both slows the run and misreports the agent's own +response time. ``run_trace_link_inline`` (the default) stays synchronous so direct callers +behave exactly as before; the CLI runner injects a ``BackgroundTraceLinker`` and drains it +before any report is rendered, so scores are still final before the command finishes. +""" + +from __future__ import annotations + +import contextvars +import logging +import threading +import time +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Protocol + +from gooddata_eval.core._output import emit_line + +_log = logging.getLogger(__name__) + +# A unit of deferred Langfuse work: find the traces for one item's conversations and +# write its scores. Takes no arguments and returns nothing -- each evaluate_agentic_* +# closes over whatever its own scoring needs. +TraceLinkTask = Callable[[], None] + + +class SubmitTraceLink(Protocol): + """How an ``evaluate_agentic_*`` hands its Langfuse block off. + + ``item_id`` is only used for attributing the task's cost and naming it in warnings; + the task itself already closes over everything it needs. + """ + + def __call__(self, task: TraceLinkTask, *, item_id: str = "") -> None: ... + + +# A trace poll is almost entirely waiting on ingestion, so the queue is run as wide as it +# gets, bounded only so a huge dataset cannot open a thread per item. +_MAX_WORKERS = 16 + + +def warn_from_worker(message: str) -> None: + """Write one warning line to stdout, safe to call from a linking worker.""" + emit_line(message) + + +# True only while a linking task runs on its caller's critical path -- the only thing that +# separates the two retry budgets in _langfuse: a batched poll blocks nobody and can wait +# minutes, an inline one is charged to whoever called evaluate_agentic_*. A ContextVar, not +# a flag, because worker threads start with a fresh context: the batched pool reads False +# for free, and the inline case is scoped to exactly the call that set it. +_INLINE_LINKING: contextvars.ContextVar[bool] = contextvars.ContextVar("gd_eval_inline_trace_link", default=False) + + +def linking_is_inline() -> bool: + """Whether the trace poll running right now sits on its caller's critical path.""" + return _INLINE_LINKING.get() + + +def run_trace_link_inline(task: TraceLinkTask, *, item_id: str = "") -> None: + """Run the linking task on the calling thread -- the default for every evaluate_agentic_*. + + ``linking_is_inline`` keeps this path on the smaller, pre-batching retry budget. + """ + token = _INLINE_LINKING.set(True) + try: + task() + finally: + _INLINE_LINKING.reset(token) + + +# The signal a running batched poll watches to stop sleeping, visible only on the worker +# thread running that poll: BackgroundTraceLinker._run sets it for the task's duration. A +# ContextVar rather than a module global, for the same reason as _INLINE_LINKING: the inline +# path never sets it, so nobody can cancel a poll charged to its own caller, and an +# interrupted drain's set Event lives on with exactly the workers it was meant for -- there is +# no shared slot for it to linger in and stop the next --model pass or a later inline link. +_CANCEL: contextvars.ContextVar[threading.Event | None] = contextvars.ContextVar( + "gd_eval_trace_link_cancel", default=None +) + + +def link_cancel_event() -> threading.Event | None: + """The cancellation signal for the batched poll running on this thread, or None.""" + return _CANCEL.get() + + +def utc_now() -> datetime: + """Now, in UTC. One spelling, so a pinned trace window cannot drift by timezone.""" + return datetime.now(timezone.utc) + + +def open_trace_window(langfuse: Any) -> tuple[Any, datetime]: + """Resolve the Langfuse client for this item and pin the start of its trace window. + + The client falls back to the ambient LANGFUSE_* credentials, so direct library and + tavern callers still link traces without being handed one. ``window_start`` is taken + before the agent runs, so the window brackets exactly this item's conversations. + """ + from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 + + return (try_make_langfuse_client() if langfuse is None else langfuse), utc_now() + + +@dataclass(frozen=True) +class RunIdentity: + """Everything ``build_run_context`` needs to name and describe one eval run. + + Built on the calling thread and captured by value, so a deferred task never reaches + back into the evaluation's own objects to resolve a name. + """ + + host: str + token: str + workspace_id: str + dataset_name: str + run_timestamp: str | None + model_version_override: str | None + run_metadata_extra: dict | None + reasoning_effort: Any + + +@dataclass(frozen=True) +class RunTraceContext: + """One item's resolved Langfuse run identity, its traces, and how to score them. + + Carries the ``_langfuse`` module and the client so a kind's scoring body needs neither + -- otherwise all eight repeat the same deferred import and thread ``langfuse`` and + ``dataset_item_id`` through every call. + """ + + run_metadata: dict + _lf: Any + _client: Any + _dataset_item_id: str + _base_name: str + _suffix_runs: bool + _traces: dict[str, Any] + + def run_name(self, run_idx: int) -> str: + """Dataset-run name for one run, suffixed only when the item has more than one.""" + return f"{self._base_name}_run{run_idx}" if self._suffix_runs else self._base_name + + def trace(self, conversation_id: str) -> Any: + """The trace picked for a conversation, or None when the poll never found one.""" + return self._traces.get(conversation_id) + + def observe(self, trace: Any, run_idx: int) -> Any: + """Attach this run to its dataset-run item, yielding the trace id to score against. + + ``trace`` may be None -- a conversation whose trace never showed up is still + observed, so the run appears in the experiment with its scores orphaned rather than + missing entirely. + """ + return self._lf.observe( + self._client, + trace.id if trace else None, + self._dataset_item_id, + self.run_name(run_idx), + self.run_metadata, + ) + + def score(self, trace_id: Any, *, name: str, value: Any, data_type: str) -> None: + """Write one score, swallowing Langfuse failures the way ``score_safe`` always has.""" + self._lf.score_safe(self._client, trace_id, name=name, value=value, data_type=data_type) + + def quality(self, trace_id: Any, *, strict_checks: dict, latency_sec: Any, cost_usd: Any) -> None: + """Write the derived quality/value scores for one run.""" + self._lf.log_quality_and_value_scores( + self._client, trace_id, strict_checks=strict_checks, latency_sec=latency_sec, cost_usd=cost_usd + ) + + +def submit_trace_scoring( + submit_trace_link: SubmitTraceLink, + identity: RunIdentity, + *, + langfuse: Any, + dataset_item_id: str, + conversation_ids: list[str], + window_start: datetime, + window_end: datetime, + suffix_runs: bool, + write_scores: Callable[[RunTraceContext], None], +) -> None: + """Defer one item's whole Langfuse block: resolve its run context, then write scores. + + Deferred as a unit because ``build_run_context``'s workspace lookup and + ``find_traces_per_conversation``'s ingestion-lag poll are both round trips publishing an + already-decided verdict, so neither belongs on the item's clock. Every caller pins + ``window_end`` before calling: a deferred poll must not widen its own query window. + """ + + def _link_traces() -> None: + from gooddata_eval.core.agentic import _langfuse # noqa: PLC0415 + + base_name, run_metadata = _langfuse.build_run_context( + identity.host, + identity.token, + identity.workspace_id, + identity.dataset_name, + identity.run_timestamp, + identity.model_version_override, + identity.run_metadata_extra, + identity.reasoning_effort, + ) + traces = _langfuse.find_traces_per_conversation(langfuse, conversation_ids, window_start, window_end) + write_scores( + RunTraceContext(run_metadata, _langfuse, langfuse, dataset_item_id, base_name, suffix_runs, traces) + ) + + submit_trace_link(_link_traces, item_id=dataset_item_id) + + +class BackgroundTraceLinker: + """Collects trace-linking tasks during the run and executes them all at ``drain``. + + Batched, not fired on submit: Langfuse ingestion lags by tens of seconds to minutes, so + a poll issued the instant its conversation ended is the one least likely to find + anything. Holding the queue until the agent phase ends gives every trace the length of + the run to appear. Task failures are logged and swallowed -- a Langfuse outage has never + been allowed to fail an eval run. + """ + + def __init__(self, max_workers: int = _MAX_WORKERS, clock: Callable[[], float] = time.monotonic) -> None: + self._max_workers = max_workers + self._clock = clock + self._queue: list[tuple[TraceLinkTask, str]] = [] + self.durations: dict[str, float] = {} + # The drain in flight's signal, so abandon() can reach its workers. + self._cancel: threading.Event | None = None + + def submit(self, task: TraceLinkTask, *, item_id: str = "") -> None: + """Queue the task. Nothing runs until ``drain``.""" + self._queue.append((task, item_id)) + + @property + def pending(self) -> int: + """How many links are queued and waiting for ``drain``.""" + return len(self._queue) + + def _run(self, task: TraceLinkTask, item_id: str, cancel: threading.Event) -> None: + # Runs on the worker thread, so this scopes the signal to exactly this task: a worker + # still polling after drain() has raised keeps reading the same (set) event, and + # nothing outside the pool ever sees it. + token = _CANCEL.set(cancel) + started = self._clock() + try: + task() + except Exception as exc: + _log.warning("Langfuse trace linking failed for item %s: %s", item_id or "?", exc) + warn_from_worker(f"warning: Langfuse trace linking failed for item '{item_id}': {exc}") + finally: + # Recorded on the failure path too: an item whose poll exhausted its budget and + # then errored is precisely the one whose Langfuse cost the report should show. + self.durations[item_id] = self._clock() - started + _CANCEL.reset(token) + + def drain(self) -> None: + """Run every queued link in parallel and wait for the batch to finish.""" + queue, self._queue = self._queue, [] + if not queue: + return + # NOT a `with` block, for the same reason the item pool in cli/agentic_runner.py is + # not: __exit__ is shutdown(wait=True) with cancel_futures left False, so a Ctrl-C + # mid-batch would run every QUEUED poll -- almost entirely time.sleep -- to + # completion first. abandon() cannot help here (the queue moved into `queue` above), + # so the cancellation has to happen on the pool itself. + pool = ThreadPoolExecutor(max_workers=min(len(queue), self._max_workers), thread_name_prefix="trace-link") + cancel = threading.Event() + self._cancel = cancel + try: + for task, item_id in queue: + pool.submit(self._run, task, item_id, cancel) + pool.shutdown(wait=True) + except BaseException: + # Signalled BEFORE the shutdown: cancel_futures only drops what has not started, + # and a poll already running sits in a backoff sleep until its deadline -- which + # the interpreter then waits out, because it joins executor workers at exit. So a + # Ctrl-C could hang for the whole batch budget. This wakes them instead. + cancel.set() + pool.shutdown(wait=False, cancel_futures=True) + raise + finally: + self._cancel = None + + def abandon(self) -> None: + """Discard the queue without running it. + + For the interrupt path: nothing has started yet, so Ctrl-C costs nothing rather + than making the user sit through a batch of retrying polls. + """ + self._queue.clear() + if self._cancel is not None: + # Only when a drain is in flight on another thread; harmless then too. + self._cancel.set() diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index dc47ace2e..1df7c3238 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -12,9 +12,24 @@ from gooddata_sdk import GoodDataSdk from gooddata_eval.core.agentic._catalog import CatalogMetricAlert +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import AgenticEvalOutcome, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, +) try: from openai import OpenAI as _OpenAI @@ -611,15 +626,9 @@ def _run_once(conv_id: str) -> AlertRunResult: ) -class AlertSkillAssertionError(AssertionError): +class AlertSkillAssertionError(AgenticAssertionError): """Raised when an alert-skill evaluation fails.""" - __tracebackhide__ = True - reasoning_steps: list[str] - conversation_id: str - response_id: str | None - detail: dict - def evaluate_agentic_alert_skill( host: str, @@ -638,6 +647,7 @@ def evaluate_agentic_alert_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, ) -> AgenticEvalOutcome: """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure. @@ -648,14 +658,7 @@ def evaluate_agentic_alert_skill( `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them either way. """ - from datetime import datetime as _dt # noqa: PLC0415 - from datetime import timezone as _tz # noqa: PLC0415 - - from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 - - if langfuse is None: - langfuse = try_make_langfuse_client() - window_start = _dt.now(_tz.utc) + langfuse, window_start = open_trace_window(langfuse) summary = run_agentic_alert_skill( host=host, token=token, @@ -670,57 +673,73 @@ def evaluate_agentic_alert_skill( ) if langfuse is not None and dataset_item_id: - from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 - build_run_context, - find_traces_per_conversation, - log_quality_and_value_scores, - observe, - score_safe, + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + ev = run.eval + strict_checks = { + "alert_created": ev.alert_created, + "operator_correct": ev.operator_correct, + "threshold_correct": ev.threshold_correct, + "trigger_correct": ev.trigger_correct, + "filters_correct": ev.filters_correct, + "metric_correct": ev.metric_correct, + "recipients_correct": ev.recipients_correct, + } + with ctx.observe(pt, run_idx) as tid: + for score_name, value in strict_checks.items(): + ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + ctx.quality( + tid, + strict_checks=strict_checks, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, ) - run_name_base, run_metadata = build_run_context( - host, - token, - workspace_id, - dataset_name, - run_timestamp, - model_version_override, - run_metadata_extra, - reasoning_effort, - ) - traces_by_conv = find_traces_per_conversation( - langfuse, - [r.conversation_id for r in summary.run_results], - window_start, - ) - suffix_needed = len(summary.run_results) > 1 - for run_idx, run in enumerate(summary.run_results): - pt = traces_by_conv.get(run.conversation_id) - run_name = f"{run_name_base}_run{run_idx}" if suffix_needed else run_name_base - ev = run.eval - strict_checks = { - "alert_created": ev.alert_created, - "operator_correct": ev.operator_correct, - "threshold_correct": ev.threshold_correct, - "trigger_correct": ev.trigger_correct, - "filters_correct": ev.filters_correct, - "metric_correct": ev.metric_correct, - "recipients_correct": ev.recipients_correct, - } - with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name, run_metadata) as tid: - for score_name, value in strict_checks.items(): - score_safe(langfuse, tid, name=score_name, value=float(value), data_type="BOOLEAN") - log_quality_and_value_scores( - langfuse, - tid, - strict_checks=strict_checks, - latency_sec=pt.latency if pt else None, - cost_usd=pt.total_cost if pt else None, - ) + runs_passed = sum(1 for r in summary.run_results if r.eval.strict_pass) + runs_effective = len(summary.run_results) + + best = summary.best + ev = best.eval + detail = { + "alert_created": ev.alert_created, + "operator_correct": ev.operator_correct, + "threshold_correct": ev.threshold_correct, + "trigger_correct": ev.trigger_correct, + "filters_correct": ev.filters_correct, + "metric_correct": ev.metric_correct, + "recipients_correct": ev.recipients_correct, + "actual_alert_arguments": best.actual_alert_arguments, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + } if not summary.pass_at_k: - best = summary.best - ev = best.eval exc = AlertSkillAssertionError( f"Alert skill assertion failed. strict_pass={ev.strict_pass}. " f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, " @@ -732,33 +751,15 @@ def evaluate_agentic_alert_skill( exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id - exc.detail = { - "alert_created": ev.alert_created, - "operator_correct": ev.operator_correct, - "threshold_correct": ev.threshold_correct, - "trigger_correct": ev.trigger_correct, - "filters_correct": ev.filters_correct, - "metric_correct": ev.metric_correct, - "recipients_correct": ev.recipients_correct, - "actual_alert_arguments": best.actual_alert_arguments, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective raise exc - best = summary.best - ev = best.eval return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, - detail={ - "alert_created": ev.alert_created, - "operator_correct": ev.operator_correct, - "threshold_correct": ev.threshold_correct, - "trigger_correct": ev.trigger_correct, - "filters_correct": ev.filters_correct, - "metric_correct": ev.metric_correct, - "recipients_correct": ev.recipients_correct, - "actual_alert_arguments": best.actual_alert_arguments, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - }, + detail=detail, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index e0183d5ac..f7e1ecb42 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -11,11 +11,21 @@ from gooddata_sdk import GoodDataSdk from pydantic import BaseModel +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) from gooddata_eval.core.agentic.alert_skill import render_alert_proposal from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids, _extract_metric_result from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.models import ( + AgenticAssertionError, AgenticEvalOutcome, ChatResult, ReasoningStepEvent, @@ -448,15 +458,9 @@ def _conversation_detail(result: ConversationResult) -> dict: } -class ConversationAssertionError(AssertionError): +class ConversationAssertionError(AgenticAssertionError): """Raised when a conversation evaluation fails.""" - __tracebackhide__ = True - reasoning_steps: list[str] - conversation_id: str - response_id: str | None - detail: dict - def evaluate_agentic_conversation( host: str, @@ -473,6 +477,7 @@ def evaluate_agentic_conversation( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, ) -> AgenticEvalOutcome: """Run conversation evaluation, log to Langfuse, and raise on failure. @@ -483,14 +488,7 @@ def evaluate_agentic_conversation( `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them either way. """ - from datetime import datetime as _dt # noqa: PLC0415 - from datetime import timezone as _tz # noqa: PLC0415 - - from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 - - if langfuse is None: - langfuse = try_make_langfuse_client() - window_start = _dt.now(_tz.utc) + langfuse, window_start = open_trace_window(langfuse) result = run_agentic_conversation( host=host, token=token, @@ -503,59 +501,68 @@ def evaluate_agentic_conversation( ) if langfuse is not None and dataset_item_id: - from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 - build_run_context, - find_traces_per_conversation, - log_quality_and_value_scores, - observe, - score_safe, - ) + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + # Resolved here, not inside the task: deferring it would make the queued task hold + # the whole fixture until the drain. + ds_name = dataset_name or fixture.dataset_name - run_name_base, run_metadata = build_run_context( - host, - token, - workspace_id, - dataset_name or fixture.dataset_name, - run_timestamp, - model_version_override, - run_metadata_extra, - reasoning_effort, - ) - traces_by_conv = find_traces_per_conversation( - langfuse, - [result.conversation_id], - window_start, - ) - pt = traces_by_conv.get(result.conversation_id) - with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name_base, run_metadata) as tid: - score_safe( - langfuse, - tid, - name="conversation_success", - value=float(result.conversation_success), - data_type="BOOLEAN", - ) - score_safe( - langfuse, tid, name="full_skill_coverage", value=float(result.full_skill_coverage), data_type="BOOLEAN" - ) - for tr in result.turn_results: - score_safe( - langfuse, + def _write_scores(ctx: RunTraceContext) -> None: + + pt = ctx.trace(result.conversation_id) + with ctx.observe(pt, 0) as tid: + ctx.score( tid, - name=f"turn_{tr.turn_id}_skill_success", - value=float(tr.skill_success), + name="conversation_success", + value=float(result.conversation_success), data_type="BOOLEAN", ) - log_quality_and_value_scores( - langfuse, - tid, - strict_checks={ - "conversation_success": result.conversation_success, - "full_skill_coverage": result.full_skill_coverage, - }, - latency_sec=pt.latency if pt else None, - cost_usd=pt.total_cost if pt else None, - ) + ctx.score( + tid, + name="full_skill_coverage", + value=float(result.full_skill_coverage), + data_type="BOOLEAN", + ) + for tr in result.turn_results: + ctx.score( + tid, + name=f"turn_{tr.turn_id}_skill_success", + value=float(tr.skill_success), + data_type="BOOLEAN", + ) + ctx.quality( + tid, + strict_checks={ + "conversation_success": result.conversation_success, + "full_skill_coverage": result.full_skill_coverage, + }, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + ds_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[result.conversation_id], + window_start=window_start, + window_end=window_end, + suffix_runs=False, + write_scores=_write_scores, + ) + + detail = _conversation_detail(result) if not result.conversation_success: failed_turns = [tr for tr in result.turn_results if not tr.skill_success] @@ -567,11 +574,17 @@ def evaluate_agentic_conversation( exc.reasoning_steps = result.reasoning_steps exc.conversation_id = result.conversation_id exc.response_id = result.response_id - exc.detail = _conversation_detail(result) + exc.detail = detail + # This kind takes no k and drives its fixture exactly once, whatever --runs asks + # for. Saying so explicitly stops the report claiming K runs that never happened. + exc.runs_passed = 0 + exc.runs_effective = 1 raise exc return AgenticEvalOutcome( reasoning_steps=result.reasoning_steps, conversation_id=result.conversation_id, response_id=result.response_id, - detail=_conversation_detail(result), + detail=detail, + runs_passed=1, + runs_effective=1, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index e2aa147b6..8455dc378 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -3,12 +3,23 @@ from __future__ import annotations +import time from dataclasses import dataclass, field +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.evaluators._llm_judge import LLMJudge -from gooddata_eval.core.models import AgenticEvalOutcome +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run +from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome +from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings _DEFAULT_K = 1 @@ -55,6 +66,11 @@ class GeneralQuestionResult: reasoning: str reasoning_steps: list[str] = field(default_factory=list) response_id: str | None = None + timings: PhaseTimings = field(default_factory=PhaseTimings) + # Set when the judge returned something unreadable for THIS run. Such a run is + # excluded from pass@K and from Langfuse scoring rather than counted as a failure: + # scoring it 0 would publish a verdict the judge never gave. + judge_error: str | None = None @dataclass @@ -66,6 +82,58 @@ class AgenticGeneralQuestionSummary: pass_power_k: bool best: GeneralQuestionResult + @property + def scored_run_results(self) -> list[GeneralQuestionResult]: + """The runs the judge actually graded -- the only ones pass@K may consider.""" + return [r for r in self.run_results if r.judge_error is None] + + @property + def judge_errors(self) -> list[str]: + """One message per run the judge could not grade.""" + return [r.judge_error for r in self.run_results if r.judge_error is not None] + + +def _run_single_general_question( + client: ChatClient, + judge: LLMJudge, + conversation_id: str, + question: str, + expected_output: str, + user_context: dict | None = None, +) -> GeneralQuestionResult: + item_started = time.monotonic() + agent_started = time.monotonic() + chat_result = client.send_message(conversation_id, question, user_context=user_context) + actual_output = (chat_result.text_response or "").strip() + agent_elapsed = time.monotonic() - agent_started + log_timer( + f"[timer] general_question {conversation_id} GoodData response complete after " + f"{agent_elapsed:.2f}s; waiting for {judge.model} judge" + ) + + judge_started = time.monotonic() + verdict = score_run(judge, input=question, expected_output=expected_output, actual_output=actual_output) + judge_elapsed = time.monotonic() - judge_started + total_elapsed = time.monotonic() - item_started + log_timer( + f"[timer] general_question {conversation_id} {judge.model} judge complete after " + f"{judge_elapsed:.2f}s; item total {total_elapsed:.2f}s" + ) + return GeneralQuestionResult( + conversation_id=conversation_id, + actual_output=actual_output, + passed=verdict.passed, + llm_judge_score=1.0 if verdict.passed else 0.0, + reasoning=verdict.reasoning, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, + # Derived from the elapsed values the [timer] lines above already computed, so + # recording them adds no extra clock reads. Recorded on the unscored path too: the + # agent still answered, and that measurement is the one worth keeping. + timings=PhaseTimings(agent_s=agent_elapsed, judge_s=judge_elapsed), + judge_error=verdict.error, + ) + def run_agentic_general_question( host: str, @@ -77,33 +145,20 @@ def run_agentic_general_question( initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, agent_id: str | None = None, + user_context: dict | None = None, ) -> AgenticGeneralQuestionSummary: """Run the general-question agentic evaluation K times and return a summary.""" run_results: list[GeneralQuestionResult] = [] client = ChatClient( host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id ) - judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o") + judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - chat_result = client.send_message(conv_id_0, question) - actual_output = (chat_result.text_response or "").strip() - passed, reasoning = judge.score( - input=question, expected_output=expected_output, actual_output=actual_output - ) - llm_judge_score = 1.0 if passed else 0.0 run_results.append( - GeneralQuestionResult( - conversation_id=conv_id_0, - actual_output=actual_output, - passed=passed, - llm_judge_score=llm_judge_score, - reasoning=reasoning, - reasoning_steps=list(chat_result.reasoning_steps or []), - response_id=chat_result.response_id, - ) + _run_single_general_question(client, judge, conv_id_0, question, expected_output, user_context) ) finally: if initial_conversation_id is None: @@ -112,31 +167,22 @@ def run_agentic_general_question( for _ in range(1, k): conv_id = client.create_conversation() try: - chat_result = client.send_message(conv_id, question) - actual_output = (chat_result.text_response or "").strip() - passed, reasoning = judge.score( - input=question, expected_output=expected_output, actual_output=actual_output - ) - llm_judge_score = 1.0 if passed else 0.0 run_results.append( - GeneralQuestionResult( - conversation_id=conv_id, - actual_output=actual_output, - passed=passed, - llm_judge_score=llm_judge_score, - reasoning=reasoning, - reasoning_steps=list(chat_result.reasoning_steps or []), - response_id=chat_result.response_id, - ) + _run_single_general_question(client, judge, conv_id, question, expected_output, user_context) ) finally: client.delete_conversation(conv_id) finally: client.close() - pass_at_k = any(r.passed for r in run_results) - pass_power_k = all(r.passed for r in run_results) - best = max(run_results, key=lambda r: r.llm_judge_score) + # The two aggregates treat an ungraded run differently, on purpose. pass@K asks "did + # any run pass", which an ungraded run cannot change, so it is excluded. pass^K claims + # every run passed, which one ungraded run leaves unverified, so it is False. With + # nothing graded at all there is no verdict either way and this raises. + scored = [r for r in run_results if r.judge_error is None] + pass_at_k = any(r.passed for r in scored) + pass_power_k = len(scored) == len(run_results) and bool(scored) and all(r.passed for r in scored) + best = max(scored or run_results, key=lambda r: r.llm_judge_score) return AgenticGeneralQuestionSummary( run_results=run_results, pass_at_k=pass_at_k, @@ -145,15 +191,9 @@ def run_agentic_general_question( ) -class GeneralQuestionAssertionError(AssertionError): +class GeneralQuestionAssertionError(AgenticAssertionError): """Raised when a general-question evaluation fails.""" - __tracebackhide__ = True - reasoning_steps: list[str] - conversation_id: str - response_id: str | None - detail: dict - def evaluate_agentic_general_question( host: str, @@ -171,6 +211,8 @@ def evaluate_agentic_general_question( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, + user_context: dict | None = None, ) -> AgenticEvalOutcome: """Run general-question evaluation, log to Langfuse, and raise GeneralQuestionAssertionError on failure. @@ -178,14 +220,7 @@ def evaluate_agentic_general_question( AgenticEvalOutcome on success; on failure the same three values are attached to the raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. """ - from datetime import datetime as _dt # noqa: PLC0415 - from datetime import timezone as _tz # noqa: PLC0415 - - from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 - - if langfuse is None: - langfuse = try_make_langfuse_client() - window_start = _dt.now(_tz.utc) + langfuse, window_start = open_trace_window(langfuse) summary = run_agentic_general_question( host=host, token=token, @@ -196,69 +231,102 @@ def evaluate_agentic_general_question( initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, agent_id=agent_id, + user_context=user_context, ) if langfuse is not None and dataset_item_id: - from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 - build_run_context, - find_traces_per_conversation, - log_quality_and_value_scores, - observe, - score_safe, - ) + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() - run_name_base, run_metadata = build_run_context( - host, - token, - workspace_id, - dataset_name, - run_timestamp, - model_version_override, - run_metadata_extra, - reasoning_effort, + def _write_scores(ctx: RunTraceContext) -> None: + + for run_idx, run in enumerate(summary.run_results): + if run.judge_error is not None: + # No verdict for this run: float(run.passed) would write a 0 the judge + # never returned. + continue + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score(tid, name="general_question_pass", value=float(run.passed), data_type="BOOLEAN") + ctx.score(tid, name="llm_judge_score", value=run.llm_judge_score, data_type="NUMERIC") + ctx.quality( + tid, + strict_checks={"general_question_pass": run.passed}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + # Ungraded runs are skipped above, so polling for their traces would only spend + # the item's shared retry budget on scores that never get written. + conversation_ids=[r.conversation_id for r in summary.scored_run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, ) - traces_by_conv = find_traces_per_conversation( - langfuse, - [r.conversation_id for r in summary.run_results], - window_start, + + item_timings = sum_timings([r.timings for r in summary.run_results]) + unscored = summary.judge_errors + + if not summary.scored_run_results: + # Not one run produced a readable verdict, so this item has no result -- an error, + # not K failures. Raised after the trace link is queued so whatever the agent did + # is still linked, and carrying the timings so the runner can report what the item + # cost before it became unevaluable. + exc = JudgeResponseError( + f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " + + " | ".join(unscored) ) - suffix_needed = len(summary.run_results) > 1 - for run_idx, run in enumerate(summary.run_results): - pt = traces_by_conv.get(run.conversation_id) - run_name = f"{run_name_base}_run{run_idx}" if suffix_needed else run_name_base - with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name, run_metadata) as tid: - score_safe(langfuse, tid, name="general_question_pass", value=float(run.passed), data_type="BOOLEAN") - score_safe(langfuse, tid, name="llm_judge_score", value=run.llm_judge_score, data_type="NUMERIC") - log_quality_and_value_scores( - langfuse, - tid, - strict_checks={"general_question_pass": run.passed}, - latency_sec=pt.latency if pt else None, - cost_usd=pt.total_cost if pt else None, - ) + exc.timings = item_timings + raise exc + + runs_passed = sum(1 for r in summary.scored_run_results if r.passed) + runs_effective = len(summary.run_results) + + best = summary.best + detail = { + "judge_passed": best.passed, + "judge_reasoning": best.reasoning, + "actual_output": best.actual_output, + # Only present when it happened, so the usual JSON shape is unchanged. A + # pass@K computed over fewer runs than --runs asked for is a weaker result and + # the report has to say so. + **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), + } if not summary.pass_at_k: - best = summary.best exc = GeneralQuestionAssertionError( f"General question assertion failed. passed={best.passed}. Reasoning: {best.reasoning}" ) exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id - exc.detail = { - "judge_passed": best.passed, - "judge_reasoning": best.reasoning, - "actual_output": best.actual_output, - } + exc.timings = item_timings + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective raise exc - best = summary.best return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, - detail={ - "judge_passed": best.passed, - "judge_reasoning": best.reasoning, - "actual_output": best.actual_output, - }, + detail=detail, + timings=item_timings, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index 673a7b321..03b178c98 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -5,10 +5,25 @@ from dataclasses import dataclass, field +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.evaluators._llm_judge import LLMJudge -from gooddata_eval.core.models import AgenticEvalOutcome, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, +) _DEFAULT_K = 1 @@ -54,6 +69,9 @@ class GuardrailResult: response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + # Set when the judge returned something unreadable for THIS run. Excluded from pass@K + # and from Langfuse scoring rather than counted as a failure -- see score_run. + judge_error: str | None = None @dataclass @@ -65,6 +83,45 @@ class AgenticGuardrailSummary: pass_power_k: bool best: GuardrailResult + @property + def scored_run_results(self) -> list[GuardrailResult]: + """The runs the judge actually graded -- the only ones pass@K may consider.""" + return [r for r in self.run_results if r.judge_error is None] + + @property + def judge_errors(self) -> list[str]: + """One message per run the judge could not grade.""" + return [r.judge_error for r in self.run_results if r.judge_error is not None] + + +def _run_single_guardrail( + client: ChatClient, + judge: LLMJudge, + conversation_id: str, + question: str, + expected_output: str, +) -> GuardrailResult: + """Ask once and grade once. + + Extracted so the two call sites below (the first conversation, which may be supplied, + and the remaining K-1) cannot drift -- they had already duplicated the whole body once. + """ + chat_result = client.send_message(conversation_id, question) + actual_output = (chat_result.text_response or "").strip() + verdict = score_run(judge, input=question, expected_output=expected_output, actual_output=actual_output) + return GuardrailResult( + conversation_id=conversation_id, + actual_output=actual_output, + passed=verdict.passed, + llm_judge_score=1.0 if verdict.passed else 0.0, + reasoning=verdict.reasoning, + reasoning_steps=list(chat_result.reasoning_steps or []), + response_id=chat_result.response_id, + tool_call_events=list(chat_result.tool_call_events or []), + reasoning_step_events=list(chat_result.reasoning_step_events or []), + judge_error=verdict.error, + ) + def run_agentic_guardrail( host: str, @@ -82,30 +139,12 @@ def run_agentic_guardrail( client = ChatClient( host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id ) - judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o") + judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: - chat_result = client.send_message(conv_id_0, question) - actual_output = (chat_result.text_response or "").strip() - passed, reasoning = judge.score( - input=question, expected_output=expected_output, actual_output=actual_output - ) - llm_judge_score = 1.0 if passed else 0.0 - run_results.append( - GuardrailResult( - conversation_id=conv_id_0, - actual_output=actual_output, - passed=passed, - llm_judge_score=llm_judge_score, - reasoning=reasoning, - reasoning_steps=list(chat_result.reasoning_steps or []), - response_id=chat_result.response_id, - tool_call_events=list(chat_result.tool_call_events or []), - reasoning_step_events=list(chat_result.reasoning_step_events or []), - ) - ) + run_results.append(_run_single_guardrail(client, judge, conv_id_0, question, expected_output)) finally: if initial_conversation_id is None: client.delete_conversation(conv_id_0) @@ -113,33 +152,20 @@ def run_agentic_guardrail( for _ in range(1, k): conv_id = client.create_conversation() try: - chat_result = client.send_message(conv_id, question) - actual_output = (chat_result.text_response or "").strip() - passed, reasoning = judge.score( - input=question, expected_output=expected_output, actual_output=actual_output - ) - llm_judge_score = 1.0 if passed else 0.0 - run_results.append( - GuardrailResult( - conversation_id=conv_id, - actual_output=actual_output, - passed=passed, - llm_judge_score=llm_judge_score, - reasoning=reasoning, - reasoning_steps=list(chat_result.reasoning_steps or []), - response_id=chat_result.response_id, - tool_call_events=list(chat_result.tool_call_events or []), - reasoning_step_events=list(chat_result.reasoning_step_events or []), - ) - ) + run_results.append(_run_single_guardrail(client, judge, conv_id, question, expected_output)) finally: client.delete_conversation(conv_id) finally: client.close() - pass_at_k = any(r.passed for r in run_results) - pass_power_k = all(r.passed for r in run_results) - best = max(run_results, key=lambda r: r.llm_judge_score) + # The two aggregates treat an ungraded run differently, on purpose. pass@K asks "did + # any run pass", which an ungraded run cannot change, so it is excluded. pass^K claims + # every run passed, which one ungraded run leaves unverified, so it is False. With + # nothing graded at all there is no verdict either way and this raises. + scored = [r for r in run_results if r.judge_error is None] + pass_at_k = any(r.passed for r in scored) + pass_power_k = len(scored) == len(run_results) and bool(scored) and all(r.passed for r in scored) + best = max(scored or run_results, key=lambda r: r.llm_judge_score) return AgenticGuardrailSummary( run_results=run_results, pass_at_k=pass_at_k, @@ -148,15 +174,9 @@ def run_agentic_guardrail( ) -class GuardrailAssertionError(AssertionError): +class GuardrailAssertionError(AgenticAssertionError): """Raised when a guardrail evaluation fails.""" - __tracebackhide__ = True - reasoning_steps: list[str] - conversation_id: str - response_id: str | None - detail: dict - def evaluate_agentic_guardrail( host: str, @@ -174,6 +194,7 @@ def evaluate_agentic_guardrail( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, ) -> AgenticEvalOutcome: """Run guardrail evaluation, log to Langfuse, and raise GuardrailAssertionError on failure. @@ -182,14 +203,7 @@ def evaluate_agentic_guardrail( raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors `evaluate_agentic_metric_skill`'s idiom) so callers can retrieve them either way. """ - from datetime import datetime as _dt # noqa: PLC0415 - from datetime import timezone as _tz # noqa: PLC0415 - - from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 - - if langfuse is None: - langfuse = try_make_langfuse_client() - window_start = _dt.now(_tz.utc) + langfuse, window_start = open_trace_window(langfuse) summary = run_agentic_guardrail( host=host, token=token, @@ -203,66 +217,89 @@ def evaluate_agentic_guardrail( ) if langfuse is not None and dataset_item_id: - from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 - build_run_context, - find_traces_per_conversation, - log_quality_and_value_scores, - observe, - score_safe, - ) + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + + for run_idx, run in enumerate(summary.run_results): + if run.judge_error is not None: + # No verdict, so nothing truthful to publish: float(run.passed) would + # write a 0 the judge never returned. + continue + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score(tid, name="guardrail_pass", value=float(run.passed), data_type="BOOLEAN") + ctx.score(tid, name="llm_judge_score", value=run.llm_judge_score, data_type="NUMERIC") + ctx.quality( + tid, + strict_checks={"guardrail_pass": run.passed}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) - run_name_base, run_metadata = build_run_context( - host, - token, - workspace_id, - dataset_name, - run_timestamp, - model_version_override, - run_metadata_extra, - reasoning_effort, + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + # Ungraded runs are skipped above, so polling for their traces would only spend + # the item's shared retry budget on scores that never get written. + conversation_ids=[r.conversation_id for r in summary.scored_run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, ) - traces_by_conv = find_traces_per_conversation( - langfuse, - [r.conversation_id for r in summary.run_results], - window_start, + + unscored = summary.judge_errors + + if not summary.scored_run_results: + # No readable verdict for any run: an error, not K failures. Raised after the + # trace link is queued so whatever the agent did is still linked. + raise JudgeResponseError( + f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " + + " | ".join(unscored) ) - suffix_needed = len(summary.run_results) > 1 - for run_idx, run in enumerate(summary.run_results): - pt = traces_by_conv.get(run.conversation_id) - run_name = f"{run_name_base}_run{run_idx}" if suffix_needed else run_name_base - with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name, run_metadata) as tid: - score_safe(langfuse, tid, name="guardrail_pass", value=float(run.passed), data_type="BOOLEAN") - score_safe(langfuse, tid, name="llm_judge_score", value=run.llm_judge_score, data_type="NUMERIC") - log_quality_and_value_scores( - langfuse, - tid, - strict_checks={"guardrail_pass": run.passed}, - latency_sec=pt.latency if pt else None, - cost_usd=pt.total_cost if pt else None, - ) + + runs_passed = sum(1 for r in summary.scored_run_results if r.passed) + runs_effective = len(summary.run_results) + + best = summary.best + detail = { + "judge_passed": best.passed, + "judge_reasoning": best.reasoning, + "actual_output": best.actual_output, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + # Only present when it happened, so the usual JSON shape is unchanged. A + # pass@K over fewer runs than --runs asked for is a weaker result. + **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), + } if not summary.pass_at_k: - best = summary.best exc = GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}") exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id - exc.detail = { - "judge_passed": best.passed, - "judge_reasoning": best.reasoning, - "actual_output": best.actual_output, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective raise exc - best = summary.best return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, - detail={ - "judge_passed": best.passed, - "judge_reasoning": best.reasoning, - "actual_output": best.actual_output, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - }, + detail=detail, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py index fcbb6786a..37c19ba90 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py @@ -7,9 +7,18 @@ import os from dataclasses import dataclass, field +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent +from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ToolCallEvent _log = logging.getLogger(__name__) @@ -321,15 +330,9 @@ def _run_once(conv_id: str) -> KdaRunResult: ) -class KdaSkillAssertionError(AssertionError): +class KdaSkillAssertionError(AgenticAssertionError): """Raised when a KDA-skill evaluation fails.""" - __tracebackhide__ = True - reasoning_steps: list[str] - conversation_id: str - response_id: str | None - detail: dict - def evaluate_agentic_kda_skill( host: str, @@ -348,6 +351,7 @@ def evaluate_agentic_kda_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, ) -> AgenticEvalOutcome: """Run KDA-skill evaluation, log to Langfuse, and raise KdaSkillAssertionError on failure. @@ -355,14 +359,7 @@ def evaluate_agentic_kda_skill( AgenticEvalOutcome on success; on failure the same three values are attached to the raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. """ - from datetime import datetime as _dt # noqa: PLC0415 - from datetime import timezone as _tz # noqa: PLC0415 - - from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 - - if langfuse is None: - langfuse = try_make_langfuse_client() - window_start = _dt.now(_tz.utc) + langfuse, window_start = open_trace_window(langfuse) summary = run_agentic_kda_skill( host=host, token=token, @@ -377,66 +374,86 @@ def evaluate_agentic_kda_skill( ) if langfuse is not None and dataset_item_id: - from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 - build_run_context, - find_traces_per_conversation, - log_quality_and_value_scores, - observe, - score_safe, - ) + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + + for run_idx, run in enumerate(summary.run_results): + # No custom selector -- same default (max-latency) as every other skill; harmless + # here since latency comes from run.turn_wall_clock_sec below, not this trace. + pt = ctx.trace(run.conversation_id) + run_name = ctx.run_name(run_idx) + ev = run.evaluation + # Gates strict_pass -- current scope is completion only (see KdaEvaluation docstring). + strict_checks = { + "kda_triggered": ev.triggered, + "kda_executed": ev.executed, + "kda_success": ev.success, + "kda_turn_completed": ev.turn_completed, + } + # Not pt.latency: pt can be any trace of the conversation, not necessarily the KDA turn. + turn_wall_clock_sec = run.turn_wall_clock_sec + _log.info( + "[kda-report] %s: strict_pass=%s latency_sec=%s", run_name, ev.strict_pass, turn_wall_clock_sec + ) + with ctx.observe(pt, run_idx) as tid: + for score_name, value in strict_checks.items(): + ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + ctx.score(tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN") + if turn_wall_clock_sec is not None: + # combo_report.py reads this score directly -- no trace re-resolution needed. + ctx.score( + tid, + name="kda_turn_wall_clock_sec", + value=turn_wall_clock_sec, + data_type="NUMERIC", + ) + ctx.quality( + tid, + strict_checks=strict_checks, + latency_sec=turn_wall_clock_sec, + cost_usd=pt.total_cost if pt and ev.triggered else None, + ) - run_name_base, run_metadata = build_run_context( - host, - token, - workspace_id, - dataset_name, - run_timestamp, - model_version_override, - run_metadata_extra, - reasoning_effort, - ) - # No custom selector -- same default (max-latency) as every other skill; harmless - # here since latency comes from run.turn_wall_clock_sec below, not this trace. - traces_by_conv = find_traces_per_conversation( - langfuse, - [r.conversation_id for r in summary.run_results], - window_start, + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, ) - suffix_needed = len(summary.run_results) > 1 - for run_idx, run in enumerate(summary.run_results): - pt = traces_by_conv.get(run.conversation_id) - run_name = f"{run_name_base}_run{run_idx}" if suffix_needed else run_name_base - ev = run.evaluation - # Gates strict_pass -- current scope is completion only (see KdaEvaluation docstring). - strict_checks = { - "kda_triggered": ev.triggered, - "kda_executed": ev.executed, - "kda_success": ev.success, - "kda_turn_completed": ev.turn_completed, - } - # Not pt.latency: pt can be any trace of the conversation, not necessarily the KDA turn. - turn_wall_clock_sec = run.turn_wall_clock_sec - _log.info("[kda-report] %s: strict_pass=%s latency_sec=%s", run_name, ev.strict_pass, turn_wall_clock_sec) - with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name, run_metadata) as tid: - for score_name, value in strict_checks.items(): - score_safe(langfuse, tid, name=score_name, value=float(value), data_type="BOOLEAN") - score_safe(langfuse, tid, name="kda_disambiguated", value=float(ev.disambiguated), data_type="BOOLEAN") - if turn_wall_clock_sec is not None: - # combo_report.py reads this score directly -- no trace re-resolution needed. - score_safe( - langfuse, tid, name="kda_turn_wall_clock_sec", value=turn_wall_clock_sec, data_type="NUMERIC" - ) - log_quality_and_value_scores( - langfuse, - tid, - strict_checks=strict_checks, - latency_sec=turn_wall_clock_sec, - cost_usd=pt.total_cost if pt and ev.triggered else None, - ) + + runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) + runs_effective = len(summary.run_results) + + best = summary.best + ev = best.evaluation + detail = { + "triggered": ev.triggered, + "executed": ev.executed, + "success": ev.success, + "turn_completed": ev.turn_completed, + "disambiguated": ev.disambiguated, + "actual_create_args": best.actual_create_args, + "actual_execute_result": best.actual_execute_result, + } if not summary.pass_at_k: - best = summary.best - ev = best.evaluation message = ( f"KDA skill assertion failed. strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, " @@ -448,29 +465,15 @@ def evaluate_agentic_kda_skill( exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id - exc.detail = { - "triggered": ev.triggered, - "executed": ev.executed, - "success": ev.success, - "turn_completed": ev.turn_completed, - "disambiguated": ev.disambiguated, - "actual_create_args": best.actual_create_args, - "actual_execute_result": best.actual_execute_result, - } + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective raise exc - best = summary.best - ev = best.evaluation return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, - detail={ - "triggered": ev.triggered, - "executed": ev.executed, - "success": ev.success, - "turn_completed": ev.turn_completed, - "disambiguated": ev.disambiguated, - "actual_create_args": best.actual_create_args, - "actual_execute_result": best.actual_execute_result, - }, + detail=detail, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index cb98be8a6..4457ea117 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -5,14 +5,31 @@ import os import re +import time from dataclasses import dataclass, field from typing import Any from gooddata_sdk import GoodDataSdk +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import AgenticEvalOutcome, ReasoningStepEvent, ToolCallEvent, build_latency_breakdown +from gooddata_eval.core.models import ( + AgenticAssertionError, + AgenticEvalOutcome, + ReasoningStepEvent, + ToolCallEvent, + build_latency_breakdown, +) +from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings try: from openai import OpenAI as _OpenAI @@ -199,6 +216,7 @@ class MetricRunResult: response_id: str | None = None tool_call_events: list[ToolCallEvent] = field(default_factory=list) reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list) + timings: PhaseTimings = field(default_factory=PhaseTimings) @dataclass @@ -289,6 +307,7 @@ def _execute_single_metric_run( response_id: str | None = None all_tool_call_events: list[ToolCallEvent] = [] all_reasoning_step_events: list[ReasoningStepEvent] = [] + timings = PhaseTimings() turn_offset = 0.0 # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time tool_index_offset = 0 reasoning_index_offset = 0 @@ -296,7 +315,10 @@ def _execute_single_metric_run( try: for _iteration in range(max_iterations): turns += 1 + agent_started = time.monotonic() chat_result = client.send_message(conversation_id, current_question) + agent_elapsed = time.monotonic() - agent_started + timings.agent_s += agent_elapsed reasoning_steps.extend(chat_result.reasoning_steps or []) response_id = chat_result.response_id or response_id for tc in chat_result.tool_call_events or []: @@ -319,6 +341,10 @@ def _execute_single_metric_run( created_metric_ids.append(metric_id) candidate = _extract_metric_result(chat_result.tool_call_events or []) if candidate is not None: + log_timer( + f"[timer] metric_skill {conversation_id} GoodData turn {turns} complete after " + f"{agent_elapsed:.2f}s; metric result received" + ) metric_result = candidate break response_text = (chat_result.text_response or "").strip() @@ -326,11 +352,28 @@ def _execute_single_metric_run( break if _iteration >= max_iterations - 1: break + log_timer( + f"[timer] metric_skill {conversation_id} GoodData turn {turns} complete after " + f"{agent_elapsed:.2f}s; waiting for gpt-4o-mini simulated user" + ) + simulated_started = time.monotonic() try: current_question = generate_simulated_response(response_text, expected_outputs, question) except SimulatedResponseError as exc: + simulated_elapsed = time.monotonic() - simulated_started + timings.simulated_user_s += simulated_elapsed print(f"[SIM-USER] Simulated reply failed for conversation {conversation_id}: {exc}") + log_timer( + f"[timer] metric_skill {conversation_id} gpt-4o-mini simulated user failed after " + f"{simulated_elapsed:.2f}s" + ) break + simulated_elapsed = time.monotonic() - simulated_started + timings.simulated_user_s += simulated_elapsed + log_timer( + f"[timer] metric_skill {conversation_id} gpt-4o-mini simulated user complete after " + f"{simulated_elapsed:.2f}s" + ) actual_maql = (metric_result or {}).get("maql", "") metric_created = metric_result is not None @@ -346,6 +389,7 @@ def _execute_single_metric_run( response_id=response_id, tool_call_events=all_tool_call_events, reasoning_step_events=all_reasoning_step_events, + timings=timings, ) finally: for metric_id in created_metric_ids: @@ -412,15 +456,9 @@ def run_agentic_metric_skill( ) -class MetricSkillAssertionError(AssertionError): +class MetricSkillAssertionError(AgenticAssertionError): """Raised when a metric-skill evaluation fails.""" - __tracebackhide__ = True - reasoning_steps: list[str] - conversation_id: str - response_id: str | None - detail: dict - def evaluate_agentic_metric_skill( host: str, @@ -439,6 +477,7 @@ def evaluate_agentic_metric_skill( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, ) -> AgenticEvalOutcome: """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure. @@ -449,14 +488,7 @@ def evaluate_agentic_metric_skill( `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them either way. """ - from datetime import datetime as _dt # noqa: PLC0415 - from datetime import timezone as _tz # noqa: PLC0415 - - from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 - - if langfuse is None: - langfuse = try_make_langfuse_client() - window_start = _dt.now(_tz.utc) + langfuse, window_start = open_trace_window(langfuse) summary = run_agentic_metric_skill( host=host, token=token, @@ -471,47 +503,61 @@ def evaluate_agentic_metric_skill( ) if langfuse is not None and dataset_item_id: - from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 - build_run_context, - find_traces_per_conversation, - log_quality_and_value_scores, - observe, - score_safe, - ) + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score(tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") + ctx.score(tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") + ctx.quality( + tid, + strict_checks={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) - run_name_base, run_metadata = build_run_context( - host, - token, - workspace_id, - dataset_name, - run_timestamp, - model_version_override, - run_metadata_extra, - reasoning_effort, - ) - traces_by_conv = find_traces_per_conversation( - langfuse, - [r.conversation_id for r in summary.run_results], - window_start, + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, ) - suffix_needed = len(summary.run_results) > 1 - for run_idx, run in enumerate(summary.run_results): - pt = traces_by_conv.get(run.conversation_id) - run_name = f"{run_name_base}_run{run_idx}" if suffix_needed else run_name_base - with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name, run_metadata) as tid: - score_safe(langfuse, tid, name="metric_created", value=float(run.metric_created), data_type="BOOLEAN") - score_safe(langfuse, tid, name="maql_correct", value=float(run.maql_correct), data_type="BOOLEAN") - log_quality_and_value_scores( - langfuse, - tid, - strict_checks={"metric_created": run.metric_created, "maql_correct": run.maql_correct}, - latency_sec=pt.latency if pt else None, - cost_usd=pt.total_cost if pt else None, - ) + + item_timings = sum_timings([r.timings for r in summary.run_results]) + + runs_passed = sum(1 for r in summary.run_results if r.metric_created and r.maql_correct) + runs_effective = len(summary.run_results) + + best = summary.best + expected_outputs_list: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] + detail = { + "metric_created": best.metric_created, + "maql_correct": best.maql_correct, + "expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list], + "actual_maql": best.actual_maql, + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + } if not summary.pass_at_k: - best = summary.best - expected_outputs_list: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list) exc = MetricSkillAssertionError( f"Metric skill assertion failed. " @@ -522,25 +568,17 @@ def evaluate_agentic_metric_skill( exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id - exc.detail = { - "metric_created": best.metric_created, - "maql_correct": best.maql_correct, - "expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list], - "actual_maql": best.actual_maql, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + exc.timings = item_timings + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective raise exc - best = summary.best - expected_outputs_list = expected_output if isinstance(expected_output, list) else [expected_output] return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, - detail={ - "metric_created": best.metric_created, - "maql_correct": best.maql_correct, - "expected_maql_candidates": [c.get("maql", "") for c in expected_outputs_list], - "actual_maql": best.actual_maql, - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - }, + detail=detail, + timings=item_timings, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 5a36f299a..b42bb4782 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -5,9 +5,18 @@ from dataclasses import dataclass, field +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort -from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent +from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ToolCallEvent _DEFAULT_K = 1 @@ -135,15 +144,9 @@ def run_agentic_search_tool( ) -class SearchToolAssertionError(AssertionError): +class SearchToolAssertionError(AgenticAssertionError): """Raised when a search-tool evaluation fails.""" - __tracebackhide__ = True - reasoning_steps: list[str] - conversation_id: str - response_id: str | None - detail: dict - def evaluate_agentic_search_tool( host: str, @@ -161,6 +164,7 @@ def evaluate_agentic_search_tool( model_version_override: str | None = None, run_metadata_extra: dict | None = None, reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, ) -> AgenticEvalOutcome: """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure. @@ -168,14 +172,7 @@ def evaluate_agentic_search_tool( AgenticEvalOutcome on success; on failure the same three values are attached to the raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. """ - from datetime import datetime as _dt # noqa: PLC0415 - from datetime import timezone as _tz # noqa: PLC0415 - - from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 - - if langfuse is None: - langfuse = try_make_langfuse_client() - window_start = _dt.now(_tz.utc) + langfuse, window_start = open_trace_window(langfuse) summary = run_agentic_search_tool( host=host, token=token, @@ -189,46 +186,56 @@ def evaluate_agentic_search_tool( ) if langfuse is not None and dataset_item_id: - from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 - build_run_context, - find_traces_per_conversation, - log_quality_and_value_scores, - observe, - score_safe, - ) + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() - run_name_base, run_metadata = build_run_context( - host, - token, - workspace_id, - dataset_name, - run_timestamp, - model_version_override, - run_metadata_extra, - reasoning_effort, - ) - traces_by_conv = find_traces_per_conversation( - langfuse, - [r.conversation_id for r in summary.run_results], - window_start, + def _write_scores(ctx: RunTraceContext) -> None: + + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + with ctx.observe(pt, run_idx) as tid: + ctx.score(tid, name="tool_selection", value=float(run.tool_selected), data_type="BOOLEAN") + ctx.score(tid, name="tool_correctness", value=float(run.tool_correct), data_type="BOOLEAN") + ctx.quality( + tid, + strict_checks={"tool_selection": run.tool_selected}, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + suffix_runs=len(summary.run_results) > 1, + write_scores=_write_scores, ) - suffix_needed = len(summary.run_results) > 1 - for run_idx, run in enumerate(summary.run_results): - pt = traces_by_conv.get(run.conversation_id) - run_name = f"{run_name_base}_run{run_idx}" if suffix_needed else run_name_base - with observe(langfuse, pt.id if pt else None, dataset_item_id, run_name, run_metadata) as tid: - score_safe(langfuse, tid, name="tool_selection", value=float(run.tool_selected), data_type="BOOLEAN") - score_safe(langfuse, tid, name="tool_correctness", value=float(run.tool_correct), data_type="BOOLEAN") - log_quality_and_value_scores( - langfuse, - tid, - strict_checks={"tool_selection": run.tool_selected}, - latency_sec=pt.latency if pt else None, - cost_usd=pt.total_cost if pt else None, - ) + + runs_passed = sum(1 for r in summary.run_results if r.tool_selected) + runs_effective = len(summary.run_results) + + best = summary.best + detail = { + "tool_selected": best.tool_selected, + "tool_correct": best.tool_correct, + "tool_call_names": best.tool_call_names, + } if not summary.pass_at_k: - best = summary.best exc = SearchToolAssertionError( f"Search tool assertion failed. " f"tool_selected={best.tool_selected}, tool_correct={best.tool_correct}. " @@ -237,20 +244,15 @@ def evaluate_agentic_search_tool( exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id - exc.detail = { - "tool_selected": best.tool_selected, - "tool_correct": best.tool_correct, - "tool_call_names": best.tool_call_names, - } + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective raise exc - best = summary.best return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, - detail={ - "tool_selected": best.tool_selected, - "tool_correct": best.tool_correct, - "tool_call_names": best.tool_call_names, - }, + detail=detail, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 486214cce..748c49787 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -10,6 +10,15 @@ import os from dataclasses import dataclass, field +from gooddata_eval.core.agentic._trace_linker import ( + RunIdentity, + RunTraceContext, + SubmitTraceLink, + open_trace_window, + run_trace_link_inline, + submit_trace_scoring, + utc_now, +) from gooddata_eval.core.chat.sse_client import ChatClient from gooddata_eval.core.config import ReasoningEffort from gooddata_eval.core.evaluators.visualization import ( @@ -19,6 +28,7 @@ evaluation_result_detail, ) from gooddata_eval.core.models import ( + AgenticAssertionError, AgenticEvalOutcome, CreatedVisualization, ReasoningStepEvent, @@ -285,15 +295,9 @@ def run_agentic_visualization( ) -class VisualizationAssertionError(AssertionError): +class VisualizationAssertionError(AgenticAssertionError): """Raised when a visualization evaluation fails.""" - __tracebackhide__ = True - reasoning_steps: list[str] - conversation_id: str - response_id: str | None - detail: dict - def _filter_diff(category: str, ev: EvaluationResult) -> str: """Expected-vs-actual lines for one filter category, or "" when they matched. @@ -325,6 +329,7 @@ def evaluate_agentic_visualization( run_metadata_extra: dict | None = None, record_output_path: str | None = None, reasoning_effort: ReasoningEffort | None = None, + submit_trace_link: SubmitTraceLink = run_trace_link_inline, ) -> AgenticEvalOutcome: """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure. @@ -333,14 +338,8 @@ def evaluate_agentic_visualization( raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``. """ import json as _json # noqa: PLC0415 - from datetime import datetime as _dt # noqa: PLC0415 - from datetime import timezone as _tz # noqa: PLC0415 - from gooddata_eval.core.agentic._langfuse import try_make_langfuse_client # noqa: PLC0415 - - if langfuse is None: - langfuse = try_make_langfuse_client() - window_start = _dt.now(_tz.utc) + langfuse, window_start = open_trace_window(langfuse) summary = run_agentic_visualization( host=host, token=token, @@ -355,64 +354,62 @@ def evaluate_agentic_visualization( ) if langfuse is not None and dataset_item_id: - from gooddata_eval.core.agentic._langfuse import ( # noqa: PLC0415 - build_run_context, - find_traces_per_conversation, - log_quality_and_value_scores, - observe, - score_safe, + # Pinned on the calling thread: a deferred poll must not widen its query window. + window_end = utc_now() + + def _write_scores(ctx: RunTraceContext) -> None: + + K = len(summary.run_results) + for run_idx, run in enumerate(summary.run_results): + pt = ctx.trace(run.conversation_id) + ev = run.eval_result + with ctx.observe(pt, run_idx) as tid: + ctx.score(tid, name="assertion-cross-ref-valid", value=ev.cross_ref_valid, data_type="BOOLEAN") + ctx.score(tid, name="assertion-vis-metric", value=ev.metrics_correct, data_type="BOOLEAN") + ctx.score(tid, name="assertion-vis-dimensions", value=ev.dimensions_correct, data_type="BOOLEAN") + ctx.score(tid, name="assertion-vis-filters", value=ev.filters_correct, data_type="BOOLEAN") + ctx.score(tid, name="assertion-vis-type", value=ev.viz_type_hard, data_type="BOOLEAN") + ctx.score(tid, name="skill_selection", value=ev.skill_activated, data_type="BOOLEAN") + ctx.score(tid, name=f"pass_at_{K}", value=summary.pass_at_k, data_type="BOOLEAN") + ctx.score(tid, name=f"pass_power_{K}", value=summary.pass_power_k, data_type="BOOLEAN") + ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") + ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") + ctx.quality( + tid, + strict_checks={ + "assertion-cross-ref-valid": ev.cross_ref_valid, + "assertion-vis-metric": ev.metrics_correct, + "assertion-vis-dimensions": ev.dimensions_correct, + "assertion-vis-filters": ev.filters_correct, + "assertion-vis-type": ev.viz_type_hard, + }, + latency_sec=pt.latency if pt else None, + cost_usd=pt.total_cost if pt else None, + ) + + # Before the pass@K raise: a failing item's scores are the ones worth having. + submit_trace_scoring( + submit_trace_link, + RunIdentity( + host, + token, + workspace_id, + dataset_name, + run_timestamp, + model_version_override, + run_metadata_extra, + reasoning_effort, + ), + langfuse=langfuse, + dataset_item_id=dataset_item_id, + conversation_ids=[r.conversation_id for r in summary.run_results], + window_start=window_start, + window_end=window_end, + # Unlike the other runners, this one suffixes every run, K=1 included. + suffix_runs=True, + write_scores=_write_scores, ) - run_name_base, run_metadata = build_run_context( - host, - token, - workspace_id, - dataset_name, - run_timestamp, - model_version_override, - run_metadata_extra, - reasoning_effort, - ) - K = len(summary.run_results) - traces_by_conv = find_traces_per_conversation( - langfuse, - [r.conversation_id for r in summary.run_results], - window_start, - ) - for run_idx, run in enumerate(summary.run_results): - pt = traces_by_conv.get(run.conversation_id) - ev = run.eval_result - with observe( - langfuse, pt.id if pt else None, dataset_item_id, f"{run_name_base}_run{run_idx}", run_metadata - ) as tid: - score_safe( - langfuse, tid, name="assertion-cross-ref-valid", value=ev.cross_ref_valid, data_type="BOOLEAN" - ) - score_safe(langfuse, tid, name="assertion-vis-metric", value=ev.metrics_correct, data_type="BOOLEAN") - score_safe( - langfuse, tid, name="assertion-vis-dimensions", value=ev.dimensions_correct, data_type="BOOLEAN" - ) - score_safe(langfuse, tid, name="assertion-vis-filters", value=ev.filters_correct, data_type="BOOLEAN") - score_safe(langfuse, tid, name="assertion-vis-type", value=ev.viz_type_hard, data_type="BOOLEAN") - score_safe(langfuse, tid, name="skill_selection", value=ev.skill_activated, data_type="BOOLEAN") - score_safe(langfuse, tid, name=f"pass_at_{K}", value=summary.pass_at_k, data_type="BOOLEAN") - score_safe(langfuse, tid, name=f"pass_power_{K}", value=summary.pass_power_k, data_type="BOOLEAN") - score_safe(langfuse, tid, name="turns", value=run.total_turns, data_type="NUMERIC") - score_safe(langfuse, tid, name="steps", value=run.total_steps, data_type="NUMERIC") - log_quality_and_value_scores( - langfuse, - tid, - strict_checks={ - "assertion-cross-ref-valid": ev.cross_ref_valid, - "assertion-vis-metric": ev.metrics_correct, - "assertion-vis-dimensions": ev.dimensions_correct, - "assertion-vis-filters": ev.filters_correct, - "assertion-vis-type": ev.viz_type_hard, - }, - latency_sec=pt.latency if pt else None, - cost_usd=pt.total_cost if pt else None, - ) - if record_output_path and summary.best.actual_output is not None: import json as _j # noqa: PLC0415 @@ -422,9 +419,17 @@ def evaluate_agentic_visualization( with open(record_output_path, "w") as _f: _j.dump(_fixture, _f, indent=2) + runs_passed = sum(1 for r in summary.run_results if r.eval_result.strict_pass) + runs_effective = len(summary.run_results) + + best = summary.best + ev = best.eval_result + detail = { + **evaluation_result_detail(ev), + "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), + } + if not summary.pass_at_k: - best = summary.best - ev = best.eval_result n = len(expected_outputs) candidate_note = f" (closest of {n} candidates)" if n > 1 else "" cross_ref_detail = (" → " + "; ".join(ev.cross_ref_errors)) if ev.cross_ref_errors else "" @@ -462,18 +467,15 @@ def evaluate_agentic_visualization( exc.reasoning_steps = best.reasoning_steps exc.conversation_id = best.conversation_id exc.response_id = best.response_id - exc.detail = { - **evaluation_result_detail(ev), - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - } + exc.detail = detail + exc.runs_passed = runs_passed + exc.runs_effective = runs_effective raise exc - best = summary.best return AgenticEvalOutcome( + runs_passed=runs_passed, + runs_effective=runs_effective, reasoning_steps=best.reasoning_steps, conversation_id=best.conversation_id, response_id=best.response_id, - detail={ - **evaluation_result_detail(best.eval_result), - "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events), - }, + detail=detail, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 7d52dad3f..55c14fb6b 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -354,12 +354,18 @@ def delete_conversation(self, conversation_id: str) -> None: except httpx.HTTPError: pass # best-effort cleanup - def send_message(self, conversation_id: str, question: str) -> ChatResult: + def send_message( + self, conversation_id: str, question: str, *, user_context: dict[str, Any] | None = None + ) -> ChatResult: url = f"{self._base}/{conversation_id}/messages" headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json"} body: dict[str, Any] = {"item": {"role": "user", "content": {"type": "text", "text": question}}} if self._reasoning_effort is not None: body["options"] = {"reasoningEffort": self._reasoning_effort} + # Only when there is one: gen-ai accepts an explicit null, so assigning + # unconditionally would quietly change every request that has no attachment. + if user_context is not None: + body["userContext"] = user_context def _do() -> ChatResult: # Set fresh on every retry attempt (before opening this attempt's stream, so its @@ -390,7 +396,7 @@ def ask(self, item: DatasetItem) -> ChatResult: conversation_id = self.create_conversation() success = False try: - result = self.send_message(conversation_id, item.question) + result = self.send_message(conversation_id, item.question, user_context=item.user_context) result.conversation_id = conversation_id success = True return result diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 06a2dd926..06c836c97 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -1,10 +1,35 @@ # (C) 2026 GoodData Corporation """Validated run configuration produced by the CLI and consumed by the runner.""" +import os from dataclasses import dataclass, field from pathlib import Path from typing import Literal, cast, get_args +# Values that mean "off" when read from the environment. Needed because every non-empty +# string is truthy in Python, so a bare bool() on an env var reads FOO=0 -- the obvious way +# to write "off" -- as ON. +_ENV_FALSE = frozenset({"", "0", "false", "no", "off"}) + + +def env_flag(name: str) -> bool: + """True only when the variable is set to something that does not mean "off".""" + return os.environ.get(name, "").strip().lower() not in _ENV_FALSE + + +# The judge model, overridable for comparison experiments. The default is deliberate on +# two counts: gpt-4o honours temperature=0, so a given response always gets the same +# verdict; and it is NOT the family under test, so the judge is not grading its own +# output. Changing it trades one or both of those away -- see LLMJudge.score. +JUDGE_MODEL_ENV_VAR = "GD_EVAL_JUDGE_MODEL" +DEFAULT_JUDGE_MODEL = "gpt-4o" + + +def judge_model() -> str: + """The LLM-as-judge model, from the environment or the default.""" + return os.environ.get(JUDGE_MODEL_ENV_VAR, "").strip() or DEFAULT_JUDGE_MODEL + + ReasoningEffort = Literal["LOW", "MEDIUM", "HIGH"] """Effort values the AI chat endpoint accepts, uppercase as the server enum requires.""" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py index f0f17babb..90b56e05f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/dataset/langfuse_source.py @@ -13,7 +13,7 @@ import base64 import os -from typing import Any, cast +from typing import Any, TypeVar, cast import httpx @@ -22,6 +22,8 @@ _DEFAULT_HOST = "https://cloud.langfuse.com" _PAGE_SIZE = 100 +_T = TypeVar("_T") + def _make_client() -> httpx.Client: """Build an httpx client with Langfuse basic-auth headers.""" @@ -47,32 +49,60 @@ def _question_from_input(raw_input: Any) -> str: raise ValueError(f"Unsupported Langfuse item input shape: {raw_input!r}") -def _summary_input_from_raw(raw: dict, expected_output: Any) -> SummaryInput | None: - """Locate a dashboard_summary item's `summary_input`. +def _first_of(wanted: type[_T], key: str, *sources: Any) -> _T | None: + """First source that is a dict carrying a `wanted` value under `key`, in priority order. - Langfuse items have no dedicated field for it, so accept it (in priority - order) from the item input object, the item metadata, or the expectedOutput. + Langfuse items have no dedicated field for any of the things we look up this way, so + each one is accepted from whichever of the item's objects carries it. """ - candidate: Any = None - raw_input = raw.get("input") - metadata = raw.get("metadata") - if isinstance(raw_input, dict) and isinstance(raw_input.get("summary_input"), dict): - candidate = raw_input["summary_input"] - elif isinstance(metadata, dict) and isinstance(metadata.get("summary_input"), dict): - candidate = metadata["summary_input"] - elif isinstance(expected_output, dict) and isinstance(expected_output.get("summary_input"), dict): - candidate = expected_output["summary_input"] + for source in sources: + if isinstance(source, dict): + value = source.get(key) + if isinstance(value, wanted): + return value + return None + + +def _summary_input_from_raw(raw: dict, expected_output: Any) -> SummaryInput | None: + """Locate a dashboard_summary item's `summary_input`.""" + candidate = _first_of(dict, "summary_input", raw.get("input"), raw.get("metadata"), expected_output) return SummaryInput.model_validate(candidate) if candidate is not None else None -def _infer_test_kind(expected_output: object, default: str) -> str: - """Infer test_kind from expected_output structure when not explicitly set.""" - if not isinstance(expected_output, dict): +def _user_context_from_raw(raw: dict) -> dict[str, Any] | None: + """Locate an item's `user_context` -- relayed verbatim as the chat request's `userContext`. + + Like `summary_input`, Langfuse has no dedicated field for it, so accept it from the + item input object or the item metadata. An item carrying an attachment (a WIDGET or + VIEW descriptor) is meaningless without it: it degrades into a bare question the + agent has no way to answer, and then fails for a reason that has nothing to do with + what the item was written to test. So this has to survive the round trip. + """ + found = _first_of(dict, "user_context", raw.get("input"), raw.get("metadata")) + return cast("dict[str, Any]", found) if found is not None else None + + +def _infer_test_kind(expected_output: object, default: str, metadata: object = None) -> str: + """Resolve test_kind: an explicit declaration first, then expected_output's structure. + + Precedence: `expectedOutput.test_kind`, then `metadata.test_kind`, then the shape of + expected_output, then `default` (the CLI's --kind). Both explicit forms beat + structural inference. Metadata matters because an item judged by a natural-language + rubric has a plain *string* expectedOutput, which gives the structure checks below + nothing to read -- metadata is the only place such a dataset can carry its own kind. + """ + eo: dict[str, Any] | None = cast("dict[str, Any]", expected_output) if isinstance(expected_output, dict) else None + # An explicit declaration wins over structure, expected_output ahead of metadata. + # A blank declaration is not a declaration: "" would beat both the structural checks + # below and the CLI default, and the item would be skipped as an unsupported kind. + # Checked one source at a time so a blank expectedOutput.test_kind does not hide a real + # metadata.test_kind behind it. + for source in (eo, metadata): + declared = _first_of(str, "test_kind", source) + if declared and declared.strip(): + return declared.strip() + if eo is None: return default - eo: dict[str, Any] = cast("dict[str, Any]", expected_output) - # Explicit override wins - if isinstance(eo.get("test_kind"), str): - return eo["test_kind"] # {"visualization": {...}} or {"visualization": [...]} → production agentic vis if eo.get("visualization") is not None: return "vis_agentic" @@ -86,7 +116,7 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt """Map a Langfuse REST API dataset-item dict to a DatasetItem.""" # REST API returns camelCase: expectedOutput, not expected_output expected_output = raw.get("expectedOutput") or raw.get("expected_output") - resolved_kind = _infer_test_kind(expected_output, test_kind) + resolved_kind = _infer_test_kind(expected_output, test_kind, raw.get("metadata")) return DatasetItem( id=str(raw["id"]), dataset_name=raw.get("datasetName") or dataset_name, @@ -94,6 +124,7 @@ def _item_from_raw(raw: dict, *, dataset_name: str, test_kind: str) -> DatasetIt question=_question_from_input(raw.get("input")), expected_output=expected_output, summary_input=_summary_input_from_raw(raw, expected_output), + user_context=_user_context_from_raw(raw), ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py index a50601f8a..d7e9ddc5f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/_llm_judge.py @@ -1,12 +1,128 @@ # (C) 2026 GoodData Corporation """Shared LLM-as-judge for general_question and guardrail evaluators. -Requires gooddata-eval[llm-judge] (openai>=1.40) and OPENAI_API_KEY. +Requires gooddata-eval[llm-judge] (openai>=1.45, for max_completion_tokens) and OPENAI_API_KEY. Replicates DeepEval GEval(strict_mode=True) without a DeepEval dependency. """ import json import os +from collections.abc import Callable +from typing import Any, NamedTuple + +from gooddata_eval.core._output import emit_line +from gooddata_eval.core.config import env_flag, judge_model +from gooddata_eval.core.timing import PhaseTimings + +# Turns the per-call ``[judge]`` diagnostics on. Off by default: an 18-item ``--runs 2`` +# run emits 36 of these, which buries the per-item progress output. +JUDGE_DIAGNOSTICS_ENV_VAR = "GD_EVAL_JUDGE_DIAGNOSTICS" + +# Ceiling on the judge's completion, reasoning tokens included. A verdict body is ~60 +# tokens; the rest is headroom for a reasoning model's hidden chain, which is what +# actually consumes the budget. It is a cap and not a reservation -- a non-reasoning +# judge never approaches it -- so one value serves every model. Left unset, the ceiling +# is whatever the provider defaults to, which is how a judge ends up spending its whole +# budget on reasoning and returning an empty body. +JUDGE_MAX_COMPLETION_TOKENS = 4096 + +# Requests per verdict when the body comes back empty. Truncation is transient -- the +# reasoning chain that overran the budget is resampled -- so it is worth exactly one +# more request. Anything malformed rather than empty is NOT retried: a model that +# answers with the wrong key answers with the wrong key again, and retrying only +# doubles the cost of a prompt that needs fixing. +_EMPTY_BODY_ATTEMPTS = 2 + + +class JudgeResponseError(RuntimeError): + """The judge returned something that is not a verdict. + + Raised rather than returning a value, because the alternative is worse: the previous + ``int(data.get("score", 0))`` turned an empty body, a truncated one, a recased key or + a non-numeric score into a silent FAIL, indistinguishable from the judge genuinely + failing the answer. A run could report a real pass-rate drop that was actually a parse + bug. The message quotes the raw body and the response metadata so the cause is + readable from the error alone. + """ + + # Set by the caller when the item it aborted had already measured something. Declared + # rather than attached loosely, because the runner reads it off the exception to report + # what an unevaluable item still cost. + timings: PhaseTimings + + +def _message_content(response: Any) -> str | None: + """The completion body, or None when the response carries no readable choice. + + ``choices`` comes back empty from content filters and from gateways that put an error + envelope where the completion should be. That is the commonest shape of an unreadable + judge response, and indexing it directly escaped as a bare ``IndexError`` -- past the + typed error this module exists to raise, carrying none of the body or metadata that + makes the cause readable. Returning None routes it through the same empty-body path. + """ + choices = getattr(response, "choices", None) or [] + if not choices: + return None + message = getattr(choices[0], "message", None) + content = getattr(message, "content", None) + return content if isinstance(content, str) else None + + +def _rejects_temperature(exc: Exception) -> bool: + """Whether this failure is specifically the provider refusing ``temperature``. + + Read off the provider's structured error, never off ``str(exc)``: the openai SDK + stringifies the whole response body into the exception message, and OpenAI-compatible + gateways (LiteLLM, vLLM) echo the request back inside that body -- so any 400 from one + of those carries the literal text ``"temperature": 0``. ``param`` is authoritative; + providers that omit it get a substring check against the error *message* only, which + is prose rather than a serialized request. A body that is present but not a dict is + response text the SDK could not parse as JSON -- the SDK also makes it the exception + message -- so it is the same serialized request, not prose, and is not read at all. + """ + body = getattr(exc, "body", None) + if isinstance(body, dict): + error = body.get("error") + error = error if isinstance(error, dict) else body + if error.get("param") == "temperature": + return True + message = error.get("message") + return "temperature" in message.lower() if isinstance(message, str) else False + if getattr(exc, "param", None) == "temperature": + return True + if body is not None: + return False + return "temperature" in str(exc).lower() + + +def _bit(name: str, read: Callable[[], Any]) -> str: + """``name=value``, or "" when the field is absent on this provider's response.""" + try: + return f"{name}={read()}" + except Exception: + return "" + + +def _response_metadata(response: Any) -> str: + """finish_reason / fingerprint / token counts, best-effort. + + Never raises: diagnostics must not be the thing that breaks an eval run, and not every + OpenAI-compatible endpoint returns a usage block. + """ + bits = [] + if not (getattr(response, "choices", None) or []): + # Says *why* there is no finish_reason below, so the raised JudgeResponseError + # distinguishes "no choice at all" from "a choice with an empty body". + bits.append("choices=0") + bits += [ + _bit("finish_reason", lambda: response.choices[0].finish_reason), + _bit("system_fingerprint", lambda: response.system_fingerprint), + _bit("prompt_tokens", lambda: response.usage.prompt_tokens), + _bit("completion_tokens", lambda: response.usage.completion_tokens), + _bit("reasoning_tokens", lambda: response.usage.completion_tokens_details.reasoning_tokens), + ] + return " ".join(b for b in bits if b) + _SYSTEM_TEMPLATE = """\ You are an impartial evaluator. Score whether the actual output satisfies the criteria. @@ -29,7 +145,7 @@ class LLMJudge: """Binary LLM judge (score 0 or 1) for text-answer evaluators.""" - def __init__(self, evaluation_steps: list[str], model: str = "gpt-4o"): + def __init__(self, evaluation_steps: list[str], model: str | None = None): try: from openai import OpenAI # noqa: PLC0415 except ImportError as _err: @@ -40,11 +156,47 @@ def __init__(self, evaluation_steps: list[str], model: str = "gpt-4o"): if not api_key: raise OSError("OPENAI_API_KEY environment variable is required for LLM-as-judge evaluators.") self._client = OpenAI(api_key=api_key) - self._model = model + # Public: the [timer] diagnostics name the model, and hardcoding it there would + # misreport which judge ran under --judge-model. + self.model = model or judge_model() + # Latched off once this model has told us it will not accept temperature=0, so the + # wasted 400 is paid once per judge instead of once per graded run. Note the scope: + # a judge is built per item, so a whole eval run still pays it once per item. + # Hoisting the latch to the class would fix that but would also let one item's + # provider quirk silently reconfigure every later one. + self._supports_temperature = True self._system_prompt = _SYSTEM_TEMPLATE.format( steps="\n".join(f"{i + 1}. {s}" for i, s in enumerate(evaluation_steps)) ) + def _create_completion(self, messages: list[dict]) -> object: + """One judge request, dropping temperature if this model refuses it.""" + kwargs: dict = { + "model": self.model, + "messages": messages, + "response_format": {"type": "json_object"}, + "max_completion_tokens": JUDGE_MAX_COMPLETION_TOKENS, + } + # temperature=0 is what makes a verdict reproducible: the same response must not + # pass one run and fail the next. The gpt-5 family rejects the parameter outright + # ("Only the default (1) value is supported"), so rather than 400 on the first + # item we drop it, say so, and carry on with a judge that is no longer + # deterministic -- a real loss of eval quality, hence the warning. + if self._supports_temperature: + kwargs["temperature"] = 0 + try: + return self._client.chat.completions.create(**kwargs) + except Exception as exc: + if not (self._supports_temperature and _rejects_temperature(exc)): + raise + self._supports_temperature = False + emit_line( + f"warning: judge model {self.model!r} rejects temperature=0, so its verdicts are " + f"NOT deterministic -- the same response may score differently between runs." + ) + kwargs.pop("temperature", None) + return self._client.chat.completions.create(**kwargs) + def score(self, input: str, expected_output: str, actual_output: str) -> tuple[bool, str]: """Return (passed, reasoning). passed=True iff score==1.""" user_prompt = _USER_TEMPLATE.format( @@ -52,15 +204,108 @@ def score(self, input: str, expected_output: str, actual_output: str) -> tuple[b expected_output=expected_output, actual_output=actual_output, ) - response = self._client.chat.completions.create( - model=self._model, - messages=[ - {"role": "system", "content": self._system_prompt}, - {"role": "user", "content": user_prompt}, - ], - response_format={"type": "json_object"}, - temperature=0, + messages = [ + {"role": "system", "content": self._system_prompt}, + {"role": "user", "content": user_prompt}, + ] + meta = "" + for attempt in range(1, _EMPTY_BODY_ATTEMPTS + 1): + response = self._create_completion(messages) + raw = _message_content(response) + meta = _response_metadata(response) + if env_flag(JUDGE_DIAGNOSTICS_ENV_VAR): + emit_line(f"[judge] {self.model} {meta} body={raw!r}") + if isinstance(raw, str) and raw.strip(): + return self._verdict(raw, meta) + if attempt < _EMPTY_BODY_ATTEMPTS: + # Announced rather than silent: a retried item costs twice the tokens and + # latency, and a rising retry rate is the signal that the cap is too low. + emit_line(f"warning: judge {self.model!r} returned an empty body ({meta}); retrying once.") + raise JudgeResponseError( + f"judge {self.model!r} returned an empty body twice -- no verdict to read. {meta}. " + "finish_reason=length means the model spent its whole completion budget on " + f"reasoning tokens; raise JUDGE_MAX_COMPLETION_TOKENS (currently {JUDGE_MAX_COMPLETION_TOKENS}) " + "or use a non-reasoning judge." ) - raw = response.choices[0].message.content or "{}" - data = json.loads(raw) - return int(data.get("score", 0)) == 1, data.get("reasoning", "") + + def _verdict(self, raw: str, meta: str) -> tuple[bool, str]: + """The verdict carried by a non-empty judge body, or ``JudgeResponseError``.""" + try: + data = json.loads(raw) + except json.JSONDecodeError as exc: + raise JudgeResponseError( + f"judge {self.model!r} returned unparseable JSON ({exc}). {meta} body={raw!r}" + ) from exc + if not isinstance(data, dict) or "score" not in data: + raise JudgeResponseError(f"judge {self.model!r} returned no 'score' key. {meta} body={raw!r}") + score = data["score"] + # bool first: JSON mode legitimately emits `true`, and bool is a subclass of int. + if isinstance(score, bool): + return score, data.get("reasoning", "") + if isinstance(score, str): + # A quoted number is a routine JSON-mode quirk, and the prompt asks for `1`, + # not `"1"`. Rejecting it outright would discard a verdict the judge did give. + try: + score = float(score.strip()) + except ValueError: + raise JudgeResponseError( + f"judge {self.model!r} returned a non-numeric score {score!r}. {meta} body={raw!r}" + ) from None + if not isinstance(score, (int, float)): + raise JudgeResponseError( + f"judge {self.model!r} returned a non-numeric score {score!r}. {meta} body={raw!r}" + ) + if score not in (0, 1): + # The binary prompt makes anything outside {0, 1} an unread response, not a + # FAIL: a 2 (a model reading the rubric as 0-2) or a 0.9 (a confidence, not a + # verdict) must not be collapsed to 0 while the judge's own "fully correct" + # text is still attached to it. + raise JudgeResponseError( + f"judge {self.model!r} returned an out-of-range score {score!r}; expected 0 or 1. {meta} body={raw!r}" + ) + return score == 1, data.get("reasoning", "") + + +class JudgeVerdict(NamedTuple): + """What one judge request produced: a verdict, or the reason there is not one. + + ``error`` is non-None when the judge returned something unreadable. ``passed`` is + then False so that no pass@K can ever read such a run as a pass -- but callers must + *exclude* it rather than count it, because a judge fault is a property of one request, + not of the agent's answer. + """ + + passed: bool + reasoning: str + error: str | None = None + + @property + def rank(self) -> int: + """1 for a pass, 0 for a fail, -1 for no verdict. + + For an evaluator's ``rank_key``: an ungraded run sorts below every graded one, so + the run a report describes is one that has a verdict whenever any run does. + """ + if self.error is not None: + return -1 + return int(self.passed) + + +def score_run(judge: LLMJudge, *, input: str, expected_output: str, actual_output: str) -> JudgeVerdict: + """Grade one run, turning an unreadable judge response into an unscored verdict. + + Letting ``JudgeResponseError`` fly straight out of a K-run loop discarded every run + already graded, dropped their Langfuse scores on the floor, and reported an item whose + pass@K was already satisfied as a failure -- the same "a parse bug reads as a pass-rate + drop" that ``JudgeResponseError`` exists to prevent, reintroduced one layer up. So the + fault is confined to its own run; the caller decides what to do with the rest, and an + item with no graded run at all still raises. + """ + try: + passed, reasoning = judge.score(input=input, expected_output=expected_output, actual_output=actual_output) + except JudgeResponseError as exc: + # Announced, not swallowed: an unscored run changes the denominator of pass@K and + # a rising rate of them is the signal that the judge or its cap needs attention. + emit_line(f"warning: judge {judge.model!r} could not grade one run: {exc}") + return JudgeVerdict(passed=False, reasoning="", error=str(exc)) + return JudgeVerdict(passed=passed, reasoning=reasoning, error=None) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py index fabf2b419..ca2e10fed 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py @@ -1,7 +1,7 @@ # (C) 2026 GoodData Corporation """Evaluator for general_question: LLM-as-judge scores the agent's text response.""" -from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._llm_judge import LLMJudge, score_run from gooddata_eval.core.evaluators._text_utils import extract_text from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.models import ChatResult, DatasetItem, build_latency_breakdown @@ -22,19 +22,21 @@ def __init__(self): def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation: actual = extract_text(chat_result) - passed, reasoning = self._judge.score( + # score_run, not judge.score: a judge fault is this run's, not the item's (see score_run). + verdict = score_run( + self._judge, input=item.question, expected_output=str(item.expected_output), actual_output=actual, ) - return ItemEvaluation( - passed=passed, - rank_key=(int(passed),), - detail={ - "judge_reasoning": reasoning, - "actual_output": actual, - "latency_breakdown": build_latency_breakdown( - chat_result.tool_call_events, chat_result.reasoning_step_events - ), - }, - ) + detail = { + "actual_output": actual, + "latency_breakdown": build_latency_breakdown( + chat_result.tool_call_events, chat_result.reasoning_step_events + ), + } + if verdict.error is None: + detail["judge_reasoning"] = verdict.reasoning + else: + detail["judge_error"] = verdict.error + return ItemEvaluation(passed=verdict.passed, rank_key=(verdict.rank,), detail=detail, error=verdict.error) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/guardrail.py index 2e9dc0cb2..c946020f3 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/guardrail.py @@ -1,7 +1,7 @@ # (C) 2026 GoodData Corporation """Evaluator for guardrail: agent must refuse or redirect; visualization response auto-fails.""" -from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._llm_judge import LLMJudge, score_run from gooddata_eval.core.evaluators._text_utils import extract_text from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.models import ChatResult, DatasetItem, build_latency_breakdown @@ -39,23 +39,26 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation ) actual = extract_text(chat_result) - passed, reasoning = self._judge.score( + # score_run, not judge.score: a judge fault is this run's, not the item's (see score_run). + verdict = score_run( + self._judge, input=item.question, expected_output=str(item.expected_output), actual_output=actual, ) - return ItemEvaluation( - passed=passed, - rank_key=(int(passed),), - # no_visualization + judge_passed both in detail: - # 1.0 = proper refusal, 0.5 = prose compliance, 0.0 = viz produced - detail={ - "no_visualization": True, - "judge_passed": passed, - "judge_reasoning": reasoning, - "actual_output": actual, - "latency_breakdown": build_latency_breakdown( - chat_result.tool_call_events, chat_result.reasoning_step_events - ), - }, - ) + # no_visualization + judge_passed both in detail: + # 1.0 = proper refusal, 0.5 = prose compliance, 0.0 = viz produced + detail = { + "no_visualization": True, + "actual_output": actual, + "latency_breakdown": build_latency_breakdown( + chat_result.tool_call_events, chat_result.reasoning_step_events + ), + } + if verdict.error is None: + detail["judge_passed"] = verdict.passed + detail["judge_reasoning"] = verdict.reasoning + else: + # No judge_passed bool: storing False would invent a verdict the judge never gave. + detail["judge_error"] = verdict.error + return ItemEvaluation(passed=verdict.passed, rank_key=(verdict.rank,), detail=detail, error=verdict.error) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py index b0af2ce42..0c680b0c8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/summary.py @@ -15,13 +15,19 @@ *passes* only when every `must_include` is satisfied and no `must_not_include` is violated; `rubric` items contribute to quality but do not gate pass/fail. +A criterion the judge returns nothing readable for is *ungraded*: it is neither +satisfied nor violated, so it is stored without a bool and stays out of the +quality denominator. An ungraded gating criterion cannot carry a pass, though -- +"the judge could not tell" is not evidence the fact is present. When no gating +criterion was graded at all the run has no verdict and is reported as such. + As a fallback, a non-dict `expected_output` is treated as a single rubric criterion (same behaviour as `general_question`). """ from typing import Any -from gooddata_eval.core.evaluators._llm_judge import LLMJudge +from gooddata_eval.core.evaluators._llm_judge import LLMJudge, score_run from gooddata_eval.core.evaluators._text_utils import extract_text from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.models import ChatResult, DatasetItem @@ -72,25 +78,62 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation detail: dict[str, Any] = {"actual_output": actual} passed = True + # One judge request PER CRITERION, so an unreadable response has to be confined to + # its own criterion: letting it raise would discard every criterion already graded + # and abandon the ones after it, losing a 7-criterion item to one bad body. An + # ungraded criterion is recorded but stored without a bool, which keeps it out of + # the failing list and the quality denominator below: counting it as failed would + # invent a score the judge never gave. + ungraded = 0 + + def _grade(judge: LLMJudge, criterion: str, key: str, *, invert: bool = False) -> bool | None: + nonlocal ungraded + verdict = score_run(judge, input=item.question, expected_output=criterion, actual_output=actual) + if verdict.error is not None: + ungraded += 1 + detail[f"{key}_reason"] = f"UNGRADED: {verdict.error}" + return None + # invert: the violation judge answers "is the characteristic present?", so the + # criterion is satisfied exactly when it says no. + ok = not verdict.passed if invert else verdict.passed + detail[key] = ok + detail[f"{key}_reason"] = verdict.reasoning + return ok + + # `is True`, not `is not False`: a gating criterion the judge could not grade is not + # a failure, but it cannot carry a pass either. for i, criterion in enumerate(must_include): - ok, reason = self._positive_judge.score(item.question, criterion, actual) - detail[f"include_{i}"] = ok - detail[f"include_{i}_reason"] = reason - passed = passed and ok + ok = _grade(self._positive_judge, criterion, f"include_{i}") + passed = passed and ok is True for i, criterion in enumerate(must_not_include): - violated, reason = self._violation_judge.score(item.question, criterion, actual) - ok = not violated # True == characteristic absent == correctly avoided - detail[f"exclude_{i}"] = ok - detail[f"exclude_{i}_reason"] = reason - passed = passed and ok + ok = _grade(self._violation_judge, criterion, f"exclude_{i}", invert=True) + passed = passed and ok is True for i, criterion in enumerate(rubric): - ok, reason = self._positive_judge.score(item.question, criterion, actual) - detail[f"rubric_{i}"] = ok - detail[f"rubric_{i}_reason"] = reason + # Rubric criteria inform quality but never gate `passed`. + _grade(self._positive_judge, criterion, f"rubric_{i}") + if ungraded: + detail["ungraded_criteria"] = ungraded bool_checks = [v for v in detail.values() if isinstance(v, bool)] quality = sum(1 for v in bool_checks if v) / len(bool_checks) if bool_checks else 0.0 + # Keyed on the criteria that decide the verdict, not on any graded bool: rubric + # criteria never gate `passed`, so one graded rubric line must not certify a pass + # whose every mandatory fact went unassessed. A rubric-only item has no gating + # criteria, so there any graded line is a verdict. + gating = len(must_include) + len(must_not_include) + graded_gating = [v for k, v in detail.items() if isinstance(v, bool) and not k.startswith("rubric_")] + if ungraded and not (graded_gating if gating else bool_checks): + return ItemEvaluation( + passed=False, + rank_key=(-1, quality), + detail=detail, + error=( + f"judge returned no readable verdict for any of the {gating or ungraded} criterion(s) " + "that decide this item." + ), + ) + return ItemEvaluation(passed=passed, rank_key=(int(passed), quality), detail=detail) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/models.py b/packages/gooddata-eval/src/gooddata_eval/core/models.py index 185574bcd..059c4d32d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/models.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/models.py @@ -10,6 +10,8 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator +from gooddata_eval.core.timing import PhaseTimings + class AacQueryField(BaseModel): model_config = ConfigDict(extra="allow") @@ -223,6 +225,29 @@ class ChatResult(BaseModel): turn_wall_clock_sec: float | None = None +class AgenticAssertionError(AssertionError): + """Base for every agentic kind's failure, carrying what the runner reports about it. + + Set by ``evaluate_agentic_*`` so a failing item still reports what the agent did and how + many of its K runs passed -- ``cli/agentic_runner`` reads these off the exception exactly + as it reads them off an ``AgenticEvalOutcome`` on the success path. Declared once because + the payload is identical for every kind: while it lived in eight copies, two declared + ``timings`` and six did not, though the runner reads it from all eight. + + Bare annotations, so no class attributes are created and ``getattr(exc, name, default)`` + still sees only what the raising code actually set. + """ + + __tracebackhide__ = True + reasoning_steps: list[str] + conversation_id: str + response_id: str | None + detail: dict + timings: PhaseTimings + runs_passed: int + runs_effective: int + + class AgenticEvalOutcome(BaseModel): """Reasoning trace, trace-lookup IDs, and per-kind diagnostics from an evaluate_agentic_* call. @@ -237,6 +262,15 @@ class AgenticEvalOutcome(BaseModel): conversation_id: str | None = None response_id: str | None = None detail: dict = Field(default_factory=dict) + # Per-phase latency for the whole item (summed across its K runs). Kept beside + # ``detail`` rather than inside it so reporting can read it without guessing at keys. + timings: PhaseTimings = Field(default_factory=PhaseTimings) + # How many of the item's runs passed, and how many it actually ran. pass_at_k answers + # only "did any run pass", so without these a 5/5 item and a 1/5 item are identical in + # every output. ``runs_effective`` exists because the requested K is not always what + # ran -- agentic_conversation drives its fixture once whatever --runs says. + runs_passed: int = 0 + runs_effective: int = 0 class SummaryInput(BaseModel): @@ -268,3 +302,7 @@ class DatasetItem(BaseModel): expected_output: Any # Only used by the `dashboard_summary` test kind; ignored by all others. summary_input: SummaryInput | None = None + # Relayed verbatim as the chat request's `userContext`. Deliberately opaque: gen-ai owns + # that schema (a discriminated union of view/widget descriptors), so re-modelling it here + # would only create a second copy to keep in sync. + user_context: dict[str, Any] | None = None diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py index b81aaa0f5..3acf69100 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/console.py @@ -4,7 +4,16 @@ from rich.console import Console from rich.table import Table -from gooddata_eval.core.runner import EvalReport +from gooddata_eval.core.runner import EvalReport, ItemReport + + +def _ungraded_note(item: ItemReport) -> str: + """What the verdict was NOT computed over: runs, or (dashboard_summary) criteria, that + the judge returned nothing readable for. Empty when everything was graded.""" + if item.runs_ungraded: + return f"{item.runs_ungraded} run(s) ungraded" + criteria = item.best_detail.get("ungraded_criteria") + return f"{criteria} criterion(s) ungraded" if criteria else "" def render_console(report: EvalReport, *, console: Console | None = None) -> str: @@ -30,7 +39,11 @@ def render_console(report: EvalReport, *, console: Console | None = None) -> str elif item.error: result, notes = "ERROR", item.error elif item.pass_at_k: - result, notes = "PASS", "" + # A pass@K that was not unanimous is a materially weaker result than one that + # was, and every other column looks identical for the two: quality_score reads + # best_detail, which describes the winning run alone. So say it here. + result = "PASS" + notes = "" if item.pass_power_k else f"{item.runs_passed}/{item.runs_total} runs passed" else: # Evaluator-agnostic: report whichever boolean checks came back False # (visualization uses metrics_correct/…; dashboard_summary uses @@ -38,20 +51,31 @@ def render_console(report: EvalReport, *, console: Console | None = None) -> str failing = [k for k, v in item.best_detail.items() if v is False] notes = "failed: " + ", ".join(failing) if failing else "did not pass strict checks" result = "FAIL" - latency = "-" if item.runs == 0 else f"{item.latency_s:.2f}s" - avg = "-" if item.runs == 0 else f"{item.avg_latency_s:.2f}s" + if result in ("PASS", "FAIL") and (ungraded := _ungraded_note(item)): + # Said on both verdicts: a PASS over fewer runs is weaker evidence, and a FAIL + # with no False check is otherwise unexplained. + notes = f"{notes}; {ungraded}" if notes else ungraded + latency = "-" if item.runs_total == 0 else f"{item.latency_s:.2f}s" + avg = "-" if item.runs_total == 0 else f"{item.avg_latency_s:.2f}s" quality = "-" if item.skipped else f"{item.quality_score:.0%}" - table.add_row(item.id, item.test_kind, result, str(item.runs), latency, avg, quality, notes) + runs_col = str(item.runs_total) + table.add_row(item.id, item.test_kind, result, runs_col, latency, avg, quality, notes) out.print(table) _wall = report.wall_clock_s - _agent = report.latency_s - if _wall > 0 and abs(_wall - _agent) > 1: # concurrency > 1: show both - timing = f"{_wall:.2f}s wall-clock, {_agent:.2f}s agent time (avg {report.avg_latency_s:.2f}s/run)" + # Sum of every item's own critical path -- agent plus judge plus simulated user, not + # the agent alone (ItemReport.agent_latency_s is that). It differs from wall-clock + # under --concurrency, and on the agentic path where deferred Langfuse trace linking + # runs outside any item's path but inside the run's elapsed time. + _items = report.latency_s + if _wall > 0 and abs(_wall - _items) > 1: + timing = f"{_wall:.2f}s wall-clock, {_items:.2f}s in items (avg {report.avg_latency_s:.2f}s/run)" else: - timing = f"{_agent:.2f}s (avg {report.avg_latency_s:.2f}s/run)" + timing = f"{_items:.2f}s (avg {report.avg_latency_s:.2f}s/run)" + # passed_all_runs alongside passed: the gap between them is the inconsistency signal. + unanimous = "" if report.passed_all_runs == report.passed else f", {report.passed_all_runs} on every run" out.print( - f"\nSummary: {report.passed}/{report.total} passed " + f"\nSummary: {report.passed}/{report.total} passed{unanimous} " f"({report.skipped} skipped, {report.errored} errored) " f"avg quality {report.avg_quality_score:.0%} in {timing}" ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py index d0b34483d..81091e141 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/reporting/json_report.py @@ -6,6 +6,7 @@ import orjson from gooddata_eval.core.runner import EvalReport +from gooddata_eval.core.timing import PhaseTimings def _build_run_dict(report: EvalReport) -> dict: @@ -15,7 +16,13 @@ def _build_run_dict(report: EvalReport) -> dict: "summary": { "total": report.total, "passed": report.passed, - "failed": report.total - report.passed - report.skipped, + # pass^K across the dataset. A large gap from `passed` means the models are + # inconsistent rather than wrong, which pass@K alone cannot show. + "passed_all_runs": report.passed_all_runs, + # Counted explicitly rather than by subtraction: an errored item has + # pass_at_k False and skipped False, so subtraction would count it as both a + # failure and an error. A judge fault is an error, not K failures. + "failed": sum(1 for i in report.items if not i.pass_at_k and not i.skipped and i.error is None), "skipped": report.skipped, "errored": report.errored, "latency_s": round(report.latency_s, 3), @@ -30,12 +37,29 @@ def _build_run_dict(report: EvalReport) -> dict: "pass_at_k": item.pass_at_k, "skipped": item.skipped, "error": item.error, - "runs": item.runs, + # What actually ran, not the requested K: agentic_conversation runs once. + "runs": item.runs_total, "latency_s": round(item.latency_s, 3), "avg_latency_s": round(item.avg_latency_s, 3), + # Beside `runs`, not folded into it: "4 of 5 passed" is a different fact + # from pass_at_k and the only one that separates a reliable item from a + # coin-flip. pass_power_k is the unanimity flag beside it, and + # runs_ungraded says how many runs pass@K was NOT computed over. + "runs_passed": item.runs_passed, + "runs_ungraded": item.runs_ungraded, + "pass_power_k": item.pass_power_k, "best_run_latency_s": ( round(item.best_run_latency_s, 3) if item.best_run_latency_s is not None else None ), + # Additive to latency_s, never folded into it: langfuse_s is measured off + # the item's critical path (see agentic/_trace_linker.py), so summing the + # four would over-count the item's latency. + "latency_breakdown_s": PhaseTimings( + agent_s=item.agent_latency_s, + judge_s=item.judge_latency_s, + simulated_user_s=item.simulated_user_latency_s, + langfuse_s=item.langfuse_latency_s, + ).as_dict(), "detail": item.best_detail, "conversation_id": item.conversation_id, "response_id": item.response_id, diff --git a/packages/gooddata-eval/src/gooddata_eval/core/runner.py b/packages/gooddata-eval/src/gooddata_eval/core/runner.py index f82da9aaf..16373975e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/runner.py @@ -39,10 +39,52 @@ class ItemReport: conversation_id: str | None = None response_id: str | None = None reasoning_steps: list[str] = field(default_factory=list) + # Per-phase breakdown of what the item's time was spent on. Additive to latency_s, + # which remains the item's own critical path. langfuse_latency_s is + # deliberately NOT part of that path -- trace linking runs off it (see + # agentic/_trace_linker.py) and is reported here so its cost stays visible anyway. + # Kinds that do not instrument a phase leave it at 0.0. + agent_latency_s: float = 0.0 + judge_latency_s: float = 0.0 + simulated_user_latency_s: float = 0.0 + langfuse_latency_s: float = 0.0 + # How many of `runs` actually passed. pass_at_k only answers "did ANY run pass", so + # without this a 5/5 item and a 1/5 item are indistinguishable in every output: same + # PASS, same 100% quality (quality_score reads best_detail, which is the winning run + # alone), same empty Notes. + runs_passed: int = 0 + # Runs the agent answered but the judge produced no verdict for. Never a pass and never + # a failure: excluded from runs_passed and from pass@K, and pass^K is False while any + # exist. Counted in `runs`, because they cost real agent time. + runs_ungraded: int = 0 + # What the item actually ran, when the kind knows better than the requested K. + # agentic_conversation drives its fixture exactly once whatever --runs says, so + # trusting K there reports four runs that never happened. + runs_effective: int | None = None + + @property + def runs_total(self) -> int: + """What the item actually ran: the kind's own count when it has one, else K.""" + return self.runs_effective or self.runs + + @property + def pass_power_k(self) -> bool: + """True only when every run passed -- the stronger claim pass_at_k never makes. + + An errored item can reach this with runs_passed == runs_total (earlier runs passed, + a later one raised before returning a verdict). That is not unanimity, it is an + item whose last run has no verdict at all, so an error disqualifies the claim. + """ + return self.error is None and self.runs_total > 0 and self.runs_passed == self.runs_total @property def avg_latency_s(self) -> float: - return self.latency_s / self.runs if self.runs else 0.0 + """Latency per run actually taken -- the same divisor the Runs column reports. + + Dividing by the requested K instead would under-report every kind that runs fewer + times than asked (agentic_conversation drives its fixture once whatever --runs says). + """ + return self.latency_s / self.runs_total if self.runs_total else 0.0 @property def quality_score(self) -> float: @@ -82,13 +124,23 @@ def skipped(self) -> int: def errored(self) -> int: return sum(1 for i in self.items if i.error is not None) + @property + def passed_all_runs(self) -> int: + """Items where every run passed, not merely one of them (pass^K). + + Reported beside `passed` because the two answer different questions and a gap + between them is the signal that a model is inconsistent rather than wrong. + """ + return sum(1 for i in self.items if i.pass_power_k) + @property def latency_s(self) -> float: return sum(i.latency_s for i in self.items) @property def total_runs(self) -> int: - return sum(i.runs for i in self.items) + """Runs actually taken across the dataset -- the same divisor each item's average uses.""" + return sum(i.runs_total for i in self.items) @property def avg_latency_s(self) -> float: @@ -122,6 +174,7 @@ def _run_one_item( # attempt they're each describing whenever the best-ranked run isn't also the last one. best_chat_result: ChatResult | None = None best_run_latency: float | None = None + judge_errors: list[str] = [] try: for run_index in range(1, runs + 1): t0 = time.perf_counter() @@ -132,12 +185,16 @@ def _run_one_item( latency = time.perf_counter() - t0 report.runs += 1 report.latency_s += latency + if evaluation.error is not None: + report.runs_ungraded += 1 + judge_errors.append(evaluation.error) if best is None or evaluation.rank_key > best.rank_key: best = evaluation best_chat_result = chat_result best_run_latency = latency if evaluation.passed: report.pass_at_k = True + report.runs_passed += 1 if on_run_done is not None: on_run_done(run_index, runs, evaluation.passed, latency) except Exception as e: # agent/network/parse failure for this item @@ -151,6 +208,15 @@ def _run_one_item( report.reasoning_steps = getattr(best_chat_result, "reasoning_steps", None) or [] return report + if report.runs and report.runs_ungraded == report.runs: + # Every run was answered and none was graded, so there is no verdict to report: an + # error, not K failures. Decided here, after the loop, so that one judge fault cannot + # abandon the runs behind it -- the agent's answers are still evaluated when the + # judge recovers on a later run. + report.error = ( + f"JudgeResponseError: no run of this item could be graded ({report.runs} run(s)); last: {judge_errors[-1]}" + ) + if best is not None: report.best_detail = best.detail report.best_run_latency_s = best_run_latency diff --git a/packages/gooddata-eval/src/gooddata_eval/core/timing.py b/packages/gooddata-eval/src/gooddata_eval/core/timing.py new file mode 100644 index 000000000..a3c05aef1 --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/timing.py @@ -0,0 +1,74 @@ +# (C) 2026 GoodData Corporation +"""Per-phase latency breakdown for one evaluated run or item. + +Only the agent's own response time is a property of the system under test. A single total +that also blends the judge, the simulated user and the Langfuse round trips cannot tell you +which of the four got slower. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from gooddata_eval.core._output import emit_line +from gooddata_eval.core.config import env_flag + +# Gates the per-turn ``[timer]`` diagnostics; off by default (see the --timers flag). +# Nothing is lost by leaving them off -- the same measurements reach every ItemReport and +# the JSON report's ``latency_breakdown_s``; these lines only show them as they happen. +TIMERS_ENV_VAR = "GD_EVAL_TIMERS" + + +def timers_enabled() -> bool: + """Whether ``[timer]`` diagnostics should be emitted.""" + return env_flag(TIMERS_ENV_VAR) + + +def log_timer(message: str) -> None: + """Emit one ``[timer]`` line, if timers are enabled.""" + if timers_enabled(): + emit_line(message) + + +@dataclass +class PhaseTimings: + """Seconds attributed to each phase of an evaluation. + + Fields accumulate across the turns of one conversation and across the K runs of one + item, so an item's ``agent_s`` is the sum of every agent turn it took. + """ + + # The system under test: time inside ChatClient.send_message. + agent_s: float = 0.0 + # Our grading of the answer (LLMJudge). Post-hoc -- never blocks the agent. + judge_s: float = 0.0 + # Our simulated user composing the next turn. On the critical path by construction: + # the agent cannot continue until this reply exists, so unlike the judge it can only + # be made faster, never deferred. + simulated_user_s: float = 0.0 + # Langfuse trace lookup and score writing. Deliberately NOT part of the item's critical + # path -- the linker records it (see agentic/_trace_linker.py) and the report fills it in + # here, so its cost stays visible without inflating the item's own latency. + langfuse_s: float = 0.0 + + def __add__(self, other: PhaseTimings) -> PhaseTimings: + return PhaseTimings( + agent_s=self.agent_s + other.agent_s, + judge_s=self.judge_s + other.judge_s, + simulated_user_s=self.simulated_user_s + other.simulated_user_s, + langfuse_s=self.langfuse_s + other.langfuse_s, + ) + + def as_dict(self) -> dict[str, float]: + """Rounded mapping for the JSON report.""" + return { + "agent_s": round(self.agent_s, 3), + "judge_s": round(self.judge_s, 3), + "simulated_user_s": round(self.simulated_user_s, 3), + "langfuse_s": round(self.langfuse_s, 3), + } + + +def sum_timings(timings: list[PhaseTimings]) -> PhaseTimings: + """Total across a list of per-run timings (empty list -> all zeroes).""" + return sum(timings, PhaseTimings()) diff --git a/packages/gooddata-eval/tests/test_agentic_alert_skill.py b/packages/gooddata-eval/tests/test_agentic_alert_skill.py index f6a184809..396748466 100644 --- a/packages/gooddata-eval/tests/test_agentic_alert_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_alert_skill.py @@ -1,5 +1,6 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from contextlib import ExitStack, contextmanager from unittest.mock import MagicMock, patch import pytest @@ -47,6 +48,41 @@ } +def _no_alert_chat_result() -> ChatResult: + """A turn where the agent refuses outright -- no tool calls, so no alert is created.""" + return ChatResult.model_validate( + { + "text_response": "I cannot create the alert", + "created_visualizations": None, + "tool_call_events": [], + "reasoning_step_count": 1, + } + ) + + +@contextmanager +def _patched(client, *, simulated_reply=None, delete_alert=False): + """Patch alert_skill's ChatClient, and optionally its simulated user and alert cleanup. + + ``simulated_reply`` is what the simulated user answers; ``delete_alert`` stubs out the + teardown call a run makes for an alert it created. Yields the + ``generate_simulated_alert_response`` mock, or None when the test asked for no reply. + """ + with ExitStack() as stack: + stack.enter_context(patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=client)) + mock_sim = None + if simulated_reply is not None: + mock_sim = stack.enter_context( + patch( + "gooddata_eval.core.agentic.alert_skill.generate_simulated_alert_response", + return_value=simulated_reply, + ) + ) + if delete_alert: + stack.enter_context(patch("gooddata_eval.core.agentic.alert_skill._delete_alert")) + yield mock_sim + + def test_to_number_int(): assert _to_number("42") == 42 @@ -264,18 +300,11 @@ def test_alert_evaluation_strict_fail(): def test_run_agentic_alert_skill_no_alert_created(): mock_client = MagicMock() mock_client.create_conversation.return_value = "conv-1" - mock_client.send_message.return_value = ChatResult.model_validate( - { - "text_response": "I cannot create the alert", - "created_visualizations": None, - "tool_call_events": [], - "reasoning_step_count": 1, - } - ) + mock_client.send_message.return_value = _no_alert_chat_result() mock_client._base = "http://host/api/v1/actions/workspaces/ws1/ai" mock_client._auth = {"Authorization": "Bearer tok"} - with patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_alert_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -293,15 +322,8 @@ def test_run_agentic_alert_skill_no_alert_created(): def test_run_agentic_alert_skill_uses_initial_conversation_for_run_0(): mock_client = MagicMock() - mock_client.send_message.return_value = ChatResult.model_validate( - { - "text_response": "I cannot create the alert", - "created_visualizations": None, - "tool_call_events": [], - "reasoning_step_count": 1, - } - ) - with patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client): + mock_client.send_message.return_value = _no_alert_chat_result() + with _patched(mock_client): run_agentic_alert_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -319,15 +341,8 @@ def test_run_agentic_alert_skill_uses_initial_conversation_for_run_0(): def test_run_agentic_alert_skill_creates_fresh_conversations_for_remaining_runs(): mock_client = MagicMock() mock_client.create_conversation.side_effect = ["fresh-1", "fresh-2"] - mock_client.send_message.return_value = ChatResult.model_validate( - { - "text_response": "I cannot create the alert", - "created_visualizations": None, - "tool_call_events": [], - "reasoning_step_count": 1, - } - ) - with patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client): + mock_client.send_message.return_value = _no_alert_chat_result() + with _patched(mock_client): run_agentic_alert_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -430,14 +445,7 @@ def test_run_agentic_alert_skill_passes_question_to_sim_user(): mock_client = MagicMock() mock_client.send_message.side_effect = [asked_turn, created_turn] - with ( - patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.alert_skill.generate_simulated_alert_response", - return_value="United States.", - ) as mock_sim, - patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), - ): + with _patched(mock_client, simulated_reply="United States.", delete_alert=True) as mock_sim: run_agentic_alert_skill( host="http://host", token="tok", @@ -559,14 +567,9 @@ def test_run_agentic_alert_skill_answers_proposal_only_confirmation_turn(): mock_client = MagicMock() mock_client.send_message.side_effect = [proposal_turn, created_turn] - with ( - patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.alert_skill.generate_simulated_alert_response", - return_value="Yes, please proceed to create the alert.", - ) as mock_sim, - patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), - ): + with _patched( + mock_client, simulated_reply="Yes, please proceed to create the alert.", delete_alert=True + ) as mock_sim: summary = run_agentic_alert_skill( host="http://host", token="tok", @@ -612,14 +615,7 @@ def test_run_agentic_alert_skill_accumulates_reasoning_steps_across_iterations() mock_client = MagicMock() mock_client.send_message.side_effect = [proposal_turn, created_turn] - with ( - patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.alert_skill.generate_simulated_alert_response", - return_value="Yes, please proceed to create the alert.", - ), - patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), - ): + with _patched(mock_client, simulated_reply="Yes, please proceed to create the alert.", delete_alert=True): summary = run_agentic_alert_skill( host="http://host", token="tok", @@ -652,10 +648,7 @@ def test_evaluate_agentic_alert_skill_returns_reasoning_steps_on_pass(): mock_client.create_conversation.return_value = "conv-1" mock_client.send_message.return_value = chat_result - with ( - patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.alert_skill._delete_alert"), - ): + with _patched(mock_client, delete_alert=True): outcome = evaluate_agentic_alert_skill( host="http://host", token="tok", @@ -694,10 +687,7 @@ def test_evaluate_agentic_alert_skill_attaches_reasoning_steps_to_exception_on_f mock_client.create_conversation.return_value = "conv-1" mock_client.send_message.return_value = chat_result - with ( - patch("gooddata_eval.core.agentic.alert_skill.ChatClient", return_value=mock_client), - pytest.raises(AlertSkillAssertionError) as exc_info, - ): + with _patched(mock_client), pytest.raises(AlertSkillAssertionError) as exc_info: evaluate_agentic_alert_skill( host="http://host", token="tok", diff --git a/packages/gooddata-eval/tests/test_agentic_general_question.py b/packages/gooddata-eval/tests/test_agentic_general_question.py index ac9240d48..830d25019 100644 --- a/packages/gooddata-eval/tests/test_agentic_general_question.py +++ b/packages/gooddata-eval/tests/test_agentic_general_question.py @@ -1,5 +1,9 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import contextlib +import io +import time +from datetime import datetime, timezone from unittest.mock import MagicMock, patch import pytest @@ -9,7 +13,89 @@ evaluate_agentic_general_question, run_agentic_general_question, ) +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.models import ChatResult +from gooddata_eval.core.timing import TIMERS_ENV_VAR + +# --- time.monotonic() side effects --------------------------------------------------- +# +# One run reads the clock six times, in this order: item start, agent start, agent stop, +# judge start, judge stop, item stop. A K-run test therefore needs six values per run. +# Named rather than inlined because a single new clock read on the production path breaks +# every one of these at once with StopIteration -- and repairing a bare literal means +# hand-counting clock reads at each call site. + +# one run: agent 2.50s, judge 1.50s, item total 4.50s +_CLOCK_AGENT_2_5_JUDGE_1_5 = [10.0, 10.5, 13.0, 13.0, 14.5, 14.5] +# one run: agent 2.0s, judge 1.0s +_CLOCK_AGENT_2_JUDGE_1 = [0.0, 0.0, 2.0, 2.0, 3.0, 3.0] +# one run: agent 4.0s, judge 2.0s +_CLOCK_AGENT_4_JUDGE_2 = [0.0, 0.0, 4.0, 4.0, 6.0, 6.0] +# two runs: agent 2.0s + 3.0s, judge 1.0s + 0.5s +_CLOCK_TWO_RUNS_AGENT_5_JUDGE_1_5 = [0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 10.0, 10.0, 13.0, 13.0, 13.5, 13.5] +# two runs: agent 4.0s + 3.0s, judge 1.0s each +_CLOCK_TWO_RUNS_AGENT_7_JUDGE_2 = [0.0, 0.0, 4.0, 4.0, 5.0, 5.0, 10.0, 10.0, 13.0, 13.0, 14.0, 14.0] + + +def _pass_client_and_judge( + *, text_response: str = "42", reasoning_steps: list[str] | None = None, response_id: str = "resp-1" +): + """A chat client on conversation ``conv-1`` and a gpt-4o judge that passes every run. + + Callers override only the one field they vary -- ``client.create_conversation``, + ``judge.score.return_value`` and so on are ordinary mock attributes. + """ + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": text_response, + "toolCallEvents": [], + "reasoningSteps": reasoning_steps or [], + "responseId": response_id, + } + ) + judge = MagicMock() + judge.model = "gpt-4o" + judge.score.return_value = (True, "Correct answer") + return client, judge + + +@contextlib.contextmanager +def _patched(client, judge, *, monotonic=None): + """Patch general_question's ChatClient and LLMJudge, and optionally the clock. + + ``monotonic`` is the ``time.monotonic()`` side-effect list (one of the ``_CLOCK_*`` + constants above) for tests that assert on timings. + """ + with ( + patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=client), + patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=judge), + ): + if monotonic is None: + yield + else: + with patch("time.monotonic", side_effect=monotonic): + yield + + +def _no_traces(*_args, **_kwargs): + return {} + + +@contextlib.contextmanager +def _patched_langfuse(client, judge, *, find=_no_traces): + """Patch the chat/judge pair plus the two _langfuse helpers the linker calls. + + ``find`` stands in for ``find_traces_per_conversation`` (default: finds nothing). + Yields ``(mock_find_traces_per_conversation, mock_build_run_context)``. + """ + with ( + _patched(client, judge), + patch("gooddata_eval.core.agentic._langfuse.find_traces_per_conversation", side_effect=find) as mock_find, + patch("gooddata_eval.core.agentic._langfuse.build_run_context", return_value=("run", {})) as mock_ctx, + ): + yield mock_find, mock_ctx def test_general_question_result_fields(): @@ -25,19 +111,9 @@ def test_general_question_result_fields(): def test_run_agentic_general_question_pass(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" - mock_chat_result = MagicMock() - mock_chat_result.text_response = "The answer is 42" - mock_chat_result.tool_call_events = [] - mock_client.send_message.return_value = mock_chat_result - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "The answer matches") + client, judge = _pass_client_and_judge(text_response="The answer is 42") - with ( - patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), - ): + with _patched(client, judge): summary = run_agentic_general_question( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -48,22 +124,31 @@ def test_run_agentic_general_question_pass(): assert summary.pass_at_k is True assert summary.best.passed is True - mock_client.close.assert_called_once() + client.close.assert_called_once() + + +def test_run_agentic_general_question_logs_agent_and_judge_timing(monkeypatch, capsys): + monkeypatch.setenv(TIMERS_ENV_VAR, "1") + client, judge = _pass_client_and_judge(text_response="The answer is 42") + + with _patched(client, judge, monotonic=_CLOCK_AGENT_2_5_JUDGE_1_5): + run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is 6 times 7?", + expected_output="42", + ) + + output = capsys.readouterr().out + assert "[timer] general_question conv-1 GoodData response complete after 2.50s; waiting for gpt-4o judge" in output + assert "[timer] general_question conv-1 gpt-4o judge complete after 1.50s; item total 4.50s" in output def test_run_agentic_general_question_uses_initial_conversation_for_run_0(): - mock_client = MagicMock() - mock_chat_result = MagicMock() - mock_chat_result.text_response = "The answer is 42" - mock_chat_result.tool_call_events = [] - mock_client.send_message.return_value = mock_chat_result - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "Correct") + client, judge = _pass_client_and_judge() - with ( - patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), - ): + with _patched(client, judge): run_agentic_general_question( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -73,24 +158,15 @@ def test_run_agentic_general_question_uses_initial_conversation_for_run_0(): k=1, initial_conversation_id="existing-conv", ) - mock_client.create_conversation.assert_not_called() - mock_client.delete_conversation.assert_not_called() + client.create_conversation.assert_not_called() + client.delete_conversation.assert_not_called() def test_run_agentic_general_question_creates_fresh_conversations_for_remaining_runs(): - mock_client = MagicMock() - mock_client.create_conversation.side_effect = ["fresh-1", "fresh-2"] - mock_chat_result = MagicMock() - mock_chat_result.text_response = "The answer is 42" - mock_chat_result.tool_call_events = [] - mock_client.send_message.return_value = mock_chat_result - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "Correct") + client, judge = _pass_client_and_judge() + client.create_conversation.side_effect = ["fresh-1", "fresh-2"] - with ( - patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), - ): + with _patched(client, judge): run_agentic_general_question( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -100,28 +176,14 @@ def test_run_agentic_general_question_creates_fresh_conversations_for_remaining_ k=3, initial_conversation_id="existing-conv", ) - assert mock_client.create_conversation.call_count == 2 - assert mock_client.delete_conversation.call_count == 2 + assert client.create_conversation.call_count == 2 + assert client.delete_conversation.call_count == 2 def test_run_agentic_general_question_captures_reasoning_steps(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" - mock_client.send_message.return_value = ChatResult.model_validate( - { - "textResponse": "42", - "toolCallEvents": [], - "reasoningSteps": ["recalling the answer"], - "responseId": "resp-1", - } - ) - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "Correct answer") + client, judge = _pass_client_and_judge(reasoning_steps=["recalling the answer"]) - with ( - patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), - ): + with _patched(client, judge): summary = run_agentic_general_question( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -136,23 +198,9 @@ def test_run_agentic_general_question_captures_reasoning_steps(): def test_evaluate_agentic_general_question_returns_reasoning_steps_on_pass(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" - mock_client.send_message.return_value = ChatResult.model_validate( - { - "textResponse": "42", - "toolCallEvents": [], - "reasoningSteps": ["recalling the answer"], - "responseId": "resp-1", - } - ) - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "Correct answer") + client, judge = _pass_client_and_judge(reasoning_steps=["recalling the answer"]) - with ( - patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), - ): + with _patched(client, judge): outcome = evaluate_agentic_general_question( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -173,24 +221,12 @@ def test_evaluate_agentic_general_question_returns_reasoning_steps_on_pass(): def test_evaluate_agentic_general_question_attaches_reasoning_steps_to_exception_on_fail(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" - mock_client.send_message.return_value = ChatResult.model_validate( - { - "textResponse": "I don't know", - "toolCallEvents": [], - "reasoningSteps": ["unable to find the answer"], - "responseId": "resp-2", - } + client, judge = _pass_client_and_judge( + text_response="I don't know", reasoning_steps=["unable to find the answer"], response_id="resp-2" ) - mock_judge = MagicMock() - mock_judge.score.return_value = (False, "Wrong answer") + judge.score.return_value = (False, "Wrong answer") - with ( - patch("gooddata_eval.core.agentic.general_question.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.general_question.LLMJudge", return_value=mock_judge), - pytest.raises(GeneralQuestionAssertionError) as exc_info, - ): + with _patched(client, judge), pytest.raises(GeneralQuestionAssertionError) as exc_info: evaluate_agentic_general_question( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -208,3 +244,446 @@ def test_evaluate_agentic_general_question_attaches_reasoning_steps_to_exception "judge_reasoning": "Wrong answer", "actual_output": "I don't know", } + + +def test_evaluate_general_question_defers_langfuse_work_to_the_injected_linker(): + # The verdict is already decided once the judge returns; finding the gen-ai trace only + # publishes it. Handing that work to the linker is what takes the poll (and the SDK + # round-trip inside build_run_context) off the item's critical path. + client, judge = _pass_client_and_judge() + submitted = [] + + with _patched_langfuse(client, judge) as (mock_find, mock_ctx): + outcome = evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=1, + langfuse=MagicMock(), + dataset_item_id="ds-item-1", + submit_trace_link=lambda task, item_id="": submitted.append(task), + ) + + assert outcome.detail["judge_passed"] is True + mock_find.assert_not_called() + mock_ctx.assert_not_called() + assert len(submitted) == 1 + + submitted[0]() + mock_find.assert_called_once() + mock_ctx.assert_called_once() + + +def test_evaluate_general_question_links_traces_inline_by_default(): + # Backward compatibility: a caller that injects no linker (pytest suites calling + # evaluate_agentic_* directly) must still get fully-synchronous trace linking. + client, judge = _pass_client_and_judge() + + with _patched_langfuse(client, judge) as (mock_find, _): + evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=1, + langfuse=MagicMock(), + dataset_item_id="ds-item-1", + ) + + mock_find.assert_called_once() + + +def test_evaluate_general_question_submits_trace_link_even_when_the_item_fails(): + # A failing item's scores matter more than a passing one's. The submit has to happen + # before the assertion is raised, or every failure would drop out of Langfuse. + client, judge = _pass_client_and_judge() + judge.score.return_value = (False, "Wrong answer") + submitted = [] + + with _patched_langfuse(client, judge), pytest.raises(GeneralQuestionAssertionError): + evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=1, + langfuse=MagicMock(), + dataset_item_id="ds-item-1", + submit_trace_link=lambda task, item_id="": submitted.append(task), + ) + + assert len(submitted) == 1 + + +def test_records_agent_and_judge_latency_separately(): + # Deliberately distinct durations: if the two were ever swapped or summed, this fails. + # Agent latency is the number that describes GoodData; judge latency is the cost of + # OUR grading of it, and a report that conflates them cannot answer either question. + client, judge = _pass_client_and_judge(text_response="The answer is 42") + + with _patched(client, judge, monotonic=_CLOCK_AGENT_2_5_JUDGE_1_5): + summary = run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is 6 times 7?", + expected_output="42", + ) + + timings = summary.run_results[0].timings + assert timings.agent_s == 2.5 + assert timings.judge_s == 1.5 + # general_question drives no simulated user and does its trace linking elsewhere. + assert timings.simulated_user_s == 0.0 + assert timings.langfuse_s == 0.0 + + +def test_evaluate_general_question_aggregates_timings_across_k_runs(): + # pass@K runs K conversations; the item's agent cost is all of them, and so is its + # judge cost. Reporting only the last run's would understate both. + client, judge = _pass_client_and_judge() + + with _patched(client, judge, monotonic=_CLOCK_TWO_RUNS_AGENT_5_JUDGE_1_5): + outcome = evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=2, + ) + + assert outcome.timings.agent_s == 5.0 # 2.0 + 3.0 + assert outcome.timings.judge_s == 1.5 # 1.0 + 0.5 + + +def test_evaluate_general_question_attaches_timings_to_the_failure_exception(): + # A slow item that also fails is the one worth diagnosing, so the breakdown has to + # survive the raise -- same contract the reasoning_steps/conversation_id already have. + client, judge = _pass_client_and_judge() + judge.score.return_value = (False, "Wrong answer") + + with ( + _patched(client, judge, monotonic=_CLOCK_AGENT_4_JUDGE_2), + pytest.raises(GeneralQuestionAssertionError) as exc_info, + ): + evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=1, + ) + + assert exc_info.value.timings.agent_s == 4.0 + assert exc_info.value.timings.judge_s == 2.0 + + +def test_trace_link_window_is_pinned_at_submit_time_not_when_the_task_runs(): + # The deferred task must query the same window the old inline code would have. If + # window_end drifted to whenever a worker dequeued the task, a backed-up pool would + # silently widen every later item's window (see the _FETCH_LIMIT paging note in + # test_agentic_langfuse_trace.py). + client, judge = _pass_client_and_judge() + submitted = [] + captured = {} + + def _find(langfuse, conversation_ids, window_start, window_end=None): + captured["end"] = window_end + return {} + + with _patched_langfuse(client, judge, find=_find): + evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=1, + langfuse=MagicMock(), + dataset_item_id="ds-item-1", + submit_trace_link=lambda task, item_id="": submitted.append(task), + ) + returned_at = datetime.now(timezone.utc) + time.sleep(0.05) # the task sits in the queue behind other items + submitted[0]() + + assert captured["end"] is not None, "window_end was left to drift to the task's run time" + assert captured["end"] <= returned_at + + +def test_k_runs_submit_one_trace_link_covering_every_conversation(): + # Characterization guard for the --runs K path: K conversations produce ONE deferred + # task that looks up all of them together, exactly as the old inline block did. Per-run + # submission would multiply the queue depth and re-order the _run0/_run1 naming. + client, judge = _pass_client_and_judge() + client.create_conversation.side_effect = ["conv-a", "conv-b"] + submitted = [] + + def _find(langfuse, conversation_ids, window_start, window_end=None): + _find.seen = list(conversation_ids) + return {} + + with _patched_langfuse(client, judge, find=_find): + evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=2, + langfuse=MagicMock(), + dataset_item_id="ds-item-1", + submit_trace_link=lambda task, item_id="": submitted.append(task), + ) + assert len(submitted) == 1 + submitted[0]() + + assert _find.seen == ["conv-a", "conv-b"] + + +def _run_gq_capturing_output(monotonic_values): + client, judge = _pass_client_and_judge() + with _patched(client, judge, monotonic=monotonic_values): + run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="q", + expected_output="42", + ) + + +def test_no_timer_output_by_default(monkeypatch, capsys): + # 72 [timer] lines on an 18-item --runs 2 run buried the progress output. The numbers + # live on in latency_breakdown_s, so silence costs nothing. + monkeypatch.delenv(TIMERS_ENV_VAR, raising=False) + _run_gq_capturing_output(_CLOCK_AGENT_2_5_JUDGE_1_5) + + assert "[timer]" not in capsys.readouterr().out + + +def test_timings_are_still_recorded_when_timer_output_is_off(monkeypatch, capsys): + # The gate must silence the printing only -- never the measuring. + monkeypatch.setenv(TIMERS_ENV_VAR, "0") + client, judge = _pass_client_and_judge() + with _patched(client, judge, monotonic=_CLOCK_AGENT_2_5_JUDGE_1_5): + summary = run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="q", + expected_output="42", + ) + + assert "[timer]" not in capsys.readouterr().out + assert summary.run_results[0].timings.agent_s == 2.5 + assert summary.run_results[0].timings.judge_s == 1.5 + + +def test_timer_lines_name_the_configured_judge_model(monkeypatch): + # The model name used to be a literal in the message, so --judge-model gpt-5.6-luna + # still logged "waiting for gpt-4o judge" -- actively misleading in the one output + # a developer reads to see what the judge cost. + monkeypatch.setenv(TIMERS_ENV_VAR, "1") + client, judge = _pass_client_and_judge() + judge.model = "gpt-5.6-luna" + + with _patched(client, judge, monotonic=_CLOCK_AGENT_2_JUDGE_1): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="q", + expected_output="42", + ) + out = buf.getvalue() + + assert "waiting for gpt-5.6-luna judge" in out + assert "gpt-5.6-luna judge complete" in out + assert "gpt-4o" not in out + + +# --- a judge fault on one run must not cost the whole item (H1) --- + + +def _client_and_flaky_judge(verdicts): + """A client that answers every turn, and a judge that returns `verdicts` in order. + + A JudgeResponseError in the list is raised for that run instead of returning. + """ + client, judge = _pass_client_and_judge() + client.create_conversation.side_effect = [f"conv-{i}" for i in range(1, 9)] + it = iter(verdicts) + + def score(**_kw): + v = next(it) + if isinstance(v, Exception): + raise v + return v + + judge.score.side_effect = score + return client, judge + + +def test_a_judge_fault_on_the_last_run_keeps_the_pass_that_run_one_earned(): + """pass@2 is satisfied the moment run 0 passes. + + Letting the error out of the K-run loop discarded run 0 entirely, so an item that had + already passed was reported as an error -- and `EvalReport` counts an errored item + against the pass rate, which is the "parse bug reads as a pass-rate drop" that + JudgeResponseError was introduced to end. + """ + client, judge = _client_and_flaky_judge([(True, "run 0 is correct"), JudgeResponseError("empty body twice")]) + + with _patched(client, judge): + summary = run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=2, + ) + + assert summary.pass_at_k is True + assert len(summary.run_results) == 2, "the ungraded run is still recorded" + assert len(summary.scored_run_results) == 1, "but only the graded one counts" + assert summary.judge_errors == ["empty body twice"] + # pass^K cannot be claimed over a run nobody graded. + assert summary.pass_power_k is False + assert summary.best.passed is True, "best must be a run that actually has a verdict" + + +def test_an_ungraded_run_is_not_counted_as_a_failure(): + # The other half of the same rule: with run 0 failing and run 1 ungraded the item + # fails on run 0's verdict alone -- not because two runs "failed". + client, judge = _client_and_flaky_judge([(False, "wrong"), JudgeResponseError("unparseable JSON")]) + + with _patched(client, judge): + summary = run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=2, + ) + + assert summary.pass_at_k is False + assert [r.judge_error is None for r in summary.run_results] == [True, False] + + +def test_the_agent_timing_of_an_ungraded_run_is_still_recorded(): + # The agent answered. That measurement is independent of whether our judge could read + # its own reply, and it is the number the report is actually for. + client, judge = _client_and_flaky_judge([JudgeResponseError("empty body twice"), (True, "ok")]) + + with _patched(client, judge, monotonic=_CLOCK_TWO_RUNS_AGENT_7_JUDGE_2): + summary = run_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=2, + ) + + assert summary.run_results[0].judge_error is not None + assert summary.run_results[0].timings.agent_s == 4.0 + + +def test_an_ungraded_run_is_never_published_to_langfuse_as_a_zero(): + """Writing float(run.passed) for an ungraded run moves the silent FAIL into Langfuse. + + Scores are what the dashboards read, so a 0 the judge never returned there is worse + than no score at all. + """ + client, judge = _client_and_flaky_judge([(True, "run 0 is correct"), JudgeResponseError("empty body twice")]) + submitted = [] + scored_conversations = [] + polled = {} + + def _find(langfuse, conversation_ids, window_start, window_end=None): + polled["ids"] = list(conversation_ids) + return dict.fromkeys(conversation_ids) + + with ( + _patched_langfuse(client, judge, find=_find), + patch( + "gooddata_eval.core.agentic._langfuse.score_safe", + side_effect=lambda lf, tid, **kw: scored_conversations.append(kw["name"]), + ), + ): + outcome = evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=2, + langfuse=MagicMock(), + dataset_item_id="ds-item-1", + submit_trace_link=lambda task, item_id="": submitted.append(task), + ) + submitted[0]() + + # Exactly one run's worth of scores, from the run that had a verdict. + assert scored_conversations.count("general_question_pass") == 1 + # And the ungraded conversation is not even polled for -- that would spend the item's + # shared retry budget on a score that is never written. + assert polled["ids"] == ["conv-1"] + # The weaker pass is visible in the report rather than passed off as a clean pass@2. + assert outcome.detail["unscored_runs"] == 1 + + +def test_an_item_with_no_gradeable_run_raises_instead_of_reporting_failures(): + # Nothing was graded, so there is no verdict for this item at all: an error, not K + # failures. This is the case JudgeResponseError is genuinely for. + client, judge = _client_and_flaky_judge( + [JudgeResponseError("empty body twice"), JudgeResponseError("no 'score' key")] + ) + + with ( + _patched(client, judge, monotonic=_CLOCK_TWO_RUNS_AGENT_7_JUDGE_2), + pytest.raises(JudgeResponseError) as err, + ): + evaluate_agentic_general_question( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="What is the answer?", + expected_output="42", + k=2, + ) + + assert "no readable verdict for any of the 2 run(s)" in str(err.value) + assert "no 'score' key" in str(err.value), "both causes are quoted" + # Carried so the runner can still report what the item cost before it became + # unevaluable. + assert err.value.timings.agent_s == 7.0 # 4.0 + 3.0 + + +def test_run_agentic_general_question_forwards_the_user_context_to_the_chat_client(): + attachment = {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "campaign_spend"}]}]} + client, judge = _pass_client_and_judge(text_response="It shows campaign spend by channel.") + + with _patched(client, judge): + run_agentic_general_question( + host="https://h", + token="tok", + workspace_id="ws1", + question="What does the visualization I attached show?", + expected_output="Describes the attached chart.", + k=1, + user_context=attachment, + ) + + assert client.send_message.call_args.kwargs["user_context"] == attachment diff --git a/packages/gooddata-eval/tests/test_agentic_guardrail.py b/packages/gooddata-eval/tests/test_agentic_guardrail.py index 4897d3e77..320ba7b9a 100644 --- a/packages/gooddata-eval/tests/test_agentic_guardrail.py +++ b/packages/gooddata-eval/tests/test_agentic_guardrail.py @@ -1,5 +1,7 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import itertools +from contextlib import contextmanager from unittest.mock import MagicMock, patch import pytest @@ -12,6 +14,44 @@ from gooddata_eval.core.models import ChatResult +@contextmanager +def _patched(client, judge): + """Patch guardrail's ChatClient and LLMJudge with the given mocks.""" + with ( + patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=client), + patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=judge), + ): + yield + + +def _client_and_judge( + *, + text_response: str = "I cannot help with that", + reasoning_steps: list[str] | None = None, + response_id: str = "resp-1", + verdict: tuple[bool, str] = (True, "Correctly refused"), +): + """A chat client on conversation ``conv-1`` and a gpt-4o judge returning ``verdict``. + + Callers override only the one field they vary -- ``client.create_conversation`` and the + rest are ordinary mock attributes. + """ + client = MagicMock() + client.create_conversation.return_value = "conv-1" + client.send_message.return_value = ChatResult.model_validate( + { + "textResponse": text_response, + "toolCallEvents": [], + "reasoningSteps": reasoning_steps or [], + "responseId": response_id, + } + ) + judge = MagicMock() + judge.model = "gpt-4o" + judge.score.return_value = verdict + return client, judge + + def test_guardrail_result_fields(): r = GuardrailResult( conversation_id="c1", @@ -24,19 +64,11 @@ def test_guardrail_result_fields(): def test_run_agentic_guardrail_fail(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" - mock_chat_result = MagicMock() - mock_chat_result.text_response = "Sure, here is how to do it" - mock_chat_result.tool_call_events = [] - mock_client.send_message.return_value = mock_chat_result - mock_judge = MagicMock() - mock_judge.score.return_value = (False, "Should have refused") + mock_client, mock_judge = _client_and_judge( + text_response="Sure, here is how to do it", verdict=(False, "Should have refused") + ) - with ( - patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), - ): + with _patched(mock_client, mock_judge): summary = run_agentic_guardrail( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -50,18 +82,9 @@ def test_run_agentic_guardrail_fail(): def test_run_agentic_guardrail_uses_initial_conversation_for_run_0(): - mock_client = MagicMock() - mock_chat_result = MagicMock() - mock_chat_result.text_response = "I cannot help with that" - mock_chat_result.tool_call_events = [] - mock_client.send_message.return_value = mock_chat_result - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "Correctly refused") + mock_client, mock_judge = _client_and_judge() - with ( - patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), - ): + with _patched(mock_client, mock_judge): run_agentic_guardrail( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -76,19 +99,10 @@ def test_run_agentic_guardrail_uses_initial_conversation_for_run_0(): def test_run_agentic_guardrail_creates_fresh_conversations_for_remaining_runs(): - mock_client = MagicMock() + mock_client, mock_judge = _client_and_judge() mock_client.create_conversation.side_effect = ["fresh-1", "fresh-2"] - mock_chat_result = MagicMock() - mock_chat_result.text_response = "I cannot help with that" - mock_chat_result.tool_call_events = [] - mock_client.send_message.return_value = mock_chat_result - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "Correctly refused") - with ( - patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), - ): + with _patched(mock_client, mock_judge): run_agentic_guardrail( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -103,23 +117,9 @@ def test_run_agentic_guardrail_creates_fresh_conversations_for_remaining_runs(): def test_run_agentic_guardrail_captures_reasoning_steps(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" - mock_client.send_message.return_value = ChatResult.model_validate( - { - "textResponse": "I cannot help with that", - "toolCallEvents": [], - "reasoningSteps": ["deciding whether this is harmful"], - "responseId": "resp-1", - } - ) - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "Correctly refused") + mock_client, mock_judge = _client_and_judge(reasoning_steps=["deciding whether this is harmful"]) - with ( - patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), - ): + with _patched(mock_client, mock_judge): summary = run_agentic_guardrail( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -134,23 +134,9 @@ def test_run_agentic_guardrail_captures_reasoning_steps(): def test_evaluate_agentic_guardrail_returns_reasoning_steps_on_pass(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" - mock_client.send_message.return_value = ChatResult.model_validate( - { - "textResponse": "I cannot help with that", - "toolCallEvents": [], - "reasoningSteps": ["deciding whether this is harmful"], - "responseId": "resp-1", - } - ) - mock_judge = MagicMock() - mock_judge.score.return_value = (True, "Correctly refused") + mock_client, mock_judge = _client_and_judge(reasoning_steps=["deciding whether this is harmful"]) - with ( - patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), - ): + with _patched(mock_client, mock_judge): outcome = evaluate_agentic_guardrail( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -172,24 +158,14 @@ def test_evaluate_agentic_guardrail_returns_reasoning_steps_on_pass(): def test_evaluate_agentic_guardrail_attaches_reasoning_steps_to_exception_on_fail(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" - mock_client.send_message.return_value = ChatResult.model_validate( - { - "textResponse": "Sure, here is how to do it", - "toolCallEvents": [], - "reasoningSteps": ["treating this as an ordinary request"], - "responseId": "resp-2", - } + mock_client, mock_judge = _client_and_judge( + text_response="Sure, here is how to do it", + reasoning_steps=["treating this as an ordinary request"], + response_id="resp-2", + verdict=(False, "Should have refused"), ) - mock_judge = MagicMock() - mock_judge.score.return_value = (False, "Should have refused") - with ( - patch("gooddata_eval.core.agentic.guardrail.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.guardrail.LLMJudge", return_value=mock_judge), - pytest.raises(GuardrailAssertionError) as exc_info, - ): + with _patched(mock_client, mock_judge), pytest.raises(GuardrailAssertionError) as exc_info: evaluate_agentic_guardrail( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -208,3 +184,53 @@ def test_evaluate_agentic_guardrail_attaches_reasoning_steps_to_exception_on_fai "actual_output": "Sure, here is how to do it", "latency_breakdown": [], } + + +# --- the run counts have to reach the report (same predicate pass_at_k uses) --- + + +def _guardrail_client_and_judge(verdicts): + client = MagicMock() + client.create_conversation.side_effect = (f"conv-{i}" for i in itertools.count(1)) + client.send_message.side_effect = lambda c, q, **k: ChatResult.model_validate( + {"textResponse": f"answer {c}", "toolCallEvents": [], "reasoningSteps": [], "responseId": "r"} + ) + it = iter(verdicts) + judge = MagicMock() + judge.model = "gpt-4o" + judge.score.side_effect = lambda **kw: next(it) + return client, judge + + +@pytest.mark.parametrize( + ("verdicts", "expected_passed", "expected_unanimous"), + [ + ([(True, "ok")] * 3, 3, True), + ([(True, "ok"), (True, "ok"), (False, "no")], 2, False), + ([(True, "ok"), (False, "no"), (False, "no")], 1, False), + ], +) +def test_the_summary_counts_how_many_runs_passed(verdicts, expected_passed, expected_unanimous): + client, judge = _guardrail_client_and_judge(verdicts) + + with _patched(client, judge): + summary = run_agentic_guardrail(host="h", token="t", workspace_id="ws", question="q", expected_output="e", k=3) + + assert sum(1 for r in summary.run_results if r.passed) == expected_passed + # pass_at_k stays "did any run pass"; pass^K is the unanimity claim. + assert summary.pass_at_k is (expected_passed > 0) + assert summary.pass_power_k is expected_unanimous + + +def test_a_non_unanimous_pass_reaches_the_outcome(): + """pass@K is satisfied by run 0, so this item PASSes -- but the report must be able to + say it only passed 2 of 3, which every other column hides. + """ + client, judge = _guardrail_client_and_judge([(True, "ok"), (True, "ok"), (False, "no")]) + + with _patched(client, judge): + outcome = evaluate_agentic_guardrail( + host="h", token="t", workspace_id="ws", question="q", expected_output="e", k=3 + ) + + assert (outcome.runs_passed, outcome.runs_effective) == (2, 3) diff --git a/packages/gooddata-eval/tests/test_agentic_kda_skill.py b/packages/gooddata-eval/tests/test_agentic_kda_skill.py index 13606414a..a9578bdda 100644 --- a/packages/gooddata-eval/tests/test_agentic_kda_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_kda_skill.py @@ -1,6 +1,7 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise import json +from contextlib import contextmanager from unittest.mock import MagicMock, patch import httpx @@ -75,6 +76,66 @@ def _no_kda_chat_result( ) +def _client() -> MagicMock: + """A chat client whose conversations are all ``conv-1``. + + Callers override only what they vary -- ``send_message`` and the rest are ordinary + mock attributes. + """ + client = MagicMock() + client.create_conversation.return_value = "conv-1" + return client + + +@contextmanager +def _patched(client, *, simulated_reply=None, simulated_error=None): + """Patch kda_skill's ChatClient, and optionally its simulated-user helper. + + ``simulated_reply`` is what the simulated user answers; ``simulated_error`` makes it + raise instead. Yields the ``generate_simulated_kda_response`` mock, or None when the + test asked for neither. + """ + with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=client): + if simulated_reply is None and simulated_error is None: + yield None + else: + with patch( + "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", + return_value=simulated_reply, + side_effect=simulated_error, + ) as mock_simulate: + yield mock_simulate + + +@contextmanager +def _patched_without_langfuse(client): + """Patch kda_skill's ChatClient and stub Langfuse discovery out (`langfuse=None` path).""" + with ( + _patched(client), + patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), + ): + yield + + +@contextmanager +def _patched_with_langfuse_scores(client, trace): + """Patch kda_skill's ChatClient plus every _langfuse helper the scoring block calls. + + ``trace`` is the trace ``find_traces_per_conversation`` returns for ``conv-1``, and the + one ``observe`` hands back. Yields ``(mock_score_safe, mock_log_quality_and_value_scores)``. + """ + with ( + _patched(client), + patch("gooddata_eval.core.agentic._langfuse.build_run_context", return_value=("run-base", {})), + patch("gooddata_eval.core.agentic._langfuse.find_traces_per_conversation", return_value={"conv-1": trace}), + patch("gooddata_eval.core.agentic._langfuse.observe") as mock_observe, + patch("gooddata_eval.core.agentic._langfuse.score_safe") as mock_score_safe, + patch("gooddata_eval.core.agentic._langfuse.log_quality_and_value_scores") as mock_log_scores, + ): + mock_observe.return_value.__enter__.return_value = trace.id + yield mock_score_safe, mock_log_scores + + # --------------------------------------------------------------------------- # # _build_clarification_prompt # --------------------------------------------------------------------------- # @@ -228,11 +289,10 @@ def test_strict_pass_false_when_any_core_check_fails(): # run_agentic_kda_skill # --------------------------------------------------------------------------- # def test_run_agentic_kda_skill_triggers_and_succeeds(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _kda_chat_result(success=True) - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -255,11 +315,10 @@ def test_run_agentic_kda_skill_fails_on_sse_cutoff_despite_nonempty_text(): # suite) can still have emitted a partial, non-empty text_response before dying. Using # "text_response is non-empty" as the completion signal would wrongly call this turn # completed; only gen-ai's own response_ended event may. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _kda_chat_result(success=True, stream_ended=False) - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -283,11 +342,10 @@ def test_run_agentic_kda_skill_survives_send_message_error(): # Langfuse-logging loop entirely for this run, leaving nothing but a bare JUnit # failure to diagnose from. It must instead surface as a normal (failed) run result, # so triggered/executed/success/turn_completed all still get scored as False. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = TransientChatError("gen-ai returned 503") - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -310,11 +368,10 @@ def test_run_agentic_kda_skill_survives_a_raw_httpx_transport_error(): # which _is_retryable_exc does not recognize as retryable and re-raises as-is -- a # narrower `except ChatError` (an earlier version of this fix) would NOT catch this # and would still propagate out of run_agentic_kda_skill uncaught. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = httpx.RemoteProtocolError("peer closed connection") - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -335,13 +392,12 @@ def test_run_agentic_kda_skill_recovers_kda_calls_from_a_chat_errors_partial_res # through (e.g. a later, unrelated final-summary generation failing with a 500) must # not misreport as "the agent never called KDA at all" -- the partial_result attached # to the exception is exactly the tool_call_events already seen. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = ChatError( "SSE error 500: boom", status_code=500, partial_result=_kda_chat_result(success=True) ) - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -363,20 +419,13 @@ def test_run_agentic_kda_skill_resets_turn_completed_when_a_later_iteration_cras # iteration 0 asks a clarifying question and ends cleanly (turn_completed=True for # THAT iteration); iteration 1 then crashes. Without resetting, the stale True from # iteration 0 would still be logged for a run that never actually finished. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result("Could you clarify which measure?", stream_ended=True), httpx.RemoteProtocolError("peer closed connection"), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Use the revenue metric.", - ), - ): + with _patched(mock_client, simulated_reply="Use the revenue metric."): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -394,11 +443,10 @@ def test_run_agentic_kda_skill_resets_turn_completed_when_a_later_iteration_cras def test_run_agentic_kda_skill_turn_not_completed_when_stream_ends_with_empty_text(): # stream_ended alone is not enough: a turn that ends cleanly but delivers nothing to # the user hasn't "delivered a final answer" either (see KdaEvaluation docstring). - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _kda_chat_result(success=True, text=" ", stream_ended=True) - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -416,20 +464,13 @@ def test_run_agentic_kda_skill_turn_not_completed_when_stream_ends_with_empty_te def test_run_agentic_kda_skill_marks_disambiguated_after_a_simulated_reply(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result("Could you clarify which measure?"), _kda_chat_result(success=True), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Use the revenue metric.", - ), - ): + with _patched(mock_client, simulated_reply="Use the revenue metric."): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -451,8 +492,7 @@ def test_run_agentic_kda_skill_disambiguates_on_question_followed_by_option_list # visualization.py/alert_skill.py), a heuristic that only matched "?" endings gave # up after turn 1 (triggered=False) instead of ever nudging the simulated user to # pick one. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result( 'I found two different "Total Net Revenue" metrics in your data model. ' @@ -463,13 +503,7 @@ def test_run_agentic_kda_skill_disambiguates_on_question_followed_by_option_list _kda_chat_result(success=True), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Use metric_l1_sql_net_sales_summary_net_revenue.", - ) as mock_simulate, - ): + with _patched(mock_client, simulated_reply="Use metric_l1_sql_net_sales_summary_net_revenue.") as mock_simulate: summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -493,20 +527,13 @@ def test_run_agentic_kda_skill_retries_on_bold_markdown_option_list_with_no_spac # classification entirely (see run_agentic_kda_skill's docstring) makes this -- and any # other future response shape -- a non-issue: a non-triggering, non-empty response # always gets a simulated reply now, regardless of how it's formatted. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result("Which one should I analyze?\n**Option 1**: revenue\n**Option 2**: gross profit"), _kda_chat_result(success=True), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Use revenue.", - ) as mock_simulate, - ): + with _patched(mock_client, simulated_reply="Use revenue.") as mock_simulate: summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -533,20 +560,13 @@ def test_run_agentic_kda_skill_disambiguates_on_period_clarification(): "Analyzed Period": "2026-2", "Reference Period": "2026-1", } - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result("Which period would you like to compare?"), _kda_chat_result(success=True), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Compare 2026-2 to 2026-1.", - ) as mock_simulate, - ): + with _patched(mock_client, simulated_reply="Compare 2026-2 to 2026-1.") as mock_simulate: summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -572,20 +592,13 @@ def test_run_agentic_kda_skill_disambiguates_when_expected_output_is_not_a_dict( # swallowed by the broad except around generate_simulated_kda_response and disabling # disambiguation with only a WARNING. Guard so the call still happens, with None # candidates, instead of crashing. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result("Could you clarify which measure?"), _kda_chat_result(success=True), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Use the revenue metric.", - ) as mock_generate, - ): + with _patched(mock_client, simulated_reply="Use the revenue metric.") as mock_generate: summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -605,20 +618,13 @@ def test_run_agentic_kda_skill_latency_is_only_the_turn_that_completed_kda(): # The disambiguation turn's own time, and the simulated-reply generation between # turns, must NOT be counted -- only the turn where KDA actually completed reflects # gen-ai's own latency; the rest is test-harness overhead (an unrelated OpenAI call). - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result("Could you clarify which measure?", turn_wall_clock_sec=5.0), _kda_chat_result(success=True, turn_wall_clock_sec=8.0), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Use the revenue metric.", - ), - ): + with _patched(mock_client, simulated_reply="Use the revenue metric."): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -638,8 +644,7 @@ def test_run_agentic_kda_skill_triggered_but_not_executed_when_execute_tool_is_u # isn't registered as a tool at all, so create can succeed alone within a single turn # with no execute_result. Must be scored as triggered but not executed immediately, # not treated as "execute is coming in a later turn". - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() create_only = ChatResult.model_validate( { "textResponse": "The analysis is ready. Open it above to review the results.", @@ -652,7 +657,7 @@ def test_run_agentic_kda_skill_triggered_but_not_executed_when_execute_tool_is_u ) mock_client.send_message.return_value = create_only - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -671,11 +676,10 @@ def test_run_agentic_kda_skill_triggered_but_not_executed_when_execute_tool_is_u def test_run_agentic_kda_skill_not_disambiguated_when_kda_triggers_immediately(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _kda_chat_result(success=True) - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -690,11 +694,10 @@ def test_run_agentic_kda_skill_not_disambiguated_when_kda_triggers_immediately() def test_run_agentic_kda_skill_no_tool_call(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _no_kda_chat_result() - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -710,20 +713,13 @@ def test_run_agentic_kda_skill_no_tool_call(): def test_run_agentic_kda_skill_resolves_after_clarification_turn(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result("Could you clarify which revenue measure you mean?"), _kda_chat_result(success=True), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="The revenue metric is fine.", - ) as mock_simulate, - ): + with _patched(mock_client, simulated_reply="The revenue metric is fine.") as mock_simulate: summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -740,17 +736,10 @@ def test_run_agentic_kda_skill_resolves_after_clarification_turn(): def test_run_agentic_kda_skill_gives_up_after_max_iterations_of_clarification(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _no_kda_chat_result("Could you clarify which measure?") - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Please use revenue.", - ), - ): + with _patched(mock_client, simulated_reply="Please use revenue."): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -770,8 +759,7 @@ def test_run_agentic_kda_skill_disambiguation_then_create_without_execute(): # create succeeds but execute_result is None (e.g. execute is unavailable for this # org). Confirms this still scores correctly (disambiguated + triggered, not executed) # instead of being mistaken for "still waiting on more turns". - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() create_only = ChatResult.model_validate( { "textResponse": "The analysis is ready. Open it above to review the results.", @@ -786,13 +774,7 @@ def test_run_agentic_kda_skill_disambiguation_then_create_without_execute(): create_only, ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="Use the revenue metric.", - ), - ): + with _patched(mock_client, simulated_reply="Use the revenue metric."): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -820,13 +802,7 @@ def test_run_agentic_kda_skill_survives_simulated_reply_failure(): _no_kda_chat_result("Could you clarify which measure?"), # run 1: asks, then helper blows up ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - side_effect=RuntimeError("openai down"), - ), - ): + with _patched(mock_client, simulated_error=RuntimeError("openai down")): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -846,7 +822,7 @@ def test_run_agentic_kda_skill_survives_simulated_reply_failure(): def test_run_agentic_kda_skill_uses_initial_conversation_for_run_0(): mock_client = MagicMock() mock_client.send_message.return_value = _kda_chat_result(success=True) - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -865,7 +841,7 @@ def test_run_agentic_kda_skill_creates_fresh_conversations_for_remaining_runs(): mock_client = MagicMock() mock_client.create_conversation.side_effect = ["fresh-1", "fresh-2"] mock_client.send_message.return_value = _kda_chat_result(success=True) - with patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -897,15 +873,10 @@ def test_run_agentic_kda_skill_rejects_non_positive_k(bad_k): # evaluate_agentic_kda_skill # --------------------------------------------------------------------------- # def test_evaluate_agentic_kda_skill_raises_on_failure(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _no_kda_chat_result() - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), - pytest.raises(KdaSkillAssertionError), - ): + with _patched_without_langfuse(mock_client), pytest.raises(KdaSkillAssertionError): evaluate_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -919,14 +890,10 @@ def test_evaluate_agentic_kda_skill_raises_on_failure(): def test_evaluate_agentic_kda_skill_does_not_raise_on_success(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _kda_chat_result(success=True) - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), - ): + with _patched_without_langfuse(mock_client): evaluate_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -943,26 +910,16 @@ def test_evaluate_agentic_kda_skill_never_treats_fallback_trace_latency_as_kda_l # Regression test: when KDA never triggered, whatever trace find_traces_per_conversation's # default (max-latency) selector picks is NOT a real KDA turn -- its latency/cost must not # be logged as the KDA run's own value_score inputs. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _no_kda_chat_result() fallback_trace = MagicMock(id="fallback-trace", latency=999.0, total_cost=5.0) mock_langfuse = MagicMock() with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic._langfuse.build_run_context", return_value=("run-base", {})), - patch( - "gooddata_eval.core.agentic._langfuse.find_traces_per_conversation", - return_value={"conv-1": fallback_trace}, - ), - patch("gooddata_eval.core.agentic._langfuse.observe") as mock_observe, - patch("gooddata_eval.core.agentic._langfuse.score_safe"), - patch("gooddata_eval.core.agentic._langfuse.log_quality_and_value_scores") as mock_log_scores, + _patched_with_langfuse_scores(mock_client, fallback_trace) as (_, mock_log_scores), pytest.raises(KdaSkillAssertionError), ): - mock_observe.return_value.__enter__.return_value = "fallback-trace" evaluate_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -985,25 +942,13 @@ def test_evaluate_agentic_kda_skill_reports_trace_latency_when_kda_triggered(): # set by ChatClient around its send_message() call), not from the trace find_traces_per_ # conversation happens to return -- that trace isn't necessarily the KDA turn at all (see # kda_skill.py's comment on `pt`). Only total_cost still comes from the trace. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _kda_chat_result(success=True, turn_wall_clock_sec=76.0) found_trace = MagicMock(id="trace-1", latency=999.0, total_cost=0.02) mock_langfuse = MagicMock() - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic._langfuse.build_run_context", return_value=("run-base", {})), - patch( - "gooddata_eval.core.agentic._langfuse.find_traces_per_conversation", - return_value={"conv-1": found_trace}, - ), - patch("gooddata_eval.core.agentic._langfuse.observe") as mock_observe, - patch("gooddata_eval.core.agentic._langfuse.score_safe") as mock_score_safe, - patch("gooddata_eval.core.agentic._langfuse.log_quality_and_value_scores") as mock_log_scores, - ): - mock_observe.return_value.__enter__.return_value = "trace-1" + with _patched_with_langfuse_scores(mock_client, found_trace) as (mock_score_safe, mock_log_scores): evaluate_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -1031,25 +976,13 @@ def test_evaluate_agentic_kda_skill_does_not_log_pass_at_k_or_pass_power_k(): # silently splitting any Langfuse view built on the old name. Only visualization.py # logs this pair, with a real consumer at k=2 (combo_report.py's viz_flaky) that # justifies it. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _kda_chat_result(success=True) found_trace = MagicMock(id="trace-1", total_cost=0.01) mock_langfuse = MagicMock() - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic._langfuse.build_run_context", return_value=("run-base", {})), - patch( - "gooddata_eval.core.agentic._langfuse.find_traces_per_conversation", - return_value={"conv-1": found_trace}, - ), - patch("gooddata_eval.core.agentic._langfuse.observe") as mock_observe, - patch("gooddata_eval.core.agentic._langfuse.score_safe") as mock_score_safe, - patch("gooddata_eval.core.agentic._langfuse.log_quality_and_value_scores"), - ): - mock_observe.return_value.__enter__.return_value = "trace-1" + with _patched_with_langfuse_scores(mock_client, found_trace) as (mock_score_safe, _): evaluate_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -1072,20 +1005,13 @@ def test_evaluate_agentic_kda_skill_does_not_log_pass_at_k_or_pass_power_k(): def test_run_agentic_kda_skill_accumulates_reasoning_steps_across_iterations(): """A clarification turn's reasoning is retained even though only the final turn's create/execute calls determine the KDA outcome.""" - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [ _no_kda_chat_result("Which metric do you mean?", reasoning_steps=["step one"], response_id="resp-1"), _kda_chat_result(reasoning_steps=["step two"], response_id="resp-2"), ] - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.kda_skill.generate_simulated_kda_response", - return_value="I mean revenue.", - ), - ): + with _patched(mock_client, simulated_reply="I mean revenue."): summary = run_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -1101,16 +1027,12 @@ def test_run_agentic_kda_skill_accumulates_reasoning_steps_across_iterations(): def test_evaluate_agentic_kda_skill_returns_reasoning_steps_on_pass(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _kda_chat_result( success=True, reasoning_steps=["analyzing drivers"], response_id="resp-1" ) - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), - ): + with _patched_without_langfuse(mock_client): outcome = evaluate_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -1137,17 +1059,12 @@ def test_evaluate_agentic_kda_skill_returns_reasoning_steps_on_pass(): def test_evaluate_agentic_kda_skill_attaches_reasoning_steps_to_exception_on_fail(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _no_kda_chat_result( reasoning_steps=["could not find a measure"], response_id="resp-2" ) - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), - pytest.raises(KdaSkillAssertionError) as exc_info, - ): + with _patched_without_langfuse(mock_client), pytest.raises(KdaSkillAssertionError) as exc_info: evaluate_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -1178,8 +1095,7 @@ def test_evaluate_agentic_kda_skill_preserves_reasoning_from_a_chat_error_partia # (KDA create/execute already streamed before an unrelated later failure), but checking that # the partial_result's own reasoning_steps/response_id survive onto the exception too, not # just the tool-call data. - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = ChatError( "SSE error 500: boom", status_code=500, @@ -1188,11 +1104,7 @@ def test_evaluate_agentic_kda_skill_preserves_reasoning_from_a_chat_error_partia ), ) - with ( - patch("gooddata_eval.core.agentic.kda_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic._langfuse.try_make_langfuse_client", return_value=None), - pytest.raises(KdaSkillAssertionError) as exc_info, - ): + with _patched_without_langfuse(mock_client), pytest.raises(KdaSkillAssertionError) as exc_info: evaluate_agentic_kda_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", diff --git a/packages/gooddata-eval/tests/test_agentic_langfuse_trace.py b/packages/gooddata-eval/tests/test_agentic_langfuse_trace.py index af64a78d7..56f252dca 100644 --- a/packages/gooddata-eval/tests/test_agentic_langfuse_trace.py +++ b/packages/gooddata-eval/tests/test_agentic_langfuse_trace.py @@ -1,9 +1,41 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise -from datetime import datetime, timezone +import os +import time +from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, patch -from gooddata_eval.core.agentic._langfuse import find_traces_per_conversation +import httpx +import pytest +from gooddata_eval.core.agentic import _langfuse as lf_module +from gooddata_eval.core.agentic._langfuse import ( + _INLINE_LINK_BUDGET_SEC, + _LINK_BUDGET_SEC, + _MAX_DELAY, + SKIP_ENV_VAR, + _fetch_traces_for_session, + find_traces_per_conversation, + make_langfuse_client, + observe, +) +from gooddata_eval.core.agentic._trace_linker import ( + BackgroundTraceLinker, + linking_is_inline, + run_trace_link_inline, +) + + +@pytest.fixture(autouse=True) +def _neutral_skip_switch(monkeypatch): + """Run every test in this module with TAVERN_E2E_SKIP_TRACE_LINK unset. + + Two things this prevents. Tests here that exercise the polling ladder used to pass only + because an earlier test in the file popped the variable globally -- so a developer with + it exported (which the README tells them to do) saw real failures, and every module + collected afterwards silently ran with trace linking re-enabled. Tests that WANT the + switch on still set it themselves with monkeypatch; this only removes the ambient value. + """ + monkeypatch.delenv(SKIP_ENV_VAR, raising=False) def test_find_traces_per_conversation_is_none_for_a_conversation_with_no_trace(): @@ -24,3 +56,497 @@ def _fetch(langfuse, cid, window_start, window_end, pad): assert result["conv-found"] is found_trace assert result["conv-missing"] is None + + +def test_find_traces_per_conversation_tries_before_sleeping(): + # The poll used to sleep _INITIAL_DELAY *before* its first fetch, so every conversation + # paid a mandatory 0.5s even when Langfuse had already ingested the trace. Attempting + # first makes the already-there case free, which is the common case once the agent turn + # itself has taken several seconds. + found_trace = MagicMock(latency=12.0) + sleeps: list[float] = [] + + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", return_value=[found_trace]), + patch("gooddata_eval.core.agentic._langfuse.time.sleep", side_effect=sleeps.append), + ): + result = find_traces_per_conversation(MagicMock(), ["conv-1"], datetime.now(timezone.utc)) + + assert result["conv-1"] is found_trace + assert sleeps == [] + + +def test_find_traces_per_conversation_accepts_an_explicit_window_end(): + # Deferred trace linking runs the poll on a worker thread, possibly well after the + # conversation ended. Letting it default window_end to "now" would widen the query + # window by however long the pool was backed up -- and _fetch_traces_for_session pages + # at _FETCH_LIMIT and filters by session locally, so a wide enough window can push the + # wanted trace off the page entirely. Callers must be able to pin the window. + captured = {} + + def _fetch(langfuse, cid, window_start, window_end, pad): + captured["end"] = window_end + return [MagicMock(latency=1.0)] + + pinned = datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", side_effect=_fetch), + patch("gooddata_eval.core.agentic._langfuse.time.sleep"), + ): + find_traces_per_conversation(MagicMock(), ["c1"], datetime.now(timezone.utc), window_end=pinned) + + assert captured["end"] == pinned + + +class _FakeClock: + """A clock where sleeping actually advances time. + + The retry budget is wall-clock, so a no-op time.sleep leaves monotonic() frozen and the + deadline never arrives -- the loop would run to its sanity cap and the test would prove + nothing about the budget. + """ + + def __init__(self) -> None: + self.now = 1000.0 + self.sleeps: list[float] = [] + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + +def test_find_traces_per_conversation_keeps_retrying_within_its_budget(): + # Langfuse ingestion lag on us.cloud runs from ~35s to several minutes, so a fixed + # 8-attempt ladder that gives up after 19s orphans every score. Trace linking is off + # the critical path now, so patience is nearly free -- the loop retries until its + # budget is spent rather than until an attempt count is reached. + clock = _FakeClock() + + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", return_value=[]), + patch("gooddata_eval.core.agentic._langfuse.time", clock), + ): + result = find_traces_per_conversation(MagicMock(), ["conv-missing"], datetime.now(timezone.utc)) + + assert result["conv-missing"] is None + assert max(clock.sleeps) <= _MAX_DELAY + # Long enough to outlast real ingestion lag, but bounded so the drain cannot run away. + assert 60.0 <= sum(clock.sleeps) <= _LINK_BUDGET_SEC + + +def test_find_traces_per_conversation_always_makes_one_attempt_even_past_the_budget(): + # Batched linking polls every conversation after the whole agent phase, by which point + # most traces are minutes old and hit on the first try. That first try must happen even + # for the last conversation in a queue that has already burnt the budget, or the items + # at the back would be skipped without ever being looked up. + attempts = [] + + def _fetch(langfuse, cid, window_start, window_end, pad): + attempts.append(cid) + return [MagicMock(latency=1.0)] + + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", side_effect=_fetch), + patch("gooddata_eval.core.agentic._langfuse.time.sleep"), + ): + result = find_traces_per_conversation( + MagicMock(), ["c1"], datetime.now(timezone.utc), deadline=time.monotonic() - 1.0 + ) + + assert attempts == ["c1"] + assert result["c1"] is not None + + +def test_the_retry_budget_is_shared_across_one_items_conversations(): + # The budget bounds the batch's tail, so it has to cover a whole item, not each of its + # K conversations. Computed per conversation, a --runs 2 item spends twice the budget + # and --runs 3 three times, so the tail grows with K exactly where it should not. + clock = _FakeClock() + + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", return_value=[]), + patch("gooddata_eval.core.agentic._langfuse.time", clock), + ): + result = find_traces_per_conversation(MagicMock(), ["c1", "c2", "c3"], datetime.now(timezone.utc)) + + assert all(v is None for v in result.values()) + assert sum(clock.sleeps) <= _LINK_BUDGET_SEC + + +def test_every_conversation_is_still_looked_up_once_after_the_budget_is_spent(): + # Sharing the budget must not mean later conversations get skipped: by the time the + # first one has exhausted it, the rest are minutes older and likely to resolve on their + # single attempt. + looked_up: list[str] = [] + clock = _FakeClock() + + def _fetch(langfuse, cid, window_start, window_end, pad): + looked_up.append(cid) + return [] + + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", side_effect=_fetch), + patch("gooddata_eval.core.agentic._langfuse.time", clock), + ): + find_traces_per_conversation(MagicMock(), ["c1", "c2", "c3"], datetime.now(timezone.utc)) + + assert set(looked_up) == {"c1", "c2", "c3"} + + +def test_the_skip_switch_announces_itself_instead_of_silently_orphaning_scores(monkeypatch): + # TAVERN_E2E_SKIP_TRACE_LINK returns all-None before any polling, so every score is + # orphaned and the only symptom is observe()'s generic "No trace found for dataset run" + # -- indistinguishable from a genuine lookup failure. That ambiguity cost a long + # debugging detour on a real run: Langfuse was healthy and every trace was present. + # If linking is switched off, say so. + monkeypatch.setenv(SKIP_ENV_VAR, "1") + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session") as mock_fetch, + patch("gooddata_eval.core.agentic._langfuse.warn_from_worker") as mock_warn, + ): + result = find_traces_per_conversation(MagicMock(), ["c1", "c2"], datetime.now(timezone.utc)) + + assert result == {"c1": None, "c2": None} + mock_fetch.assert_not_called() + said = " ".join(str(c.args[0]) for c in mock_warn.call_args_list) + assert SKIP_ENV_VAR in said + assert "2" in said # how many conversations it gave up on + + +def test_no_skip_announcement_when_the_switch_is_off(): + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", return_value=[MagicMock(latency=1.0)]), + patch("gooddata_eval.core.agentic._langfuse.warn_from_worker") as mock_warn, + ): + find_traces_per_conversation(MagicMock(), ["c1"], datetime.now(timezone.utc)) + + mock_warn.assert_not_called() + + +@pytest.mark.parametrize( + ("value", "should_skip"), + [ + ("1", True), + ("true", True), + ("TRUE", True), + ("yes", True), + # The whole point: a non-empty string is truthy in Python, so `bool(os.environ[...])` + # reads "0" -- the natural way to write "off" -- as ON, silently disabling trace + # linking and orphaning every score in the run. + ("0", False), + ("false", False), + ("False", False), + ("no", False), + ("", False), + ], +) +def test_skip_switch_treats_explicit_off_values_as_off(monkeypatch, value, should_skip): + monkeypatch.setenv(SKIP_ENV_VAR, value) + with ( + patch( + "gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", + return_value=[MagicMock(latency=1.0)], + ) as mock_fetch, + patch("gooddata_eval.core.agentic._langfuse.warn_from_worker"), + ): + result = find_traces_per_conversation(MagicMock(), ["c1"], datetime.now(timezone.utc)) + + if should_skip: + mock_fetch.assert_not_called() + assert result["c1"] is None + else: + mock_fetch.assert_called() + assert result["c1"] is not None + + +# --- the session filter has to reach the server (M8) --- + + +def _stub_langfuse_http(monkeypatch, captured: list[dict]): + """A HttpxLangfuseClient whose httpx GET is recorded instead of sent.""" + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") + monkeypatch.setenv("LANGFUSE_HOST", "https://lf.test") + client = make_langfuse_client() + + def _get(url, params=None, **_kw): + captured.append({"url": url, "params": params}) + return MagicMock(raise_for_status=lambda: None, json=lambda: {"data": []}) + + client._http = MagicMock(get=_get) + client.api = type(client.api)(client._http) + return client + + +def test_the_trace_lookup_filters_by_session_server_side(monkeypatch): + """Without sessionId the endpoint returns the whole window newest-first, capped at + limit, and the caller filters locally -- so an item's own (oldest) trace is evicted + once the window holds more than limit traces, which --concurrency makes likely by + overlapping every item's window. The score then orphans after a full retry budget + spent on a page that could never contain it. + """ + captured: list[dict] = [] + client = _stub_langfuse_http(monkeypatch, captured) + + _fetch_traces_for_session( + client, "conv-abc", datetime.now(timezone.utc), datetime.now(timezone.utc), timedelta(seconds=2) + ) + + assert len(captured) == 1 + assert captured[0]["params"]["sessionId"] == "conv-abc", "the session filter never reached the server" + + +def test_a_server_that_ignores_the_session_filter_cannot_hand_over_a_foreign_trace(monkeypatch): + """The Langfuse API drops a query parameter it does not know rather than rejecting it. + + The httpx client declares ``session_id``, which used to switch the local filter off for + it -- so a server that ignored the parameter returned the whole window, and the + max-latency pick attached this item's scores to a stranger's trace with no warning. + The server-side filter is still sent (paging); the local one is a post-check, not a + fallback. + """ + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk") + monkeypatch.setenv("LANGFUSE_HOST", "https://lf.test") + client = make_langfuse_client() + page = { + "data": [ + {"id": "t-other", "sessionId": "conv-zzz", "latency": 9.0}, + {"id": "t-mine", "sessionId": "conv-abc", "latency": 1.0}, + ] + } + client._http = MagicMock( + get=lambda url, params=None, **_kw: MagicMock(raise_for_status=lambda: None, json=lambda: page) + ) + client.api = type(client.api)(client._http) + now = datetime.now(timezone.utc) + + found = _fetch_traces_for_session(client, "conv-abc", now, now, timedelta(seconds=2)) + + assert [t.id for t in found] == ["t-mine"] + + +def test_a_client_without_the_session_parameter_is_filtered_locally_too(): + # A client whose trace.list cannot take session_id still gets correct results, just by + # filtering the page itself -- that path must keep working. + wanted = MagicMock(session_id="conv-abc", latency=1.0) + other = MagicMock(session_id="conv-zzz", latency=1.0) + legacy = MagicMock() + legacy.api.trace.list = lambda from_timestamp, to_timestamp, limit: MagicMock(data=[other, wanted]) + + found = _fetch_traces_for_session( + legacy, "conv-abc", datetime.now(timezone.utc), datetime.now(timezone.utc), timedelta(seconds=2) + ) + + assert found == [wanted] + + +# --- a 404 from dataset-run-items is one fact about the dataset, not N failures --- + + +@pytest.fixture +def _fresh_unlinkable_runs(): + """Reset the module-level dedup set, which otherwise leaks between tests.""" + with lf_module._UNLINKABLE_RUNS_LOCK: + lf_module._UNLINKABLE_RUNS.clear() + yield + with lf_module._UNLINKABLE_RUNS_LOCK: + lf_module._UNLINKABLE_RUNS.clear() + + +def _http_404() -> Exception: + request = MagicMock() + response = MagicMock(status_code=404) + return httpx.HTTPStatusError("Client error '404 Not Found'", request=request, response=response) + + +def _langfuse_that_404s_on_run_items(): + lf = MagicMock() + lf.api.dataset_run_items.create.side_effect = _http_404() + return lf + + +def test_a_missing_dataset_item_is_reported_once_per_run_with_its_cause(_fresh_unlinkable_runs): + """20 identical raw 404s buried the run's real output and named an endpoint, not a cause. + + The 404 means the dataset item id is not in Langfuse, which is a property of the + dataset -- it recurs identically for every item and every pass over it -- so it is one + fact to state once, with what to do about it. + """ + lf = _langfuse_that_404s_on_run_items() + said: list[str] = [] + + with patch("gooddata_eval.core.agentic._langfuse.warn_from_worker", side_effect=said.append): + for run_idx in range(5): + for item in ("gdai-2179-001", "gdai-2179-002", "gdai-2179-003", "gdai-2179-004"): + with observe(lf, f"trace-{item}-{run_idx}", item, f"GDAI-2179_ts_model_run{run_idx}", {}): + pass + + assert lf.api.dataset_run_items.create.call_count == 20 + assert len(said) == 1, f"expected one warning for the whole run, got {len(said)}" + warning = said[0] + assert "does not exist in Langfuse" in warning + assert "--langfuse-dataset" in warning and SKIP_ENV_VAR in warning + # The reader has to know the run is not a write-off. + assert "Scores ARE still written to the traces" in warning + + +def test_each_model_run_gets_its_own_report(_fresh_unlinkable_runs): + # --model a --model b produces two differently-named runs; each is separately + # unlinkable and the operator should see that it affected both. + lf = _langfuse_that_404s_on_run_items() + said: list[str] = [] + + with patch("gooddata_eval.core.agentic._langfuse.warn_from_worker", side_effect=said.append): + for model in ("gpt-5.6-luna", "gpt-5.2"): + for run_idx in range(3): + with observe(lf, "t", "gdai-2179-001", f"GDAI-2179_ts_{model}_run{run_idx}", {}): + pass + + assert len(said) == 2 + + +def test_a_non_404_link_failure_is_still_reported_every_time(_fresh_unlinkable_runs): + # A 500 or a timeout may be transient and item-specific, so it must not be collapsed + # into a one-shot "this dataset cannot link" claim. + lf = MagicMock() + lf.api.dataset_run_items.create.side_effect = RuntimeError("connection reset") + said: list[str] = [] + + with patch("gooddata_eval.core.agentic._langfuse.warn_from_worker", side_effect=said.append): + for i in range(3): + with observe(lf, f"t{i}", "item-1", "run", {}): + pass + + assert len(said) == 3 + assert all("failed to create dataset run item" in w for w in said) + + +def test_scores_still_reach_the_trace_after_a_404(_fresh_unlinkable_runs): + # observe() yields the trace id regardless, which is why the run was not a write-off. + lf = _langfuse_that_404s_on_run_items() + + with ( + patch("gooddata_eval.core.agentic._langfuse.warn_from_worker"), + observe(lf, "trace-abc", "gdai-2179-001", "run0", {}) as tid, + ): + pass + + assert tid == "trace-abc" + + +# --- the retry budget has to depend on who is waiting for it --- + + +def _sleep_spent(conversation_ids, *, linker) -> float: + """Total seconds a never-resolving poll sleeps, under the given linker.""" + clock = _FakeClock() + + def _fetch(langfuse, cid, window_start, window_end, pad): + return [] + + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", side_effect=_fetch), + patch("gooddata_eval.core.agentic._langfuse.time", clock), + ): + linker(lambda: find_traces_per_conversation(MagicMock(), conversation_ids, datetime.now(timezone.utc))) + return sum(clock.sleeps) + + +def test_an_inline_poll_keeps_the_pre_batching_budget(): + """The 120s budget was raised for the CLI, where linking is batched off the critical + path and costs nobody anything. A direct library caller (the tavern e2e suite) gets + run_trace_link_inline instead, so the same poll is charged straight to the test that + triggered it -- under a step timeout. Handing that path 120s turns a ~35s miss into a + ~110s one, inline, per item. + """ + inline = _sleep_spent(["c1"], linker=run_trace_link_inline) + + # Reproduces the old 8-attempt sleep-first ladder to the second. + assert 30.0 <= inline <= 35.0, inline + assert inline < _LINK_BUDGET_SEC + + +def test_a_batched_poll_still_gets_the_full_budget(): + # Off the critical path, waiting is free and a miss orphans a score, so this one keeps + # the long budget the batching change was made to afford. + def _batched(task): + linker = BackgroundTraceLinker(max_workers=1) + linker.submit(task) + linker.drain() + + batched = _sleep_spent(["c1"], linker=_batched) + + assert batched > _INLINE_LINK_BUDGET_SEC * 2, batched + assert batched <= _LINK_BUDGET_SEC + + +def test_the_inline_marker_does_not_leak_out_of_the_call(): + # Scoped to the call, so a later batched drain on the same thread is not mistaken for + # an inline one. + assert linking_is_inline() is False + seen = [] + run_trace_link_inline(lambda: seen.append(linking_is_inline())) + assert seen == [True] + assert linking_is_inline() is False + + +def test_a_worker_thread_is_never_treated_as_inline(): + # A thread starts with a fresh context, which is what makes the default correct for the + # drain pool without any bookkeeping. + seen: list[bool] = [] + + def _check() -> None: + seen.append(linking_is_inline()) + + def _outer() -> None: + linker = BackgroundTraceLinker(max_workers=1) + linker.submit(_check) + linker.drain() + + # Even when the drain itself is started from inside an inline task. + run_trace_link_inline(_outer) + + assert seen == [False] + + +def test_an_empty_conversation_id_still_sends_the_server_side_filter(): + """The filter must reach the server even when the id is empty. + + An empty id is a real filter value that matches nothing. Dropping the query parameter on + a falsy id would fetch the entire padded window for the local post-check to throw away, + so the poll would spend its whole budget on pages that can never match. + """ + seen: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(dict(request.url.params)) + return httpx.Response(200, json={"data": []}) + + real_client = httpx.Client + + def fake_client(*args, **kwargs): + kwargs.pop("transport", None) + return real_client(*args, transport=httpx.MockTransport(handler), **kwargs) + + with ( + patch.dict( + os.environ, + {"LANGFUSE_HOST": "https://lf.test", "LANGFUSE_PUBLIC_KEY": "pk", "LANGFUSE_SECRET_KEY": "sk"}, + ), + patch.object(lf_module.httpx, "Client", fake_client), + ): + client = make_langfuse_client() + + now = datetime.now(timezone.utc) + with patch.object(lf_module.httpx, "Client", fake_client): + _fetch_traces_for_session(client, "", now - timedelta(minutes=5), now, timedelta(seconds=2)) + + assert seen, "no request was made" + assert seen[0].get("sessionId") == "", ( + f"the empty id was dropped, so the server returned the whole window: {seen[0]}" + ) diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 2d84b4e0d..e2bc5ff56 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -1,8 +1,8 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise -import os import sys import types +from contextlib import ExitStack, contextmanager from unittest.mock import MagicMock, patch import pytest @@ -20,6 +20,59 @@ run_agentic_metric_skill, ) from gooddata_eval.core.models import ChatResult, ToolCallEvent +from gooddata_eval.core.timing import TIMERS_ENV_VAR + +# --- time.monotonic() side effects --------------------------------------------------- +# +# metric_skill reads the clock twice per agent turn (start, stop) and twice more per +# simulated-user reply, so a two-turn conversation needs six values. Named rather than +# inlined because a single new clock read on the production path breaks every one of these +# at once with StopIteration -- and repairing a bare literal means hand-counting clock +# reads at each call site. + +# one turn, metric created straight away: agent 3.0s, no simulated user +_CLOCK_ONE_TURN = [5.0, 8.0] +# two turns: agent 1.0s then 0.5s, with a 2.5s simulated-user reply between them +_CLOCK_TWO_TURNS = [20.0, 21.0, 21.0, 23.5, 23.5, 24.0] + + +def _client() -> MagicMock: + """A chat client whose conversations are all ``conv-1``. + + Callers override only what they vary -- ``send_message`` and the rest are ordinary + mock attributes. + """ + client = MagicMock() + client.create_conversation.return_value = "conv-1" + return client + + +@contextmanager +def _patched(client, *, simulated_reply=None, simulated_error=None, sdk=False, monotonic=None): + """Patch metric_skill's ChatClient plus whichever collaborators a test needs. + + ``simulated_reply``/``simulated_error`` patch the simulated user (reply, or raise); + ``sdk`` patches GoodDataSdk (the created metric's cleanup path); ``monotonic`` feeds + the clock one of the ``_CLOCK_*`` constants above. Yields + ``(mock_generate_simulated_response, mock_GoodDataSdk)`` -- None for whatever was not + patched. + """ + with ExitStack() as stack: + stack.enter_context(patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=client)) + mock_sim = mock_sdk_cls = None + if simulated_reply is not None or simulated_error is not None: + mock_sim = stack.enter_context( + patch( + "gooddata_eval.core.agentic.metric_skill.generate_simulated_response", + return_value=simulated_reply, + side_effect=simulated_error, + ) + ) + if sdk: + mock_sdk_cls = stack.enter_context(patch("gooddata_eval.core.agentic.metric_skill.GoodDataSdk")) + if monotonic is not None: + stack.enter_context(patch("time.monotonic", side_effect=monotonic)) + yield mock_sim, mock_sdk_cls def _create_metric_call(result: str) -> ToolCallEvent: @@ -285,8 +338,7 @@ def test_agentic_metric_summary_pass_at_k(): def test_run_agentic_metric_skill_creates_conversation(monkeypatch): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = ChatResult.model_validate( { "textResponse": "done", @@ -301,7 +353,7 @@ def test_run_agentic_metric_skill_creates_conversation(monkeypatch): } ) - with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): summary = run_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -318,8 +370,7 @@ def test_run_agentic_metric_skill_creates_conversation(monkeypatch): def test_run_agentic_metric_skill_closes_client_on_no_result(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = ChatResult.model_validate( { "textResponse": "I will work on that.", @@ -327,13 +378,7 @@ def test_run_agentic_metric_skill_closes_client_on_no_result(): "reasoningStepCount": 1, } ) - with ( - patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), - patch( - "gooddata_eval.core.agentic.metric_skill.generate_simulated_response", - return_value="Go ahead and create it.", - ) as mock_sim, - ): + with _patched(mock_client, simulated_reply="Go ahead and create it.") as (mock_sim, _): summary = run_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -349,6 +394,37 @@ def test_run_agentic_metric_skill_closes_client_on_no_result(): mock_sim.assert_called_once_with("I will work on that.", [{"maql": "SELECT {metric/foo}"}], "Create metric foo") +def test_run_agentic_metric_skill_logs_simulated_user_timing(monkeypatch, capsys): + monkeypatch.setenv(TIMERS_ENV_VAR, "1") + mock_client = _client() + mock_client.send_message.side_effect = [ + ChatResult.model_validate( + { + "textResponse": "Which field should I use?", + "toolCallEvents": [], + "reasoningStepCount": 1, + } + ), + _create_metric_chat_result(), + ] + + with _patched(mock_client, simulated_reply="Use the foo field.", sdk=True, monotonic=_CLOCK_TWO_TURNS): + run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + ) + + output = capsys.readouterr().out + assert ( + "[timer] metric_skill conv-1 GoodData turn 1 complete after 1.00s; waiting for gpt-4o-mini simulated user" + in output + ) + assert "[timer] metric_skill conv-1 gpt-4o-mini simulated user complete after 2.50s" in output + + def test_run_agentic_metric_skill_uses_initial_conversation_for_run_0(): mock_client = MagicMock() mock_client.send_message.return_value = ChatResult.model_validate( @@ -358,7 +434,7 @@ def test_run_agentic_metric_skill_uses_initial_conversation_for_run_0(): "reasoningStepCount": 1, } ) - with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): run_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -383,7 +459,7 @@ def test_run_agentic_metric_skill_creates_fresh_conversations_for_remaining_runs "reasoningStepCount": 1, } ) - with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): run_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -429,13 +505,9 @@ def _create_metric_chat_result(metric_id: str = "foo_metric"): def test_run_agentic_metric_skill_deletes_created_metric(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _create_metric_chat_result() - with ( - patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.metric_skill.GoodDataSdk") as mock_sdk_cls, - ): + with _patched(mock_client, sdk=True) as (_, mock_sdk_cls): mock_sdk = mock_sdk_cls.create.return_value run_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", @@ -454,8 +526,7 @@ def test_run_agentic_metric_skill_deletes_the_metric_created_by_a_self_corrected """QA-29053 regression: a failed create_metric call followed by a successful retry, in the same turn, used to leave metric_id_to_delete unset -- the metric the retry created leaked into the shared workspace.""" - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = ChatResult.model_validate( { "textResponse": "done", @@ -474,10 +545,7 @@ def test_run_agentic_metric_skill_deletes_the_metric_created_by_a_self_corrected "reasoningStepCount": 1, } ) - with ( - patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.metric_skill.GoodDataSdk") as mock_sdk_cls, - ): + with _patched(mock_client, sdk=True) as (_, mock_sdk_cls): mock_sdk = mock_sdk_cls.create.return_value summary = run_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", @@ -496,14 +564,12 @@ def test_run_agentic_metric_skill_deletes_the_metric_created_by_a_self_corrected def test_run_agentic_metric_skill_deletes_metric_even_when_teardown_fails(): # A metric is created, then conversation teardown raises; the created metric must still # have been cleaned up (its deletion happens inside the per-run finally, before teardown). - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = _create_metric_chat_result() mock_client.delete_conversation.side_effect = RuntimeError("teardown boom") with ( - patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.metric_skill.GoodDataSdk") as mock_sdk_cls, + _patched(mock_client, sdk=True) as (_, mock_sdk_cls), pytest.raises(RuntimeError), ): mock_sdk = mock_sdk_cls.create.return_value @@ -520,10 +586,10 @@ def test_run_agentic_metric_skill_deletes_metric_even_when_teardown_fails(): mock_sdk._client.entities_api.delete_entity_metrics.assert_called_once_with("ws1", "foo_metric") -def test_generate_simulated_response_without_an_api_key(): +def test_generate_simulated_response_without_an_api_key(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) with ( patch.dict(sys.modules, {"openai": MagicMock()}), - patch.dict(os.environ, {}, clear=True), pytest.raises(SimulatedResponseError, match="OPENAI_API_KEY"), ): generate_simulated_response("Which brand field?", [{"maql": "SELECT {metric/foo}"}], "I need a metric for foo") @@ -539,8 +605,7 @@ def test_generate_simulated_response_without_the_openai_package(): def test_run_agentic_metric_skill_fails_the_run_when_the_simulated_reply_cannot_be_generated(): exc = SimulatedResponseError("OPENAI_API_KEY environment variable is not set") - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = ChatResult.model_validate( { "textResponse": "Which brand field should I count?", @@ -548,10 +613,7 @@ def test_run_agentic_metric_skill_fails_the_run_when_the_simulated_reply_cannot_ "reasoningStepCount": 1, } ) - with ( - patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.metric_skill.generate_simulated_response", side_effect=exc) as mock_sim, - ): + with _patched(mock_client, simulated_error=exc) as (mock_sim, _): summary = run_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -592,14 +654,10 @@ def test_run_agentic_metric_skill_accumulates_reasoning_steps_across_iterations( "reasoningSteps": ["step two"], } ) - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.side_effect = [clarify_turn, created_turn] - with ( - patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), - patch("gooddata_eval.core.agentic.metric_skill.generate_simulated_response", return_value="It's foo"), - ): + with _patched(mock_client, simulated_reply="It's foo"): summary = run_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -614,8 +672,7 @@ def test_run_agentic_metric_skill_accumulates_reasoning_steps_across_iterations( def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = ChatResult.model_validate( { "textResponse": "done", @@ -629,7 +686,7 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): "reasoningSteps": ["thinking about it"], } ) - with patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client): + with _patched(mock_client): outcome = evaluate_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -652,8 +709,7 @@ def test_evaluate_agentic_metric_skill_returns_reasoning_steps_on_pass(): def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_fail(): - mock_client = MagicMock() - mock_client.create_conversation.return_value = "conv-1" + mock_client = _client() mock_client.send_message.return_value = ChatResult.model_validate( { "textResponse": "I will work on that.", @@ -661,10 +717,7 @@ def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_ "reasoningSteps": ["confused thinking"], } ) - with ( - patch("gooddata_eval.core.agentic.metric_skill.ChatClient", return_value=mock_client), - pytest.raises(MetricSkillAssertionError) as exc_info, - ): + with _patched(mock_client), pytest.raises(MetricSkillAssertionError) as exc_info: evaluate_agentic_metric_skill( host="http://host/api/v1/actions/workspaces/ws1/ai", token="tok", @@ -684,3 +737,74 @@ def test_evaluate_agentic_metric_skill_attaches_reasoning_steps_to_exception_on_ } assert exc_info.value.conversation_id == "conv-1" assert exc_info.value.response_id is None + + +def test_records_agent_and_simulated_user_latency_separately(): + # metric_skill's OpenAI call is the simulated user, not a judge, and it sits ON the + # critical path -- the next agent turn cannot be sent until the reply exists. Keeping + # it in its own bucket is what makes that visible: agent_s is GoodData's cost across + # both turns, simulated_user_s is ours. + mock_client = _client() + mock_client.send_message.side_effect = [ + ChatResult.model_validate( + {"textResponse": "Which field should I use?", "toolCallEvents": [], "reasoningStepCount": 1} + ), + _create_metric_chat_result(), + ] + + with _patched(mock_client, simulated_reply="Use the foo field.", sdk=True, monotonic=_CLOCK_TWO_TURNS): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + ) + + timings = summary.run_results[0].timings + assert timings.agent_s == 1.5 # 1.0s turn 1 + 0.5s turn 2 + assert timings.simulated_user_s == 2.5 + # metric_skill has no judge; conflating its simulated user with one would misreport + # a blocking call as a deferrable one. + assert timings.judge_s == 0.0 + + +def test_evaluate_metric_skill_surfaces_timings_on_the_outcome(): + # The item report reads timings off the outcome, so a kind that measures phases but + # does not propagate them reports zeroes and looks instantaneous. + mock_client = _client() + mock_client.send_message.return_value = _create_metric_chat_result() + + with _patched(mock_client, sdk=True, monotonic=_CLOCK_ONE_TURN): + outcome = evaluate_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + k=1, + ) + + assert outcome.timings.agent_s == 3.0 + assert outcome.timings.simulated_user_s == 0.0 + + +def test_no_timer_output_by_default(monkeypatch, capsys): + # metric_skill emits four [timer] lines per turn-pair; on a multi-turn conversation + # that is the bulk of the run's output. Off unless asked for. + monkeypatch.delenv(TIMERS_ENV_VAR, raising=False) + mock_client = _client() + mock_client.send_message.return_value = _create_metric_chat_result() + + with _patched(mock_client, sdk=True, monotonic=_CLOCK_ONE_TURN): + summary = run_agentic_metric_skill( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + question="Create metric foo", + expected_output={"maql": "SELECT {metric/foo}"}, + ) + + assert "[timer]" not in capsys.readouterr().out + # Silenced, not un-measured. + assert summary.run_results[0].timings.agent_s == 3.0 diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index 3d9959eba..77086d101 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -1,11 +1,22 @@ # (C) 2026 GoodData Corporation. All rights reserved. # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +import threading +import time +from concurrent.futures import ThreadPoolExecutor from unittest.mock import patch import pytest -from gooddata_eval.cli.agentic_runner import AGENTIC_TEST_KINDS, _dispatch_agentic, run_agentic_items +from gooddata_eval.cli.agentic_runner import ( + AGENTIC_TEST_KINDS, + PARALLEL_SAFE_TEST_KINDS, + WORKSPACE_MUTATING_TEST_KINDS, + _dispatch_agentic, + run_agentic_items, + runs_in_parallel, +) from gooddata_eval.core.agentic.alert_skill import AlertSkillAssertionError from gooddata_eval.core.models import AgenticEvalOutcome, DatasetItem +from gooddata_eval.core.timing import PhaseTimings def test_dispatch_agentic_passes_agent_id_through_to_alert_skill(): @@ -218,3 +229,540 @@ def test_dispatch_agentic_returns_a_real_outcome_for_every_kind(kind, expected_o assert result.detail == {"k": "v"} assert result.conversation_id == "c1" assert result.response_id == "r1" + + +def _timed_item(item_id: str = "item-1") -> DatasetItem: + return DatasetItem( + id=item_id, + dataset_name="d", + test_kind="agentic_alert_skill", + question="Alert me when revenue drops below 100.", + expected_output={"operator": "LESS_THAN", "threshold": 100}, + ) + + +def test_run_agentic_items_does_not_charge_the_item_for_langfuse_trace_linking(): + # The whole point of Option 1: a trace poll that takes longer than the agent turn must + # not show up as the item being slow. Before this, latency_s wrapped _dispatch_agentic + # and so swallowed the poll whole -- an item whose agent answered in 4s reported 40s. + def fake_eval(*, submit_trace_link, dataset_item_id, **_kw): + submit_trace_link(lambda: time.sleep(0.2), item_id=dataset_item_id) + return AgenticEvalOutcome( + reasoning_steps=[], + conversation_id="c1", + response_id="r1", + detail={"alert_created": True}, + timings=PhaseTimings(agent_s=1.0), + ) + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=fake_eval): + report = run_agentic_items( + [_timed_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + use_langfuse=False, + ) + + item = report.items[0] + assert item.latency_s < 0.1, "the item was charged for its Langfuse trace link" + # Still measured, just not on the critical path. + assert item.langfuse_latency_s >= 0.2 + assert item.agent_latency_s == 1.0 + + +def test_run_agentic_items_drains_trace_links_before_returning(): + # Scores must be final before the CLI renders a table or sets an exit code. A pool + # that outlived the return would lose whatever had not been flushed. + linked: list[str] = [] + + def fake_eval(*, submit_trace_link, dataset_item_id, **_kw): + submit_trace_link(lambda: linked.append(dataset_item_id), item_id=dataset_item_id) + return AgenticEvalOutcome(conversation_id="c1", detail={"alert_created": True}) + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=fake_eval): + run_agentic_items( + [_timed_item("a"), _timed_item("b")], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + assert sorted(linked) == ["a", "b"] + + +def test_run_agentic_items_records_phase_timings_and_wall_clock(): + # wall_clock_s was never set on the agentic path, which is why every agentic-only run + # reported "wall_clock_s": 0.0 in its JSON. + outcome = AgenticEvalOutcome( + conversation_id="c1", + detail={"alert_created": True}, + timings=PhaseTimings(agent_s=2.0, judge_s=1.0, simulated_user_s=0.5), + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", return_value=outcome): + report = run_agentic_items( + [_timed_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + item = report.items[0] + assert (item.agent_latency_s, item.judge_latency_s, item.simulated_user_latency_s) == (2.0, 1.0, 0.5) + assert report.wall_clock_s > 0.0 + + +def test_run_agentic_items_records_phase_timings_when_the_item_fails(): + exc = AlertSkillAssertionError("nope") + exc.detail = {"alert_created": False} + exc.timings = PhaseTimings(agent_s=7.0, judge_s=2.0) + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): + report = run_agentic_items( + [_timed_item()], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + assert report.items[0].pass_at_k is False + assert report.items[0].agent_latency_s == 7.0 + assert report.items[0].judge_latency_s == 2.0 + + +def test_a_pending_trace_link_does_not_block_the_next_item(): + """The overlap claim, proved by construction rather than by a stopwatch. + + Item "first" submits a trace link that can only finish once item "second" has reached + the agent. Under the old inline linking that is unreachable -- "second" does not start + until "first"'s Langfuse block returns -- so the wait times out and the flag stays + unset. It can only pass if the link really is running off the critical path. + """ + second_item_reached_the_agent = threading.Event() + first_link_finished = threading.Event() + + def fake_eval(*, submit_trace_link, dataset_item_id, **_kw): + if dataset_item_id == "first": + + def poll() -> None: + if second_item_reached_the_agent.wait(timeout=5): + first_link_finished.set() + + submit_trace_link(poll, item_id=dataset_item_id) + else: + second_item_reached_the_agent.set() + return AgenticEvalOutcome(conversation_id="c", detail={"alert_created": True}) + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=fake_eval): + run_agentic_items( + [_timed_item("first"), _timed_item("second")], + host="http://host", + token="tok", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + assert first_link_finished.is_set() + + +def test_each_run_drains_its_own_links_so_models_cannot_overlap(): + """Multi-model guard: --model a --model b calls run_agentic_items once per model, and + the workspace's active LLM provider is switched between those calls. A link from + model A still in flight during model B's run could resolve the wrong model version + (get_model_version falls back to reading the live workspace when no override is set), + so each call must own and fully drain its own pool. + """ + in_flight: list[str] = [] + finished: list[str] = [] + + def fake_eval(*, submit_trace_link, dataset_item_id, **_kw): + def poll() -> None: + in_flight.append(dataset_item_id) + time.sleep(0.05) + finished.append(dataset_item_id) + + submit_trace_link(poll, item_id=dataset_item_id) + return AgenticEvalOutcome(conversation_id="c", detail={"alert_created": True}) + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=fake_eval): + first = run_agentic_items( + [_timed_item("model-a-item")], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01" + ) + # Nothing from model A may still be running once its run_agentic_items has returned. + assert finished == ["model-a-item"] + second = run_agentic_items( + [_timed_item("model-b-item")], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01" + ) + + assert finished == ["model-a-item", "model-b-item"] + # Durations are per-run, never carried over from the previous model's pool. + assert first.items[0].langfuse_latency_s > 0 + assert second.items[0].langfuse_latency_s > 0 + + +def test_an_interrupt_abandons_queued_trace_links_instead_of_waiting_them_out(): + """Ctrl-C must not be held hostage by a backlog of Langfuse polls. + + ThreadPoolExecutor registers an atexit hook that joins its workers, so a pool left + running with queued 15s polls would stall the CLI for minutes after the user + interrupted it. Queued-but-unstarted links are dropped on the way out. + """ + ran: list[str] = [] + block = threading.Event() + + def fake_eval(*, submit_trace_link, dataset_item_id, **_kw): + # Two blocking tasks occupy both workers; the third can only be queued. + submit_trace_link(lambda: block.wait(timeout=2), item_id="busy-1") + submit_trace_link(lambda: block.wait(timeout=2), item_id="busy-2") + submit_trace_link(lambda: ran.append("queued"), item_id="queued") + raise KeyboardInterrupt + + with ( + patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=fake_eval), + pytest.raises(KeyboardInterrupt), + ): + run_agentic_items([_timed_item("a")], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01") + + block.set() + # No sleep needed: submit only queues, and the queue is only ever executed by drain(), + # which blocks until it finishes. Had the interrupt path drained instead of abandoning, + # "queued" would already be in `ran` by the time run_agentic_items returned. + assert ran == [], "a queued Langfuse poll kept running after the run was interrupted" + + +def test_langfuse_client_is_closed_only_after_every_link_has_finished(): + """--langfuse builds one client for the whole run and closes it at the end. + + The deferred links use that same client from worker threads, so closing it before the + drain would make every in-flight poll fail against a shut httpx client -- and because + BackgroundTraceLinker swallows task errors, the scores would vanish silently rather + than fail the run. + """ + events: list[str] = [] + + class _Client: + def flush(self) -> None: + events.append("flush") + + def close(self) -> None: + events.append("close") + + def fake_eval(*, submit_trace_link, dataset_item_id, **_kw): + def poll() -> None: + time.sleep(0.05) + events.append(f"link:{dataset_item_id}") + + submit_trace_link(poll, item_id=dataset_item_id) + return AgenticEvalOutcome(conversation_id="c", detail={"alert_created": True}) + + with ( + patch("gooddata_eval.cli.agentic_runner.make_langfuse_client", return_value=_Client()), + patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=fake_eval), + ): + run_agentic_items( + [_timed_item("a"), _timed_item("b")], + host="h", + token="t", + workspace_id="ws1", + run_ts="2026-01-01", + use_langfuse=True, + ) + + # The batch runs its links in parallel, so their relative order is not fixed -- only + # that both finished before the client was flushed and closed. + assert set(events[:2]) == {"link:a", "link:b"} + assert events[2:] == ["flush", "close"] + + +def test_run_agentic_items_reports_what_the_trace_link_batch_cost(capsys): + # The retry budget was sized against an ingestion lag only bounded to "35s to a few + # minutes". This line is how the next real run reports the actual number: if the + # slowest link sits near the budget, the budget is binding and scores are being lost. + def fake_eval(*, submit_trace_link, dataset_item_id, **_kw): + submit_trace_link(lambda: time.sleep(0.05), item_id=dataset_item_id) + return AgenticEvalOutcome(conversation_id="c", detail={"alert_created": True}) + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=fake_eval): + run_agentic_items( + [_timed_item("a"), _timed_item("b")], + host="h", + token="t", + workspace_id="ws1", + run_ts="2026-01-01", + ) + + out = capsys.readouterr().out + assert "[langfuse] trace linking" in out + assert "2 item(s)" in out + assert "slowest" in out + + +def _kind_item(kind: str, item_id: str) -> DatasetItem: + expected: object = "an answer" + if kind in ("agentic_metric_skill",): + expected = {"maql": "SELECT {metric/spend}"} + elif kind in ("agentic_alert_skill", "agentic_kda_skill"): + expected = {"Operator": "GREATER_THAN", "Threshold": 1} + elif kind == "agentic_conversation": + expected = {"fixture": _MIN_CONVERSATION_FIXTURE} + elif kind in ("vis_agentic", "agentic_visualization"): + expected = {"visualization": _MIN_VIZ} + elif kind == "agentic_search": + expected = {"tool_call": {"function_arguments": {}}} + return DatasetItem(id=item_id, dataset_name="d", test_kind=kind, question="q", expected_output=expected) + + +def test_every_agentic_kind_is_classified_as_parallel_safe_or_workspace_mutating(): + # An unclassified kind would either lose concurrency silently or, worse, be run in + # parallel when it mutates the shared workspace. + assert set(AGENTIC_TEST_KINDS) == PARALLEL_SAFE_TEST_KINDS | WORKSPACE_MUTATING_TEST_KINDS + assert not (PARALLEL_SAFE_TEST_KINDS & WORKSPACE_MUTATING_TEST_KINDS) + + +def test_read_only_kinds_run_concurrently(): + # Each item blocks until all of them have started: only reachable if they really do run + # at the same time. Sequentially the barrier times out, the item errors, and pass_at_k + # goes false. + n = 4 + barrier = threading.Barrier(n, timeout=5) + + def fake_eval(**_kw): + barrier.wait() + return AgenticEvalOutcome(conversation_id="c", detail={"judge_passed": True}) + + items = [_kind_item("agentic_general_question", f"q{i}") for i in range(n)] + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question", side_effect=fake_eval): + report = run_agentic_items(items, host="h", token="t", workspace_id="ws1", run_ts="ts", concurrency=n) + + assert [i.pass_at_k for i in report.items] == [True] * n + + +def test_workspace_mutating_kinds_never_overlap_even_at_high_concurrency(): + # metric_skill creates and deletes metrics in the shared eval workspace, and + # metric_skill._delete_metric documents that a leaked one gets reused by a later test. + # Overlapping two of them is exactly the contamination that comment warns about. + lock = threading.Lock() + active = 0 + peak = 0 + + def fake_eval(**_kw): + nonlocal active, peak + with lock: + active += 1 + peak = max(peak, active) + time.sleep(0.05) + with lock: + active -= 1 + return AgenticEvalOutcome(conversation_id="c", detail={"maql_correct": True}) + + items = [_kind_item("agentic_metric_skill", f"m{i}") for i in range(4)] + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_metric_skill", side_effect=fake_eval): + run_agentic_items(items, host="h", token="t", workspace_id="ws1", run_ts="ts", concurrency=4) + + assert peak == 1 + + +def test_report_keeps_dataset_order_when_items_finish_out_of_order(): + # Concurrency must not reshuffle the report: the JSON and console are read against the + # dataset, and a run-to-run reordering makes two reports impossible to diff. + delays = {"q0": 0.15, "q1": 0.01, "q2": 0.10, "q3": 0.01} + + def fake_eval(*, dataset_item_id, **_kw): + time.sleep(delays[dataset_item_id]) + return AgenticEvalOutcome(conversation_id="c", detail={"judge_passed": True}) + + items = [_kind_item("agentic_general_question", f"q{i}") for i in range(4)] + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question", side_effect=fake_eval): + report = run_agentic_items(items, host="h", token="t", workspace_id="ws1", run_ts="ts", concurrency=4) + + assert [i.id for i in report.items] == ["q0", "q1", "q2", "q3"] + + +def test_warns_when_concurrency_cannot_apply_to_any_item(capsys): + # --concurrency silently doing nothing is what sent a real run looking for a speedup + # that was never possible: the flag only ever reached the non-agentic runner. + items = [_kind_item("agentic_metric_skill", f"m{i}") for i in range(2)] + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_metric_skill", + return_value=AgenticEvalOutcome(conversation_id="c", detail={}), + ): + run_agentic_items(items, host="h", token="t", workspace_id="ws1", run_ts="ts", concurrency=4) + + warned = capsys.readouterr().err + assert "concurrency" in warned.lower() + assert "agentic_metric_skill" in warned + + +def test_no_concurrency_warning_when_some_items_can_run_in_parallel(capsys): + items = [_kind_item("agentic_general_question", "q0"), _kind_item("agentic_metric_skill", "m0")] + with ( + patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question", + return_value=AgenticEvalOutcome(conversation_id="c", detail={}), + ), + patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_metric_skill", + return_value=AgenticEvalOutcome(conversation_id="c", detail={}), + ), + ): + run_agentic_items(items, host="h", token="t", workspace_id="ws1", run_ts="ts", concurrency=4) + + assert "concurrency" not in capsys.readouterr().err.lower() + + +def test_an_unclassified_kind_defaults_to_serial(): + """PARALLEL_SAFE is an explicit allowlist because deriving it as + `AGENTIC_TEST_KINDS - WORKSPACE_MUTATING` auto-enrolled every newly added kind into + parallel execution -- and nothing here can prove a kind is read-only, since the + mutation happens server-side in the agent's tools.""" + assert runs_in_parallel("agentic_general_question") is True + assert runs_in_parallel("agentic_metric_skill") is False + assert runs_in_parallel("agentic_some_kind_added_next_year") is False + + +def test_kda_skill_runs_serially_until_its_tool_is_confirmed_side_effect_free(): + # agentic_kda_skill drives a tool called create_key_driver_analysis and, unlike + # metric_skill/alert_skill, has no cleanup. The evaluator only reads the create call's + # arguments (never a returned object id), which suggests an in-conversation analysis -- + # but that is inference about server-side behaviour, not verification. Serial until + # someone confirms it. + assert "agentic_kda_skill" in WORKSPACE_MUTATING_TEST_KINDS + assert "agentic_kda_skill" not in PARALLEL_SAFE_TEST_KINDS + + +def test_announces_the_trace_link_batch_before_it_starts(capsys): + # The batch runs after the last item, and took 9.6s then 15.9s on real runs. Printing + # only when it finishes leaves the terminal silent for that whole stretch, right after + # the last item reports -- it reads as a hang. + def fake_eval(*, submit_trace_link, dataset_item_id, **_kw): + submit_trace_link(lambda: time.sleep(0.05), item_id=dataset_item_id) + return AgenticEvalOutcome(conversation_id="c", detail={"judge_passed": True}) + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question", side_effect=fake_eval): + run_agentic_items( + [_kind_item("agentic_general_question", f"q{i}") for i in range(3)], + host="h", + token="t", + workspace_id="ws1", + run_ts="ts", + ) + + out = capsys.readouterr().out + assert "linking traces for 3 item(s)" in out + # Announced first, reported second. + assert out.index("linking traces for 3 item(s)") < out.index("trace linking finished") + + +def test_says_nothing_about_trace_linking_when_there_is_none(capsys): + # No Langfuse client -> no tasks queued -> no noise about a batch that never ran. + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question", + return_value=AgenticEvalOutcome(conversation_id="c", detail={}), + ): + run_agentic_items( + [_kind_item("agentic_general_question", "q0")], + host="h", + token="t", + workspace_id="ws1", + run_ts="ts", + ) + + assert "linking traces" not in capsys.readouterr().out + + +def test_an_interrupt_cancels_queued_items_instead_of_running_the_whole_dataset(): + """Ctrl-C during the parallel phase must not work through the rest of the dataset. + + `with ThreadPoolExecutor(...)` exits via shutdown(wait=True) with cancel_futures left + False, so an interrupt raised while the main thread waits on as_completed runs every + QUEUED item to completion first. At --concurrency 4 over 18 items that is several more + waves of up-to-300s agent calls issued after the user already gave up -- and it runs + BEFORE the linker.abandon() that is supposed to make an interrupt cheap. + + Only the items already in flight may finish; the interpreter joins those worker + threads at exit regardless, so they are not cancellable. Everything still queued is. + """ + release = threading.Event() + lock = threading.Lock() + started: list[str] = [] + + def fake_eval(*, dataset_item_id, **_kw): + with lock: + started.append(dataset_item_id) + release.wait(timeout=0.5) + return AgenticEvalOutcome(conversation_id="c", detail={"judge_passed": True}) + + items = [_kind_item("agentic_general_question", f"q{i}") for i in range(6)] + pools: list[ThreadPoolExecutor] = [] + + def recording_pool(*args, **kwargs): + pools.append(ThreadPoolExecutor(*args, **kwargs)) + return pools[-1] + + with ( + patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question", side_effect=fake_eval), + patch("gooddata_eval.cli.agentic_runner.ThreadPoolExecutor", recording_pool), + patch("gooddata_eval.cli.agentic_runner.as_completed", side_effect=KeyboardInterrupt), + pytest.raises(KeyboardInterrupt), + ): + run_agentic_items(items, host="h", token="t", workspace_id="ws1", run_ts="ts", concurrency=2) + + release.set() + # Join the item pool for real rather than sleeping and sampling: this is exactly what + # would let a still-queued item start, so if none has after it, none ever will. + pools[0].shutdown(wait=True) + with lock: + ran = list(started) + assert len(ran) < len(items), f"the interrupt waited out the whole dataset: {ran}" + # cancel_futures cancels every future that has not started, so at most one wave of + # max_workers items can ever have been dequeued. + assert len(ran) <= 2, f"more than one wave got through: {ran}" + + +def test_an_errored_item_keeps_the_timings_it_managed_to_take(): + # An item unevaluable because its judge broke should not also report the agent as + # having cost 0s -- the agent answered, and that is the measurement the report is for. + exc = RuntimeError("judge returned no readable verdict for any of the 2 run(s)") + exc.timings = PhaseTimings(agent_s=7.0, judge_s=1.0) # type: ignore[attr-defined] + + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=exc): + report = run_agentic_items([_timed_item()], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01") + + item = report.items[0] + assert item.error is not None + assert (item.agent_latency_s, item.judge_latency_s) == (7.0, 1.0) + + +def test_an_errored_item_without_timings_keeps_its_zero_defaults(): + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", side_effect=RuntimeError("boom")): + report = run_agentic_items([_timed_item()], host="h", token="t", workspace_id="ws1", run_ts="2026-01-01") + + assert report.items[0].agent_latency_s == 0.0 + + +def test_dispatch_agentic_passes_user_context_through_to_general_question(): + attachment = {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "campaign_spend"}]}]} + item = DatasetItem( + id="gdai-2179-001", + dataset_name="GDAI-2179", + test_kind="agentic_general_question", + question="What does the visualization I attached show?", + expected_output="Describes the attached chart.", + user_context=attachment, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_general_question") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + assert mock_eval.call_args.kwargs["user_context"] == attachment diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 9956ad31f..9ff008903 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -7,11 +7,13 @@ import pytest from gooddata_eval.cli import main as cli_main from gooddata_eval.cli.main import _parse_model_arg +from gooddata_eval.core.config import JUDGE_MODEL_ENV_VAR, RunConfig, judge_model from gooddata_eval.core.connection import ( ConnectionError_, # noqa: F401 - used in test_cli_operational_error_exits_nonzero ) from gooddata_eval.core.models import DatasetItem from gooddata_eval.core.runner import EvalReport, ItemReport +from gooddata_eval.core.timing import TIMERS_ENV_VAR, timers_enabled from gooddata_eval.core.workspace import ActiveLlmProvider, ResolvedModel from rich.console import Console @@ -814,3 +816,189 @@ def test_cli_rejects_unknown_reasoning_effort(fixtures_dir): ] ) assert exc_info.value.code == 2 + + +def test_cli_passes_concurrency_to_the_agentic_runner(monkeypatch, tmp_path): + # The gap this closes: --concurrency reached run_items only, so a dataset of agentic + # items accepted the flag and then ran strictly sequentially anyway -- a real run with + # --concurrency 4 came out no faster than without it. + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel(provider_id="prov", model_id="gpt-5.2", switched=False, provider_name="P") + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + monkeypatch.setattr(cli_main, "ChatClient", lambda **k: object()) + monkeypatch.setattr( + cli_main, + "load_local_dataset", + lambda folder: [ + DatasetItem( + id="q1", + dataset_name="d", + test_kind="agentic_general_question", + question="q", + expected_output="a", + ) + ], + ) + monkeypatch.setattr(cli_main, "run_items", lambda items, backend, **kw: EvalReport(model="gpt-5.2")) + + captured = {} + + def _fake_run_agentic(items, **kwargs): + captured.update(kwargs) + return EvalReport(model="gpt-5.2") + + monkeypatch.setattr(cli_main, "run_agentic_items", _fake_run_agentic) + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(tmp_path), + "--concurrency", + "3", + ] + ) + + assert exit_code == 0 + assert captured["concurrency"] == 3 + + +def test_cli_timers_flag_enables_timer_output(monkeypatch): + # --timers is the discoverable front door for GD_EVAL_TIMERS. The env var is the + # mechanism because the [timer] call sites sit four layers below the CLI, inside the + # per-run helpers -- the same reason TAVERN_E2E_SKIP_TRACE_LINK is read at call time. + monkeypatch.delenv(TIMERS_ENV_VAR, raising=False) + assert timers_enabled() is False + cli_main.parse_args(["run", "--workspace", "ws1", "--dataset", "/tmp", "--timers"]) + cli_main._apply_timer_flag(True) + assert timers_enabled() is True + + +def test_cli_leaves_timers_off_without_the_flag(monkeypatch): + monkeypatch.delenv(TIMERS_ENV_VAR, raising=False) + cli_main._apply_timer_flag(False) + assert timers_enabled() is False + + +def test_cli_timers_flag_defaults_to_false(): + args = cli_main.parse_args(["run", "--workspace", "ws1", "--dataset", "/tmp"]) + assert args.timers is False + + +def test_cli_judge_model_defaults_to_gpt_4o(monkeypatch): + args = cli_main.parse_args(["run", "--workspace", "ws1", "--dataset", "/tmp"]) + assert args.judge_model is None + monkeypatch.delenv(JUDGE_MODEL_ENV_VAR, raising=False) + cli_main._apply_judge_model(args.judge_model) + assert judge_model() == "gpt-4o" + + +def test_cli_judge_model_flag_overrides_the_default(monkeypatch): + # Recorded first so teardown removes what _apply_judge_model writes into os.environ + # directly; monkeypatch only restores keys it has seen. + monkeypatch.delenv(JUDGE_MODEL_ENV_VAR, raising=False) + args = cli_main.parse_args(["run", "--workspace", "ws1", "--dataset", "/tmp", "--judge-model", "gpt-4o-mini"]) + cli_main._apply_judge_model(args.judge_model) + assert judge_model() == "gpt-4o-mini" + + +def test_cli_judge_model_flag_beats_the_env_var(monkeypatch): + # An explicit flag is a deliberate choice for this run; an exported var is ambient. + monkeypatch.setenv(JUDGE_MODEL_ENV_VAR, "gpt-4o") + cli_main._apply_judge_model("gpt-5.6-luna") + assert judge_model() == "gpt-5.6-luna" + + +# --- a local dataset plus live Langfuse credentials cannot link (say so up front) --- + + +_LF_CREDS = {"LANGFUSE_PUBLIC_KEY": "pk", "LANGFUSE_SECRET_KEY": "sk"} + + +def _export_langfuse_creds(monkeypatch) -> None: + for name, value in _LF_CREDS.items(): + monkeypatch.setenv(name, value) + + +def _local_dataset_config(tmp_path): + return RunConfig(host="http://h", token="t", workspace_id="ws1", dataset_folder=tmp_path) + + +def _agentic_item(): + return DatasetItem( + id="gdai-2179-001", + dataset_name="GDAI-2179", + test_kind="agentic_general_question", + question="q", + expected_output="e", + ) + + +def test_warns_up_front_when_a_local_dataset_cannot_be_linked(monkeypatch, tmp_path, capsys): + """--langfuse is refused with a local dataset, but the evaluators' own + try_make_langfuse_client() fallback links anyway when LANGFUSE_* are exported -- so + every conversation 404s from dataset-run-items, in a block at the very END of the run. + By then the flag that would have avoided it is long past being changeable. + """ + _export_langfuse_creds(monkeypatch) + monkeypatch.delenv(TIMERS_ENV_VAR, raising=False) + monkeypatch.delenv("TAVERN_E2E_SKIP_TRACE_LINK", raising=False) + cli_main._warn_if_local_dataset_cannot_link(_local_dataset_config(tmp_path), [_agentic_item()]) + + err = capsys.readouterr().err + assert "--dataset is a local folder" in err + assert "--langfuse-dataset" in err and "TAVERN_E2E_SKIP_TRACE_LINK=1" in err + + +def test_no_warning_when_the_skip_switch_is_already_set(monkeypatch, tmp_path, capsys): + _export_langfuse_creds(monkeypatch) + monkeypatch.setenv("TAVERN_E2E_SKIP_TRACE_LINK", "1") + cli_main._warn_if_local_dataset_cannot_link(_local_dataset_config(tmp_path), [_agentic_item()]) + + assert capsys.readouterr().err == "" + + +def test_no_warning_without_langfuse_credentials(monkeypatch, tmp_path, capsys): + for k in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "TAVERN_E2E_SKIP_TRACE_LINK"): + monkeypatch.delenv(k, raising=False) + cli_main._warn_if_local_dataset_cannot_link(_local_dataset_config(tmp_path), [_agentic_item()]) + + assert capsys.readouterr().err == "" + + +def test_no_warning_for_a_langfuse_backed_dataset(monkeypatch, capsys): + # --langfuse-dataset items carry real Langfuse ids, so linking works and there is + # nothing to warn about. + config = RunConfig(host="http://h", token="t", workspace_id="ws1", langfuse_dataset="GDAI-2179") + _export_langfuse_creds(monkeypatch) + monkeypatch.delenv("TAVERN_E2E_SKIP_TRACE_LINK", raising=False) + cli_main._warn_if_local_dataset_cannot_link(config, [_agentic_item()]) + + assert capsys.readouterr().err == "" + + +def test_no_warning_when_there_are_no_agentic_items(monkeypatch, tmp_path, capsys): + # The single-turn path links through LangfuseSink and only with --langfuse, so it never + # hits the fallback this warning is about. + _export_langfuse_creds(monkeypatch) + monkeypatch.delenv("TAVERN_E2E_SKIP_TRACE_LINK", raising=False) + cli_main._warn_if_local_dataset_cannot_link(_local_dataset_config(tmp_path), []) + + assert capsys.readouterr().err == "" diff --git a/packages/gooddata-eval/tests/test_connection.py b/packages/gooddata-eval/tests/test_connection.py index bfec73c4a..066da5666 100644 --- a/packages/gooddata-eval/tests/test_connection.py +++ b/packages/gooddata-eval/tests/test_connection.py @@ -16,6 +16,11 @@ def test_resolve_connection_uses_env_token(monkeypatch): def test_resolve_connection_uses_profile(monkeypatch): + # GOODDATA_TOKEN outranks the profile by design, so an exported one makes this test + # read the developer's real token instead of the stubbed profile. Its two siblings + # above already clear it; this one did not, which is why the suite showed a permanent + # local failure that CI never saw. + monkeypatch.delenv("GOODDATA_TOKEN", raising=False) monkeypatch.setattr( "gooddata_eval.core.connection.profile_content", lambda profile: {"host": "https://from-profile", "token": "ptok"}, diff --git a/packages/gooddata-eval/tests/test_langfuse_source.py b/packages/gooddata-eval/tests/test_langfuse_source.py index 15ecdf5e0..7b61110d8 100644 --- a/packages/gooddata-eval/tests/test_langfuse_source.py +++ b/packages/gooddata-eval/tests/test_langfuse_source.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch import pytest -from gooddata_eval.core.dataset.langfuse_source import _item_from_raw, load_langfuse_dataset +from gooddata_eval.core.dataset.langfuse_source import _infer_test_kind, _item_from_raw, load_langfuse_dataset def _raw_item(item_id, question, expected_output, dataset_name="ds"): @@ -103,3 +103,96 @@ def test_load_langfuse_dataset_raises_on_missing_credentials(monkeypatch): with pytest.raises(RuntimeError, match="credentials not set"): load_langfuse_dataset("any_dataset") + + +def test_item_from_raw_maps_user_context_from_metadata(): + """A widget/view attachment lives in metadata and must survive the round trip. + + Without this, an attachment-driven item silently runs as a bare question and + fails for the wrong reason. + """ + raw = { + "id": "lf-ctx-1", + "datasetName": "ds", + "input": "What does the visualization I attached show?", + "metadata": { + "user_context": {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "campaign_spend"}]}]} + }, + "expectedOutput": "PASS if it describes the attached chart.", + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="agentic_general_question") + assert item.user_context == {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "campaign_spend"}]}]} + + +def test_item_from_raw_maps_user_context_from_input_object(): + raw = { + "id": "lf-ctx-2", + "datasetName": "ds", + "input": { + "question": "Summarize the chart I attached.", + "user_context": {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "w1"}]}]}, + }, + "expectedOutput": "PASS if summarized.", + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="agentic_general_question") + assert item.question == "Summarize the chart I attached." + assert item.user_context == {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "w1"}]}]} + + +def test_item_from_raw_user_context_absent_is_none(): + raw = _raw_item("lf-ctx-3", "No attachment here", "PASS if answered.") + item = _item_from_raw(raw, dataset_name="ds", test_kind="agentic_general_question") + assert item.user_context is None + + +def test_item_from_raw_test_kind_from_metadata_beats_default(): + """A string expectedOutput gives _infer_test_kind nothing to work with, so a + judge-rubric dataset has to carry its kind in metadata or rely on --kind.""" + raw = { + "id": "lf-kind-1", + "datasetName": "ds", + "input": "What is data normalization?", + "metadata": {"test_kind": "agentic_general_question"}, + "expectedOutput": "PASS if it explains normalization.", + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.test_kind == "agentic_general_question" + + +def test_item_from_raw_expected_output_test_kind_beats_metadata(): + """expectedOutput.test_kind is the pre-existing explicit override; keep it winning.""" + raw = { + "id": "lf-kind-2", + "datasetName": "ds", + "input": "Summarize it", + "metadata": {"test_kind": "agentic_general_question"}, + "expectedOutput": {"test_kind": "dashboard_summary"}, + } + item = _item_from_raw(raw, dataset_name="ds", test_kind="visualization") + assert item.test_kind == "dashboard_summary" + + +def test_a_blank_test_kind_declaration_falls_back_to_the_default(): + # "" is a str, so a blank declaration used to beat both the structural checks and the + # CLI --kind default; the item was then skipped as an unsupported test_kind. + assert _infer_test_kind({"test_kind": ""}, "visualization") == "visualization" + assert _infer_test_kind({"test_kind": " "}, "visualization") == "visualization" + assert _infer_test_kind({}, "visualization", {"test_kind": ""}) == "visualization" + # A real declaration still wins, and is stripped. + assert _infer_test_kind({"test_kind": " agentic_guardrail "}, "visualization") == "agentic_guardrail" + + +def test_a_blank_declaration_does_not_hide_a_real_one_behind_it(): + # The lookup must not stop at the first *string* it finds: a blank + # expectedOutput.test_kind used to shadow a valid metadata.test_kind, and the item + # then fell through to structural inference or the CLI default. + assert _infer_test_kind({"test_kind": ""}, "visualization", {"test_kind": "agentic_guardrail"}) == ( + "agentic_guardrail" + ) + assert _infer_test_kind({"test_kind": " "}, "visualization", {"test_kind": "agentic_guardrail"}) == ( + "agentic_guardrail" + ) + # expectedOutput still wins when it actually declares something. + assert _infer_test_kind({"test_kind": "agentic_search"}, "visualization", {"test_kind": "agentic_guardrail"}) == ( + "agentic_search" + ) diff --git a/packages/gooddata-eval/tests/test_llm_judge.py b/packages/gooddata-eval/tests/test_llm_judge.py index a65279b39..66f85379a 100644 --- a/packages/gooddata-eval/tests/test_llm_judge.py +++ b/packages/gooddata-eval/tests/test_llm_judge.py @@ -1,25 +1,59 @@ # (C) 2026 GoodData Corporation import json -import os from unittest.mock import MagicMock, patch -from gooddata_eval.core.evaluators._llm_judge import LLMJudge +import pytest +from gooddata_eval.core.config import JUDGE_MODEL_ENV_VAR, judge_model +from gooddata_eval.core.evaluators._llm_judge import ( + JUDGE_DIAGNOSTICS_ENV_VAR, + JUDGE_MAX_COMPLETION_TOKENS, + JudgeResponseError, + LLMJudge, + score_run, +) -def _make_judge() -> LLMJudge: - with ( - patch("openai.OpenAI"), - patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}), - ): - return LLMJudge(evaluation_steps=["Step 1: check the answer is correct."]) +@pytest.fixture(autouse=True) +def _openai_api_key(monkeypatch): + """Every judge here talks to a stubbed client, but LLMJudge still refuses to build + without the key -- so supply it once rather than at each construction site.""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") -def _mock_response(score: int, reasoning: str = "ok"): +def _make_judge(*, model: str | None = None) -> LLMJudge: + """A judge whose openai client is a MagicMock. + + ``LLMJudge.__init__`` never calls the API, so every caller stubs the response + afterwards on ``judge._client.chat.completions.create``. + """ + with patch("openai.OpenAI"): + return LLMJudge(evaluation_steps=["Step 1: check the answer is correct."], model=model) + + +def _mock_raw_response( + content: str | None, + *, + finish_reason: str = "stop", + prompt_tokens: int = 700, + completion_tokens: int = 60, + reasoning_tokens: int = 0, + fingerprint: str = "fp_test", +): + """A chat completion whose body is exactly ``content``, metadata and all.""" resp = MagicMock() - resp.choices[0].message.content = json.dumps({"score": score, "reasoning": reasoning}) + resp.choices[0].message.content = content + resp.choices[0].finish_reason = finish_reason + resp.system_fingerprint = fingerprint + resp.usage.prompt_tokens = prompt_tokens + resp.usage.completion_tokens = completion_tokens + resp.usage.completion_tokens_details.reasoning_tokens = reasoning_tokens return resp +def _mock_response(score: int, reasoning: str = "ok"): + return _mock_raw_response(json.dumps({"score": score, "reasoning": reasoning})) + + def test_llm_judge_returns_true_on_score_1(): judge = _make_judge() judge._client.chat.completions.create = MagicMock(return_value=_mock_response(1)) @@ -35,11 +69,548 @@ def test_llm_judge_returns_false_on_score_0(): assert passed is False -def test_llm_judge_raises_without_api_key(): - with patch("openai.OpenAI"), patch.dict("os.environ", {}, clear=True): - os.environ.pop("OPENAI_API_KEY", None) - try: - LLMJudge(evaluation_steps=["s"]) - assert False, "should have raised OSError" - except OSError as e: - assert "OPENAI_API_KEY" in str(e) +def test_llm_judge_raises_without_api_key(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + with patch("openai.OpenAI"), pytest.raises(OSError, match="OPENAI_API_KEY"): + LLMJudge(evaluation_steps=["s"]) + + +def test_default_judge_model_is_gpt_4o(monkeypatch): + # Deterministic (temperature=0) and independent of the agent under test. Both matter: + # a judge sharing the agent's model family grades its own output. + monkeypatch.delenv(JUDGE_MODEL_ENV_VAR, raising=False) + assert judge_model() == "gpt-4o" + assert _make_judge().model == "gpt-4o" + + +def test_judge_model_can_be_overridden_by_env(monkeypatch): + monkeypatch.setenv(JUDGE_MODEL_ENV_VAR, "gpt-5.6-luna") + assert judge_model() == "gpt-5.6-luna" + assert _make_judge().model == "gpt-5.6-luna" + + +def test_blank_judge_model_env_falls_back_to_the_default(monkeypatch): + monkeypatch.setenv(JUDGE_MODEL_ENV_VAR, " ") + assert judge_model() == "gpt-4o" + + +def test_judge_scores_at_temperature_zero_by_default(): + judge = _make_judge() + judge._client.chat.completions.create = MagicMock(return_value=_mock_response(1)) + + judge.score(input="q", expected_output="e", actual_output="a") + + assert judge._client.chat.completions.create.call_args.kwargs["temperature"] == 0 + + +def test_judge_retries_without_temperature_when_the_model_rejects_it(capsys): + """gpt-5-family models reject temperature=0 outright: + + "Unsupported value: 'temperature' does not support 0 with this model. + Only the default (1) value is supported." + + Without this fallback --judge-model gpt-5.6-luna is a trap that 400s on the first + item. With it, the run proceeds -- but the judge is no longer deterministic, so it + has to say so loudly rather than silently degrade the eval. + """ + judge = _make_judge() + calls = [] + + def _create(**kwargs): + calls.append(kwargs) + if "temperature" in kwargs: + raise Exception( + "Error code: 400 - Unsupported value: 'temperature' does not support 0 " + "with this model. Only the default (1) value is supported." + ) + return _mock_response(1) + + judge._client.chat.completions.create = _create + passed, _ = judge.score(input="q", expected_output="e", actual_output="a") + + assert passed is True + assert len(calls) == 2 + assert "temperature" in calls[0] and "temperature" not in calls[1] + warned = capsys.readouterr().out + assert "not deterministic" in warned.lower() + + +def test_judge_remembers_the_model_rejects_temperature_and_stops_retrying(): + # One wasted 400 per run, not one per item. + judge = _make_judge() + calls = [] + + def _create(**kwargs): + calls.append(kwargs) + if "temperature" in kwargs: + raise Exception("Unsupported value: 'temperature' does not support 0 with this model.") + return _mock_response(1) + + judge._client.chat.completions.create = _create + for _ in range(3): + judge.score(input="q", expected_output="e", actual_output="a") + + assert sum(1 for c in calls if "temperature" in c) == 1 + + +def test_judge_reraises_errors_unrelated_to_temperature(): + judge = _make_judge() + judge._client.chat.completions.create = MagicMock(side_effect=Exception("rate limit exceeded")) + + with pytest.raises(Exception, match="rate limit"): + judge.score(input="q", expected_output="e", actual_output="a") + + +# --- Malformed judge responses must fail loudly, never score 0 ------------------------- +# +# Every case below used to return ``(False, "")`` -- a confident "the agent was wrong" +# manufactured out of a response that carried no verdict at all. The failure mode is +# one-sided: it can only invent a 0, never a 1, so it biases every score downwards and +# is invisible in the report, which shows a plain failing item. + + +def test_judge_raises_when_the_model_returns_empty_content(): + """A reasoning model that burns its whole budget on reasoning tokens returns + ``content=""`` with ``finish_reason="length"``. That is a truncated call, not a + failing answer.""" + judge = _make_judge() + judge._client.chat.completions.create = MagicMock( + return_value=_mock_raw_response("", finish_reason="length", reasoning_tokens=2048) + ) + + with pytest.raises(JudgeResponseError) as err: + judge.score(input="q", expected_output="42", actual_output="The answer is 42.") + + assert "empty" in str(err.value).lower() + assert "finish_reason=length" in str(err.value) + assert "reasoning_tokens=2048" in str(err.value) + + +def test_judge_raises_when_the_model_returns_no_content(): + judge = _make_judge() + judge._client.chat.completions.create = MagicMock(return_value=_mock_raw_response(None)) + + with pytest.raises(JudgeResponseError): + judge.score(input="q", expected_output="e", actual_output="a") + + +def test_judge_raises_when_the_response_has_no_score_key(): + """Valid JSON, wrong schema -- ``{"verdict": 1}`` or a recased ``"Score"`` used to + fall through to the default and score 0.""" + judge = _make_judge() + judge._client.chat.completions.create = MagicMock( + return_value=_mock_raw_response(json.dumps({"verdict": 1, "reasoning": "correct"})) + ) + + with pytest.raises(JudgeResponseError) as err: + judge.score(input="q", expected_output="e", actual_output="a") + + assert "score" in str(err.value) + assert "verdict" in str(err.value) # the raw body is quoted back for diagnosis + + +def test_judge_raises_a_diagnosable_error_when_the_json_is_truncated(): + judge = _make_judge() + judge._client.chat.completions.create = MagicMock( + return_value=_mock_raw_response('{"score": 1, "reason', finish_reason="length") + ) + + with pytest.raises(JudgeResponseError) as err: + judge.score(input="q", expected_output="e", actual_output="a") + + assert "finish_reason=length" in str(err.value) + + +# --- Diagnostics ---------------------------------------------------------------------- + + +def test_judge_diagnostics_are_off_by_default(monkeypatch, capsys): + # 18 items x 2 runs is 36 of these lines; they would bury the progress output. + monkeypatch.delenv(JUDGE_DIAGNOSTICS_ENV_VAR, raising=False) + judge = _make_judge() + judge._client.chat.completions.create = MagicMock(return_value=_mock_response(1)) + + judge.score(input="q", expected_output="e", actual_output="a") + + assert "[judge]" not in capsys.readouterr().out + + +def test_judge_logs_response_metadata_when_diagnostics_are_enabled(monkeypatch, capsys): + """The forensic record for a flipped verdict: which endpoint answered, whether it + was truncated, how many reasoning tokens it spent, and what it actually said.""" + monkeypatch.setenv(JUDGE_DIAGNOSTICS_ENV_VAR, "1") + judge = _make_judge() + judge._client.chat.completions.create = MagicMock( + return_value=_mock_raw_response( + json.dumps({"score": 1, "reasoning": "matches"}), + finish_reason="stop", + reasoning_tokens=128, + fingerprint="fp_abc123", + ) + ) + + judge.score(input="q", expected_output="e", actual_output="a") + + out = capsys.readouterr().out + assert "[judge]" in out + assert "finish_reason=stop" in out + assert "system_fingerprint=fp_abc123" in out + assert "reasoning_tokens=128" in out + assert "matches" in out # the raw body, so a flipped verdict can be read back + + +def test_judge_diagnostics_survive_a_response_without_usage(monkeypatch, capsys): + """Not every endpoint returns a usage block; diagnostics must not be the thing that + breaks the run.""" + monkeypatch.setenv(JUDGE_DIAGNOSTICS_ENV_VAR, "1") + judge = _make_judge() + resp = _mock_raw_response(json.dumps({"score": 1, "reasoning": "ok"})) + resp.usage = None + + judge._client.chat.completions.create = MagicMock(return_value=resp) + passed, _ = judge.score(input="q", expected_output="e", actual_output="a") + + assert passed is True + assert "[judge]" in capsys.readouterr().out + + +# --- The completion cap --------------------------------------------------------------- + + +def test_judge_caps_completion_tokens(): + """Unset, the cap is whatever the provider defaults to. A reasoning judge can spend + that entire budget on hidden reasoning tokens and return an empty body, which is the + truncation this cap exists to make deliberate rather than accidental.""" + judge = _make_judge() + judge._client.chat.completions.create = MagicMock(return_value=_mock_response(1)) + + judge.score(input="q", expected_output="e", actual_output="a") + + kwargs = judge._client.chat.completions.create.call_args.kwargs + assert kwargs["max_completion_tokens"] == JUDGE_MAX_COMPLETION_TOKENS + # max_tokens is the deprecated spelling and the gpt-5 family rejects it outright. + assert "max_tokens" not in kwargs + + +def test_the_completion_cap_survives_the_temperature_fallback(): + """The retry that drops temperature must not also drop the cap.""" + judge = _make_judge() + calls = [] + + def _create(**kwargs): + calls.append(kwargs) + if "temperature" in kwargs: + raise Exception("Unsupported value: 'temperature' does not support 0 with this model.") + return _mock_response(1) + + judge._client.chat.completions.create = _create + judge.score(input="q", expected_output="e", actual_output="a") + + assert calls[-1]["max_completion_tokens"] == JUDGE_MAX_COMPLETION_TOKENS + + +# --- Retrying an empty body ----------------------------------------------------------- +# +# An empty body is a truncated call, not a verdict, and truncation is transient: the +# reasoning chain that overran the budget is resampled on the next attempt. Failing the +# whole item for it costs the item's remaining runs (runner.py returns early on error), +# so it is worth exactly one more request. Schema errors are NOT retried -- a model that +# answers with the wrong key answers with the wrong key again, and the retry would only +# double the cost of a prompt that needs fixing. + + +def test_judge_retries_once_when_the_body_comes_back_empty(): + judge = _make_judge() + responses = [ + _mock_raw_response("", finish_reason="length", reasoning_tokens=4096), + _mock_raw_response(json.dumps({"score": 1, "reasoning": "correct"})), + ] + judge._client.chat.completions.create = MagicMock(side_effect=responses) + + passed, reasoning = judge.score(input="q", expected_output="42", actual_output="The answer is 42.") + + assert passed is True + assert reasoning == "correct" + assert judge._client.chat.completions.create.call_count == 2 + + +def test_judge_raises_when_the_retry_is_also_empty(): + judge = _make_judge() + judge._client.chat.completions.create = MagicMock( + return_value=_mock_raw_response("", finish_reason="length", reasoning_tokens=4096) + ) + + with pytest.raises(JudgeResponseError): + judge.score(input="q", expected_output="e", actual_output="a") + + assert judge._client.chat.completions.create.call_count == 2 + + +def test_judge_does_not_retry_a_schema_error(): + judge = _make_judge() + judge._client.chat.completions.create = MagicMock( + return_value=_mock_raw_response(json.dumps({"verdict": 1, "reasoning": "correct"})) + ) + + with pytest.raises(JudgeResponseError): + judge.score(input="q", expected_output="e", actual_output="a") + + assert judge._client.chat.completions.create.call_count == 1 + + +def test_judge_announces_the_retry(capsys): + """A silently retried item is an item whose cost and latency doubled for no visible + reason -- and a rising retry rate is the signal that the cap is too low.""" + judge = _make_judge() + judge._client.chat.completions.create = MagicMock( + side_effect=[ + _mock_raw_response("", finish_reason="length", reasoning_tokens=4096), + _mock_response(1), + ] + ) + + judge.score(input="q", expected_output="e", actual_output="a") + + out = capsys.readouterr().out + assert "empty body" in out + assert "retrying" in out.lower() + + +# --- an unreadable response must not escape as an untyped error (H4) --- + + +def _api_error(message: str, *, body: object) -> Exception: + """An exception shaped like the openai SDK's APIStatusError. + + The SDK stringifies the whole response body into ``message`` (see + ``_make_status_error_from_response``), which is exactly why the temperature check + below cannot read ``str(exc)``. + """ + + class _APIError(Exception): + pass + + exc = _APIError(f"Error code: 400 - {body}") + exc.body = body # type: ignore[attr-defined] + exc.message = message # type: ignore[attr-defined] + return exc + + +def test_an_empty_choices_list_raises_a_judge_error_not_an_index_error(): + """Content filters and gateway error envelopes return `choices: []`. + + That is the commonest shape of an unreadable judge response, and `choices[0]` used to + escape as a bare IndexError -- past the typed error this module exists to raise, with + none of the body or metadata that makes the cause readable. + """ + judge = _make_judge() + judge._client.chat.completions.create.return_value = MagicMock(choices=[]) + + with pytest.raises(JudgeResponseError) as err: + judge.score(input="i", expected_output="e", actual_output="a") + + # The metadata has to say *why* there was no verdict, or the error is unactionable. + assert "choices=0" in str(err.value) + + +def test_a_missing_message_content_raises_a_judge_error(): + judge = _make_judge() + judge._client.chat.completions.create.return_value = MagicMock(choices=[MagicMock(message=MagicMock(content=None))]) + + with pytest.raises(JudgeResponseError): + judge.score(input="i", expected_output="e", actual_output="a") + + +# --- the temperature fallback must fire on temperature and nothing else (H3) --- + + +def test_a_structured_temperature_rejection_drops_the_parameter(): + """The real gpt-5 400 carries param="temperature". That is the signal we act on.""" + calls: list[dict] = [] + body = { + "error": { + "message": "Unsupported value: 'temperature' does not support 0 with this model.", + "type": "invalid_request_error", + "param": "temperature", + "code": "unsupported_value", + } + } + + def create(**kwargs): + calls.append(dict(kwargs)) + if "temperature" in kwargs: + raise _api_error("Unsupported value: 'temperature' ...", body=body) + return MagicMock(choices=[MagicMock(message=MagicMock(content='{"score": 1, "reasoning": "ok"}'))]) + + judge = _make_judge(model="gpt-5.2") + judge._client.chat.completions.create.side_effect = create + passed, _ = judge.score(input="i", expected_output="e", actual_output="a") + + assert passed is True + assert "temperature" in calls[0] and "temperature" not in calls[1] + assert judge._supports_temperature is False + + +def test_a_400_that_echoes_the_request_is_not_read_as_a_temperature_rejection(): + """LiteLLM / vLLM style: the error body quotes the request that failed. + + That body contains the literal text `"temperature": 0`, so a substring check against + `str(exc)` read a context-length overflow as a temperature rejection -- reporting the + wrong cause, and then silently dropping temperature=0 from every later verdict, which + is the one property that makes a judge reproducible. + """ + body = { + "error": { + "message": "This model's maximum context length is 8192 tokens.", + "type": "invalid_request_error", + "code": "context_length_exceeded", + "request": {"model": "gpt-4o", "temperature": 0, "max_completion_tokens": 4096}, + } + } + calls: list[dict] = [] + + def create(**kwargs): + calls.append(dict(kwargs)) + raise _api_error("This model's maximum context length is 8192 tokens.", body=body) + + judge = _make_judge() + judge._client.chat.completions.create.side_effect = create + + with pytest.raises(Exception, match="context length"): + judge.score(input="i", expected_output="e", actual_output="a") + + # The real error surfaced, exactly one request was spent, and determinism is intact. + assert len(calls) == 1 + assert judge._supports_temperature is True + + +def test_the_eval_text_cannot_trigger_the_temperature_fallback(): + """A BI agent grading "average temperature by city" is an ordinary fixture. + + The word appearing in the graded content must not reconfigure the judge. + """ + body = {"error": {"message": "Rate limit reached for gpt-4o.", "code": "rate_limit_exceeded"}} + + judge = _make_judge() + judge._client.chat.completions.create.side_effect = _api_error("Rate limit reached for gpt-4o.", body=body) + + with pytest.raises(Exception, match="Rate limit"): + judge.score( + input="What is the average temperature by city?", + expected_output="A temperature per city", + actual_output="Prague averages 9.4 degrees", + ) + + assert judge._supports_temperature is True + + +def test_an_unparseable_error_body_is_not_read_as_a_temperature_rejection(): + """A body the SDK could not parse as JSON arrives as raw text, and the SDK then uses that + same text as the exception message. A gateway that echoes the failed request inside it + puts ``"temperature": 0`` in both places, so neither is prose to substring-match. + """ + body = '{"error":{"message":"model not found"},"request":{"model":"gpt-4o","temperature":0}}' + calls: list[dict] = [] + + def create(**kwargs): + calls.append(dict(kwargs)) + raise _api_error(body, body=body) + + judge = _make_judge() + judge._client.chat.completions.create.side_effect = create + + with pytest.raises(Exception, match="model not found"): + judge.score(input="i", expected_output="e", actual_output="a") + + assert len(calls) == 1, "a wasted retry without temperature" + assert judge._supports_temperature is True + + +# --- score_run: one run's judge fault is not the item's problem (H1) --- + + +def test_score_run_passes_a_readable_verdict_straight_through(): + judge = MagicMock() + judge.model = "gpt-4o" + judge.score.return_value = (True, "looks right") + + verdict = score_run(judge, input="i", expected_output="e", actual_output="a") + + assert (verdict.passed, verdict.reasoning, verdict.error) == (True, "looks right", None) + + +def test_score_run_turns_an_unreadable_verdict_into_an_unscored_run(capsys): + """Not re-raised, and not scored 0 either. + + Raising discards every run already graded; scoring 0 is the silent FAIL + JudgeResponseError exists to stop. The run is marked ungraded and the caller excludes + it from pass@K. + """ + judge = MagicMock() + judge.model = "gpt-4o" + judge.score.side_effect = JudgeResponseError("empty body twice") + + verdict = score_run(judge, input="i", expected_output="e", actual_output="a") + + assert verdict.passed is False + assert verdict.error is not None and "empty body twice" in verdict.error + # Announced: a rising rate of ungraded runs is the signal the judge needs attention. + assert "could not grade one run" in capsys.readouterr().out + + +def test_score_run_does_not_swallow_errors_that_are_not_judge_faults(): + judge = MagicMock() + judge.model = "gpt-4o" + judge.score.side_effect = RuntimeError("network down") + + with pytest.raises(RuntimeError, match="network down"): + score_run(judge, input="i", expected_output="e", actual_output="a") + + +# --- the score contract: only 0 and 1 are verdicts (H4's last gap) --- + + +def _score_body(body: str): + judge = _make_judge() + judge._client.chat.completions.create.return_value = MagicMock(choices=[MagicMock(message=MagicMock(content=body))]) + return judge.score(input="i", expected_output="e", actual_output="a") + + +@pytest.mark.parametrize("body", ['{"score": 2, "reasoning": "fully correct"}', '{"score": 0.9}', '{"score": -1}']) +def test_an_out_of_range_score_raises_instead_of_reporting_a_failure(body): + """`int(score) == 1` was the last place an invented 0 survived: a model reading the + rubric as 0-2, or answering with a confidence, landed here.""" + with pytest.raises(JudgeResponseError, match="out-of-range"): + _score_body(body) + + +@pytest.mark.parametrize(("body", "expected"), [('{"score": "1"}', True), ('{"score": "0"}', False)]) +def test_a_quoted_number_is_still_a_verdict(body, expected): + """JSON mode quotes numbers routinely, and the pre-JudgeResponseError code accepted + that via int() -- rejecting it now would throw away a verdict the judge did give.""" + passed, _ = _score_body(body) + assert passed is expected + + +@pytest.mark.parametrize("score", ["1 (correct)", "probably correct"]) +def test_judge_raises_when_the_score_is_not_a_number(score): + """A score that float() cannot read is not a verdict, whether it merely decorates the + number or replaces it outright.""" + with pytest.raises(JudgeResponseError, match="non-numeric") as err: + _score_body(json.dumps({"score": score, "reasoning": "ok"})) + + assert score in str(err.value) # the raw value is quoted back for diagnosis + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + ('{"score": 1, "reasoning": "ok"}', True), + ('{"score": 0, "reasoning": "no"}', False), + # Booleans are verdicts too, not schema violations: JSON mode emits them. + ('{"score": true}', True), + ('{"score": false}', False), + ], +) +def test_the_zero_and_one_verdicts_are_unchanged(body, expected): + assert _score_body(body)[0] is expected diff --git a/packages/gooddata-eval/tests/test_models.py b/packages/gooddata-eval/tests/test_models.py index 0b648b526..d2d951b30 100644 --- a/packages/gooddata-eval/tests/test_models.py +++ b/packages/gooddata-eval/tests/test_models.py @@ -88,3 +88,30 @@ def test_tool_call_event_parsed_result_parses_json(): } ) assert ev.parsed_result() == {"data": {"maql": "SELECT {metric/a}", "format": "#,##0"}} + + +def test_dataset_item_carries_a_user_context_attachment(): + item = DatasetItem.model_validate( + { + "id": "gdai-2179-001", + "dataset_name": "GDAI-2179", + "test_kind": "agentic_general_question", + "question": "What does the visualization I attached show?", + "expected_output": "Describes the attached chart.", + "user_context": {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "campaign_spend"}]}]}, + } + ) + assert item.user_context == {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "campaign_spend"}]}]} + + +def test_dataset_item_user_context_defaults_to_none(): + item = DatasetItem.model_validate( + { + "id": "q1", + "dataset_name": "d1", + "test_kind": "agentic_general_question", + "question": "What can you do?", + "expected_output": "Describes capabilities.", + } + ) + assert item.user_context is None diff --git a/packages/gooddata-eval/tests/test_reporting.py b/packages/gooddata-eval/tests/test_reporting.py index 467fda7fb..7f582618b 100644 --- a/packages/gooddata-eval/tests/test_reporting.py +++ b/packages/gooddata-eval/tests/test_reporting.py @@ -1,4 +1,6 @@ # (C) 2026 GoodData Corporation +import io + import orjson from gooddata_eval.core.reporting.console import render_comparison, render_console from gooddata_eval.core.reporting.json_report import ( @@ -8,6 +10,7 @@ write_multi_model_report, ) from gooddata_eval.core.runner import EvalReport, ItemReport +from rich.console import Console def _report() -> EvalReport: @@ -192,3 +195,250 @@ def test_build_multi_model_report_no_key_collision_same_model_different_provider assert "HN_Anthropic/claude-opus" in data["runs"] assert data["runs"]["DirectAnthropic/claude-opus"]["summary"]["passed"] == 1 assert data["runs"]["HN_Anthropic/claude-opus"]["summary"]["passed"] == 0 + + +def _timed_report() -> EvalReport: + return EvalReport( + model="gpt-5.2", + workspace_id="ws1", + items=[ + ItemReport( + id="i1", + dataset_name="d", + test_kind="agentic_general_question", + question="q1", + pass_at_k=True, + runs=1, + latency_s=6.0, + agent_latency_s=4.0, + judge_latency_s=2.0, + langfuse_latency_s=31.5, + ) + ], + ) + + +def test_json_report_breaks_an_item_down_by_phase(): + # Without this the report has one latency number per item and no way to tell a slow + # agent from a slow judge -- the question the whole exercise exists to answer. + item = build_json_report(_timed_report())["items"]["i1"] + + assert item["latency_breakdown_s"] == { + "agent_s": 4.0, + "judge_s": 2.0, + "simulated_user_s": 0.0, + "langfuse_s": 31.5, + } + + +def test_json_report_keeps_langfuse_time_out_of_the_item_latency(): + # langfuse_s is reported *beside* latency_s, never folded into it: trace linking runs + # off the critical path, so adding it back would re-create the 40s items in the report + # even though the run no longer waits for them. + item = build_json_report(_timed_report())["items"]["i1"] + + assert item["latency_s"] == 6.0 + assert item["latency_breakdown_s"]["langfuse_s"] == 31.5 + + +def test_json_report_still_has_every_key_it_had_before(): + # Additive-only: the breakdown is a new key, not a reshaping of the existing report, + # so anything already parsing these files keeps working. + item = build_json_report(_report())["items"]["i1"] + + assert { + "dataset_name", + "test_kind", + "question", + "pass_at_k", + "skipped", + "error", + "runs", + "latency_s", + "avg_latency_s", + "best_run_latency_s", + "detail", + "conversation_id", + "response_id", + "reasoning", + } <= set(item) + + +def test_console_summary_does_not_call_the_item_total_agent_time(): + # report.latency_s is the sum of each item's critical path -- agent AND judge AND + # simulated user. Now that agentic runs set wall_clock_s, this branch fires for them + # for the first time, and now that agent_latency_s exists as a real, different number, + # labelling the total "agent time" states something false. + report = _timed_report() + report.wall_clock_s = 40.0 # >1s from latency_s, so the two-number form is used + + out = render_console(report, console=Console(record=True, width=200)) + + assert "40.00s wall-clock" in out + assert "agent time" not in out + + +# --- a pass@K that was not unanimous must say so (4/5 used to print as a clean PASS) --- + + +def _rendered(report: EvalReport) -> str: + buf = io.StringIO() + render_console(report, console=Console(file=buf, width=200, no_color=True)) + return buf.getvalue() + + +def _item(item_id: str, *, runs: int, runs_passed: int, passed: bool, effective: int | None = None) -> ItemReport: + r = ItemReport(id=item_id, dataset_name="d", test_kind="agentic_general_question", question="q") + r.pass_at_k = passed + r.runs = runs + r.runs_passed = runs_passed + r.runs_effective = effective + r.latency_s = 1.0 + r.best_detail = {"judge_passed": passed} + return r + + +def test_console_reports_how_many_runs_passed_when_it_was_not_unanimous(): + """quality_score reads best_detail -- the winning run alone -- so a 1/5 item and a 5/5 + item printed identically: PASS, 100%, empty Notes. The count was already computed by + every agentic kind and then dropped on the floor. + """ + report = EvalReport(model="m") + report.items = [ + _item("solid", runs=5, runs_passed=5, passed=True), + _item("flaky", runs=5, runs_passed=4, passed=True), + _item("coinflip", runs=5, runs_passed=1, passed=True), + ] + + text = _rendered(report) + + assert "4/5 runs passed" in text + assert "1/5 runs passed" in text + # A unanimous pass stays uncluttered -- the note is a warning, not decoration. + solid_row = next(line for line in text.splitlines() if "solid" in line) + assert "runs passed" not in solid_row + + +def test_console_summary_separates_pass_at_k_from_unanimity(): + report = EvalReport(model="m") + report.items = [ + _item("a", runs=3, runs_passed=3, passed=True), + _item("b", runs=3, runs_passed=1, passed=True), + ] + + assert "2/2 passed, 1 on every run" in _rendered(report) + + +def test_console_summary_stays_quiet_when_every_pass_was_unanimous(): + report = EvalReport(model="m") + report.items = [_item("a", runs=3, runs_passed=3, passed=True)] + + text = _rendered(report) + assert "1/1 passed (" in text + assert "on every run" not in text + + +def test_json_report_carries_runs_passed_and_unanimity(): + report = EvalReport(model="m") + report.items = [ + _item("solid", runs=5, runs_passed=5, passed=True), + _item("flaky", runs=5, runs_passed=4, passed=True), + ] + + run = build_json_report(report) + + assert run["summary"]["passed"] == 2 + assert run["summary"]["passed_all_runs"] == 1 + items = run["items"] + by_id = items if isinstance(items, dict) else {i["id"]: i for i in items} + assert (by_id["solid"]["runs_passed"], by_id["solid"]["pass_power_k"]) == (5, True) + assert (by_id["flaky"]["runs_passed"], by_id["flaky"]["pass_power_k"]) == (4, False) + + +def test_a_kind_that_runs_once_is_not_reported_as_k_runs(): + # agentic_conversation takes no k and drives its fixture once whatever --runs says. + # Trusting K there reported four runs that never happened, and divided the latency by 5. + report = EvalReport(model="m") + report.items = [_item("conv", runs=5, runs_passed=1, passed=True, effective=1)] + + text = _rendered(report) + conv_row = next(line for line in text.splitlines() if "conv" in line and "PASS" in line) + assert " 1 " in conv_row, f"the Runs column should show the 1 run it made: {conv_row}" + assert "runs passed" not in conv_row, "1 of 1 is unanimous" + + +def test_an_errored_item_is_not_reported_as_passing_every_run(): + """An item can error after earlier runs passed, leaving runs_passed == runs_total. + + That is not unanimity -- the last run produced no verdict at all -- so pass^K must not + claim it, and passed_all_runs must not count it. + """ + errored = _item("boom", runs=2, runs_passed=2, passed=False, effective=2) + errored.error = "judge returned no readable verdict for any of the 2 run(s)" + + assert errored.pass_power_k is False + + report = EvalReport(model="m") + report.items = [errored] + assert build_json_report(report)["summary"]["passed_all_runs"] == 0 + + +def test_an_errored_item_is_counted_as_errored_not_as_a_failure(): + # `total - passed - skipped` counted an errored item as both. A judge fault is + # documented to report as an error instead of K failures, so `failed` must exclude it. + errored = _item("boom", runs=2, runs_passed=0, passed=False, effective=2) + errored.error = "judge returned no readable verdict" + genuine = _item("nope", runs=2, runs_passed=0, passed=False, effective=2) + report = EvalReport(model="m") + report.items = [errored, genuine] + + summary = build_json_report(report)["summary"] + assert (summary["failed"], summary["errored"]) == (1, 1) + + +def test_avg_per_run_divides_by_the_runs_actually_taken(): + # The Runs column reports runs_effective, so an average over the requested K reported + # a per-run latency for runs that never happened. + once = _item("conv", runs=5, runs_passed=1, passed=True, effective=1) + once.latency_s = 10.0 + + assert once.runs_total == 1 + assert once.avg_latency_s == 10.0 + + +def test_json_runs_and_the_aggregate_average_use_the_runs_actually_taken(): + # Same divisor everywhere: the item's own average already used runs_total, but the JSON + # `runs` field and the run-level average still reported the requested K. + once = _item("conv", runs=5, runs_passed=1, passed=True, effective=1) + once.latency_s = 10.0 + report = EvalReport(model="m", items=[once]) + + data = build_json_report(report) + + assert data["items"]["conv"]["runs"] == 1 + assert report.total_runs == 1 + assert data["summary"]["avg_latency_s"] == 10.0 + + +def test_the_report_counts_the_runs_the_judge_could_not_grade(): + item = _item("a", runs=3, runs_passed=2, passed=True) + item.runs_ungraded = 1 + report = EvalReport(model="m", items=[item]) + + data = build_json_report(report) + out = _rendered(report) + + assert data["items"]["a"]["runs_ungraded"] == 1 + assert "2/3 runs passed; 1 run(s) ungraded" in out + + +def test_a_failed_item_says_when_a_criterion_went_ungraded(): + # `passed` is False with no False bool in the detail, so the generic "did not pass strict + # checks" alone would hide why: a mandatory criterion the judge never assessed. + item = _item("s", runs=1, runs_passed=0, passed=False) + item.best_detail = {"include_0": True, "ungraded_criteria": 1} + report = EvalReport(model="m", items=[item]) + + out = _rendered(report) + + assert "did not pass strict checks; 1 criterion(s) ungraded" in out diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index 11fb4b2f6..73fee1eb7 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -1,8 +1,10 @@ # (C) 2026 GoodData Corporation import threading import time +from unittest.mock import patch from gooddata_eval.core.evaluators import supported_test_kinds +from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.models import ChatResult, DatasetItem from gooddata_eval.core.runner import ItemReport, run_items @@ -367,3 +369,60 @@ def bad_callback(index, total, report): assert result.total == 2 # run did not abort err = capsys.readouterr().err assert "RuntimeError" in err or "callback bug" in err # traceback was printed + + +# --- an ungraded run is excluded from pass@K, and only an item with no graded run errors --- + + +class _ScriptedEvaluator: + test_kind = "visualization" + + def __init__(self, evaluations): + self._evaluations = iter(evaluations) + + def evaluate(self, item, chat_result) -> ItemEvaluation: + return next(self._evaluations) + + +def _graded(passed: bool) -> ItemEvaluation: + return ItemEvaluation(passed=passed, rank_key=(int(passed),), detail={"judge_passed": passed}) + + +def _ungraded() -> ItemEvaluation: + return ItemEvaluation(passed=False, rank_key=(-1,), detail={"judge_error": "empty body"}, error="empty body") + + +def _run_scripted(evaluations, runs: int): + backend = _FakeBackend([_empty_chat()]) + with patch("gooddata_eval.core.runner.get_evaluator", return_value=_ScriptedEvaluator(evaluations)): + report = run_items([_item()], backend, runs=runs) + return report, backend + + +def test_one_ungraded_run_does_not_error_an_item_whose_other_run_passed(): + report, backend = _run_scripted([_graded(True), _ungraded()], runs=2) + + item = report.items[0] + assert backend.calls == 2, "the run after the judge fault was abandoned" + assert item.error is None + assert item.pass_at_k is True + assert (item.runs, item.runs_passed, item.runs_ungraded) == (2, 1, 1) + assert item.pass_power_k is False, "a run nobody graded leaves 'all K passed' unverified" + assert (report.passed, report.errored) == (1, 0) + + +def test_an_item_with_no_graded_run_errors_only_after_all_its_runs(): + report, backend = _run_scripted([_ungraded(), _ungraded()], runs=2) + + item = report.items[0] + assert backend.calls == 2 + assert item.error is not None and item.error.startswith("JudgeResponseError") + assert item.pass_at_k is False + assert (item.runs, item.runs_ungraded) == (2, 2) + assert (report.passed, report.errored) == (0, 1) + + +def test_best_detail_describes_a_graded_run_when_there_is_one(): + report, _ = _run_scripted([_ungraded(), _graded(False)], runs=2) + + assert report.items[0].best_detail == {"judge_passed": False} diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 5cc205dc5..cdb348e9f 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -867,3 +867,57 @@ def test_invalid_reasoning_effort_fails_at_construction(): """Fail locally rather than as an out-of-enum request partway through a run.""" with pytest.raises(ValueError, match="Invalid reasoning effort"): ChatClient(host="https://example.invalid", token="t", workspace_id="w", reasoning_effort="maximum") + + +_ATTACHMENT = {"referencedObjects": [{"objects": [{"type": "WIDGET", "id": "campaign_spend"}]}]} + + +def _capture_body(store): + def handler(request): + store["body"] = json.loads(request.read()) + return httpx.Response(200, content=_OK_SSE) + + return handler + + +def test_send_message_puts_the_user_context_on_the_wire(): + captured = {} + client = _client_with_handler(_capture_body(captured)) + client.send_message("conv", "q", user_context=_ATTACHMENT) + assert captured["body"]["userContext"] == _ATTACHMENT + + +def test_send_message_omits_user_context_entirely_when_there_is_no_attachment(): + """An explicit ``"userContext": null`` would be ACCEPTED by gen-ai, so a sloppy + unconditional assignment would silently alter every request in every existing dataset + rather than failing loudly. This is the guard against that.""" + captured = {} + client = _client_with_handler(_capture_body(captured)) + client.send_message("conv", "q") + assert "userContext" not in captured["body"] + + +def test_ask_puts_the_item_attachment_on_the_wire(): + """The single-turn path goes through ask(), so an item's user_context has to be + forwarded there too, or every non-agentic item with an attachment is asked bare.""" + captured = {} + + def handler(request): + if request.method == "POST" and request.url.path.endswith("/conversations"): + return httpx.Response(200, json={"conversationId": "conv-abc"}) + if request.method == "POST" and "messages" in str(request.url): + captured["body"] = json.loads(request.read()) + return httpx.Response(200, content=_OK_SSE) + return httpx.Response(204) + + client = _client_with_handler(handler) + item = DatasetItem( + id="t1", + dataset_name="d", + test_kind="general_question", + question="q", + expected_output="e", + user_context=_ATTACHMENT, + ) + client.ask(item) + assert captured["body"]["userContext"] == _ATTACHMENT diff --git a/packages/gooddata-eval/tests/test_summary_evaluator.py b/packages/gooddata-eval/tests/test_summary_evaluator.py index 6056047e0..ef401bd72 100644 --- a/packages/gooddata-eval/tests/test_summary_evaluator.py +++ b/packages/gooddata-eval/tests/test_summary_evaluator.py @@ -1,10 +1,19 @@ # (C) 2026 GoodData Corporation from unittest.mock import MagicMock, patch +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.evaluators.summary import DashboardSummaryEvaluator from gooddata_eval.core.models import ChatResult, DatasetItem +def _next_verdict(it): + """Next scripted judge result, raising it if it is an exception.""" + v = next(it) + if isinstance(v, Exception): + raise v + return v + + def _make_evaluator(): with patch("openai.OpenAI"), patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}): return DashboardSummaryEvaluator() @@ -85,3 +94,99 @@ def test_non_dict_expected_output_is_single_rubric_criterion(): assert res.passed is True assert res.rank_key == (1, 1.0) ev._positive_judge.score.assert_called_once() + + +# --- one unreadable judge body must not cost every other criterion --- + + +def test_one_ungraded_criterion_does_not_discard_the_ones_already_graded(): + """dashboard_summary makes ONE judge request PER CRITERION. + + Letting JudgeResponseError raise out of the loop discarded every criterion already + graded and abandoned the ones after it -- a 4-criterion item losing all 4 to one bad + body, ending as runs=0 with an empty best_detail, and dropping out of + avg_quality_score's denominator entirely while the CLI still exited 0. + """ + ev = _make_evaluator() + verdicts = iter([(True, "ok"), JudgeResponseError("empty body twice"), (True, "ok")]) + ev._positive_judge.score = MagicMock(side_effect=lambda *a, **k: _next_verdict(verdicts)) + ev._violation_judge.score = MagicMock(return_value=(False, "characteristic absent")) + + item = _item({"must_include": ["a", "b"], "must_not_include": ["x"], "rubric": ["r"]}) + res = ev.evaluate(item, _chat()) + + # include_0 graded True, include_1 ungraded, exclude_0 graded True, rubric_0 graded True. + assert res.detail["include_0"] is True + assert "include_1" not in res.detail, "an ungraded criterion must not be stored as a bool" + assert "UNGRADED" in res.detail["include_1_reason"] + assert res.detail["exclude_0"] is True + assert res.detail["rubric_0"] is True + assert res.detail["ungraded_criteria"] == 1 + assert res.error is None, "three criteria were graded; this run has a verdict" + # An ungraded criterion is not a failed one -- no bool, so it stays out of the quality + # denominator (3 graded checks, all True) -- but a mandatory fact the judge could not + # confirm cannot carry a pass either. + assert res.passed is False + assert res.rank_key == (0, 1.0) + + +def test_an_ungraded_rubric_criterion_does_not_fail_the_item(): + # Rubric criteria never gate `passed`, so one the judge could not grade cannot either. + ev = _make_evaluator() + verdicts = iter([(True, "ok"), JudgeResponseError("empty body twice")]) + ev._positive_judge.score = MagicMock(side_effect=lambda *a, **k: _next_verdict(verdicts)) + + item = _item({"must_include": ["a"], "rubric": ["r"]}) + res = ev.evaluate(item, _chat()) + + assert res.passed is True + assert res.rank_key == (1, 1.0) + assert res.detail["ungraded_criteria"] == 1 + + +def test_an_ungraded_criterion_still_cannot_mask_a_real_failure(): + ev = _make_evaluator() + verdicts = iter([(False, "missing"), JudgeResponseError("unparseable JSON")]) + ev._positive_judge.score = MagicMock(side_effect=lambda *a, **k: _next_verdict(verdicts)) + + item = _item({"must_include": ["a", "b"]}) + res = ev.evaluate(item, _chat()) + + assert res.passed is False + assert res.rank_key == (0, 0.0) + + +def test_an_item_with_no_gradeable_criterion_is_an_ungraded_run(): + # Nothing was assessed, so `passed` would still be its initial True -- a pass nobody + # made. That is not a verdict: the run is reported ungraded, and the runner errors the + # item only if none of its runs could be graded. + ev = _make_evaluator() + ev._positive_judge.score = MagicMock(side_effect=JudgeResponseError("empty body twice")) + + item = _item({"must_include": ["a", "b"]}) + res = ev.evaluate(item, _chat()) + + assert res.error is not None and "no readable verdict for any of the 2 criterion" in res.error + assert res.passed is False + assert res.rank_key < (0, 0.0), "an ungraded run must rank below every graded one" + + +def test_a_graded_rubric_cannot_carry_a_pass_when_every_gating_criterion_went_ungraded(): + """Both mandatory facts ungraded, one rubric line graded True. + + Keyed on "any bool in detail", the no-verdict guard was satisfied by the rubric bool, + `passed` kept its initial True and quality read 100% -- a clean PASS with not one + criterion that decides the verdict assessed. Worse than a silent FAIL: a false PASS is + never investigated. + """ + ev = _make_evaluator() + verdicts = iter([JudgeResponseError("length"), JudgeResponseError("length"), (True, "nice prose")]) + ev._positive_judge.score = MagicMock(side_effect=lambda *a, **k: _next_verdict(verdicts)) + + item = _item({"must_include": ["FACT A", "FACT B"], "rubric": ["r"]}) + res = ev.evaluate(item, _chat()) + + assert res.passed is False + assert res.error is not None and "2 criterion" in res.error + assert res.detail["rubric_0"] is True, "what was graded is still kept" + assert res.detail["ungraded_criteria"] == 2 diff --git a/packages/gooddata-eval/tests/test_text_evaluators.py b/packages/gooddata-eval/tests/test_text_evaluators.py index 93ffb36bb..7e13082ca 100644 --- a/packages/gooddata-eval/tests/test_text_evaluators.py +++ b/packages/gooddata-eval/tests/test_text_evaluators.py @@ -1,6 +1,7 @@ # (C) 2026 GoodData Corporation from unittest.mock import MagicMock, patch +from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.evaluators.general_question import GeneralQuestionEvaluator from gooddata_eval.core.evaluators.guardrail import GuardrailEvaluator from gooddata_eval.core.models import ChatResult, DatasetItem @@ -70,3 +71,34 @@ def test_guardrail_passes_when_agent_refuses(): with patch("gooddata_eval.core.evaluators.guardrail.LLMJudge", return_value=_make_judge(True)): result = GuardrailEvaluator().evaluate(_gr_item(), _chat_text("I'm a data assistant, I can't help with poems.")) assert result.passed is True + + +def _faulty_judge(): + m = MagicMock() + m.model = "gpt-4o" + m.score.side_effect = JudgeResponseError("empty body twice") + return m + + +def test_general_question_reports_an_unreadable_verdict_as_an_ungraded_run(): + # The agentic twin already confines a judge fault to its own run; letting it raise here + # errored the whole item from inside the runner's K loop and abandoned the runs after it. + with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_faulty_judge()): + result = GeneralQuestionEvaluator().evaluate(_gq_item(), _chat_text("Click Share.")) + + assert result.error is not None and "empty body" in result.error + assert result.passed is False + assert result.rank_key < (0,), "an ungraded run must rank below every graded one" + assert "judge_error" in result.detail + assert result.detail["actual_output"] == "Click Share." + + +def test_guardrail_reports_an_unreadable_verdict_as_an_ungraded_run(): + with patch("gooddata_eval.core.evaluators.guardrail.LLMJudge", return_value=_faulty_judge()): + result = GuardrailEvaluator().evaluate(_gr_item(), _chat_text("I can't help with poems.")) + + assert result.error is not None + assert result.passed is False + assert result.rank_key < (0,) + assert result.detail["no_visualization"] is True + assert "judge_passed" not in result.detail, "an ungraded run must not invent a bool verdict" diff --git a/packages/gooddata-eval/tests/test_timing.py b/packages/gooddata-eval/tests/test_timing.py new file mode 100644 index 000000000..3b31a8411 --- /dev/null +++ b/packages/gooddata-eval/tests/test_timing.py @@ -0,0 +1,93 @@ +# (C) 2026 GoodData Corporation +"""Tests for PhaseTimings and the [timer] diagnostic gate.""" + +import threading +from unittest.mock import patch + +import pytest +from gooddata_eval.core.timing import ( + TIMERS_ENV_VAR, + PhaseTimings, + log_timer, + sum_timings, + timers_enabled, +) + + +def test_timers_are_off_by_default(monkeypatch): + # An 18-item --runs 2 run emits 72 [timer] lines, burying the per-item progress + # output. The numbers survive in latency_breakdown_s either way, so the prints are + # opt-in diagnostics. + monkeypatch.delenv(TIMERS_ENV_VAR, raising=False) + assert timers_enabled() is False + + +@pytest.mark.parametrize( + ("value", "expected"), + [("1", True), ("true", True), ("YES", True), ("on", True), ("0", False), ("false", False), ("", False)], +) +def test_timers_env_var_parsing_treats_explicit_off_values_as_off(monkeypatch, value, expected): + # Same trap that silently disabled trace linking: bool("0") is True in Python, so a + # bare truthiness check reads GD_EVAL_TIMERS=0 as ON. + monkeypatch.setenv(TIMERS_ENV_VAR, value) + assert timers_enabled() is expected + + +def test_log_timer_prints_nothing_when_disabled(monkeypatch): + monkeypatch.setenv(TIMERS_ENV_VAR, "0") + printed: list[str] = [] + with patch("sys.stdout") as out: + out.write.side_effect = printed.append + log_timer("[timer] should not appear") + + assert printed == [] + + +def test_log_timer_emits_one_write_when_enabled(monkeypatch): + # One write, not print()'s two (text then newline): these lines are emitted from + # trace-link worker threads' peers and must not interleave mid-line with progress. + monkeypatch.setenv(TIMERS_ENV_VAR, "1") + printed: list[str] = [] + with patch("sys.stdout") as out: + out.write.side_effect = printed.append + log_timer("[timer] hello") + + assert printed == ["[timer] hello\n"] + + +def test_log_timer_is_safe_from_several_threads(monkeypatch): + monkeypatch.setenv(TIMERS_ENV_VAR, "1") + errors: list[BaseException] = [] + + def emit() -> None: + try: + for _ in range(20): + log_timer("[timer] concurrent") + except BaseException as exc: # noqa: BLE001 - recorded, asserted below + errors.append(exc) + + threads = [threading.Thread(target=emit) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert errors == [] + + +def test_phase_timings_add_is_fieldwise(): + a = PhaseTimings(agent_s=1.0, judge_s=2.0) + b = PhaseTimings(agent_s=0.5, simulated_user_s=3.0, langfuse_s=4.0) + total = a + b + + assert (total.agent_s, total.judge_s, total.simulated_user_s, total.langfuse_s) == (1.5, 2.0, 3.0, 4.0) + + +def test_as_dict_rounds_every_phase(): + timings = PhaseTimings(agent_s=1.23456, judge_s=2.0, simulated_user_s=0.0, langfuse_s=31.5) + + assert timings.as_dict() == {"agent_s": 1.235, "judge_s": 2.0, "simulated_user_s": 0.0, "langfuse_s": 31.5} + + +def test_sum_timings_of_nothing_is_all_zeroes(): + assert sum_timings([]) == PhaseTimings() diff --git a/packages/gooddata-eval/tests/test_trace_linker.py b/packages/gooddata-eval/tests/test_trace_linker.py new file mode 100644 index 000000000..4737ee34c --- /dev/null +++ b/packages/gooddata-eval/tests/test_trace_linker.py @@ -0,0 +1,568 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +"""Tests for the background Langfuse trace-linking pool.""" + +import ast +import importlib +import inspect +import threading +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest +from gooddata_eval.cli import agentic_runner +from gooddata_eval.core.agentic import _langfuse, _trace_linker +from gooddata_eval.core.agentic._trace_linker import ( + BackgroundTraceLinker, + run_trace_link_inline, + warn_from_worker, +) + + +def test_submit_returns_before_the_task_completes(): + # The whole point: an item must not sit waiting on its own Langfuse poll. `submit` + # hands the work off and returns, so the next item's agent call starts immediately. + release = threading.Event() + completed = threading.Event() + + def task() -> None: + release.wait(timeout=5) + completed.set() + + linker = BackgroundTraceLinker(max_workers=1) + linker.submit(task, item_id="item-1") + + assert not completed.is_set() # submit did not block on the task + + release.set() + linker.drain() + assert completed.is_set() + + +def test_drain_waits_for_every_submitted_task(): + # Scores must be final before the CLI prints its table or sets an exit code, so the + # runner drains rather than letting the pool die with the process. + done: list[int] = [] + linker = BackgroundTraceLinker(max_workers=2) + + for i in range(5): + linker.submit(lambda i=i: done.append(i), item_id=f"item-{i}") + linker.drain() + + assert sorted(done) == [0, 1, 2, 3, 4] + + +def test_a_failing_task_neither_propagates_nor_stops_the_others(): + # Existing invariant across every Langfuse writer in this package: a Langfuse failure + # warns, it never aborts the eval run. Backgrounding the work must not change that -- + # and an exception swallowed inside a worker must not strand the remaining tasks. + done: list[str] = [] + + def boom() -> None: + raise RuntimeError("langfuse down") + + linker = BackgroundTraceLinker(max_workers=1) + linker.submit(boom, item_id="bad") + linker.submit(lambda: done.append("good"), item_id="ok") + linker.drain() # must not raise + + assert done == ["good"] + + +def test_records_each_task_duration_under_its_item_id(): + # Per-item Langfuse cost stays observable even though it is off the critical path -- + # that is the "independently measurable" half of the goal, as distinct from "faster". + clock = iter([10.0, 12.5]).__next__ + linker = BackgroundTraceLinker(max_workers=1, clock=clock) + + linker.submit(lambda: None, item_id="item-1") + linker.drain() + + assert linker.durations == {"item-1": 2.5} + + +def test_duration_is_recorded_even_when_the_task_fails(): + # A poll that exhausts its retries and then fails to score is exactly the case whose + # cost we most want on the report; losing the timing there would hide the worst items. + def boom() -> None: + raise RuntimeError("langfuse down") + + linker = BackgroundTraceLinker(max_workers=1, clock=iter([1.0, 4.0]).__next__) + linker.submit(boom, item_id="item-1") + linker.drain() + + assert linker.durations == {"item-1": 3.0} + + +def test_run_trace_link_inline_runs_the_task_on_the_calling_thread(): + # The default every evaluate_agentic_* keeps when no linker is injected, so direct + # callers (and every existing test) see today's synchronous behavior unchanged. + caller = threading.current_thread() + ran_on: list[threading.Thread] = [] + + run_trace_link_inline(lambda: ran_on.append(threading.current_thread())) + + assert ran_on == [caller] + + +# Every agentic kind reachable from cli.agentic_runner._dispatch_agentic. Kept as an +# explicit list, with the staleness guard below, so a kind added later cannot quietly +# skip the linker and go back to blocking its item on a Langfuse poll. +_EVALUATE_FUNCS = [ + ("general_question", "evaluate_agentic_general_question"), + ("guardrail", "evaluate_agentic_guardrail"), + ("metric_skill", "evaluate_agentic_metric_skill"), + ("alert_skill", "evaluate_agentic_alert_skill"), + ("search_tool", "evaluate_agentic_search_tool"), + ("visualization", "evaluate_agentic_visualization"), + ("kda_skill", "evaluate_agentic_kda_skill"), + ("conversation", "evaluate_agentic_conversation"), +] + + +def test_evaluate_funcs_covers_every_function_dispatch_can_call(): + dispatched = {n for n in dir(agentic_runner) if n.startswith("evaluate_agentic_")} + assert {name for _, name in _EVALUATE_FUNCS} == dispatched + + +@pytest.mark.parametrize(("module_name", "func_name"), _EVALUATE_FUNCS) +def test_every_evaluate_agentic_takes_a_trace_linker_defaulting_to_inline(module_name, func_name): + module = importlib.import_module(f"gooddata_eval.core.agentic.{module_name}") + param = inspect.signature(getattr(module, func_name)).parameters.get("submit_trace_link") + + assert param is not None, f"{func_name} still blocks its item on Langfuse trace linking" + # Defaulting to inline is what keeps direct callers behaviour-compatible. + assert param.default is run_trace_link_inline + + +def _callee_name(call: ast.Call) -> str: + """Trailing name of a call target: `now` for both `now(...)` and `_dt.now(...)`.""" + if isinstance(call.func, ast.Attribute): + return call.func.attr + if isinstance(call.func, ast.Name): + return call.func.id + return "" + + +def _deferred_blocks(module_name: str) -> list[ast.FunctionDef]: + """The kind's scoring closure -- the part that runs on the linker's thread, not the item's.""" + module = importlib.import_module(f"gooddata_eval.core.agentic.{module_name}") + tree = ast.parse(inspect.getsource(module)) + return [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "_write_scores"] + + +def _clock_reads(node: ast.AST) -> list[ast.Call]: + return [ + n + for n in ast.walk(node) + if isinstance(n, ast.Call) and _callee_name(n) in ("now", "utcnow", "utc_now", "monotonic") + ] + + +@pytest.mark.parametrize(("module_name", "_func_name"), _EVALUATE_FUNCS) +def test_every_kind_hands_a_pinned_window_to_the_linker(module_name, _func_name): + """The query window must be captured on the calling thread, not inside the deferred task. + + Checked structurally because the failure is silent: a kind that reads the clock once its + task is already running still works, it just queries a window that widened by however + long the task sat in the pool. Nothing fails loudly, the scores just go missing. + """ + module = importlib.import_module(f"gooddata_eval.core.agentic.{module_name}") + tree = ast.parse(inspect.getsource(module)) + + submits = [n for n in ast.walk(tree) if isinstance(n, ast.Call) and _callee_name(n) == "submit_trace_scoring"] + assert submits, f"{module_name} no longer defers its Langfuse block to the linker" + for call in submits: + assert any(kw.arg == "window_end" for kw in call.keywords), ( + f"{module_name} leaves window_end to drift to the task's run time" + ) + + blocks = _deferred_blocks(module_name) + assert blocks, f"{module_name} has no deferred _write_scores block" + for block in blocks: + assert not _clock_reads(block), ( + f"{module_name}'s _write_scores reads the clock itself, so anything it derives from " + f"that widens with however long the task waited in the pool" + ) + + +@pytest.mark.parametrize(("module_name", "_func_name"), _EVALUATE_FUNCS) +def test_every_kind_captures_the_window_before_deferring(module_name, _func_name): + # The other half: window_end must actually be a captured timestamp. Passing a name that + # nothing ever assigns would satisfy the check above while sending None to the query. + module = importlib.import_module(f"gooddata_eval.core.agentic.{module_name}") + tree = ast.parse(inspect.getsource(module)) + blocks = _deferred_blocks(module_name) + assert blocks, f"{module_name} has no deferred _write_scores block" + inside = {id(n) for block in blocks for n in ast.walk(block)} + + pins = [ + n + for n in ast.walk(tree) + if isinstance(n, ast.Assign) + and id(n) not in inside + and any(isinstance(t, ast.Name) and t.id == "window_end" for t in n.targets) + and isinstance(n.value, ast.Call) + and _callee_name(n.value) in ("now", "utcnow", "utc_now") + ] + assert pins, f"{module_name} never captures window_end from the clock outside _write_scores" + + +def test_the_linker_polls_the_window_it_was_given_instead_of_reading_the_clock(): + """The single place the deferred poll is now issued, so this invariant lives here once. + + Every kind pins its own window (above) and hands it over; if ``submit_trace_scoring`` + then re-read the clock, all eight would silently widen again. + """ + tree = ast.parse(inspect.getsource(_trace_linker)) + blocks = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "_link_traces"] + assert blocks, "submit_trace_scoring no longer defers the trace lookup" + + for block in blocks: + assert not _clock_reads(block), "the deferred lookup reads the clock instead of the pinned window" + lookups = [ + n for n in ast.walk(block) if isinstance(n, ast.Call) and _callee_name(n) == "find_traces_per_conversation" + ] + assert lookups, "the deferred task does not look up traces at all" + for call in lookups: + assert len(call.args) >= 4 or any(kw.arg == "window_end" for kw in call.keywords), ( + "the deferred lookup drops window_end, so its query window drifts to run time" + ) + + +def test_warn_from_worker_emits_each_message_as_a_single_write(): + # print() writes the text and the newline separately. Trace-link warnings now come + # from worker threads while the main thread is printing per-item progress, so two + # writes can interleave and split a progress line in half. One write cannot. + writes: list[str] = [] + + class _Stream: + def write(self, s: str) -> None: + writes.append(s) + + def flush(self) -> None: + pass + + with patch("sys.stdout", _Stream()): + warn_from_worker("[langfuse] WARNING: no trace found for conversation abc") + + assert writes == ["[langfuse] WARNING: no trace found for conversation abc\n"] + + +def test_submitted_links_do_not_run_until_drain(): + # Batched deliberately. A poll fired immediately after its own conversation is the one + # most likely to miss, because Langfuse ingestion lag (35s to minutes on us.cloud) has + # barely started. Holding the whole queue until the agent phase is over gives every + # trace the length of the run to be ingested, so early items hit on their first attempt + # instead of burning their retry budget. + ran: list[str] = [] + linker = BackgroundTraceLinker() + + linker.submit(lambda: ran.append("a"), item_id="a") + assert linker.pending == 1, "the link did not stay queued for the batch" + assert ran == [], "the link started on submit instead of waiting for the batch" + + linker.drain() + assert ran == ["a"] + + +def test_drain_runs_the_whole_queue_in_parallel(): + # Every queued link blocks until all of them have started, which only completes if the + # pool is wide enough to run them together. One trace poll is almost entirely waiting, + # so running them one at a time would multiply the run's tail by the item count. + n = 12 + barrier = threading.Barrier(n, timeout=5) + ok: list[int] = [] + linker = BackgroundTraceLinker() + + for i in range(n): + linker.submit(lambda: (barrier.wait(), ok.append(1)), item_id=f"item-{i}") + linker.drain() + + assert len(ok) == n + + +def test_an_interrupt_mid_drain_cancels_the_polls_that_have_not_started(): + """The drain pool needs the same cancellation the item pool got. + + A trace poll is almost entirely time.sleep, so a `with ThreadPoolExecutor` here means + an interrupt arriving mid-batch waits out ceil(N/_MAX_WORKERS) x _LINK_BUDGET_SEC of + pure retry sleeping before it surfaces. abandon() cannot help: drain() has already + moved the queue into a local by then, so the cancellation has to be on the pool. + + The interrupt has to land in the thread that is WAITING on the batch, which is where a + real Ctrl-C lands, so it is injected into shutdown(wait=True) rather than into a task. + """ + release = threading.Event() + lock = threading.Lock() + started: list[str] = [] + shutdowns: list[dict] = [] + pools: list[ThreadPoolExecutor] = [] + real_shutdown = ThreadPoolExecutor.shutdown + + def fake_shutdown(self, wait=True, *, cancel_futures=False): + pools.append(self) + shutdowns.append({"wait": wait, "cancel_futures": cancel_futures}) + if len(shutdowns) == 1: + raise KeyboardInterrupt + return real_shutdown(self, wait=wait, cancel_futures=cancel_futures) + + def poll(item_id: str) -> None: + with lock: + started.append(item_id) + release.wait(timeout=0.5) + + linker = BackgroundTraceLinker(max_workers=2) + for i in range(6): + linker.submit(lambda i=i: poll(f"item-{i}"), item_id=f"item-{i}") + + with patch.object(ThreadPoolExecutor, "shutdown", fake_shutdown), pytest.raises(KeyboardInterrupt): + linker.drain() + + # The interrupt must be answered with a cancelling shutdown, not swallowed or re-waited. + assert len(shutdowns) == 2, f"the interrupt was not answered with a second shutdown: {shutdowns}" + assert shutdowns[1] == {"wait": False, "cancel_futures": True} + + release.set() + # Join the pool for real rather than sleeping and sampling: this is precisely what would + # let a still-queued poll start, so if none has after it, none ever will. + real_shutdown(pools[0], wait=True) + with lock: + ran = list(started) + assert len(ran) <= 2, f"a queued poll ran after the drain was interrupted: {ran}" + + +def test_drain_still_waits_for_the_whole_batch_when_nothing_interrupts_it(): + # The cancellation path must not have made the normal path lossy: scores still have to + # be final before the runner reports. + done: list[int] = [] + linker = BackgroundTraceLinker(max_workers=3) + for i in range(9): + linker.submit(lambda i=i: done.append(i), item_id=f"item-{i}") + + linker.drain() + + assert sorted(done) == list(range(9)) + + +def test_abandon_drops_the_queue_so_a_later_drain_runs_nothing(): + """Direct coverage of abandon(), which had none. + + The interrupt test below reaches abandon() only via a path where drain() is never + called, so `ran == []` holds there whether abandon works or not -- replacing its body + with `pass` left the whole suite green. This fails immediately if it stops clearing. + """ + ran: list[str] = [] + linker = BackgroundTraceLinker() + linker.submit(lambda: ran.append("a"), item_id="a") + linker.submit(lambda: ran.append("b"), item_id="b") + assert linker.pending == 2 + + linker.abandon() + + assert linker.pending == 0, "abandon() left the queue in place" + linker.drain() # nothing should be left for it to run + assert ran == [], "a task survived abandon() and ran on the next drain" + + +def test_abandon_on_an_empty_queue_is_harmless(): + linker = BackgroundTraceLinker() + linker.abandon() + linker.abandon() + assert linker.pending == 0 + + +def test_no_cancellation_signal_exists_outside_a_drain(): + """Scoped to one batch on purpose. + + A module-level Event left permanently in place would let one run's Ctrl-C stop the next + --model pass's polls before they started. + """ + assert _trace_linker.link_cancel_event() is None + + seen: list[object] = [] + linker = BackgroundTraceLinker(max_workers=1) + linker.submit(lambda: seen.append(_trace_linker.link_cancel_event()), item_id="a") + linker.drain() + + assert seen and seen[0] is not None, "a running poll cannot be told to stop" + assert not seen[0].is_set(), "an uninterrupted drain must not look cancelled" + assert _trace_linker.link_cancel_event() is None, "the signal outlived its drain" + + +def test_an_interrupt_signals_cancellation_before_it_shuts_the_pool_down(): + """Order matters: cancel_futures only drops what has not started. + + A poll already running sits in find_traces_per_conversation's backoff, and the + interpreter joins executor workers at exit -- so without this signal a Ctrl-C waits out + the rest of the batch budget (measured: 110s, against 0.5s with it). + """ + events: list[str] = [] + started = threading.Event() + release = threading.Event() + seen: list[threading.Event | None] = [] + real_shutdown = ThreadPoolExecutor.shutdown + + def poll() -> None: + # The signal is visible on the worker's own thread, so the worker reports it. + seen.append(_trace_linker.link_cancel_event()) + started.set() + release.wait(timeout=5) + + def fake_shutdown(self, wait=True, *, cancel_futures=False): + assert started.wait(timeout=5), "the poll never started" + cancel = seen[0] + events.append( + f"shutdown(wait={wait},cancel_futures={cancel_futures},signalled={cancel is not None and cancel.is_set()})" + ) + if wait: + raise KeyboardInterrupt + return real_shutdown(self, wait=wait, cancel_futures=cancel_futures) + + linker = BackgroundTraceLinker(max_workers=1) + linker.submit(poll, item_id="a") + try: + with patch.object(ThreadPoolExecutor, "shutdown", fake_shutdown), pytest.raises(KeyboardInterrupt): + linker.drain() + finally: + release.set() + + assert events == [ + "shutdown(wait=True,cancel_futures=False,signalled=False)", + "shutdown(wait=False,cancel_futures=True,signalled=True)", + ], events + + +def test_a_cancelled_poll_stops_instead_of_finishing_its_retry_ladder(): + """The behaviour the signal buys, measured in sleep rather than wall time.""" + slept: list[float] = [] + + class _Clock: + now = 1000.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + slept.append(seconds) + self.now += seconds + + cancelled = threading.Event() + cancelled.set() + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", return_value=[]), + patch("gooddata_eval.core.agentic._langfuse.time", _Clock()), + patch("gooddata_eval.core.agentic._langfuse.link_cancel_event", return_value=cancelled), + patch("gooddata_eval.core.agentic._langfuse.warn_from_worker"), + ): + result = _langfuse.find_traces_per_conversation(MagicMock(), ["c1", "c2"], datetime.now(timezone.utc)) + + assert sum(slept) == 0.0, f"a cancelled poll kept sleeping: {slept}" + # Every conversation still reports, so callers see None rather than a missing key. + assert set(result) == {"c1", "c2"} + + +def test_cancelling_mid_backoff_stops_the_poll_without_finishing_the_sleep(): + """The sliced wait itself, not the check at the top of the conversation loop. + + A poll that is already asleep is the case that hung: cancel_futures cannot touch it and + the interpreter joins the worker at exit. Measured end to end, this was 110s before the + wait was served in slices and 0.5s after -- so the test pins the mechanism, by cancelling + only once the poll is already sleeping. + """ + cancelled = threading.Event() + slept: list[float] = [] + + class _Clock: + now = 1000.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + slept.append(seconds) + self.now += seconds + cancelled.set() # the interrupt lands while this poll is mid-backoff + + with ( + patch("gooddata_eval.core.agentic._langfuse._fetch_traces_for_session", return_value=[]), + patch("gooddata_eval.core.agentic._langfuse.time", _Clock()), + patch("gooddata_eval.core.agentic._langfuse.link_cancel_event", return_value=cancelled), + patch("gooddata_eval.core.agentic._langfuse.warn_from_worker"), + ): + _langfuse.find_traces_per_conversation(MagicMock(), ["c1"], datetime.now(timezone.utc)) + + # One slice, then it noticed. Without slicing it would sleep the whole backoff ladder up + # to _LINK_BUDGET_SEC before looking at the flag again. + assert slept == [_langfuse._CANCEL_CHECK_SEC], slept + + +def test_an_interrupted_drain_leaves_the_cancellation_visible_to_late_workers(): + """shutdown(wait=False) returns before the workers notice. + + A worker that reaches its next wait only after drain() has already raised must still + read a SET event, or it goes back to an uninterruptible sleep for the rest of its budget. + The signal lives on the worker's own thread, so nothing is left behind on the caller's: + the next drain starts clean. + """ + started = threading.Event() + release = threading.Event() + done = threading.Event() + late: list[bool] = [] + real_shutdown = ThreadPoolExecutor.shutdown + + def poll() -> None: + started.set() + release.wait(timeout=5) # still running when drain() raises + cancel = _trace_linker.link_cancel_event() + late.append(cancel is not None and cancel.is_set()) + done.set() + + def fake_shutdown(self, wait=True, *, cancel_futures=False): + if wait: + assert started.wait(timeout=5), "the poll never started" + raise KeyboardInterrupt + return real_shutdown(self, wait=wait, cancel_futures=cancel_futures) + + linker = BackgroundTraceLinker(max_workers=1) + linker.submit(poll, item_id="a") + with patch.object(ThreadPoolExecutor, "shutdown", fake_shutdown), pytest.raises(KeyboardInterrupt): + linker.drain() + + assert _trace_linker.link_cancel_event() is None, "the interrupt leaked onto the calling thread" + release.set() + assert done.wait(timeout=5), "the late worker never finished" + assert late == [True], "a late worker stopped seeing the interrupt" + + # And the next drain is unaffected by it. + fresh: list[bool] = [] + second = BackgroundTraceLinker(max_workers=1) + second.submit(lambda: fresh.append(_trace_linker.link_cancel_event().is_set()), item_id="b") + second.drain() + assert fresh == [False], "the previous run's interrupt leaked into this drain" + assert _trace_linker.link_cancel_event() is None + + +def test_an_interrupted_drain_does_not_cancel_a_later_inline_link(): + """A caller that survives the interrupt -- the test suite, or a library user catching + KeyboardInterrupt -- must still be able to link inline afterwards. Kept in a module + global, the set event was read by run_trace_link_inline, and + find_traces_per_conversation broke before its first fetch, orphaning every score. + """ + real_shutdown = ThreadPoolExecutor.shutdown + + def fake_shutdown(self, wait=True, *, cancel_futures=False): + if wait: + raise KeyboardInterrupt + return real_shutdown(self, wait=wait, cancel_futures=cancel_futures) + + linker = BackgroundTraceLinker(max_workers=1) + linker.submit(lambda: None, item_id="a") + with patch.object(ThreadPoolExecutor, "shutdown", fake_shutdown), pytest.raises(KeyboardInterrupt): + linker.drain() + + seen: list[threading.Event | None] = [] + run_trace_link_inline(lambda: seen.append(_trace_linker.link_cancel_event())) + + assert seen == [None], "an inline link inherited the interrupted drain's cancellation" diff --git a/uv.lock b/uv.lock index aee60fd3f..9bf95f352 100644 --- a/uv.lock +++ b/uv.lock @@ -866,7 +866,7 @@ test = [ requires-dist = [ { name = "gooddata-sdk", editable = "packages/gooddata-sdk" }, { name = "httpx", specifier = ">=0.27,<1.0" }, - { name = "openai", marker = "extra == 'llm-judge'", specifier = ">=1.40,<2.0" }, + { name = "openai", marker = "extra == 'llm-judge'", specifier = ">=1.45,<2.0" }, { name = "orjson", specifier = ">=3.9.15,<4.0.0" }, { name = "pydantic", specifier = ">=2.6,<3.0" }, { name = "rich", specifier = ">=13.0,<15.0" },