From 79a3d1de0c0ac9e69ec6b4999c5defd40d2fee31 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 27 Aug 2026 12:05:08 +0200 Subject: [PATCH 1/2] feat(gooddata-eval): register knowledge_question evaluator knowledge_question had reclassified fixtures (commit 658b9ea6) but no registered evaluator, so get_evaluator() raised KeyError and every item was silently skipped -- data/test_kinds.yaml had to keep it disabled. Reuses GeneralQuestionEvaluator directly: both are free-text-rubric, LLM-judged prose answers, and ItemReport.test_kind is tagged from the dataset item's own field rather than the evaluator class, so sharing one class across both kinds doesn't mislabel results. --- .../gooddata_eval/core/evaluators/__init__.py | 12 ++++++-- packages/gooddata-eval/tests/test_runner.py | 1 + .../tests/test_text_evaluators.py | 28 +++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py index f2b71955a..9ef6816ab 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py @@ -20,18 +20,26 @@ ) } -# LLM-judge evaluators (general_question, guardrail, dashboard_summary) require the -# [llm-judge] extra. Their modules are imported lazily on first use so the CLI +# LLM-judge evaluators (general_question, guardrail, dashboard_summary, knowledge_question) +# require the [llm-judge] extra. Their modules are imported lazily on first use so the CLI # starts without openai. +# +# knowledge_question reuses GeneralQuestionEvaluator directly: both are free-text-rubric, +# LLM-judged prose answers -- knowledge_question just covers platform/product/policy facts +# instead of LDM-grounded ones. `ItemReport.test_kind` is tagged from the dataset item's own +# `test_kind` field (see runner.py), not from the evaluator class, so sharing one class +# across both kinds does not mislabel results. _LAZY_EVALUATOR_MODULES: dict[str, str] = { "general_question": "gooddata_eval.core.evaluators.general_question", "guardrail": "gooddata_eval.core.evaluators.guardrail", "dashboard_summary": "gooddata_eval.core.evaluators.summary", + "knowledge_question": "gooddata_eval.core.evaluators.general_question", } _LAZY_EVALUATOR_CLASSES: dict[str, str] = { "general_question": "GeneralQuestionEvaluator", "guardrail": "GuardrailEvaluator", "dashboard_summary": "DashboardSummaryEvaluator", + "knowledge_question": "GeneralQuestionEvaluator", } diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index 73fee1eb7..64c5c3567 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -122,6 +122,7 @@ def test_run_items_routes_all_supported_kinds(): "general_question", "guardrail", "dashboard_summary", + "knowledge_question", } assert expected_kinds == supported_test_kinds() diff --git a/packages/gooddata-eval/tests/test_text_evaluators.py b/packages/gooddata-eval/tests/test_text_evaluators.py index 7e13082ca..c465c0237 100644 --- a/packages/gooddata-eval/tests/test_text_evaluators.py +++ b/packages/gooddata-eval/tests/test_text_evaluators.py @@ -1,6 +1,7 @@ # (C) 2026 GoodData Corporation from unittest.mock import MagicMock, patch +from gooddata_eval.core.evaluators import get_evaluator from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError from gooddata_eval.core.evaluators.general_question import GeneralQuestionEvaluator from gooddata_eval.core.evaluators.guardrail import GuardrailEvaluator @@ -17,6 +18,16 @@ def _gq_item() -> DatasetItem: ) +def _kq_item() -> DatasetItem: + return DatasetItem( + id="kq-001", + dataset_name="d", + test_kind="knowledge_question", + question="How can I get access to more ICAs?", + expected_output="Must explain the process for requesting additional ICAs, e.g. contacting an account manager.", + ) + + def _gr_item() -> DatasetItem: return DatasetItem( id="gr-001", @@ -60,6 +71,23 @@ def test_general_question_fails_when_judge_scores_0(): assert result.passed is False +def test_knowledge_question_dispatches_to_general_question_evaluator(): + with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(True)): + assert isinstance(get_evaluator("knowledge_question"), GeneralQuestionEvaluator) + + +def test_knowledge_question_passes_when_judge_scores_1(): + with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(True)): + result = GeneralQuestionEvaluator().evaluate(_kq_item(), _chat_text("Contact your account manager.")) + assert result.passed is True + + +def test_knowledge_question_fails_when_judge_scores_0(): + with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(False)): + result = GeneralQuestionEvaluator().evaluate(_kq_item(), _chat_text("I don't know.")) + assert result.passed is False + + def test_guardrail_fails_when_viz_returned(): with patch("gooddata_eval.core.evaluators.guardrail.LLMJudge", return_value=_make_judge(True)): result = GuardrailEvaluator().evaluate(_gr_item(), _chat_viz()) From f213331295617b5530239c0210f928bab141fe9e Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 3 Sep 2026 16:25:02 +0200 Subject: [PATCH 2/2] docs(gooddata-eval): fix stale supported_test_kinds docstring, note dual test_kind use Addresses two of hkad98's non-blocking nits on #1763: - supported_test_kinds()'s docstring only listed (general_question, guardrail) as the LLM-judge kinds; dashboard_summary and knowledge_question were already excluded by the same check but never mentioned. - GeneralQuestionEvaluator.test_kind is now inaccurate for half its registrations (knowledge_question shares the class) -- added a pointer to the registry comment explaining why, so a future reader doesn't derive an output label from it without realizing. Third nit (no agentic_knowledge_question dispatch branch) left as scope, per the review comment's own "fine as scope, likely next request." --- .../src/gooddata_eval/core/evaluators/__init__.py | 6 +++--- .../src/gooddata_eval/core/evaluators/general_question.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py index 9ef6816ab..30b154821 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py @@ -65,9 +65,9 @@ def _openai_available() -> bool: def supported_test_kinds() -> set[str]: """Return all supported test_kind values. - LLM-judge kinds (general_question, guardrail) are excluded when the - [llm-judge] extra (openai) is not installed — those items are skipped - rather than erroring out mid-run. + LLM-judge kinds (general_question, guardrail, dashboard_summary, + knowledge_question) are excluded when the [llm-judge] extra (openai) is + not installed — those items are skipped rather than erroring out mid-run. """ kinds = set(_EAGER_EVALUATORS) if _openai_available(): diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py index ca2e10fed..dac16c01a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py @@ -15,6 +15,10 @@ class GeneralQuestionEvaluator: + # Also registered under "knowledge_question" (core/evaluators/__init__.py) -- this + # attribute is unused by that registration (which keys off item.test_kind, not this + # class attribute, see the comment there), so it's harmless as-is, but don't derive + # an output label from it without accounting for the second kind it now backs. test_kind = "general_question" def __init__(self):