From 0c3e65ec620c4db3c4ec0628105473d243b0cf16 Mon Sep 17 00:00:00 2001 From: John Pangas Date: Wed, 19 Aug 2026 02:52:35 -0600 Subject: [PATCH] Submit the agent's execution results to TestRail Submitting a test plan created a suite of cases and stopped there, so TestRail recorded what the agent generated but never what happened when it ran them, leaving "Test Runs and Results" empty. After the cases are created, create a run over exactly those case ids and post one result per case. Case statuses map onto TestRail's own: passed and failed match by name, and unsuitable posts as Blocked, the closest status a result can carry. The status ids are resolved before anything is created, so an unmapped status fails the submission rather than leaving a half-populated suite behind. The action's summary becomes the run description, and the case summary and failure reason become the result comment. The run id joins the other created ids on the action result. Fixes #6436 --- .../actions/handlers/testrail_handler.py | 90 +++++++++++++- .../tests/test_testrail_handler.py | 114 +++++++++++++++++- .../testrail-client/testrail_client/client.py | 19 +++ libs/testrail-client/tests/test_client.py | 19 +++ 4 files changed, 238 insertions(+), 4 deletions(-) diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py index 25227bc207..bbdd361ce5 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +from datetime import UTC, datetime from functools import lru_cache from typing import Any @@ -16,6 +17,7 @@ _CASE_TEMPLATE_NAME = "Test Case (Steps)" _CASE_LABEL = "AI Generated" _SECTION_NAME = "Test Cases" +_CASE_STATUSES = ("passed", "failed", "unsuitable") @lru_cache(maxsize=1) @@ -65,6 +67,22 @@ async def _resolve_template_id(client: TestRailClient) -> int: raise RuntimeError(f'TestRail has no template named "{_CASE_TEMPLATE_NAME}"') +async def _resolve_status_ids(client: TestRailClient) -> dict[str, int]: + """Case status -> id of the TestRail status it is posted as. + + An unsuitable case is posted as Blocked, the closest status a result can + carry; the other two match TestRail's own status names. + """ + response = await client.get_statuses() + statuses = response.get("statuses", []) if isinstance(response, dict) else response + ids = {status["name"]: int(status["id"]) for status in statuses} + return { + "passed": ids["passed"], + "failed": ids["failed"], + "unsuitable": ids["blocked"], + } + + def _case_payload( test_case: dict[str, Any], case_type_id: int, @@ -92,20 +110,79 @@ def _separated_steps(test_case: dict[str, Any]) -> list[dict[str, str]]: ] +def _run_name() -> str: + stamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC") + return f"[Hackbot] Run Results - {stamp}" + + +def _result_comment(result: dict[str, Any]) -> str: + summary = str(result.get("summary") or "") + failure_reason = str(result.get("failure_reason") or "") + if failure_reason: + return f"{summary}\n\nFailure reason:\n{failure_reason}" + return summary + + +def _executed_results( + params: dict[str, Any], + created_case_ids: dict[int, int], + status_ids: dict[str, int], +) -> list[dict[str, Any]]: + """One TestRail result per generated case, in generation order. + + ``SubmitTestPlanHandler.apply`` has already checked that every case carries a + recognised status, so indexing here cannot silently drop one. + """ + return [ + { + "case_id": created_case_ids[int(test_case["id"])], + "status_id": status_ids[test_case["result"]["status"]], + "comment": _result_comment(test_case["result"]), + } + for test_case in params["generated_test_cases"] + ] + + +def _run_payload( + suite_id: int, case_ids: list[int], description: str +) -> dict[str, Any]: + payload: dict[str, Any] = { + "name": _run_name(), + "include_all": False, + "case_ids": case_ids, + "suite_id": suite_id, + } + if description: + payload["description"] = description + return payload + + class SubmitTestPlanHandler: async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult: feature = str(params.get("feature") or "").strip() test_cases = params.get("generated_test_cases") or [] + summary = str(params.get("summary") or "") if not feature: return ActionResult.failed("TestRail submission requires a feature name") if not test_cases: return ActionResult.failed("TestRail submission requires test cases") + if any(not test_case.get("result") for test_case in test_cases): + return ActionResult.failed("TestRail submission requires execution results") + if any( + str(test_case["result"].get("status") or "") not in _CASE_STATUSES + for test_case in test_cases + ): + return ActionResult.failed( + "TestRail submission requires all cases to have executed results" + ) try: client = _client() case_type_id = await _resolve_case_type_id(client) template_id = await _resolve_template_id(client) + # Before anything is created, so a missing status leaves no suite behind. + status_ids = await _resolve_status_ids(client) suite_name = f"[Hackbot] - {feature}" suite_id = _require_id( await client.add_suite(suite_name), @@ -132,6 +209,16 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult ) created_case_ids[generated_id] = case_id + case_ids = list(created_case_ids.values()) + run_id = _require_id( + await client.add_run(_run_payload(suite_id, case_ids, summary)), + "run", + ) + await client.add_results_for_cases( + run_id, + _executed_results(params, created_case_ids, status_ids), + ) + except Exception as exc: log.exception( "Failed to submit test plan to TestRail for run %s", ctx.run_id @@ -143,6 +230,7 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult "suite_id": suite_id, "url": client.suite_url(suite_id), "section_id": section_id, - "case_ids": list(created_case_ids.values()), + "case_ids": case_ids, + "run_id": run_id, } ) diff --git a/libs/hackbot-runtime/tests/test_testrail_handler.py b/libs/hackbot-runtime/tests/test_testrail_handler.py index 3640b65673..c7be486867 100644 --- a/libs/hackbot-runtime/tests/test_testrail_handler.py +++ b/libs/hackbot-runtime/tests/test_testrail_handler.py @@ -52,6 +52,17 @@ def _plan(): } +# Trimmed GET get_statuses: matched on the lowercase ``name``, not ``label``. +_STATUSES = [ + {"id": 1, "name": "passed", "label": "Passed"}, + {"id": 2, "name": "blocked", "label": "Blocked"}, + {"id": 3, "name": "untested", "label": "Untested"}, + {"id": 4, "name": "retest", "label": "Retest"}, + {"id": 5, "name": "failed", "label": "Failed"}, + {"id": 6, "name": "custom_status1", "label": "Needs Triage"}, +] + + class _FakeClient: def __init__(self, responses=None): self.calls = [] @@ -60,10 +71,13 @@ def __init__(self, responses=None): or [ [{"id": 6, "name": "Functional"}], [{"id": 2, "name": "Test Case (Steps)"}], + _STATUSES, {"id": 10}, {"id": 20}, {"id": 101}, {"id": 102}, + {"id": 301}, + {"results": [{"id": 401}, {"id": 402}]}, ] ) @@ -81,6 +95,10 @@ async def get_templates(self): self.calls.append(("get_templates",)) return self._next() + async def get_statuses(self): + self.calls.append(("get_statuses",)) + return self._next() + async def add_suite(self, name): self.calls.append(("add_suite", name)) return self._next() @@ -93,6 +111,14 @@ async def add_case(self, section_id, payload): self.calls.append(("add_case", section_id, payload)) return self._next() + async def add_run(self, payload): + self.calls.append(("add_run", payload)) + return self._next() + + async def add_results_for_cases(self, run_id, results): + self.calls.append(("add_results_for_cases", run_id, results)) + return self._next() + def suite_url(self, suite_id): return f"https://testrail.example/index.php?/suites/view/{suite_id}" @@ -109,14 +135,16 @@ async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch): "url": "https://testrail.example/index.php?/suites/view/10", "section_id": 20, "case_ids": [101, 102], + "run_id": 301, } assert client.calls[0] == ("get_case_types",) assert client.calls[1] == ("get_templates",) - assert client.calls[2] == ("add_suite", "[Hackbot] - PDF Improvements") - assert client.calls[3] == ("add_section", 10, "Test Cases") + assert client.calls[2] == ("get_statuses",) + assert client.calls[3] == ("add_suite", "[Hackbot] - PDF Improvements") + assert client.calls[4] == ("add_section", 10, "Test Cases") - first_case = client.calls[4] + first_case = client.calls[5] assert first_case[0:2] == ("add_case", 20) assert first_case[2] == { "title": "The PDF opens", @@ -133,6 +161,63 @@ async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch): ], } + add_run_call = client.calls[7] + assert add_run_call[0] == "add_run" + assert add_run_call[1]["include_all"] is False + assert add_run_call[1]["case_ids"] == [101, 102] + assert add_run_call[1]["suite_id"] == 10 + assert add_run_call[1]["name"].startswith("[Hackbot] Run Results - ") + assert add_run_call[1]["description"] == "One passed and one was unsuitable." + + assert client.calls[8] == ( + "add_results_for_cases", + 301, + [ + { + "case_id": 101, + "status_id": 1, + "comment": "The PDF behaved as expected.", + }, + { + "case_id": 102, + "status_id": 2, + "comment": ( + "The toolbar could not be inspected.\n\n" + "Failure reason:\nNo available tool can inspect it." + ), + }, + ], + ) + + +async def test_submit_test_plan_requires_execution_results(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + plan = _plan() + del plan["generated_test_cases"][0]["result"] + + result = await testrail_handler.SubmitTestPlanHandler().apply(plan, _ctx()) + + assert result.status == "failed" + assert result.error == "TestRail submission requires execution results" + assert client.calls == [] + + +async def test_submit_test_plan_requires_all_cases_to_be_executed(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + plan = _plan() + plan["generated_test_cases"][1]["result"]["status"] = "not_run" + + result = await testrail_handler.SubmitTestPlanHandler().apply(plan, _ctx()) + + assert result.status == "failed" + assert ( + result.error + == "TestRail submission requires all cases to have executed results" + ) + assert client.calls == [] + def test_separated_steps_maps_expectations_from_step_objects(): assert testrail_handler._separated_steps( @@ -150,6 +235,29 @@ def test_separated_steps_maps_expectations_from_step_objects(): ] +async def test_submit_test_plan_fails_when_the_run_cannot_be_created(monkeypatch): + # The whole submission fails even though the suite and cases were created, so + # a retry duplicates the suite. Accepted for now; see the de-duplication work. + client = _FakeClient( + responses=[ + [{"id": 6, "name": "Functional"}], + [{"id": 2, "name": "Test Case (Steps)"}], + _STATUSES, + {"id": 10}, + {"id": 20}, + {"id": 101}, + {"id": 102}, + RuntimeError("TestRail rejected the run"), + ] + ) + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + + result = await testrail_handler.SubmitTestPlanHandler().apply(_plan(), _ctx()) + + assert result.status == "failed" + assert result.error == "TestRail rejected the run" + + async def test_submit_test_plan_reports_api_failure(monkeypatch): client = _FakeClient(responses=[RuntimeError("TestRail is unavailable")]) monkeypatch.setattr(testrail_handler, "_client", lambda: client) diff --git a/libs/testrail-client/testrail_client/client.py b/libs/testrail-client/testrail_client/client.py index c8634d8cbb..87a0b1bdac 100644 --- a/libs/testrail-client/testrail_client/client.py +++ b/libs/testrail-client/testrail_client/client.py @@ -48,6 +48,9 @@ async def get_case_types(self) -> dict[str, Any] | list[Any]: async def get_templates(self) -> dict[str, Any] | list[Any]: return await self.request("GET", f"get_templates/{self.settings.project_id}") + async def get_statuses(self) -> dict[str, Any] | list[Any]: + return await self.request("GET", "get_statuses") + async def add_suite(self, name: str) -> dict[str, Any] | list[Any]: return await self.request( "POST", @@ -66,3 +69,19 @@ async def add_case( self, section_id: int, payload: dict[str, Any] ) -> dict[str, Any] | list[Any]: return await self.request("POST", f"add_case/{section_id}", payload) + + async def add_run(self, payload: dict[str, Any]) -> dict[str, Any] | list[Any]: + return await self.request( + "POST", + f"add_run/{self.settings.project_id}", + payload, + ) + + async def add_results_for_cases( + self, run_id: int, results: list[dict[str, Any]] + ) -> dict[str, Any] | list[Any]: + return await self.request( + "POST", + f"add_results_for_cases/{run_id}", + {"results": results}, + ) diff --git a/libs/testrail-client/tests/test_client.py b/libs/testrail-client/tests/test_client.py index 850cac30fd..3728adb560 100644 --- a/libs/testrail-client/tests/test_client.py +++ b/libs/testrail-client/tests/test_client.py @@ -110,6 +110,9 @@ async def test_endpoint_wrappers(monkeypatch): await client.get_templates() assert captured["url"].endswith("/get_templates/73") + await client.get_statuses() + assert captured["url"].endswith("/get_statuses") + await client.add_suite("Suite") assert captured["url"].endswith("/add_suite/73") assert captured["json"] == {"name": "Suite"} @@ -122,6 +125,22 @@ async def test_endpoint_wrappers(monkeypatch): assert captured["url"].endswith("/add_case/20") assert captured["json"] == {"title": "Case"} + await client.add_run({"name": "Run", "include_all": False, "case_ids": [101]}) + assert captured["url"].endswith("/add_run/73") + assert captured["json"] == { + "name": "Run", + "include_all": False, + "case_ids": [101], + } + + await client.add_results_for_cases( + 301, [{"case_id": 101, "status_id": 1, "comment": "ok"}] + ) + assert captured["url"].endswith("/add_results_for_cases/301") + assert captured["json"] == { + "results": [{"case_id": 101, "status_id": 1, "comment": "ok"}] + } + def test_suite_url_default_base(): assert (