diff --git a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py index 25227bc207..bbdd361ce5 100644 --- a/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py +++ b/libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +from datetime import UTC, datetime from functools import lru_cache from typing import Any @@ -16,6 +17,7 @@ _CASE_TEMPLATE_NAME = "Test Case (Steps)" _CASE_LABEL = "AI Generated" _SECTION_NAME = "Test Cases" +_CASE_STATUSES = ("passed", "failed", "unsuitable") @lru_cache(maxsize=1) @@ -65,6 +67,22 @@ async def _resolve_template_id(client: TestRailClient) -> int: raise RuntimeError(f'TestRail has no template named "{_CASE_TEMPLATE_NAME}"') +async def _resolve_status_ids(client: TestRailClient) -> dict[str, int]: + """Case status -> id of the TestRail status it is posted as. + + An unsuitable case is posted as Blocked, the closest status a result can + carry; the other two match TestRail's own status names. + """ + response = await client.get_statuses() + statuses = response.get("statuses", []) if isinstance(response, dict) else response + ids = {status["name"]: int(status["id"]) for status in statuses} + return { + "passed": ids["passed"], + "failed": ids["failed"], + "unsuitable": ids["blocked"], + } + + def _case_payload( test_case: dict[str, Any], case_type_id: int, @@ -92,20 +110,79 @@ def _separated_steps(test_case: dict[str, Any]) -> list[dict[str, str]]: ] +def _run_name() -> str: + stamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC") + return f"[Hackbot] Run Results - {stamp}" + + +def _result_comment(result: dict[str, Any]) -> str: + summary = str(result.get("summary") or "") + failure_reason = str(result.get("failure_reason") or "") + if failure_reason: + return f"{summary}\n\nFailure reason:\n{failure_reason}" + return summary + + +def _executed_results( + params: dict[str, Any], + created_case_ids: dict[int, int], + status_ids: dict[str, int], +) -> list[dict[str, Any]]: + """One TestRail result per generated case, in generation order. + + ``SubmitTestPlanHandler.apply`` has already checked that every case carries a + recognised status, so indexing here cannot silently drop one. + """ + return [ + { + "case_id": created_case_ids[int(test_case["id"])], + "status_id": status_ids[test_case["result"]["status"]], + "comment": _result_comment(test_case["result"]), + } + for test_case in params["generated_test_cases"] + ] + + +def _run_payload( + suite_id: int, case_ids: list[int], description: str +) -> dict[str, Any]: + payload: dict[str, Any] = { + "name": _run_name(), + "include_all": False, + "case_ids": case_ids, + "suite_id": suite_id, + } + if description: + payload["description"] = description + return payload + + class SubmitTestPlanHandler: async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult: feature = str(params.get("feature") or "").strip() test_cases = params.get("generated_test_cases") or [] + summary = str(params.get("summary") or "") if not feature: return ActionResult.failed("TestRail submission requires a feature name") if not test_cases: return ActionResult.failed("TestRail submission requires test cases") + if any(not test_case.get("result") for test_case in test_cases): + return ActionResult.failed("TestRail submission requires execution results") + if any( + str(test_case["result"].get("status") or "") not in _CASE_STATUSES + for test_case in test_cases + ): + return ActionResult.failed( + "TestRail submission requires all cases to have executed results" + ) try: client = _client() case_type_id = await _resolve_case_type_id(client) template_id = await _resolve_template_id(client) + # Before anything is created, so a missing status leaves no suite behind. + status_ids = await _resolve_status_ids(client) suite_name = f"[Hackbot] - {feature}" suite_id = _require_id( await client.add_suite(suite_name), @@ -132,6 +209,16 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult ) created_case_ids[generated_id] = case_id + case_ids = list(created_case_ids.values()) + run_id = _require_id( + await client.add_run(_run_payload(suite_id, case_ids, summary)), + "run", + ) + await client.add_results_for_cases( + run_id, + _executed_results(params, created_case_ids, status_ids), + ) + except Exception as exc: log.exception( "Failed to submit test plan to TestRail for run %s", ctx.run_id @@ -143,6 +230,7 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult "suite_id": suite_id, "url": client.suite_url(suite_id), "section_id": section_id, - "case_ids": list(created_case_ids.values()), + "case_ids": case_ids, + "run_id": run_id, } ) diff --git a/libs/hackbot-runtime/tests/test_testrail_handler.py b/libs/hackbot-runtime/tests/test_testrail_handler.py index 3640b65673..c7be486867 100644 --- a/libs/hackbot-runtime/tests/test_testrail_handler.py +++ b/libs/hackbot-runtime/tests/test_testrail_handler.py @@ -52,6 +52,17 @@ def _plan(): } +# Trimmed GET get_statuses: matched on the lowercase ``name``, not ``label``. +_STATUSES = [ + {"id": 1, "name": "passed", "label": "Passed"}, + {"id": 2, "name": "blocked", "label": "Blocked"}, + {"id": 3, "name": "untested", "label": "Untested"}, + {"id": 4, "name": "retest", "label": "Retest"}, + {"id": 5, "name": "failed", "label": "Failed"}, + {"id": 6, "name": "custom_status1", "label": "Needs Triage"}, +] + + class _FakeClient: def __init__(self, responses=None): self.calls = [] @@ -60,10 +71,13 @@ def __init__(self, responses=None): or [ [{"id": 6, "name": "Functional"}], [{"id": 2, "name": "Test Case (Steps)"}], + _STATUSES, {"id": 10}, {"id": 20}, {"id": 101}, {"id": 102}, + {"id": 301}, + {"results": [{"id": 401}, {"id": 402}]}, ] ) @@ -81,6 +95,10 @@ async def get_templates(self): self.calls.append(("get_templates",)) return self._next() + async def get_statuses(self): + self.calls.append(("get_statuses",)) + return self._next() + async def add_suite(self, name): self.calls.append(("add_suite", name)) return self._next() @@ -93,6 +111,14 @@ async def add_case(self, section_id, payload): self.calls.append(("add_case", section_id, payload)) return self._next() + async def add_run(self, payload): + self.calls.append(("add_run", payload)) + return self._next() + + async def add_results_for_cases(self, run_id, results): + self.calls.append(("add_results_for_cases", run_id, results)) + return self._next() + def suite_url(self, suite_id): return f"https://testrail.example/index.php?/suites/view/{suite_id}" @@ -109,14 +135,16 @@ async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch): "url": "https://testrail.example/index.php?/suites/view/10", "section_id": 20, "case_ids": [101, 102], + "run_id": 301, } assert client.calls[0] == ("get_case_types",) assert client.calls[1] == ("get_templates",) - assert client.calls[2] == ("add_suite", "[Hackbot] - PDF Improvements") - assert client.calls[3] == ("add_section", 10, "Test Cases") + assert client.calls[2] == ("get_statuses",) + assert client.calls[3] == ("add_suite", "[Hackbot] - PDF Improvements") + assert client.calls[4] == ("add_section", 10, "Test Cases") - first_case = client.calls[4] + first_case = client.calls[5] assert first_case[0:2] == ("add_case", 20) assert first_case[2] == { "title": "The PDF opens", @@ -133,6 +161,63 @@ async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch): ], } + add_run_call = client.calls[7] + assert add_run_call[0] == "add_run" + assert add_run_call[1]["include_all"] is False + assert add_run_call[1]["case_ids"] == [101, 102] + assert add_run_call[1]["suite_id"] == 10 + assert add_run_call[1]["name"].startswith("[Hackbot] Run Results - ") + assert add_run_call[1]["description"] == "One passed and one was unsuitable." + + assert client.calls[8] == ( + "add_results_for_cases", + 301, + [ + { + "case_id": 101, + "status_id": 1, + "comment": "The PDF behaved as expected.", + }, + { + "case_id": 102, + "status_id": 2, + "comment": ( + "The toolbar could not be inspected.\n\n" + "Failure reason:\nNo available tool can inspect it." + ), + }, + ], + ) + + +async def test_submit_test_plan_requires_execution_results(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + plan = _plan() + del plan["generated_test_cases"][0]["result"] + + result = await testrail_handler.SubmitTestPlanHandler().apply(plan, _ctx()) + + assert result.status == "failed" + assert result.error == "TestRail submission requires execution results" + assert client.calls == [] + + +async def test_submit_test_plan_requires_all_cases_to_be_executed(monkeypatch): + client = _FakeClient() + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + plan = _plan() + plan["generated_test_cases"][1]["result"]["status"] = "not_run" + + result = await testrail_handler.SubmitTestPlanHandler().apply(plan, _ctx()) + + assert result.status == "failed" + assert ( + result.error + == "TestRail submission requires all cases to have executed results" + ) + assert client.calls == [] + def test_separated_steps_maps_expectations_from_step_objects(): assert testrail_handler._separated_steps( @@ -150,6 +235,29 @@ def test_separated_steps_maps_expectations_from_step_objects(): ] +async def test_submit_test_plan_fails_when_the_run_cannot_be_created(monkeypatch): + # The whole submission fails even though the suite and cases were created, so + # a retry duplicates the suite. Accepted for now; see the de-duplication work. + client = _FakeClient( + responses=[ + [{"id": 6, "name": "Functional"}], + [{"id": 2, "name": "Test Case (Steps)"}], + _STATUSES, + {"id": 10}, + {"id": 20}, + {"id": 101}, + {"id": 102}, + RuntimeError("TestRail rejected the run"), + ] + ) + monkeypatch.setattr(testrail_handler, "_client", lambda: client) + + result = await testrail_handler.SubmitTestPlanHandler().apply(_plan(), _ctx()) + + assert result.status == "failed" + assert result.error == "TestRail rejected the run" + + async def test_submit_test_plan_reports_api_failure(monkeypatch): client = _FakeClient(responses=[RuntimeError("TestRail is unavailable")]) monkeypatch.setattr(testrail_handler, "_client", lambda: client) diff --git a/libs/testrail-client/testrail_client/client.py b/libs/testrail-client/testrail_client/client.py index c8634d8cbb..87a0b1bdac 100644 --- a/libs/testrail-client/testrail_client/client.py +++ b/libs/testrail-client/testrail_client/client.py @@ -48,6 +48,9 @@ async def get_case_types(self) -> dict[str, Any] | list[Any]: async def get_templates(self) -> dict[str, Any] | list[Any]: return await self.request("GET", f"get_templates/{self.settings.project_id}") + async def get_statuses(self) -> dict[str, Any] | list[Any]: + return await self.request("GET", "get_statuses") + async def add_suite(self, name: str) -> dict[str, Any] | list[Any]: return await self.request( "POST", @@ -66,3 +69,19 @@ async def add_case( self, section_id: int, payload: dict[str, Any] ) -> dict[str, Any] | list[Any]: return await self.request("POST", f"add_case/{section_id}", payload) + + async def add_run(self, payload: dict[str, Any]) -> dict[str, Any] | list[Any]: + return await self.request( + "POST", + f"add_run/{self.settings.project_id}", + payload, + ) + + async def add_results_for_cases( + self, run_id: int, results: list[dict[str, Any]] + ) -> dict[str, Any] | list[Any]: + return await self.request( + "POST", + f"add_results_for_cases/{run_id}", + {"results": results}, + ) diff --git a/libs/testrail-client/tests/test_client.py b/libs/testrail-client/tests/test_client.py index 850cac30fd..3728adb560 100644 --- a/libs/testrail-client/tests/test_client.py +++ b/libs/testrail-client/tests/test_client.py @@ -110,6 +110,9 @@ async def test_endpoint_wrappers(monkeypatch): await client.get_templates() assert captured["url"].endswith("/get_templates/73") + await client.get_statuses() + assert captured["url"].endswith("/get_statuses") + await client.add_suite("Suite") assert captured["url"].endswith("/add_suite/73") assert captured["json"] == {"name": "Suite"} @@ -122,6 +125,22 @@ async def test_endpoint_wrappers(monkeypatch): assert captured["url"].endswith("/add_case/20") assert captured["json"] == {"title": "Case"} + await client.add_run({"name": "Run", "include_all": False, "case_ids": [101]}) + assert captured["url"].endswith("/add_run/73") + assert captured["json"] == { + "name": "Run", + "include_all": False, + "case_ids": [101], + } + + await client.add_results_for_cases( + 301, [{"case_id": 101, "status_id": 1, "comment": "ok"}] + ) + assert captured["url"].endswith("/add_results_for_cases/301") + assert captured["json"] == { + "results": [{"case_id": 101, "status_id": 1, "comment": "ok"}] + } + def test_suite_url_default_base(): assert (