Submit the agent's execution results to TestRail

libs/hackbot-runtime/hackbot_runtime/actions/handlers/testrail_handler.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+from datetime import UTC, datetime
 from functools import lru_cache
 from typing import Any
 
@@ -16,6 +17,7 @@
 _CASE_TEMPLATE_NAME = "Test Case (Steps)"
 _CASE_LABEL = "AI Generated"
 _SECTION_NAME = "Test Cases"
+_CASE_STATUSES = ("passed", "failed", "unsuitable")
 
 
 @lru_cache(maxsize=1)
@@ -65,6 +67,22 @@ async def _resolve_template_id(client: TestRailClient) -> int:
     raise RuntimeError(f'TestRail has no template named "{_CASE_TEMPLATE_NAME}"')
 
 
+async def _resolve_status_ids(client: TestRailClient) -> dict[str, int]:
+    """Case status -> id of the TestRail status it is posted as.
+
+    An unsuitable case is posted as Blocked, the closest status a result can
+    carry; the other two match TestRail's own status names.
+    """
+    response = await client.get_statuses()
+    statuses = response.get("statuses", []) if isinstance(response, dict) else response
+    ids = {status["name"]: int(status["id"]) for status in statuses}
+    return {
+        "passed": ids["passed"],
+        "failed": ids["failed"],
+        "unsuitable": ids["blocked"],
+    }
+
+
 def _case_payload(
     test_case: dict[str, Any],
     case_type_id: int,
@@ -92,20 +110,79 @@ def _separated_steps(test_case: dict[str, Any]) -> list[dict[str, str]]:
     ]
 
 
+def _run_name() -> str:
+    stamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC")
+    return f"[Hackbot] Run Results - {stamp}"
+
+
+def _result_comment(result: dict[str, Any]) -> str:
+    summary = str(result.get("summary") or "")
+    failure_reason = str(result.get("failure_reason") or "")
+    if failure_reason:
+        return f"{summary}\n\nFailure reason:\n{failure_reason}"
+    return summary
+
+
+def _executed_results(
+    params: dict[str, Any],
+    created_case_ids: dict[int, int],
+    status_ids: dict[str, int],
+) -> list[dict[str, Any]]:
+    """One TestRail result per generated case, in generation order.
+
+    ``SubmitTestPlanHandler.apply`` has already checked that every case carries a
+    recognised status, so indexing here cannot silently drop one.
+    """
+    return [
+        {
+            "case_id": created_case_ids[int(test_case["id"])],
+            "status_id": status_ids[test_case["result"]["status"]],
+            "comment": _result_comment(test_case["result"]),
+        }
+        for test_case in params["generated_test_cases"]
+    ]
+
+
+def _run_payload(
+    suite_id: int, case_ids: list[int], description: str
+) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "name": _run_name(),
+        "include_all": False,
+        "case_ids": case_ids,
+        "suite_id": suite_id,
+    }
+    if description:
+        payload["description"] = description
+    return payload
+
+
 class SubmitTestPlanHandler:
     async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult:
         feature = str(params.get("feature") or "").strip()
         test_cases = params.get("generated_test_cases") or []
+        summary = str(params.get("summary") or "")
 
         if not feature:
             return ActionResult.failed("TestRail submission requires a feature name")
         if not test_cases:
             return ActionResult.failed("TestRail submission requires test cases")
+        if any(not test_case.get("result") for test_case in test_cases):
+            return ActionResult.failed("TestRail submission requires execution results")
+        if any(
+            str(test_case["result"].get("status") or "") not in _CASE_STATUSES
+            for test_case in test_cases
+        ):
+            return ActionResult.failed(
+                "TestRail submission requires all cases to have executed results"
+            )
 
         try:
             client = _client()
             case_type_id = await _resolve_case_type_id(client)
             template_id = await _resolve_template_id(client)
+            # Before anything is created, so a missing status leaves no suite behind.
+            status_ids = await _resolve_status_ids(client)
             suite_name = f"[Hackbot] - {feature}"
             suite_id = _require_id(
                 await client.add_suite(suite_name),
@@ -132,6 +209,16 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult
                 )
                 created_case_ids[generated_id] = case_id
 
+            case_ids = list(created_case_ids.values())
+            run_id = _require_id(
+                await client.add_run(_run_payload(suite_id, case_ids, summary)),
+                "run",
+            )
+            await client.add_results_for_cases(
+                run_id,
+                _executed_results(params, created_case_ids, status_ids),
+            )
+
         except Exception as exc:
             log.exception(
                 "Failed to submit test plan to TestRail for run %s", ctx.run_id
@@ -143,6 +230,7 @@ async def apply(self, params: dict[str, Any], ctx: ApplyContext) -> ActionResult
                 "suite_id": suite_id,
                 "url": client.suite_url(suite_id),
                 "section_id": section_id,
-                "case_ids": list(created_case_ids.values()),
+                "case_ids": case_ids,
+                "run_id": run_id,
             }
         )

libs/hackbot-runtime/tests/test_testrail_handler.py

@@ -52,6 +52,17 @@ def _plan():
     }
 
 
+# Trimmed GET get_statuses: matched on the lowercase ``name``, not ``label``.
+_STATUSES = [
+    {"id": 1, "name": "passed", "label": "Passed"},
+    {"id": 2, "name": "blocked", "label": "Blocked"},
+    {"id": 3, "name": "untested", "label": "Untested"},
+    {"id": 4, "name": "retest", "label": "Retest"},
+    {"id": 5, "name": "failed", "label": "Failed"},
+    {"id": 6, "name": "custom_status1", "label": "Needs Triage"},
+]
+
+
 class _FakeClient:
     def __init__(self, responses=None):
         self.calls = []
@@ -60,10 +71,13 @@ def __init__(self, responses=None):
             or [
                 [{"id": 6, "name": "Functional"}],
                 [{"id": 2, "name": "Test Case (Steps)"}],
+                _STATUSES,
                 {"id": 10},
                 {"id": 20},
                 {"id": 101},
                 {"id": 102},
+                {"id": 301},
+                {"results": [{"id": 401}, {"id": 402}]},
             ]
         )
 
@@ -81,6 +95,10 @@ async def get_templates(self):
         self.calls.append(("get_templates",))
         return self._next()
 
+    async def get_statuses(self):
+        self.calls.append(("get_statuses",))
+        return self._next()
+
     async def add_suite(self, name):
         self.calls.append(("add_suite", name))
         return self._next()
@@ -93,6 +111,14 @@ async def add_case(self, section_id, payload):
         self.calls.append(("add_case", section_id, payload))
         return self._next()
 
+    async def add_run(self, payload):
+        self.calls.append(("add_run", payload))
+        return self._next()
+
+    async def add_results_for_cases(self, run_id, results):
+        self.calls.append(("add_results_for_cases", run_id, results))
+        return self._next()
+
     def suite_url(self, suite_id):
         return f"https://testrail.example/index.php?/suites/view/{suite_id}"
 
@@ -109,14 +135,16 @@ async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch):
         "url": "https://testrail.example/index.php?/suites/view/10",
         "section_id": 20,
         "case_ids": [101, 102],
+        "run_id": 301,
     }
 
     assert client.calls[0] == ("get_case_types",)
     assert client.calls[1] == ("get_templates",)
-    assert client.calls[2] == ("add_suite", "[Hackbot] - PDF Improvements")
-    assert client.calls[3] == ("add_section", 10, "Test Cases")
+    assert client.calls[2] == ("get_statuses",)
+    assert client.calls[3] == ("add_suite", "[Hackbot] - PDF Improvements")
+    assert client.calls[4] == ("add_section", 10, "Test Cases")
 
-    first_case = client.calls[4]
+    first_case = client.calls[5]
     assert first_case[0:2] == ("add_case", 20)
     assert first_case[2] == {
         "title": "The PDF opens",
@@ -133,6 +161,63 @@ async def test_submit_test_plan_creates_suite_section_and_cases(monkeypatch):
         ],
     }
 
+    add_run_call = client.calls[7]
+    assert add_run_call[0] == "add_run"
+    assert add_run_call[1]["include_all"] is False
+    assert add_run_call[1]["case_ids"] == [101, 102]
+    assert add_run_call[1]["suite_id"] == 10
+    assert add_run_call[1]["name"].startswith("[Hackbot] Run Results - ")
+    assert add_run_call[1]["description"] == "One passed and one was unsuitable."
+
+    assert client.calls[8] == (
+        "add_results_for_cases",
+        301,
+        [
+            {
+                "case_id": 101,
+                "status_id": 1,
+                "comment": "The PDF behaved as expected.",
+            },
+            {
+                "case_id": 102,
+                "status_id": 2,
+                "comment": (
+                    "The toolbar could not be inspected.\n\n"
+                    "Failure reason:\nNo available tool can inspect it."
+                ),
+            },
+        ],
+    )
+
+
+async def test_submit_test_plan_requires_execution_results(monkeypatch):
+    client = _FakeClient()
+    monkeypatch.setattr(testrail_handler, "_client", lambda: client)
+    plan = _plan()
+    del plan["generated_test_cases"][0]["result"]
+
+    result = await testrail_handler.SubmitTestPlanHandler().apply(plan, _ctx())
+
+    assert result.status == "failed"
+    assert result.error == "TestRail submission requires execution results"
+    assert client.calls == []
+
+
+async def test_submit_test_plan_requires_all_cases_to_be_executed(monkeypatch):
+    client = _FakeClient()
+    monkeypatch.setattr(testrail_handler, "_client", lambda: client)
+    plan = _plan()
+    plan["generated_test_cases"][1]["result"]["status"] = "not_run"
+
+    result = await testrail_handler.SubmitTestPlanHandler().apply(plan, _ctx())
+
+    assert result.status == "failed"
+    assert (
+        result.error
+        == "TestRail submission requires all cases to have executed results"
+    )
+    assert client.calls == []
+
 
 def test_separated_steps_maps_expectations_from_step_objects():
     assert testrail_handler._separated_steps(
@@ -150,6 +235,29 @@ def test_separated_steps_maps_expectations_from_step_objects():
     ]
 
 
+async def test_submit_test_plan_fails_when_the_run_cannot_be_created(monkeypatch):
+    # The whole submission fails even though the suite and cases were created, so
+    # a retry duplicates the suite. Accepted for now; see the de-duplication work.
+    client = _FakeClient(
+        responses=[
+            [{"id": 6, "name": "Functional"}],
+            [{"id": 2, "name": "Test Case (Steps)"}],
+            _STATUSES,
+            {"id": 10},
+            {"id": 20},
+            {"id": 101},
+            {"id": 102},
+            RuntimeError("TestRail rejected the run"),
+        ]
+    )
+    monkeypatch.setattr(testrail_handler, "_client", lambda: client)
+
+    result = await testrail_handler.SubmitTestPlanHandler().apply(_plan(), _ctx())
+
+    assert result.status == "failed"
+    assert result.error == "TestRail rejected the run"
+
+
 async def test_submit_test_plan_reports_api_failure(monkeypatch):
     client = _FakeClient(responses=[RuntimeError("TestRail is unavailable")])
     monkeypatch.setattr(testrail_handler, "_client", lambda: client)

libs/testrail-client/testrail_client/client.py

@@ -48,6 +48,9 @@ async def get_case_types(self) -> dict[str, Any] | list[Any]:
     async def get_templates(self) -> dict[str, Any] | list[Any]:
         return await self.request("GET", f"get_templates/{self.settings.project_id}")
 
+    async def get_statuses(self) -> dict[str, Any] | list[Any]:
+        return await self.request("GET", "get_statuses")
+
     async def add_suite(self, name: str) -> dict[str, Any] | list[Any]:
         return await self.request(
             "POST",
@@ -66,3 +69,19 @@ async def add_case(
         self, section_id: int, payload: dict[str, Any]
     ) -> dict[str, Any] | list[Any]:
         return await self.request("POST", f"add_case/{section_id}", payload)
+
+    async def add_run(self, payload: dict[str, Any]) -> dict[str, Any] | list[Any]:
+        return await self.request(
+            "POST",
+            f"add_run/{self.settings.project_id}",
+            payload,
+        )
+
+    async def add_results_for_cases(
+        self, run_id: int, results: list[dict[str, Any]]
+    ) -> dict[str, Any] | list[Any]:
+        return await self.request(
+            "POST",
+            f"add_results_for_cases/{run_id}",
+            {"results": results},
+        )

libs/testrail-client/tests/test_client.py

@@ -110,6 +110,9 @@ async def test_endpoint_wrappers(monkeypatch):
     await client.get_templates()
     assert captured["url"].endswith("/get_templates/73")
 
+    await client.get_statuses()
+    assert captured["url"].endswith("/get_statuses")
+
     await client.add_suite("Suite")
     assert captured["url"].endswith("/add_suite/73")
     assert captured["json"] == {"name": "Suite"}
@@ -122,6 +125,22 @@ async def test_endpoint_wrappers(monkeypatch):
     assert captured["url"].endswith("/add_case/20")
     assert captured["json"] == {"title": "Case"}
 
+    await client.add_run({"name": "Run", "include_all": False, "case_ids": [101]})
+    assert captured["url"].endswith("/add_run/73")
+    assert captured["json"] == {
+        "name": "Run",
+        "include_all": False,
+        "case_ids": [101],
+    }
+
+    await client.add_results_for_cases(
+        301, [{"case_id": 101, "status_id": 1, "comment": "ok"}]
+    )
+    assert captured["url"].endswith("/add_results_for_cases/301")
+    assert captured["json"] == {
+        "results": [{"case_id": 101, "status_id": 1, "comment": "ok"}]
+    }
+
 
 def test_suite_url_default_base():
     assert (