Use the TestRail action directly in test-plan-generator

agents/test-plan-generator/hackbot_agents/test_plan_generator/__main__.py

@@ -1,5 +1,4 @@
 from hackbot_runtime import HackbotContext, run_async
-from hackbot_runtime.actions.testrail import record_test_plan
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from .agent import TestPlanGeneratorResult, run_test_plan_generator
@@ -32,8 +31,8 @@ async def main(ctx: HackbotContext) -> TestPlanGeneratorResult:
         firefox_path=firefox_path,
         log=ctx.log_path,
         verbose=True,
+        actions_recorder=ctx.actions,
     )
-    record_test_plan(ctx.actions, result.result.model_dump(mode="json"))
     return result
 
 

agents/test-plan-generator/hackbot_agents/test_plan_generator/agent.py

@@ -11,26 +11,22 @@
     McpServerConfig,
     ResultMessage,
 )
-from hackbot_runtime import AgentError, HackbotAgentResult
+from hackbot_runtime import ActionsRecorder, AgentError, HackbotAgentResult
+from hackbot_runtime.actions import ACTIONS_SERVER_NAME
+from hackbot_runtime.actions.claude_sdk import actions_server_for, actions_to_tool_names
+from hackbot_runtime.actions.testrail import ACTION_TYPE as TESTRAIL_SUBMIT_TEST_PLAN
 from hackbot_runtime.claude import Reporter
 
-from .config import DEVTOOLS_TOOLS
+from .config import DEVTOOLS_TOOLS, ENABLED_ACTION_TYPES
 from .devtools_mcp import build_devtools_server
-from .result import (
-    RESULT_SERVER_NAME,
-    SUBMIT_RESULT_TOOL,
-    ResultCollector,
-    TestPlanResult,
-    build_result_server,
-)
 
 HERE = Path(__file__).resolve().parent
 
 logger = logging.getLogger("test-plan-generator")
 
 
 class TestPlanGeneratorResult(HackbotAgentResult):
-    result: TestPlanResult | None = None
+    result: str | None = None
 
 
 def load_system_prompt() -> str:
@@ -45,12 +41,12 @@ def build_user_prompt(
         f"Feature name:\n{feature_name}\n\n"
         f"Feature description:\n{feature_description}\n\n"
         f"Test scope:\n{test_scope}\n\n"
-        "Use the provided feature name as the structured result feature. "
+        "Use the provided feature name as the TestRail action feature. "
         "Keep all generated test cases within the provided test scope.\n\n"
         "Follow the required workflow exactly: before execution, generate no more than 30 "
         "test cases to cover all distinct behaviors, meaningful variations, and "
         "negative scenarios. Run the cases in order, stop each case after its first "
-        "failed step, and submit exactly one structured result."
+        "failed step, and record exactly one TestRail test plan action."
     )
 
 
@@ -65,6 +61,7 @@ async def run_test_plan_generator(
     firefox_path: str | None = None,
     verbose: bool = False,
     log: Path | None = None,
+    actions_recorder: ActionsRecorder | None = None,
 ) -> TestPlanGeneratorResult:
     """Generate and run a Firefox QA test plan for one feature."""
     subject = feature_name
@@ -76,12 +73,14 @@ async def run_test_plan_generator(
         enable_script=True,
     )
 
-    result_collector = ResultCollector()
-    result_server = build_result_server(result_collector)
+    actions_recorder, actions_server = actions_server_for(
+        actions_recorder, types=ENABLED_ACTION_TYPES
+    )
+    enabled_action_tools = actions_to_tool_names(ENABLED_ACTION_TYPES)
 
     mcp_servers: dict[str, McpServerConfig] = {
         "firefox-devtools": devtools_server,
-        RESULT_SERVER_NAME: result_server,
+        ACTIONS_SERVER_NAME: actions_server,
     }
 
     options = ClaudeAgentOptions(
@@ -90,7 +89,7 @@ async def run_test_plan_generator(
         permission_mode="bypassPermissions",
         allowed_tools=[
             *DEVTOOLS_TOOLS,
-            SUBMIT_RESULT_TOOL,
+            *enabled_action_tools,
         ],
         model=model,
         max_turns=max_turns,
@@ -118,13 +117,16 @@ async def run_test_plan_generator(
             f"{subject} test-plan generation failed: "
             f"{result_msg.result or result_msg.subtype}"
         )
-    if result_collector.result is None:
+    if not any(
+        action["type"] == TESTRAIL_SUBMIT_TEST_PLAN
+        for action in actions_recorder.actions
+    ):
         raise AgentError(
-            f"{subject}: agent finished without submitting a result via submit_result"
+            f"{subject}: agent finished without recording a TestRail test plan action"
         )
 
     return TestPlanGeneratorResult(
-        result=result_collector.result,
+        result=result_msg.result,
         num_turns=result_msg.num_turns,
         total_cost_usd=result_msg.total_cost_usd,
     )

agents/test-plan-generator/hackbot_agents/test_plan_generator/config.py

@@ -1,3 +1,9 @@
+# Recordable action types the agent may take, by dotted id. It generates and runs
+# test cases, then records them for TestRail; it takes no other action.
+ENABLED_ACTION_TYPES = [
+    "testrail.submit_test_plan",
+]
+
 # Firefox DevTools MCP tools (@mozilla/firefox-devtools-mcp-moz), exposed under
 # the "firefox-devtools" server name. Keep this focused on tools needed to
 # generate and execute Firefox QA cases: page interaction, observation,

agents/test-plan-generator/hackbot_agents/test_plan_generator/prompts/system.md

@@ -2,27 +2,27 @@ You are a Firefox QA test-plan generation and execution agent.
 
 Generate test cases from the provided Firefox feature name, feature description,
 and test scope, run them in Firefox with the available DevTools MCP tools, and
-report only pass/fail/unsuitable results. Do not try to fix, patch or make changes.
+record the generated test plan for TestRail. Do not try to fix, patch or make
+changes.
 
 ## Required workflow
 
 1. Generate no more than 30 test cases to cover all distinct behaviors,
    variations, and negative scenarios before running any case.
 2. Each test case must have:
    - A title.
-   - A primary execution context label: `chrome` or `content`.
    - Ordered test steps, each with an `action` and optional `expectation`.
 3. Run the generated cases and steps in order.
-4. Submit one final structured result with `submit_result`.
-   - Use the provided feature name as the structured result feature.
+4. Record one final TestRail action with `testrail_submit_test_plan`.
+   - Use the provided feature name as the action feature.
 
 ## Context guidance
 
-Choose a primary context label per case: `content` for normal web page or
-document behavior; `chrome` for Firefox UI, browser state, preferences, toolbar,
-menus, panels, downloads, history, bookmarks, PDF viewer chrome behavior, or
-uncertainty. The label describes what the case mainly exercises; it does not
-restrict per-step tool choice.
+Decide which context each case mainly exercises and pick tools accordingly:
+`content` for normal web page or document behavior; `chrome` for Firefox UI,
+browser state, preferences, toolbar, menus, panels, downloads, history,
+bookmarks, PDF viewer chrome behavior, or uncertainty. This judgment guides your
+tool selection and it does not restrict per-step tool choice.
 
 Use the most appropriate DevTools MCP tool for each step. Prefer content tools
 for page/DOM interaction and privileged-context tools for browser UI/state or
@@ -68,9 +68,8 @@ Mark a case as `unsuitable` only if it requires:
 
 ## Reporting
 
-The final answer must be submitted through `submit_result` exactly once. A prose
-message is not enough. Include one case result for every generated test case.
+Record the generated test plan through `testrail_submit_test_plan` exactly once.
+A prose message is not enough.
 
-For failed steps, set `failure_reason` to a short explanation of the observed
-failure. For failed or unsuitable cases, set the case-level `failure_reason` as
-well. Leave `failure_reason` empty for passed steps and passed cases.
+Then close with a write-up of the execution: which cases passed, failed, or were
+unsuitable, with concise observations for the failed and unsuitable ones.