From 89c5145de2f53dcd6e69737c17092c02fbb440c2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 06:45:13 +0000 Subject: [PATCH 1/8] scoring 1.2: replace coordination/communication_efficiency with plan_coherence (#51) Both removed metrics were ~1.0 by construction (the resolver marks every detected conflict resolved; empty message traffic scored 1.0) and read Felix-specific counters, so they inflated every composite by ~20% and could not score a non-Felix harness fairly. - add scoring/coherence.py: contradiction detection over a tick's executed writes (draft/undraft, move+job, work_priority clash, overlapping zones, duplicate blueprint cell, competing research targets) - plan_coherence metric (weight 0.08); efficiency and plan_coherence return a neutral 0.5 for ticks with no writes so the unmanaged baseline earns no free process points - drop conflicts_*/messages_* from MetricContext; keep resolver + CentralPost counts on the CONFLICT event as diagnostics - redistribute weights (outcomes dominate); sync all 6 scenario YAMLs - SCORING_VERSION 1.1 -> 1.2 (pinned baseline sidecars now require recalibration) - README/CLAUDE/ADR-003 now match the code Co-authored-by: Jason --- CLAUDE.md | 19 +- README.md | 19 +- .../003-automated-benchmark-infrastructure.md | 2 + src/rle/orchestration/game_loop.py | 17 +- .../definitions/01_crashlanded_survival.yaml | 9 +- .../definitions/02_first_winter.yaml | 9 +- .../definitions/03_toxic_fallout.yaml | 9 +- .../definitions/04_raid_defense.yaml | 7 +- .../definitions/05_plague_response.yaml | 7 +- .../scenarios/definitions/06_ship_launch.yaml | 7 +- src/rle/scoring/coherence.py | 152 +++++++++++++++ src/rle/scoring/composite.py | 17 +- src/rle/scoring/metrics.py | 46 ++--- src/rle/tracking/metadata.py | 14 +- tests/unit/test_composite_scorer.py | 42 ++++- tests/unit/test_metadata.py | 2 +- tests/unit/test_metrics.py | 177 ++++++++++++------ tests/unit/test_scenario_loader.py | 2 +- uv.lock | 2 +- 19 files changed, 405 insertions(+), 154 deletions(-) create mode 100644 src/rle/scoring/coherence.py diff --git a/CLAUDE.md b/CLAUDE.md index b1006b8..4a1576d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -268,22 +268,21 @@ Macro helix: `t = min(1.0, game_day / expected_duration_days)` drives agent beha - **Analysis** (0.4 <= t < 0.7): Medium temp, evaluate trade-offs - **Synthesis** (t >= 0.7): Low temperature, decisive actions -## Scoring (10 metrics, weighted composite) +## Scoring (9 metrics, weighted composite, SCORING_VERSION 1.2) | Metric | Default Weight | Source | |--------|---------------|--------| -| survival | 0.25 | alive/started colonists | -| threat_response | 0.15 | draft response speed | -| mood | 0.15 | avg colonist mood (from real RIMAPI data) | +| survival | 0.24 | alive/started colonists | +| threat_response | 0.14 | draft response speed | +| mood | 0.12 | avg colonist mood (from real RIMAPI data) | | food_security | 0.10 | food count / 10 (from /api/v1/resources/summary) | -| wealth | 0.10 | wealth growth ratio | -| research | 0.10 | % research tree completed | +| wealth | 0.08 | wealth growth ratio | +| research | 0.08 | % research tree completed | | self_sufficiency | 0.10 | power + food + population stability | -| efficiency | 0.05 | action execution rate | -| coordination | 0.00* | conflicts resolved / total conflicts | -| communication_efficiency | 0.00* | messages acted on / total messages | +| efficiency | 0.06 | executed / proposed writes per tick (neutral 0.5 when no writes) | +| plan_coherence | 0.08 | 1 − contradictory executed writes / executed writes per tick (neutral 0.5 when no writes); see `scoring/coherence.py` | -*Process metrics have 0.0 weight until game loop wires MetricContext counters. Target: coordination=0.12, communication_efficiency=0.08. +Process metrics are harness-agnostic: they read the executed write stream (`ExecutionResult.outcomes`), never CentralPost or resolver counters. Those Felix-specific counts are still emitted on the `CONFLICT` event as diagnostics. `coordination` / `communication_efficiency` were removed in 1.2 (issue #51) because they were ≈1.0 by construction. Bump `SCORING_VERSION` in `tracking/metadata.py` whenever weights or metric implementations change; pinned `.baseline.json` sidecars are rejected on mismatch until recalibrated with `scripts/calibrate_baseline.py`. Scenarios can override weights. TimeSeriesRecorder exports per-tick CSV. diff --git a/README.md b/README.md index b8a9f7b..9ba7a0a 100644 --- a/README.md +++ b/README.md @@ -194,22 +194,21 @@ Measured against a pinned no-agent baseline (4 seeds, mean time-to-end 8.0 days) ## Scoring -10 metrics, weighted composite (scenarios can override weights): +9 metrics, weighted composite (`SCORING_VERSION = "1.2"`; scenarios can override weights): | Metric | Default Weight | What it measures | |--------|---------------|------------------| -| survival | 0.25 | alive / started colonists | -| threat_response | 0.15 | draft response speed | -| mood | 0.15 | avg colonist mood | +| survival | 0.24 | alive / started colonists | +| threat_response | 0.14 | draft response speed | +| mood | 0.12 | avg colonist mood | | food_security | 0.10 | days of food (10+ = 1.0) | -| wealth | 0.10 | wealth growth ratio | -| research | 0.10 | % research tree completed | +| wealth | 0.08 | wealth growth ratio | +| research | 0.08 | % research tree completed | | self_sufficiency | 0.10 | power + food + population stability | -| efficiency | 0.05 | action execution rate | -| coordination | 0.00* | conflicts resolved / total conflicts | -| communication_efficiency | 0.00* | messages acted on / total messages | +| efficiency | 0.06 | executed / proposed writes per tick | +| plan_coherence | 0.08 | 1 − contradictory executed writes / executed writes per tick | -*Process metrics weighted 0.0 until game loop wires MetricContext counters. +Both process metrics (`efficiency`, `plan_coherence`) are computed from the writes that actually reached RIMAPI, so any harness is scored the same way, and both return a neutral 0.5 for ticks with no writes so the unmanaged baseline earns no free points. The pre-1.2 `coordination` / `communication_efficiency` metrics were removed because they were ≈1.0 by construction (issue #51). ## Development diff --git a/docs/adr/003-automated-benchmark-infrastructure.md b/docs/adr/003-automated-benchmark-infrastructure.md index aa476b6..2b345d9 100644 --- a/docs/adr/003-automated-benchmark-infrastructure.md +++ b/docs/adr/003-automated-benchmark-infrastructure.md @@ -44,6 +44,8 @@ Two new metrics folded into the composite score: Process metrics get 20% combined weight. No historical data to protect (pre-release). All 6 scenario YAMLs updated. +> **Superseded (scoring 1.2, issue #51, ADR-004).** Both metrics turned out to be ≈1.0 by construction (the resolver always "resolves" every conflict it detects; empty message traffic scored 1.0) and were Felix-specific. They were replaced by a single harness-agnostic `plan_coherence` metric computed from the executed write stream, with neutral 0.5 defaults for ticks that issue no writes. + ### 3. Bootstrap confidence intervals (stdlib-only) We evaluated `resample` (scikit-hep, best modern option) but it pulls in scipy (~150MB) + numpy (~50MB). For percentile bootstrap CIs and Welch's t-test at N≥4, stdlib `random.choices()` + `math` is mathematically correct and keeps the install lightweight. The existing hand-rolled Welch's t-test in `delta.py` uses a normal CDF approximation (Abramowitz & Stegun) documented as "very accurate for df > 30, approximate for smaller df." Our N≥4 minimum guarantees sufficient accuracy. diff --git a/src/rle/orchestration/game_loop.py b/src/rle/orchestration/game_loop.py index 8b36458..da9bfe8 100644 --- a/src/rle/orchestration/game_loop.py +++ b/src/rle/orchestration/game_loop.py @@ -600,12 +600,13 @@ async def run_tick(self) -> TickResult: results = await self._deliberate_sequential(state, current_time, tick_num) # Collect plans, update visualizer, send via CentralPost + agents_acted_with_messages = 0 for agent, plan in results: if plan is None: continue plans.append(plan) if agent.agent_id in agents_with_messages: - self._metric_context.messages_acted_on += 1 + agents_acted_with_messages += 1 self._update_visualizer_agent(agent, plan, current_time) spoke = self._spoke_manager.get_spoke(agent.agent_id) if spoke and spoke.is_connected: @@ -631,23 +632,21 @@ async def run_tick(self) -> TickResult: for gen_id in any_agent.drain_generation_ids(): self._cost_tracker.record_generation_id(gen_id) - # Resolve conflicts + # Resolve conflicts. Resolver + CentralPost counts are diagnostics + # in the event log only — they are Felix-specific and no longer + # feed the composite (#51). resolved, resolver_stats = self._resolver.resolve(plans, state) - self._metric_context.conflicts_total += resolver_stats.conflicts_total - self._metric_context.conflicts_resolved += resolver_stats.conflicts_resolved self._emit( EventType.CONFLICT, tick_num, input_plans=len(plans), output_actions=len(resolved.actions), conflicts_detected=resolver_stats.conflicts_total, conflicts_resolved=resolver_stats.conflicts_resolved, + messages_routed=self._hub.total_messages_processed - messages_before, + agents_with_messages=len(agents_with_messages), + agents_acted_with_messages=agents_acted_with_messages, ) - # Track message effectiveness - messages_after = self._hub.total_messages_processed - new_messages = messages_after - messages_before - self._metric_context.messages_sent += new_messages - # 7. Execute merged plan exec_result = await self._executor.execute(resolved) for outcome in exec_result.outcomes: diff --git a/src/rle/scenarios/definitions/01_crashlanded_survival.yaml b/src/rle/scenarios/definitions/01_crashlanded_survival.yaml index e4e0079..438eb36 100644 --- a/src/rle/scenarios/definitions/01_crashlanded_survival.yaml +++ b/src/rle/scenarios/definitions/01_crashlanded_survival.yaml @@ -17,14 +17,13 @@ failure_conditions: operator: == value: 1 scoring_weights: - survival: 0.24 - food_security: 0.12 + survival: 0.28 + food_security: 0.16 mood: 0.12 threat_response: 0.08 wealth: 0.04 research: 0.04 - self_sufficiency: 0.12 + self_sufficiency: 0.16 efficiency: 0.04 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: 29530cd8e5f373b50f9ee51617bc0b036de97f95c9fd3264a2bbd5939a133d4e diff --git a/src/rle/scenarios/definitions/02_first_winter.yaml b/src/rle/scenarios/definitions/02_first_winter.yaml index def07f6..edbd71f 100644 --- a/src/rle/scenarios/definitions/02_first_winter.yaml +++ b/src/rle/scenarios/definitions/02_first_winter.yaml @@ -20,14 +20,13 @@ failure_conditions: operator: <= value: 0 scoring_weights: - survival: 0.2 - food_security: 0.16 - self_sufficiency: 0.12 + survival: 0.24 + food_security: 0.2 + self_sufficiency: 0.16 mood: 0.08 wealth: 0.08 research: 0.04 threat_response: 0.08 efficiency: 0.04 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: 22d52f88b3398f4d3c61de65a93fc809e3c946c41b1b19b8f0f6a0b99e93361d diff --git a/src/rle/scenarios/definitions/03_toxic_fallout.yaml b/src/rle/scenarios/definitions/03_toxic_fallout.yaml index 06cdc81..f9abed4 100644 --- a/src/rle/scenarios/definitions/03_toxic_fallout.yaml +++ b/src/rle/scenarios/definitions/03_toxic_fallout.yaml @@ -23,14 +23,13 @@ triggered_incidents: - tick_offset: 1 name: ToxicFallout scoring_weights: - survival: 0.28 - food_security: 0.12 - mood: 0.16 + survival: 0.32 + food_security: 0.16 + mood: 0.2 self_sufficiency: 0.08 wealth: 0.04 research: 0.04 threat_response: 0.04 efficiency: 0.04 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: c8a73b3beb3a6df4988ed4c100886584c376cd61814831ffe78fec9b30228ba5 diff --git a/src/rle/scenarios/definitions/04_raid_defense.yaml b/src/rle/scenarios/definitions/04_raid_defense.yaml index fd66f5e..766f29f 100644 --- a/src/rle/scenarios/definitions/04_raid_defense.yaml +++ b/src/rle/scenarios/definitions/04_raid_defense.yaml @@ -25,14 +25,13 @@ triggered_incidents: incident_parms: points: 500 scoring_weights: - survival: 0.2 - threat_response: 0.24 + survival: 0.26 + threat_response: 0.3 mood: 0.08 food_security: 0.04 wealth: 0.04 research: 0.04 self_sufficiency: 0.08 efficiency: 0.08 - coordination: 0.15 - communication_efficiency: 0.05 + plan_coherence: 0.08 save_sha256: c11ff2d4476d7f4da7c3bdcbb07f96610ae7588c642af3f5696278c0530666e2 diff --git a/src/rle/scenarios/definitions/05_plague_response.yaml b/src/rle/scenarios/definitions/05_plague_response.yaml index f9588e8..4f9f19d 100644 --- a/src/rle/scenarios/definitions/05_plague_response.yaml +++ b/src/rle/scenarios/definitions/05_plague_response.yaml @@ -23,14 +23,13 @@ triggered_incidents: - tick_offset: 1 name: Plague scoring_weights: - survival: 0.28 - mood: 0.12 + survival: 0.34 + mood: 0.18 food_security: 0.08 threat_response: 0.08 wealth: 0.04 research: 0.04 self_sufficiency: 0.08 efficiency: 0.08 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: a31a1eb31913fee79cb39b91009e4058029eac133a30ad3e0fc0e53a31a79e6d diff --git a/src/rle/scenarios/definitions/06_ship_launch.yaml b/src/rle/scenarios/definitions/06_ship_launch.yaml index 1735fe3..6e5b716 100644 --- a/src/rle/scenarios/definitions/06_ship_launch.yaml +++ b/src/rle/scenarios/definitions/06_ship_launch.yaml @@ -14,14 +14,13 @@ failure_conditions: operator: == value: 1 scoring_weights: - research: 0.2 + research: 0.28 survival: 0.16 - wealth: 0.12 + wealth: 0.16 self_sufficiency: 0.12 food_security: 0.04 mood: 0.08 threat_response: 0.04 efficiency: 0.04 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: 24db37fec15423b8eb5da56a9be471a4814b0e4040a436a4864d3dba234397ff diff --git a/src/rle/scoring/coherence.py b/src/rle/scoring/coherence.py new file mode 100644 index 0000000..e15022b --- /dev/null +++ b/src/rle/scoring/coherence.py @@ -0,0 +1,152 @@ +"""Contradiction detection over a tick's executed writes (``plan_coherence``). + +A harness's job is to hand the game a coherent set of writes each tick. +Whether that coherence comes from a conflict resolver (multi-agent), a single +model's judgement, or a coding agent calling tools one at a time is the +harness's business — this module only looks at what actually reached RIMAPI, +so every harness is scored on the same footing (issue #51). +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import Any + +from rle.agents.actions import resolve_endpoint +from rle.orchestration.action_executor import ActionOutcome + +Rect = tuple[int, int, int, int] + +_ZONE_ENDPOINTS = frozenset({"growing_zone", "stockpile_zone"}) + + +def _int(value: object) -> int | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, (str, float)): + try: + return int(value) + except (TypeError, ValueError): + return None + return None + + +def _rect(params: dict[str, Any]) -> Rect | None: + """Mirror the executor's rectangle defaults (x2/z2 default to +5).""" + x1 = _int(params.get("x1", params.get("x"))) + z1 = _int(params.get("z1", params.get("z"))) + if x1 is None or z1 is None: + return None + x2 = _int(params.get("x2", x1 + 5)) + z2 = _int(params.get("z2", z1 + 5)) + if x2 is None or z2 is None: + return None + return (min(x1, x2), min(z1, z2), max(x1, x2), max(z1, z2)) + + +def _rects_overlap(a: Rect, b: Rect) -> bool: + ax1, az1, ax2, az2 = a + bx1, bz1, bx2, bz2 = b + return not (ax2 < bx1 or ax1 > bx2 or az2 < bz1 or az1 > bz2) + + +def _all_conflicting(groups: Iterable[Sequence[int]], flagged: set[int]) -> None: + for group in groups: + if len(group) > 1: + flagged.update(group) + + +def count_contradictions(outcomes: Sequence[ActionOutcome]) -> tuple[int, int]: + """Return ``(contradictory, executed)`` for one tick's outcomes. + + Only successful writes count — a failed write never reached the game, so + it cannot contradict anything (it is already penalised by ``efficiency``). + A write is contradictory when, within the same tick, another executed + write targets the same thing with an incompatible intent: + + * ``draft`` on the same pawn with different ``is_drafted`` values + * ``move`` and ``job_assign`` on the same pawn + * ``work_priority`` on the same pawn setting the same work type to + different priorities + * ``growing_zone`` / ``stockpile_zone`` rectangles that overlap + * ``blueprint`` twice on the same cell + * more than one distinct ``research_target`` project + """ + executed = [(i, o) for i, o in enumerate(outcomes) if o.success] + if not executed: + return 0, 0 + + flagged: set[int] = set() + + draft_by_pawn: dict[str, dict[bool, list[int]]] = {} + move_or_job_by_pawn: dict[str, dict[str, list[int]]] = {} + work_by_pawn_type: dict[tuple[str, str], dict[int, list[int]]] = {} + zones: list[tuple[int, Rect]] = [] + blueprint_by_cell: dict[tuple[int, int], list[int]] = {} + research_by_project: dict[str, list[int]] = {} + + for idx, o in executed: + endpoint = resolve_endpoint(o.action_type) + pawn = o.target_colonist_id or "" + params = o.parameters + if endpoint == "draft": + state = bool(params.get("is_drafted", True)) + draft_by_pawn.setdefault(pawn, {}).setdefault(state, []).append(idx) + elif endpoint in ("move", "job_assign"): + move_or_job_by_pawn.setdefault(pawn, {}).setdefault(endpoint, []).append(idx) + elif endpoint == "work_priority": + for work, pri in _work_priorities(params).items(): + work_by_pawn_type.setdefault((pawn, work), {}).setdefault(pri, []).append(idx) + elif endpoint in _ZONE_ENDPOINTS: + rect = _rect(params) + if rect is not None: + zones.append((idx, rect)) + elif endpoint == "blueprint": + try: + cell = (int(params["x"]), int(params["z"])) + except (KeyError, TypeError, ValueError): + continue + blueprint_by_cell.setdefault(cell, []).append(idx) + elif endpoint == "research_target": + project = str(params.get("project", params.get("name", ""))) + research_by_project.setdefault(project, []).append(idx) + + for by_state in draft_by_pawn.values(): + if len(by_state) > 1: + for idxs in by_state.values(): + flagged.update(idxs) + for by_kind in move_or_job_by_pawn.values(): + if len(by_kind) > 1: + for idxs in by_kind.values(): + flagged.update(idxs) + for by_pri in work_by_pawn_type.values(): + if len(by_pri) > 1: + for idxs in by_pri.values(): + flagged.update(idxs) + for i, (idx_a, rect_a) in enumerate(zones): + for idx_b, rect_b in zones[i + 1:]: + if _rects_overlap(rect_a, rect_b): + flagged.add(idx_a) + flagged.add(idx_b) + _all_conflicting(blueprint_by_cell.values(), flagged) + if len(research_by_project) > 1: + for idxs in research_by_project.values(): + flagged.update(idxs) + + return len(flagged), len(executed) + + +def _work_priorities(params: dict[str, Any]) -> dict[str, int]: + """Mirror of the executor's accepted shapes, tolerant of garbage.""" + nested = params.get("work_priorities") + if isinstance(nested, dict): + return {str(w): int(p) for w, p in nested.items() if isinstance(p, int)} + if "work_type" in params: + pri = params.get("priority", 1) + return {str(params["work_type"]): int(pri)} if isinstance(pri, int) else {} + return { + str(w): p for w, p in params.items() + if isinstance(p, int) and not isinstance(p, bool) + } diff --git a/src/rle/scoring/composite.py b/src/rle/scoring/composite.py index 046c057..e1db8c8 100644 --- a/src/rle/scoring/composite.py +++ b/src/rle/scoring/composite.py @@ -7,17 +7,20 @@ from rle.rimapi.schemas import GameState from rle.scoring.metrics import ALL_METRICS, MetricContext +# Scoring 1.2 (#51): one harness-agnostic process metric (plan_coherence) +# replaces coordination + communication_efficiency; the freed weight returns +# to colony outcomes. Bump SCORING_VERSION in tracking/metadata.py whenever +# this table changes. DEFAULT_WEIGHTS: dict[str, float] = { - "survival": 0.20, - "threat_response": 0.12, + "survival": 0.24, + "threat_response": 0.14, "mood": 0.12, - "food_security": 0.08, + "food_security": 0.10, "wealth": 0.08, "research": 0.08, - "self_sufficiency": 0.08, - "efficiency": 0.04, - "coordination": 0.12, - "communication_efficiency": 0.08, + "self_sufficiency": 0.10, + "efficiency": 0.06, + "plan_coherence": 0.08, } diff --git a/src/rle/scoring/metrics.py b/src/rle/scoring/metrics.py index 4dad794..fbeb150 100644 --- a/src/rle/scoring/metrics.py +++ b/src/rle/scoring/metrics.py @@ -6,10 +6,16 @@ from typing import TYPE_CHECKING from rle.rimapi.schemas import GameState, ThreatData +from rle.scoring.coherence import count_contradictions if TYPE_CHECKING: from rle.orchestration.game_loop import TickResult +# Value returned by process metrics when there is nothing to judge (no ticks +# yet, or a tick that issued zero writes). Neutral rather than 1.0 so an +# unmanaged baseline does not bank free points on process metrics (#51). +NEUTRAL = 0.5 + @dataclass class MetricContext: @@ -24,11 +30,6 @@ class MetricContext: # Response delay in loop ticks per threat, recorded when a draft executes first_draft_tick: dict[str, int] = field(default_factory=dict) initial_wealth: float = 0.0 - # Process metrics (populated by game loop after conflict resolution) - conflicts_total: int = 0 - conflicts_resolved: int = 0 - messages_sent: int = 0 - messages_acted_on: int = 0 def survival(state: GameState, ctx: MetricContext) -> float: @@ -93,31 +94,33 @@ def self_sufficiency(state: GameState, ctx: MetricContext) -> float: def efficiency(state: GameState, ctx: MetricContext) -> float: - """Average action execution rate across all ticks.""" + """Average action execution rate across ticks. Ticks with no writes are neutral.""" if not ctx.tick_results: - return 1.0 + return NEUTRAL rates = [] for tr in ctx.tick_results: total = tr.execution.total if total > 0: rates.append(tr.execution.executed / total) else: - rates.append(1.0) + rates.append(NEUTRAL) return sum(rates) / len(rates) -def coordination(state: GameState, ctx: MetricContext) -> float: - """Ratio of conflicts resolved peacefully. 1.0 = no conflicts or all resolved.""" - if ctx.conflicts_total == 0: - return 1.0 - return min(1.0, ctx.conflicts_resolved / ctx.conflicts_total) - - -def communication_efficiency(state: GameState, ctx: MetricContext) -> float: - """Ratio of inter-agent messages that led to action changes. 1.0 = all useful.""" - if ctx.messages_sent == 0: - return 1.0 - return min(1.0, ctx.messages_acted_on / ctx.messages_sent) +def plan_coherence(state: GameState, ctx: MetricContext) -> float: + """Fraction of executed writes that did not contradict another write in + the same tick, averaged across ticks. Harness-agnostic: computed from what + reached RIMAPI, not from any harness's internal messaging (#51).""" + if not ctx.tick_results: + return NEUTRAL + scores = [] + for tr in ctx.tick_results: + contradictory, executed = count_contradictions(tr.execution.outcomes) + if executed == 0: + scores.append(NEUTRAL) + else: + scores.append(1.0 - contradictory / executed) + return sum(scores) / len(scores) ALL_METRICS = { @@ -129,6 +132,5 @@ def communication_efficiency(state: GameState, ctx: MetricContext) -> float: "research": research, "self_sufficiency": self_sufficiency, "efficiency": efficiency, - "coordination": coordination, - "communication_efficiency": communication_efficiency, + "plan_coherence": plan_coherence, } diff --git a/src/rle/tracking/metadata.py b/src/rle/tracking/metadata.py index 7e85d6d..cddea43 100644 --- a/src/rle/tracking/metadata.py +++ b/src/rle/tracking/metadata.py @@ -15,13 +15,17 @@ # metric implementations, or composite math change in a way that makes scores # from older runs not directly comparable. The leaderboard re-scores artifacts # at the current version on render; mismatches are surfaced, not silently -# elided. 1.1 (issue #25): threat_response now tracks actual draft responses +# elided. +# 1.1 (issue #25): threat_response now tracks actual draft responses # (first_draft_tick wired, was permanently 0.0 once any threat registered) # and null incident placeholders (enemy_count=0, threat_level=0.0) no longer -# count as threats — 1.0 scores with non-empty threats_seen are not -# comparable. coordination + communication_efficiency remain the broken -# implementations to be repaired in Phase C. -SCORING_VERSION = "1.1" +# count as threats. +# 1.2 (issue #51, "Phase C"): coordination + communication_efficiency removed +# (both were ~1.0 by construction and Felix-specific); plan_coherence added +# (contradictory executed writes per tick, harness-agnostic); efficiency and +# plan_coherence return a neutral 0.5 for ticks with no writes so an unmanaged +# baseline no longer banks free process points; weights redistributed. +SCORING_VERSION = "1.2" # Conventional install path for the RIMAPI Workshop mod we deploy our fork DLL # over. Best-effort — if Steam lives elsewhere set the RIMAPI_DLL_PATH env var. diff --git a/tests/unit/test_composite_scorer.py b/tests/unit/test_composite_scorer.py index e941ad3..f0c507b 100644 --- a/tests/unit/test_composite_scorer.py +++ b/tests/unit/test_composite_scorer.py @@ -4,6 +4,9 @@ import pytest +from rle.agents.actions import ActionPlan +from rle.orchestration.action_executor import ExecutionResult +from rle.orchestration.game_loop import TickResult from rle.rimapi.schemas import ( ColonyData, GameState, @@ -13,7 +16,7 @@ WeatherData, ) from rle.scoring.composite import DEFAULT_WEIGHTS, CompositeScorer, ScoreSnapshot -from rle.scoring.metrics import MetricContext +from rle.scoring.metrics import ALL_METRICS, NEUTRAL, MetricContext def _state() -> GameState: @@ -48,8 +51,35 @@ class TestDefaultWeights: def test_sum_to_one(self) -> None: assert sum(DEFAULT_WEIGHTS.values()) == pytest.approx(1.0) - def test_ten_metrics(self) -> None: - assert len(DEFAULT_WEIGHTS) == 10 + def test_nine_metrics(self) -> None: + assert len(DEFAULT_WEIGHTS) == 9 + + def test_weights_cover_exactly_the_registered_metrics(self) -> None: + assert set(DEFAULT_WEIGHTS) == set(ALL_METRICS) + + def test_legacy_process_metrics_gone(self) -> None: + """Scoring 1.2 (#51): always-1.0 Felix-specific metrics removed.""" + assert "coordination" not in DEFAULT_WEIGHTS + assert "communication_efficiency" not in DEFAULT_WEIGHTS + assert "plan_coherence" in DEFAULT_WEIGHTS + + def test_outcome_metrics_dominate(self) -> None: + process = DEFAULT_WEIGHTS["efficiency"] + DEFAULT_WEIGHTS["plan_coherence"] + assert process <= 0.15 + + +class TestBaselineNeutrality: + def test_unmanaged_baseline_gets_no_free_process_points(self) -> None: + """A run with zero writes scores NEUTRAL, not 1.0, on process metrics.""" + ctx = _ctx() + ctx.tick_results.append(TickResult( + tick=1, day=1, macro_time=0.0, + plan=ActionPlan(role="baseline", tick=1, actions=[]), + execution=ExecutionResult(executed=0, failed=0, total=0), + )) + snap = CompositeScorer().score(_state(), ctx) + assert snap.metrics["efficiency"] == pytest.approx(NEUTRAL) + assert snap.metrics["plan_coherence"] == pytest.approx(NEUTRAL) class TestCompositeScorer: @@ -59,7 +89,7 @@ def test_score_returns_snapshot(self) -> None: assert isinstance(snap, ScoreSnapshot) assert snap.tick == 600000 assert snap.day == 10 - assert len(snap.metrics) == 10 + assert len(snap.metrics) == 9 assert 0.0 <= snap.composite <= 1.0 def test_custom_weights(self) -> None: @@ -89,7 +119,7 @@ def test_averages_snapshots(self) -> None: metrics={"survival": 1.0, "mood": 0.8, "food_security": 0.6, "wealth": 0.5, "research": 0.0, "threat_response": 1.0, "self_sufficiency": 0.5, "efficiency": 1.0, - "coordination": 0.9, "communication_efficiency": 0.8}, + "plan_coherence": 0.9}, composite=0.7, ), ScoreSnapshot( @@ -97,7 +127,7 @@ def test_averages_snapshots(self) -> None: metrics={"survival": 0.5, "mood": 0.6, "food_security": 0.4, "wealth": 0.3, "research": 0.5, "threat_response": 0.5, "self_sufficiency": 0.5, "efficiency": 0.5, - "coordination": 0.7, "communication_efficiency": 0.6}, + "plan_coherence": 0.7}, composite=0.5, ), ] diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index c6a1fba..1eec201 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -17,7 +17,7 @@ def test_scoring_version_pins_a_string() -> None: that requires this test (and the dataset card) to be updated.""" assert isinstance(SCORING_VERSION, str) assert SCORING_VERSION - assert SCORING_VERSION == "1.1" + assert SCORING_VERSION == "1.2" def test_file_sha256_returns_none_for_missing_path() -> None: diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py index b893a87..10a9f8f 100644 --- a/tests/unit/test_metrics.py +++ b/tests/unit/test_metrics.py @@ -6,7 +6,7 @@ # Using the ActionPlan import for TickResult construction from rle.agents.actions import ActionPlan -from rle.orchestration.action_executor import ExecutionResult +from rle.orchestration.action_executor import ActionOutcome, ExecutionResult from rle.orchestration.game_loop import TickResult from rle.rimapi.schemas import ( ColonistData, @@ -19,12 +19,12 @@ WeatherData, ) from rle.scoring.metrics import ( + NEUTRAL, MetricContext, - communication_efficiency, - coordination, efficiency, food_security, mood, + plan_coherence, research, self_sufficiency, survival, @@ -95,14 +95,29 @@ def _ctx( ) -def _tick_result(executed: int, total: int) -> TickResult: +def _tick_result( + executed: int, total: int, outcomes: tuple[ActionOutcome, ...] = (), +) -> TickResult: return TickResult( tick=1, day=1, macro_time=0.1, plan=ActionPlan(role="test", tick=1, actions=[]), - execution=ExecutionResult(executed=executed, failed=total - executed, total=total), + execution=ExecutionResult( + executed=executed, failed=total - executed, total=total, outcomes=outcomes, + ), ) +def _ok(action_type: str, pawn: str | None = None, **params: object) -> ActionOutcome: + return ActionOutcome( + action_type=action_type, endpoint=action_type, target_colonist_id=pawn, + success=True, parameters=dict(params), + ) + + +def _coherence_tick(*outcomes: ActionOutcome) -> TickResult: + return _tick_result(len(outcomes), len(outcomes), outcomes) + + class TestSurvival: def test_all_alive(self) -> None: assert survival(_state(population=3), _ctx(initial_pop=3)) == pytest.approx(1.0) @@ -222,57 +237,109 @@ def test_mixed(self) -> None: # (0.5 + 1.0) / 2 = 0.75 assert efficiency(_state(), ctx) == pytest.approx(0.75) - def test_no_ticks(self) -> None: - assert efficiency(_state(), _ctx()) == pytest.approx(1.0) + def test_no_ticks_is_neutral(self) -> None: + assert efficiency(_state(), _ctx()) == pytest.approx(NEUTRAL) - def test_empty_plan(self) -> None: + def test_empty_plan_is_neutral(self) -> None: + """An unmanaged baseline issues no writes — it must not bank 1.0 (#51).""" ctx = _ctx(tick_results=[_tick_result(0, 0)]) - assert efficiency(_state(), ctx) == pytest.approx(1.0) + assert efficiency(_state(), ctx) == pytest.approx(NEUTRAL) + +class TestPlanCoherence: + def test_no_ticks_is_neutral(self) -> None: + assert plan_coherence(_state(), _ctx()) == pytest.approx(NEUTRAL) -class TestCoordination: - def test_no_conflicts(self) -> None: - ctx = _ctx() - assert coordination(_state(), ctx) == pytest.approx(1.0) - - def test_all_resolved(self) -> None: - ctx = _ctx() - ctx.conflicts_total = 10 - ctx.conflicts_resolved = 10 - assert coordination(_state(), ctx) == pytest.approx(1.0) - - def test_half_resolved(self) -> None: - ctx = _ctx() - ctx.conflicts_total = 8 - ctx.conflicts_resolved = 4 - assert coordination(_state(), ctx) == pytest.approx(0.5) - - def test_none_resolved(self) -> None: - ctx = _ctx() - ctx.conflicts_total = 5 - ctx.conflicts_resolved = 0 - assert coordination(_state(), ctx) == pytest.approx(0.0) - - -class TestCommunicationEfficiency: - def test_no_messages(self) -> None: - ctx = _ctx() - assert communication_efficiency(_state(), ctx) == pytest.approx(1.0) - - def test_all_acted_on(self) -> None: - ctx = _ctx() - ctx.messages_sent = 12 - ctx.messages_acted_on = 12 - assert communication_efficiency(_state(), ctx) == pytest.approx(1.0) - - def test_half_acted_on(self) -> None: - ctx = _ctx() - ctx.messages_sent = 10 - ctx.messages_acted_on = 5 - assert communication_efficiency(_state(), ctx) == pytest.approx(0.5) - - def test_none_acted_on(self) -> None: - ctx = _ctx() - ctx.messages_sent = 7 - ctx.messages_acted_on = 0 - assert communication_efficiency(_state(), ctx) == pytest.approx(0.0) + def test_no_writes_is_neutral(self) -> None: + ctx = _ctx(tick_results=[_tick_result(0, 0)]) + assert plan_coherence(_state(), ctx) == pytest.approx(NEUTRAL) + + def test_independent_writes_are_coherent(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("draft", "1", is_drafted=True), + _ok("work_priority", "2", Growing=1), + _ok("research_target", project="Electricity"), + _ok("blueprint", x=10, z=10, def_name="Wall"), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(1.0) + + def test_draft_undraft_same_pawn(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("draft", "1", is_drafted=True), + _ok("draft", "1", is_drafted=False), + _ok("work_priority", "2", Mining=1), + )]) + # 2 of 3 executed writes contradict each other + assert plan_coherence(_state(), ctx) == pytest.approx(1 / 3) + + def test_move_and_job_same_pawn(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("move", "1", x=5, z=5), + _ok("job_assign", "1", job_def="Mine"), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(0.0) + + def test_legacy_alias_resolves(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("draft_colonist", "1", is_drafted=True), + _ok("undraft_colonist", "1", is_drafted=False), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(0.0) + + def test_conflicting_work_priority(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("work_priority", "1", Growing=1), + _ok("work_priority", "1", Growing=4), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(0.0) + + def test_same_work_priority_twice_is_redundant_not_contradictory(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("work_priority", "1", Growing=1), + _ok("work_priority", "1", Growing=1), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(1.0) + + def test_overlapping_zones(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("growing_zone", x1=0, z1=0, x2=8, z2=8), + _ok("stockpile_zone", x1=4, z1=4, x2=9, z2=9), + _ok("growing_zone", x1=20, z1=20, x2=28, z2=28), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(1 / 3) + + def test_duplicate_blueprint_cell(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("blueprint", x=1, z=1, def_name="Wall"), + _ok("blueprint", x=1, z=1, def_name="Door"), + _ok("blueprint", x=2, z=1, def_name="Wall"), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(1 / 3) + + def test_competing_research_targets(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("research_target", project="Electricity"), + _ok("research_target", project="Batteries"), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(0.0) + + def test_failed_writes_do_not_count(self) -> None: + failed = ActionOutcome( + action_type="draft", endpoint="draft", target_colonist_id="1", + success=False, error="boom", parameters={"is_drafted": False}, + ) + ctx = _ctx(tick_results=[_tick_result(1, 2, ( + _ok("draft", "1", is_drafted=True), failed, + ))]) + assert plan_coherence(_state(), ctx) == pytest.approx(1.0) + + def test_averages_across_ticks(self) -> None: + ctx = _ctx(tick_results=[ + _coherence_tick(_ok("draft", "1", is_drafted=True)), + _coherence_tick( + _ok("draft", "1", is_drafted=True), + _ok("draft", "1", is_drafted=False), + ), + _tick_result(0, 0), + ]) + assert plan_coherence(_state(), ctx) == pytest.approx((1.0 + 0.0 + NEUTRAL) / 3) diff --git a/tests/unit/test_scenario_loader.py b/tests/unit/test_scenario_loader.py index 9136378..19ef1af 100644 --- a/tests/unit/test_scenario_loader.py +++ b/tests/unit/test_scenario_loader.py @@ -34,7 +34,7 @@ def test_load_ship_launch(self) -> None: def test_scoring_weights_override(self) -> None: path = DEFINITIONS_DIR / "04_raid_defense.yaml" scenario = load_scenario(path) - assert scenario.scoring_weights["threat_response"] == 0.24 + assert scenario.scoring_weights["threat_response"] == 0.3 def test_invalid_path_raises(self) -> None: with pytest.raises(FileNotFoundError): diff --git a/uv.lock b/uv.lock index fc4c818..3232f15 100644 --- a/uv.lock +++ b/uv.lock @@ -951,7 +951,7 @@ wheels = [ [[package]] name = "rimworld-learning-environment" -version = "0.3.0" +version = "0.4.1" source = { editable = "." } dependencies = [ { name = "felix-agent-sdk" }, From 05108e4c75d356e701e6a0113fef737b3511ad74 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 06:57:42 +0000 Subject: [PATCH 2/8] harness layer: BaseHarness/StepResult protocol, FelixHarness + BaselineHarness, entry-point registry RLEGameLoop now owns only the environment (pause/state/execute/score/export) and delegates decisions to a harness. The Felix stack (CentralPost wiring, MapAnalyst-first deliberation, per-agent timeouts, phase/score/error broadcasts, helix visualiser, generation-id accounting) moves verbatim into rle.harness.felix.FelixHarness; the unmanaged colony becomes BaselineHarness. The loop has no felix_agent_sdk import left. - rle.harness.protocol: BaseHarness, StepResult (plan + optional pre-applied execution for tool-using harnesses), HarnessContext, TickObserver, HarnessPlugin, Availability - rle.harness.registry: discovery via the rle.harnesses entry-point group, option validation against each plugin's pydantic schema, --harness-opt key=value parsing; baseline + felix registered in pyproject - rle.harness.compat: RLEGameLoop(agents=..., no_agent=...) keep working - FelixOptions (parallel, no_think, helix_preset, role_timeout_s, exclude_agent, provider_kwargs, visualize); provider/helix construction moves off RLEConfig into rle.harness.felix.provider_factory - RLEConfig gains harness / harness_options / tick_timeout_s; loop-level step timeout and HarnessStepError degrade to an empty scored tick - dashboard export gains harness + extras; TickResult records harness and step latency - tests: registry, compat, custom harness through the loop, pre-executed writes scored by plan_coherence, failure/timeout degradation Co-authored-by: Jason --- pyproject.toml | 7 + src/rle/config.py | 73 +-- src/rle/harness/__init__.py | 59 ++ src/rle/harness/baseline.py | 58 ++ src/rle/harness/compat.py | 40 ++ src/rle/harness/felix/__init__.py | 11 + src/rle/harness/felix/build.py | 105 ++++ src/rle/harness/felix/harness.py | 466 ++++++++++++++++ src/rle/harness/felix/options.py | 50 ++ src/rle/harness/felix/plugin.py | 60 +++ src/rle/harness/felix/provider_factory.py | 48 ++ src/rle/harness/felix/smoke.py | 70 +++ src/rle/harness/protocol.py | 195 +++++++ src/rle/harness/registry.py | 150 ++++++ src/rle/orchestration/game_loop.py | 621 +++++++--------------- tests/integration/test_game_loop.py | 14 +- tests/integration/test_harness_loop.py | 211 ++++++++ tests/unit/test_config.py | 23 +- tests/unit/test_harness_registry.py | 176 ++++++ 19 files changed, 1931 insertions(+), 506 deletions(-) create mode 100644 src/rle/harness/__init__.py create mode 100644 src/rle/harness/baseline.py create mode 100644 src/rle/harness/compat.py create mode 100644 src/rle/harness/felix/__init__.py create mode 100644 src/rle/harness/felix/build.py create mode 100644 src/rle/harness/felix/harness.py create mode 100644 src/rle/harness/felix/options.py create mode 100644 src/rle/harness/felix/plugin.py create mode 100644 src/rle/harness/felix/provider_factory.py create mode 100644 src/rle/harness/felix/smoke.py create mode 100644 src/rle/harness/protocol.py create mode 100644 src/rle/harness/registry.py create mode 100644 tests/integration/test_harness_loop.py create mode 100644 tests/unit/test_harness_registry.py diff --git a/pyproject.toml b/pyproject.toml index a11941b..3555d57 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,13 @@ tracking = [ "huggingface-hub>=0.20", ] +# Harness registry. Only RLE-authored harnesses live in this tree; harnesses +# wrapping third-party tools (OpenCode, Grok Build, ...) ship as their own +# packages and register under the same group. +[project.entry-points."rle.harnesses"] +baseline = "rle.harness.baseline:PLUGIN" +felix = "rle.harness.felix:PLUGIN" + [tool.hatch.build.targets.wheel] packages = ["src/rle"] diff --git a/src/rle/config.py b/src/rle/config.py index 89e11ef..cdc06bd 100644 --- a/src/rle/config.py +++ b/src/rle/config.py @@ -3,34 +3,19 @@ from __future__ import annotations import os +from typing import Any -from felix_agent_sdk.core import HelixConfig -from felix_agent_sdk.providers import ( - AnthropicProvider, - BaseProvider, - LocalProvider, - OpenAIProvider, -) from pydantic_settings import BaseSettings -from rle.providers.claude_code import ClaudeCodeProvider - -_HELIX_PRESETS: dict[str, HelixConfig] = { - "default": HelixConfig.default(), - "research_heavy": HelixConfig.research_heavy(), - "fast_convergence": HelixConfig.fast_convergence(), -} - -_PROVIDER_CLASSES: dict[str, type[BaseProvider]] = { - "anthropic": AnthropicProvider, - "openai": OpenAIProvider, - "local": LocalProvider, - "claude-code": ClaudeCodeProvider, -} - class RLEConfig(BaseSettings): - """Top-level configuration for the RimWorld Learning Environment.""" + """Top-level configuration for the RimWorld Learning Environment. + + Deliberately framework-free: provider/model/harness are strings, and the + harness that gets built from them (see ``rle.harness``) decides what they + mean. Felix-only knobs (helix preset, no-think, parallelism) live in + ``FelixOptions`` and arrive via ``harness_options``. + """ model_config = {"env_prefix": "", "env_file": ".env", "extra": "ignore"} @@ -41,13 +26,21 @@ class RLEConfig(BaseSettings): openrouter_api_key: str | None = None anthropic_api_key: str | None = None tick_interval: float = 1.0 + harness: str = "felix" + """Which harness decides the colony's actions. Resolved through the + ``rle.harnesses`` entry-point registry (``--harness list``).""" + harness_options: dict[str, Any] = {} + """Harness-specific options validated against the plugin's schema + (``RLE_HARNESS_OPTIONS`` as JSON, or ``--harness-opt key=value``).""" + tick_timeout_s: float | None = None + """Loop-level cap on a whole harness step. ``None`` = no cap (the Felix + harness applies its own per-agent ``role_timeout_s``).""" role_timeout_s: float = 60.0 - """Max wall-clock seconds for a single agent's deliberation. Hung LLM + """Max wall-clock seconds for a single Felix agent's deliberation. Hung LLM calls beyond this fire a deliberation_timeout ERROR event and the agent - contributes no actions for the tick. The hung thread eventually unwinds - via the provider's own timeout (Python threads can't be force-killed). - Docker-benchmark average is ~7s per deliberation; 60s leaves ~8x headroom.""" - helix_preset: str = "default" + contributes no actions for the tick. Kept on RLEConfig for the legacy + ``RLEGameLoop(agents=...)`` path; ``--harness-opt role_timeout_s=`` is the + modern spelling.""" max_agents: int = 7 log_level: str = "INFO" docker_image: str = "rle-headless:latest" @@ -57,30 +50,6 @@ class RLEConfig(BaseSettings): hf_dataset_repo: str = "AppSprout/rle-benchmarks" """Target HF dataset repo (HF_DATASET_REPO in .env to override).""" - def get_helix_config(self) -> HelixConfig: - """Return the HelixConfig preset matching ``helix_preset``.""" - try: - return _HELIX_PRESETS[self.helix_preset] - except KeyError: - raise ValueError( - f"Unknown helix preset {self.helix_preset!r}. " - f"Choose from: {list(_HELIX_PRESETS)}" - ) from None - - def get_provider(self) -> BaseProvider: - """Construct an LLM provider from the current config.""" - cls = _PROVIDER_CLASSES.get(self.provider) - if cls is None: - raise ValueError( - f"Unknown provider {self.provider!r}. " - f"Choose from: {list(_PROVIDER_CLASSES)}" - ) - kwargs: dict[str, str] = {"model": self.model} - if self.provider_base_url: - kwargs["base_url"] = self.provider_base_url - return cls(**kwargs) # type: ignore[arg-type] # subclasses accept kwargs - - def bridge_openrouter_key(config: RLEConfig) -> None: """If OPENROUTER_API_KEY is set but OPENAI_API_KEY isn't, bridge them.""" if config.openrouter_api_key and not os.environ.get("OPENAI_API_KEY"): diff --git a/src/rle/harness/__init__.py b/src/rle/harness/__init__.py new file mode 100644 index 0000000..1da916d --- /dev/null +++ b/src/rle/harness/__init__.py @@ -0,0 +1,59 @@ +"""Swappable harnesses — the decision-making side of an RLE run. + +Public API for harness authors (kept stable; external harness packages depend +on it): + +- :class:`BaseHarness`, :class:`StepResult`, :class:`HarnessContext` +- :class:`HarnessPlugin`, :class:`Availability`, :class:`EmptyOptions` +- :class:`TickObserver`, :class:`HarnessStepError` +- :func:`create_harness`, :func:`list_harnesses`, :func:`get_plugin` + +Register a harness under the ``rle.harnesses`` entry-point group; see +``docs/harness-plugins.md``. +""" + +from rle.harness.protocol import ( + Availability, + BaseHarness, + EmptyOptions, + HarnessContext, + HarnessPlugin, + HarnessStepError, + StepResult, + TickObserver, +) +from rle.harness.registry import ( + ENTRY_POINT_GROUP, + HarnessInfo, + HarnessNotFoundError, + HarnessOptionsError, + HarnessUnavailableError, + create_harness, + get_plugin, + harness_names, + list_harnesses, + parse_option_pairs, + validate_options, +) + +__all__ = [ + "ENTRY_POINT_GROUP", + "Availability", + "BaseHarness", + "EmptyOptions", + "HarnessContext", + "HarnessInfo", + "HarnessNotFoundError", + "HarnessOptionsError", + "HarnessPlugin", + "HarnessStepError", + "HarnessUnavailableError", + "StepResult", + "TickObserver", + "create_harness", + "get_plugin", + "harness_names", + "list_harnesses", + "parse_option_pairs", + "validate_options", +] diff --git a/src/rle/harness/baseline.py b/src/rle/harness/baseline.py new file mode 100644 index 0000000..952fab3 --- /dev/null +++ b/src/rle/harness/baseline.py @@ -0,0 +1,58 @@ +"""Unmanaged baseline harness — the colony runs on RimWorld's own AI.""" + +from __future__ import annotations + +from typing import ClassVar + +from pydantic import BaseModel + +from rle.agents.actions import ActionPlan +from rle.harness.protocol import ( + Availability, + BaseHarness, + EmptyOptions, + HarnessContext, + StepResult, +) +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent + + +class BaselineHarness(BaseHarness): + """Proposes nothing every tick. Paired against every other harness.""" + + name: ClassVar[str] = "baseline" + + async def step( + self, state: GameState, tick: int, macro_time: float, + events: list[RimAPIEvent], + ) -> StepResult: + return StepResult( + plan=ActionPlan( + role="baseline", tick=state.colony.tick, actions=[], + summary="No agents", + ), + ) + + +class BaselinePlugin: + name = "baseline" + description = "Unmanaged colony (RimWorld built-in AI). The paired control for every run." + + def available(self) -> Availability: + return Availability.available() + + def option_schema(self) -> type[BaseModel]: + return EmptyOptions + + def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + return BaselineHarness() + + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + return BaselineHarness() + + def describe(self) -> dict[str, str]: + return {"harness": self.name} + + +PLUGIN = BaselinePlugin() diff --git a/src/rle/harness/compat.py b/src/rle/harness/compat.py new file mode 100644 index 0000000..19c05ec --- /dev/null +++ b/src/rle/harness/compat.py @@ -0,0 +1,40 @@ +"""Backward-compatibility shims for callers that predate the harness layer. + +``RLEGameLoop(config, client, agents, no_agent=..., parallel=..., visualizer=...)`` +still works: this module turns those legacy arguments into a harness. It is +the one place in core that reaches for the Felix harness by module path — +through ``importlib`` so that importing the loop never imports the SDK +(documented exception to the no-inline-imports rule: optional dependency). +""" + +from __future__ import annotations + +from collections.abc import Sequence +from importlib import import_module +from typing import Any + +from rle.harness.baseline import BaselineHarness +from rle.harness.protocol import BaseHarness + +FELIX_HARNESS_MODULE = "rle.harness.felix.harness" + + +def build_legacy_harness( + agents: Sequence[Any] | None, + *, + no_agent: bool = False, + parallel: bool = True, + visualizer: Any | None = None, + role_timeout_s: float = 60.0, +) -> BaseHarness: + if no_agent or not agents: + return BaselineHarness() + module = import_module(FELIX_HARNESS_MODULE) + felix_cls = module.FelixHarness + harness: BaseHarness = felix_cls( + list(agents), + parallel=parallel, + role_timeout_s=role_timeout_s, + visualizer=visualizer, + ) + return harness diff --git a/src/rle/harness/felix/__init__.py b/src/rle/harness/felix/__init__.py new file mode 100644 index 0000000..73a76da --- /dev/null +++ b/src/rle/harness/felix/__init__.py @@ -0,0 +1,11 @@ +"""Felix multi-agent harness (optional extra ``felix``). + +Importing this package must stay cheap and Felix-free: the ``felix`` entry +point resolves to :data:`PLUGIN`, whose methods import the SDK-dependent +modules (``harness``, ``build``, ``provider_factory``) only when a Felix +harness is actually requested. +""" + +from rle.harness.felix.plugin import PLUGIN, FelixPlugin + +__all__ = ["PLUGIN", "FelixPlugin"] diff --git a/src/rle/harness/felix/build.py b/src/rle/harness/felix/build.py new file mode 100644 index 0000000..9828dad --- /dev/null +++ b/src/rle/harness/felix/build.py @@ -0,0 +1,105 @@ +"""Construct a fully wired FelixHarness from HarnessContext + FelixOptions.""" + +from __future__ import annotations + +from typing import Any + +from felix_agent_sdk.core import HelixGeometry +from felix_agent_sdk.providers.base import BaseProvider +from felix_agent_sdk.visualization import HelixVisualizer +from pydantic import BaseModel + +from rle.agents import AGENT_DISPLAY +from rle.agents.base_role import RimWorldRoleAgent +from rle.agents.construction_planner import ConstructionPlanner +from rle.agents.defense_commander import DefenseCommander +from rle.agents.map_analyst import MapAnalyst +from rle.agents.medical_officer import MedicalOfficer +from rle.agents.research_director import ResearchDirector +from rle.agents.resource_manager import ResourceManager +from rle.agents.social_overseer import SocialOverseer +from rle.config import bridge_anthropic_key, bridge_openrouter_key +from rle.harness.felix.harness import FelixHarness +from rle.harness.felix.options import FelixOptions +from rle.harness.felix.provider_factory import build_helix, build_provider +from rle.harness.felix.smoke import SmokeProvider +from rle.harness.protocol import HarnessContext + +# Context extras understood by this builder. +WEAVE_MODULE_EXTRA = "weave_module" + + +def create_agents( + provider: BaseProvider, + helix: HelixGeometry, + *, + exclude_agent: str | None = None, + provider_kwargs: dict[str, Any] | None = None, + no_think: bool = False, +) -> list[RimWorldRoleAgent]: + """The canonical 7-agent roster (MapAnalyst + 6 domain roles).""" + roster: list[RimWorldRoleAgent] = [ + MapAnalyst("map_analyst", provider, helix, spawn_time=0.0, velocity=1.0), + ResourceManager("resource_manager", provider, helix, spawn_time=0.0, velocity=1.0), + DefenseCommander("defense_commander", provider, helix, spawn_time=0.0, velocity=1.0), + ResearchDirector("research_director", provider, helix, spawn_time=0.0, velocity=1.0), + SocialOverseer("social_overseer", provider, helix, spawn_time=0.0, velocity=1.0), + ConstructionPlanner( + "construction_planner", provider, helix, spawn_time=0.0, velocity=1.0, + ), + MedicalOfficer("medical_officer", provider, helix, spawn_time=0.0, velocity=1.0), + ] + agents = [a for a in roster if a.agent_id != exclude_agent] + for agent in agents: + if provider_kwargs: + agent.set_provider_kwargs(**provider_kwargs) + if no_think: + agent.set_no_think(True) + return agents + + +def create_visualizer( + helix: HelixGeometry, agents: list[RimWorldRoleAgent], title: str = "R L E", +) -> HelixVisualizer: + visualizer = HelixVisualizer(helix, title=title) + for agent in agents: + display = AGENT_DISPLAY[agent.agent_id] + visualizer.register_agent( + agent.agent_id, label=display["label"], color=display["color"], + ) + return visualizer + + +def build_felix_harness( + ctx: HarnessContext, options: BaseModel, *, smoke: bool = False, +) -> FelixHarness: + opts = options if isinstance(options, FelixOptions) else FelixOptions.model_validate( + options.model_dump(), + ) + provider: BaseProvider + if smoke or ctx.smoke: + provider = SmokeProvider() + else: + bridge_openrouter_key(ctx.config) + bridge_anthropic_key(ctx.config) + provider = build_provider( + ctx.config.provider, ctx.config.model, ctx.config.provider_base_url, + ) + helix = build_helix(opts.helix_preset) + agents = create_agents( + provider, helix, + exclude_agent=opts.exclude_agent, + provider_kwargs=opts.provider_kwargs or None, + no_think=opts.no_think, + ) + weave_module = ctx.extras.get(WEAVE_MODULE_EXTRA) + if weave_module is not None: + for agent in agents: + agent.enable_weave(weave_module) + visualizer = create_visualizer(helix, agents) if opts.visualize else None + return FelixHarness( + agents, + parallel=opts.parallel, + role_timeout_s=opts.role_timeout_s, + visualizer=visualizer, + ) diff --git a/src/rle/harness/felix/harness.py b/src/rle/harness/felix/harness.py new file mode 100644 index 0000000..56b9569 --- /dev/null +++ b/src/rle/harness/felix/harness.py @@ -0,0 +1,466 @@ +"""FelixHarness — the original RLE harness: MapAnalyst-first, six role agents +deliberating in parallel over Felix SDK's CentralPost hub-spoke bus, merged by +``ActionResolver``. + +Everything Felix-specific that used to live in ``RLEGameLoop`` lives here: +hub/spoke wiring, per-agent timeouts, phase broadcasts, action-error and score +feedback, helix visualisation, generation-id / token accounting. +""" + +from __future__ import annotations + +import asyncio +import logging +import time as _time +from contextlib import AbstractContextManager, nullcontext +from importlib.metadata import PackageNotFoundError, version +from typing import Any, ClassVar + +from felix_agent_sdk.communication import CentralPost, MessageType, SpokeManager +from felix_agent_sdk.providers import ProviderError + +from rle.agents.actions import ActionPlan, ActionPlanParseError +from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.protocol import BaseHarness, HarnessContext, StepResult +from rle.orchestration.action_executor import ExecutionResult +from rle.orchestration.action_resolver import ActionResolver +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.scoring.composite import ScoreSnapshot +from rle.tracking.event_log import EventType + +logger = logging.getLogger(__name__) + +# Truncation limits for human-readable text persisted to the event log / +# deliberation log. Keep these tight so the JSONL stays grep-able and small; +# the per-scenario *_deliberations.jsonl carries the full raw reasoning. +_ACTION_REASON_CHARS = 200 +_PLAN_SUMMARY_CHARS = 300 +_PARSE_FAILURE_RAW_CHARS = 500 +# Full LLM completion text on successful deliberation (PROVIDER_CALL events). +# 16 KB headroom: frontier models emit multi-KB structured plans and the +# verbatim transcripts are first-class analysis artifacts; longer completions +# are tail-truncated so the parsed action JSON remains visible. +_RAW_OUTPUT_CHARS = 16384 + + +def felix_sdk_version() -> str: + try: + return version("felix-agent-sdk") + except PackageNotFoundError: + return "unknown" + + +def phase_for(macro_time: float) -> str: + """Helix macro phase for a normalised run position (0..1).""" + if macro_time < 0.4: + return "exploration" + if macro_time < 0.7: + return "analysis" + return "synthesis" + + +class FelixHarness(BaseHarness): + name: ClassVar[str] = "felix" + + def __init__( + self, + agents: list[RimWorldRoleAgent], + *, + parallel: bool = True, + role_timeout_s: float = 60.0, + visualizer: Any | None = None, + ) -> None: + super().__init__() + self._agents = agents + self._map_analyst: RimWorldRoleAgent | None = None + self._role_agents: list[RimWorldRoleAgent] = [] + for agent in agents: + if agent.ROLE_NAME == "map_analyst": + self._map_analyst = agent + else: + self._role_agents.append(agent) + self._parallel = parallel + self._role_timeout_s = role_timeout_s + self._visualizer = visualizer + self._resolver = ActionResolver() + self.last_phase: str = "" + + # Hub-spoke communication — agents read messages from their spokes. + # Wired at construction so callers can inspect spokes before setup(). + self.hub = CentralPost(max_agents=max(1, len(agents))) + self.spoke_manager = SpokeManager(self.hub) + for agent in agents: + spoke = self.spoke_manager.create_spoke(agent.agent_id, agent=agent) + agent.attach_spoke(spoke) + + @property + def agents(self) -> list[RimWorldRoleAgent]: + return list(self._agents) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + async def setup(self, ctx: HarnessContext) -> None: + await super().setup(ctx) + if self._visualizer is not None: + self.observers.append(_HelixRenderObserver(self._visualizer)) + + async def teardown(self) -> None: + # Final drain: a deliberation that timed out on the last tick may have + # completed after the per-tick drain. + self._drain_generation_ids() + + def run_context(self) -> AbstractContextManager[Any]: + if self._visualizer is not None: + return self._visualizer.live() # type: ignore[no-any-return] + return nullcontext() + + @property + def visualizer(self) -> Any | None: + return self._visualizer + + def describe(self) -> dict[str, str]: + return { + "harness": self.name, + "agents": ",".join(a.ROLE_NAME for a in self._agents), + "felix_agent_sdk": felix_sdk_version(), + } + + # ------------------------------------------------------------------ + # Per-tick step + # ------------------------------------------------------------------ + + async def step( + self, state: GameState, tick: int, macro_time: float, + events: list[RimAPIEvent], + ) -> StepResult: + # Route previous tick's messages to agent spokes + broadcast phase changes + messages_before = self.hub.total_messages_processed + self.spoke_manager.process_all_messages() + self._broadcast_phase_if_changed(macro_time) + + for agent in self._agents: + agent.set_pending_events(events) + for evt in events: + self.ctx.emit( + EventType.SSE_EVENT, tick, + sse_type=evt.event_type, sse_data=str(evt.data)[:200], + ) + + plans: list[ActionPlan] = [] + + # MapAnalyst deliberates FIRST (sequential, timeout-wrapped) + if self._map_analyst: + ma_agent, ma_plan = await self._deliberate_agent_with_timeout( + self._map_analyst, state, macro_time, tick, + ) + if ma_plan is not None: + plans.append(ma_plan) + self._update_visualizer_agent(ma_agent, ma_plan, macro_time) + self._send_task_complete(ma_agent, ma_plan) + # Route MapAnalyst output to role agent spokes immediately + self.spoke_manager.process_all_messages() + + # Snapshot which agents have pending spoke messages (diagnostics) + agents_with_messages: set[str] = set() + for ra in self._role_agents: + spoke = self.spoke_manager.get_spoke(ra.agent_id) + if spoke and spoke.has_pending_messages(): + agents_with_messages.add(ra.agent_id) + + if self._parallel: + results = await self._deliberate_parallel(state, macro_time, tick) + else: + results = await self._deliberate_sequential(state, macro_time, tick) + + agents_acted_with_messages = 0 + for agent, plan in results: + if plan is None: + continue + plans.append(plan) + if agent.agent_id in agents_with_messages: + agents_acted_with_messages += 1 + self._update_visualizer_agent(agent, plan, macro_time) + self._send_task_complete(agent, plan) + + # Capture generation IDs from every provider call this tick — parse + # retries and failed deliberations bill tokens too. + self._drain_generation_ids() + + # Resolve conflicts. Resolver + CentralPost counts are diagnostics in + # the event log only — they no longer feed the composite (#51). + resolved, resolver_stats = self._resolver.resolve(plans, state) + self.ctx.emit( + EventType.CONFLICT, tick, + input_plans=len(plans), + output_actions=len(resolved.actions), + conflicts_detected=resolver_stats.conflicts_total, + conflicts_resolved=resolver_stats.conflicts_resolved, + messages_routed=self.hub.total_messages_processed - messages_before, + agents_with_messages=len(agents_with_messages), + agents_acted_with_messages=agents_acted_with_messages, + ) + + return StepResult( + plan=resolved, + proposals=tuple(plans), + extras={ + "phase": self.last_phase, + "conflicts_detected": resolver_stats.conflicts_total, + "conflicts_resolved": resolver_stats.conflicts_resolved, + }, + ) + + async def on_tick_end( + self, tick: int, state: GameState, step: StepResult, + execution: ExecutionResult, score: ScoreSnapshot | None, + ) -> None: + # Surface per-action errors back to agents so they can avoid + # re-proposing the same invalid action next tick. + failed_outcomes = [o for o in execution.outcomes if not o.success and o.error] + if failed_outcomes: + error_summary = "; ".join( + f"{o.action_type}({o.target_colonist_id or '-'}): {o.error}" + for o in failed_outcomes[:10] + ) + self.spoke_manager.broadcast_message( + MessageType.STATUS_UPDATE, + { + "tick": state.colony.tick, + "summary": f"Last tick action errors — DO NOT REPEAT: {error_summary}", + "action_errors": [ + { + "action_type": o.action_type, + "target_colonist_id": o.target_colonist_id, + "error": o.error, + } + for o in failed_outcomes + ], + }, + sender_id="hub", + ) + + if score: + self.spoke_manager.broadcast_message( + MessageType.STATUS_UPDATE, + { + "tick": state.colony.tick, + "day": state.colony.day, + "composite_score": score.composite, + "metrics": score.metrics, + }, + sender_id="hub", + ) + + await super().on_tick_end(tick, state, step, execution, score) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _send_task_complete(self, agent: RimWorldRoleAgent, plan: ActionPlan) -> None: + spoke = self.spoke_manager.get_spoke(agent.agent_id) + if spoke and spoke.is_connected: + spoke.send_message( + MessageType.TASK_COMPLETE, + { + "role": plan.role, + "summary": plan.summary, + "confidence": plan.confidence, + "num_actions": len(plan.actions), + "action_types": [a.action_type for a in plan.actions], + }, + ) + + def _drain_generation_ids(self) -> None: + tracker = self._ctx.cost_tracker if self._ctx else None + if tracker is None: + return + for agent in self._agents: + for gen_id in agent.drain_generation_ids(): + tracker.record_generation_id(gen_id) + + def _broadcast_phase_if_changed(self, macro_time: float) -> None: + phase = phase_for(macro_time) + if phase != self.last_phase: + self.spoke_manager.broadcast_message( + MessageType.PHASE_ANNOUNCE, + {"phase": phase, "depth_ratio": macro_time}, + sender_id="hub", + ) + self.last_phase = phase + + def _update_visualizer_agent( + self, agent: RimWorldRoleAgent, plan: ActionPlan, macro_time: float, + ) -> None: + if self._visualizer is None: + return + self._visualizer.update( + agent.agent_id, + progress=macro_time, + confidence=plan.confidence, + phase=agent.position.phase, + status=f"{len(plan.actions)} actions", + ) + + async def _deliberate_parallel( + self, state: GameState, macro_time: float, tick: int, + ) -> list[tuple[RimWorldRoleAgent, ActionPlan | None]]: + return list(await asyncio.gather(*[ + self._deliberate_agent_with_timeout(a, state, macro_time, tick) + for a in self._role_agents + ])) + + async def _deliberate_sequential( + self, state: GameState, macro_time: float, tick: int, + ) -> list[tuple[RimWorldRoleAgent, ActionPlan | None]]: + results: list[tuple[RimWorldRoleAgent, ActionPlan | None]] = [] + for agent in self._role_agents: + results.append( + await self._deliberate_agent_with_timeout(agent, state, macro_time, tick), + ) + return results + + async def _deliberate_agent_with_timeout( + self, agent: RimWorldRoleAgent, state: GameState, macro_time: float, tick: int, + ) -> tuple[RimWorldRoleAgent, ActionPlan | None]: + """Run one agent's deliberation with a hard timeout. + + On timeout: emits a deliberation_timeout ERROR event, records it in the + deliberation log, and returns ``(agent, None)`` so the tick continues. + """ + try: + return await asyncio.wait_for( + self._deliberate_agent(agent, state, macro_time, tick), + timeout=self._role_timeout_s, + ) + except asyncio.TimeoutError: + logger.warning( + "Agent %s deliberation timed out after %.1fs (tick %d)", + agent.ROLE_NAME, self._role_timeout_s, tick, + ) + self.parse_failures += 1 + self.deliberation_log.append({ + "tick": tick, "agent": agent.ROLE_NAME, + "status": "deliberation_timeout", + "reason": f"timed out after {self._role_timeout_s}s", + }) + self.ctx.emit( + EventType.ERROR, tick, agent=agent.ROLE_NAME, + error_type="deliberation_timeout", + reason=f"timed out after {self._role_timeout_s}s", + timeout_s=self._role_timeout_s, + ) + return agent, None + + async def _deliberate_agent( + self, agent: RimWorldRoleAgent, state: GameState, macro_time: float, tick: int, + ) -> tuple[RimWorldRoleAgent, ActionPlan | None]: + t0 = _time.monotonic() + try: + plan = await agent.adeliberate(state, macro_time) + except ActionPlanParseError as e: + latency_ms = round((_time.monotonic() - t0) * 1000, 1) + logger.warning( + "Agent %s parse failure (tick %d): %s", agent.ROLE_NAME, tick, e.reason, + ) + self.parse_failures += 1 + raw_truncated = ( + e.raw_content[:_PARSE_FAILURE_RAW_CHARS] if e.raw_content else None + ) + self.deliberation_log.append({ + "tick": tick, "agent": agent.ROLE_NAME, + "status": "parse_failure", "reason": e.reason, + "raw": raw_truncated, + }) + self.ctx.emit( + EventType.ERROR, tick, agent=agent.ROLE_NAME, + error_type="parse_failure", reason=e.reason, latency_ms=latency_ms, + raw=raw_truncated, + ) + return agent, None + except ProviderError as e: + latency_ms = round((_time.monotonic() - t0) * 1000, 1) + logger.warning( + "Agent %s provider error (tick %d): %s", agent.ROLE_NAME, tick, e, + ) + self.parse_failures += 1 + self.deliberation_log.append({ + "tick": tick, "agent": agent.ROLE_NAME, + "status": "provider_error", "reason": str(e), + }) + self.ctx.emit( + EventType.ERROR, tick, agent=agent.ROLE_NAME, + error_type="provider_error", reason=str(e), latency_ms=latency_ms, + ) + return agent, None + + latency_ms = round((_time.monotonic() - t0) * 1000, 1) + self.parse_successes += 1 + actions_payload = [ + {"type": a.action_type, "target": a.target_colonist_id, + "priority": a.priority, "reason": a.reason[:_ACTION_REASON_CHARS]} + for a in plan.actions + ] + summary_truncated = plan.summary[:_PLAN_SUMMARY_CHARS] + self.deliberation_log.append({ + "tick": tick, "agent": plan.role, + "status": "success", "confidence": plan.confidence, + "num_actions": len(plan.actions), + "actions": actions_payload, + "summary": summary_truncated, + }) + self.ctx.emit( + EventType.DELIBERATION, tick, agent=plan.role, + latency_ms=latency_ms, confidence=plan.confidence, + num_actions=len(plan.actions), + actions=actions_payload, + summary=summary_truncated, + ) + + usage = agent._last_usage + if usage and isinstance(usage, dict): + pt = usage.get("prompt_tokens", 0) + ct = usage.get("completion_tokens", 0) + rt = usage.get("reasoning_tokens", 0) + if not isinstance(rt, int): + rt = 0 + if isinstance(pt, int) and isinstance(ct, int): + if self.ctx.cost_tracker: + self.ctx.cost_tracker.record_raw(pt, ct, rt) + raw_output = agent._last_raw_output + raw_output_truncated = ( + raw_output[:_RAW_OUTPUT_CHARS] if raw_output else None + ) + was_truncated = ( + raw_output is not None and len(raw_output) > _RAW_OUTPUT_CHARS + ) + self.ctx.emit( + EventType.PROVIDER_CALL, tick, agent=plan.role, + prompt_tokens=pt, completion_tokens=ct, + reasoning_tokens=rt, + raw_output=raw_output_truncated, + raw_output_truncated=was_truncated, + ) + + return agent, plan + + +class _HelixRenderObserver: + """Renders the terminal helix at the end of each tick.""" + + def __init__(self, visualizer: Any) -> None: + self._visualizer = visualizer + + def on_tick_end( + self, tick: int, day: int, step: StepResult, + execution: ExecutionResult, score: ScoreSnapshot | None, + ) -> None: + extra: dict[str, str] = { + "actions": f"{execution.executed}/{execution.total}", + } + if score: + extra["score"] = f"{score.composite:.3f}" + self._visualizer.render(tick=tick, day=day, extra_info=extra) diff --git a/src/rle/harness/felix/options.py b/src/rle/harness/felix/options.py new file mode 100644 index 0000000..f41c452 --- /dev/null +++ b/src/rle/harness/felix/options.py @@ -0,0 +1,50 @@ +"""Options accepted by the Felix harness (``--harness-opt key=value``).""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class FelixOptions(BaseModel): + """Knobs that only make sense for the Felix multi-agent harness. + + These used to be top-level CLI flags / ``RLEConfig`` fields. They live + here now so the environment and other harnesses never see them. + """ + + model_config = ConfigDict(extra="forbid") + + parallel: bool = Field( + default=True, + description="Deliberate the six role agents concurrently (MapAnalyst always first).", + ) + no_think: bool = Field( + default=False, + description="Inject a assistant prefill so thinking models skip reasoning.", + ) + helix_preset: str = Field( + default="default", + description="HelixConfig preset: default | research_heavy | fast_convergence.", + ) + role_timeout_s: float = Field( + default=60.0, + description=( + "Max wall-clock seconds for a single agent's deliberation. Hung LLM calls " + "beyond this fire a deliberation_timeout ERROR event and the agent " + "contributes no actions for the tick." + ), + ) + exclude_agent: str | None = Field( + default=None, + description="Drop one role agent by id (ablation runs).", + ) + provider_kwargs: dict[str, Any] = Field( + default_factory=dict, + description="Extra kwargs forwarded to provider.complete() (e.g. extra_body).", + ) + visualize: bool = Field( + default=False, + description="Render the terminal helix visualiser.", + ) diff --git a/src/rle/harness/felix/plugin.py b/src/rle/harness/felix/plugin.py new file mode 100644 index 0000000..224b24e --- /dev/null +++ b/src/rle/harness/felix/plugin.py @@ -0,0 +1,60 @@ +"""Entry-point plugin for the Felix multi-agent harness. + +This module is the optional-dependency boundary for ``felix-agent-sdk``: it +is imported by the plugin registry on every ``--harness list``, so it must +not import the SDK at module load. The SDK-dependent modules are imported +inside the methods that need them (documented exception to the +no-inline-imports rule). +""" + +from __future__ import annotations + +from importlib.util import find_spec +from typing import Any + +from pydantic import BaseModel + +from rle.harness.protocol import Availability, BaseHarness, HarnessContext + +FELIX_DESCRIPTION = ( + "MapAnalyst + 6 role agents over Felix SDK CentralPost, merged by ActionResolver " + "(the original RLE harness)." +) + + +class FelixPlugin: + name = "felix" + description = FELIX_DESCRIPTION + + def available(self) -> Availability: + if find_spec("felix_agent_sdk") is None: + return Availability.missing( + "felix-agent-sdk is not installed — `uv sync --extra felix`", + ) + return Availability.available() + + def option_schema(self) -> type[BaseModel]: + from rle.harness.felix.options import FelixOptions # noqa: PLC0415 - optional dep + + return FelixOptions + + def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + from rle.harness.felix.build import build_felix_harness # noqa: PLC0415 - optional dep + + return build_felix_harness(ctx, options) + + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + from rle.harness.felix.build import build_felix_harness # noqa: PLC0415 - optional dep + + return build_felix_harness(ctx, options, smoke=True) + + def describe(self) -> dict[str, str]: + info: dict[str, Any] = {"harness": self.name} + if self.available().ok: + from rle.harness.felix.harness import felix_sdk_version # noqa: PLC0415 + + info["felix_agent_sdk"] = felix_sdk_version() + return info + + +PLUGIN = FelixPlugin() diff --git a/src/rle/harness/felix/provider_factory.py b/src/rle/harness/felix/provider_factory.py new file mode 100644 index 0000000..62a9e35 --- /dev/null +++ b/src/rle/harness/felix/provider_factory.py @@ -0,0 +1,48 @@ +"""Build Felix SDK providers and helix geometry from RLE's string config.""" + +from __future__ import annotations + +from felix_agent_sdk.core import HelixConfig, HelixGeometry +from felix_agent_sdk.providers import ( + AnthropicProvider, + BaseProvider, + LocalProvider, + OpenAIProvider, +) + +from rle.providers.claude_code import ClaudeCodeProvider + +HELIX_PRESETS: dict[str, HelixConfig] = { + "default": HelixConfig.default(), + "research_heavy": HelixConfig.research_heavy(), + "fast_convergence": HelixConfig.fast_convergence(), +} + +PROVIDER_CLASSES: dict[str, type[BaseProvider]] = { + "anthropic": AnthropicProvider, + "openai": OpenAIProvider, + "local": LocalProvider, + "claude-code": ClaudeCodeProvider, +} + + +def build_provider(provider: str, model: str, base_url: str | None = None) -> BaseProvider: + """Construct a Felix provider from provider name + model (+ optional base URL).""" + cls = PROVIDER_CLASSES.get(provider) + if cls is None: + raise ValueError( + f"Unknown provider {provider!r}. Choose from: {list(PROVIDER_CLASSES)}" + ) + kwargs: dict[str, str] = {"model": model} + if base_url: + kwargs["base_url"] = base_url + return cls(**kwargs) # type: ignore[arg-type] # subclasses accept kwargs + + +def build_helix(preset: str = "default") -> HelixGeometry: + try: + return HELIX_PRESETS[preset].to_geometry() + except KeyError: + raise ValueError( + f"Unknown helix preset {preset!r}. Choose from: {list(HELIX_PRESETS)}" + ) from None diff --git a/src/rle/harness/felix/smoke.py b/src/rle/harness/felix/smoke.py new file mode 100644 index 0000000..7361d57 --- /dev/null +++ b/src/rle/harness/felix/smoke.py @@ -0,0 +1,70 @@ +"""Deterministic stand-in provider for ``--smoke-test`` runs (no LLM calls).""" + +from __future__ import annotations + +import json +from collections.abc import Iterator, Sequence +from typing import Any + +from felix_agent_sdk.providers.base import BaseProvider +from felix_agent_sdk.providers.types import ( + ChatMessage, + CompletionResult, + ProviderConfig, + StreamChunk, +) + +SMOKE_ACTION_PLAN = json.dumps({ + "actions": [ + {"action_type": "no_action", "reason": "Smoke test — no real LLM call"}, + ], + "summary": "Smoke deliberation.", + "confidence": 0.6, +}) + + +class SmokeProvider(BaseProvider): + """Returns a fixed, always-parseable action plan.""" + + def __init__(self, content: str = SMOKE_ACTION_PLAN, model: str = "smoke") -> None: + super().__init__(ProviderConfig(model=model, api_key=None)) + self._content = content + self.calls = 0 + + @property + def provider_name(self) -> str: + return "smoke" + + def _result(self) -> CompletionResult: + self.calls += 1 + return CompletionResult( + content=self._content, + model=self.config.model, + usage={"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + ) + + def complete( + self, messages: Sequence[ChatMessage], *, temperature: float | None = None, + max_tokens: int | None = None, stop_sequences: list[str] | None = None, + **kwargs: Any, + ) -> CompletionResult: + return self._result() + + async def acomplete( + self, messages: Sequence[ChatMessage], *, temperature: float | None = None, + max_tokens: int | None = None, stop_sequences: list[str] | None = None, + **kwargs: Any, + ) -> CompletionResult: + return self._result() + + def stream( + self, messages: Sequence[ChatMessage], *, temperature: float | None = None, + max_tokens: int | None = None, stop_sequences: list[str] | None = None, + **kwargs: Any, + ) -> Iterator[StreamChunk]: + result = self._result() + yield StreamChunk(text=result.content) + yield StreamChunk(text="", is_final=True, usage=result.usage) + + def count_tokens(self, messages: Sequence[ChatMessage]) -> int: + return sum(len(m.content) // 4 for m in messages) diff --git a/src/rle/harness/protocol.py b/src/rle/harness/protocol.py new file mode 100644 index 0000000..3862686 --- /dev/null +++ b/src/rle/harness/protocol.py @@ -0,0 +1,195 @@ +"""Harness protocol — the seam between the RLE environment and whatever +decides what the colony does each tick. + +The environment (``RLEGameLoop``) owns the game: pause/unpause, state +refresh, action execution, scoring, evaluation, export. A *harness* owns the +decision-making: one or many LLM agents, a coding agent attached over MCP, a +scripted policy, or nothing at all (the unmanaged baseline). Harnesses are +benchmarked side by side with models, so nothing in this module may depend on +any particular agent framework. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable +from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict + +from rle.agents.actions import ActionPlan +from rle.orchestration.action_executor import ExecutionResult +from rle.rimapi.client import RimAPIClient +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.scoring.composite import ScoreSnapshot +from rle.tracking.cost_tracker import CostTracker +from rle.tracking.event_log import EventLog, EventType + +if TYPE_CHECKING: + from rle.config import RLEConfig + from rle.scenarios.schema import ScenarioConfig + + +class HarnessStepError(Exception): + """A harness failed to produce a step. The loop records an ERROR event and + treats the tick as having no actions; the run continues.""" + + +class StepResult(BaseModel): + """What a harness hands back for one tick. + + ``plan`` is the merged set of actions to apply. ``execution`` is set when + the harness has *already* applied its writes (coding agents acting through + the RLE MCP server see tool results inside their turn, so nothing is left + for the loop to execute); when it is ``None`` the loop runs + ``ActionExecutor`` itself. ``proposals`` are optional per-sub-agent plans + kept for dashboards and post-hoc analysis; ``extras`` is a free-form bag + for harness-specific telemetry (helix phase, session ids, ...). + """ + + model_config = ConfigDict(frozen=True) + + plan: ActionPlan + execution: ExecutionResult | None = None + proposals: tuple[ActionPlan, ...] = () + extras: dict[str, Any] = {} + + +@dataclass +class HarnessContext: + """Everything the environment lends a harness for the duration of a run.""" + + config: RLEConfig + client: RimAPIClient + expected_duration_days: int = 60 + initial_population: int = 3 + scenario: ScenarioConfig | None = None + event_log: EventLog | None = None + cost_tracker: CostTracker | None = None + tick_timeout_s: float | None = None + smoke: bool = False + extras: dict[str, Any] = field(default_factory=dict) + + def emit( + self, event_type: EventType, tick: int, + agent: str | None = None, **data: object, + ) -> None: + """Emit to the run's event log if one is configured.""" + if self.event_log is not None: + self.event_log.emit(event_type, tick, agent=agent, **data) + + +@runtime_checkable +class TickObserver(Protocol): + """Receives the end-of-tick summary (visualisers, recorders, cameras).""" + + def on_tick_end( + self, tick: int, day: int, step: StepResult, + execution: ExecutionResult, score: ScoreSnapshot | None, + ) -> None: ... + + +class BaseHarness(ABC): + """Base class every harness derives from. + + Lifecycle per run: ``setup`` once, then per tick ``step`` followed by + ``on_tick_end`` (after execution + scoring), then ``teardown``. Subclasses + should keep ``deliberation_log`` / ``parse_successes`` / ``parse_failures`` + updated so run reports stay uniform across harnesses. + """ + + name: ClassVar[str] = "base" + + def __init__(self) -> None: + self.deliberation_log: list[dict[str, object]] = [] + self.parse_successes = 0 + self.parse_failures = 0 + self.observers: list[TickObserver] = [] + self._ctx: HarnessContext | None = None + + @property + def ctx(self) -> HarnessContext: + if self._ctx is None: + raise RuntimeError(f"{type(self).__name__}.setup() has not been called") + return self._ctx + + async def setup(self, ctx: HarnessContext) -> None: + self._ctx = ctx + + @abstractmethod + async def step( + self, state: GameState, tick: int, macro_time: float, + events: list[RimAPIEvent], + ) -> StepResult: + """Decide (and optionally apply) this tick's actions.""" + + async def on_tick_end( + self, tick: int, state: GameState, step: StepResult, + execution: ExecutionResult, score: ScoreSnapshot | None, + ) -> None: + """Feedback hook after execution and scoring. Default: notify observers.""" + for observer in self.observers: + observer.on_tick_end(state.colony.tick, state.colony.day, step, execution, score) + + async def teardown(self) -> None: + return None + + def run_context(self) -> AbstractContextManager[Any]: + """Context the loop enters around ``run()`` (e.g. a live terminal UI).""" + return nullcontext() + + def describe(self) -> dict[str, str]: + """Version / identity info recorded in run metadata.""" + return {"harness": self.name} + + +class Availability(BaseModel): + """Whether a plugin can run here, and if not, why.""" + + model_config = ConfigDict(frozen=True) + + ok: bool + reason: str = "" + + @classmethod + def available(cls) -> Availability: + return cls(ok=True) + + @classmethod + def missing(cls, reason: str) -> Availability: + return cls(ok=False, reason=reason) + + +class HarnessPlugin(Protocol): + """Entry-point contract under the ``rle.harnesses`` group. + + A plugin is a module-level object (conventionally ``PLUGIN``) whose + ``create`` builds a harness from validated options. ``available`` must be + cheap and must not import optional dependencies at module load — probe + for them inside the method. + """ + + name: str + description: str + + def available(self) -> Availability: ... + + def option_schema(self) -> type[BaseModel]: ... + + def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: ... + + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: ... + + def describe(self) -> dict[str, str]: ... + + +class EmptyOptions(BaseModel): + """Option schema for harnesses that take no options.""" + + model_config = ConfigDict(extra="forbid") + + +OptionsFactory = Callable[[], type[BaseModel]] diff --git a/src/rle/harness/registry.py b/src/rle/harness/registry.py new file mode 100644 index 0000000..8dd8118 --- /dev/null +++ b/src/rle/harness/registry.py @@ -0,0 +1,150 @@ +"""Harness discovery via the ``rle.harnesses`` entry-point group. + +Built-in harnesses (``baseline``, ``felix``) and third-party packages +(``rle-harness-opencode``, ...) register the same way, so adding a harness is +``pip install `` — never a change to RLE core. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from importlib.metadata import EntryPoint, entry_points +from typing import Any, cast + +from pydantic import BaseModel, ValidationError + +from rle.harness.protocol import Availability, BaseHarness, HarnessContext, HarnessPlugin + +ENTRY_POINT_GROUP = "rle.harnesses" + + +class HarnessNotFoundError(LookupError): + """No plugin registered under the requested name.""" + + +class HarnessUnavailableError(RuntimeError): + """The plugin exists but cannot run here (missing extra, binary, ...).""" + + +class HarnessOptionsError(ValueError): + """Harness options failed validation against the plugin's schema.""" + + +@dataclass(frozen=True) +class HarnessInfo: + name: str + description: str + availability: Availability + package: str + version: str + + +def _entry_points() -> dict[str, EntryPoint]: + found: dict[str, EntryPoint] = {} + for ep in entry_points(group=ENTRY_POINT_GROUP): + if ep.name in found: + other = found[ep.name] + raise RuntimeError( + f"Two packages register harness {ep.name!r}: " + f"{_dist_name(other)} and {_dist_name(ep)}. Uninstall one.", + ) + found[ep.name] = ep + return found + + +def _dist_name(ep: EntryPoint) -> str: + dist = ep.dist + return dist.metadata["Name"] if dist is not None else "?" + + +def _dist_version(ep: EntryPoint) -> str: + dist = ep.dist + return dist.version if dist is not None else "?" + + +def harness_names() -> list[str]: + return sorted(_entry_points()) + + +def get_plugin(name: str) -> HarnessPlugin: + eps = _entry_points() + ep = eps.get(name) + if ep is None: + raise HarnessNotFoundError( + f"Unknown harness {name!r}. Installed: {', '.join(sorted(eps)) or '(none)'}. " + "Install a harness package (e.g. rle-harness-opencode) or check the name.", + ) + return cast(HarnessPlugin, ep.load()) + + +def list_harnesses() -> list[HarnessInfo]: + infos: list[HarnessInfo] = [] + for name, ep in sorted(_entry_points().items()): + try: + plugin = cast(HarnessPlugin, ep.load()) + availability = plugin.available() + description = plugin.description + except Exception as exc: # a broken plugin must not break the CLI + availability = Availability.missing(f"plugin failed to load: {exc}") + description = "" + infos.append(HarnessInfo( + name=name, + description=description, + availability=availability, + package=_dist_name(ep), + version=_dist_version(ep), + )) + return infos + + +def parse_option_pairs(pairs: list[str] | None) -> dict[str, Any]: + """Turn ``["key=value", ...]`` into a dict, decoding JSON-looking values. + + ``true``/``false``/numbers/quoted strings/objects parse as JSON; anything + else is kept as the raw string. + """ + out: dict[str, Any] = {} + for pair in pairs or []: + key, sep, raw = pair.partition("=") + if not sep or not key: + raise HarnessOptionsError(f"--harness-opt expects key=value, got {pair!r}") + try: + out[key.strip()] = json.loads(raw) + except json.JSONDecodeError: + out[key.strip()] = raw + return out + + +def validate_options(plugin: HarnessPlugin, raw: dict[str, Any] | BaseModel | None) -> BaseModel: + schema = plugin.option_schema() + if isinstance(raw, BaseModel): + if isinstance(raw, schema): + return raw + raw = raw.model_dump() + try: + return schema.model_validate(raw or {}) + except ValidationError as exc: + raise HarnessOptionsError( + f"Invalid options for harness {plugin.name!r}:\n{exc}", + ) from exc + + +def create_harness( + name: str, + ctx: HarnessContext, + options: dict[str, Any] | BaseModel | None = None, + *, + smoke: bool = False, +) -> BaseHarness: + """Resolve ``name`` through the registry and build a ready-to-setup harness.""" + plugin = get_plugin(name) + availability = plugin.available() + if not availability.ok: + raise HarnessUnavailableError( + f"Harness {name!r} is installed but unavailable: {availability.reason}", + ) + opts = validate_options(plugin, options) + if smoke or ctx.smoke: + return plugin.smoke(ctx, opts) + return plugin.create(ctx, opts) diff --git a/src/rle/orchestration/game_loop.py b/src/rle/orchestration/game_loop.py index da9bfe8..2ba498b 100644 --- a/src/rle/orchestration/game_loop.py +++ b/src/rle/orchestration/game_loop.py @@ -1,4 +1,11 @@ -"""RLE game loop — turn-based orchestrator.""" +"""RLE game loop — the environment side of a run. + +The loop owns the game: pause/unpause, state refresh, action execution, +scoring, scenario evaluation, dashboard export. Deciding *what* to do each +tick is delegated to a :class:`~rle.harness.protocol.BaseHarness`, so the same +loop benchmarks the Felix multi-agent stack, an unmanaged baseline, or an +external coding agent attached over MCP without knowing which one it has. +""" from __future__ import annotations @@ -7,25 +14,29 @@ import json import logging import time as _time +from collections.abc import Sequence from pathlib import Path +from typing import Any -from felix_agent_sdk.communication import CentralPost, MessageType, SpokeManager -from felix_agent_sdk.providers import ProviderError -from felix_agent_sdk.visualization import HelixVisualizer from pydantic import BaseModel, ConfigDict -from rle.agents.actions import ActionPlan, ActionPlanParseError, resolve_endpoint -from rle.agents.base_role import RimWorldRoleAgent +from rle.agents.actions import ActionPlan, resolve_endpoint from rle.config import RLEConfig +from rle.harness.compat import build_legacy_harness +from rle.harness.protocol import ( + BaseHarness, + HarnessContext, + HarnessStepError, + StepResult, +) from rle.orchestration.action_executor import ActionExecutor, ExecutionResult -from rle.orchestration.action_resolver import ActionResolver from rle.orchestration.camera_director import CameraDirector from rle.orchestration.state_manager import GameStateManager from rle.rimapi.client import RimAPIClient from rle.rimapi.schemas import GameState from rle.rimapi.sse_client import RimAPISSEClient from rle.scenarios.evaluator import EvaluationResult, ScenarioEvaluator -from rle.scenarios.schema import TriggeredIncident +from rle.scenarios.schema import ScenarioConfig, TriggeredIncident from rle.scoring.composite import CompositeScorer, ScoreSnapshot from rle.scoring.metrics import MetricContext from rle.scoring.recorder import TimeSeriesRecorder @@ -34,19 +45,6 @@ logger = logging.getLogger(__name__) -# Truncation limits for human-readable text persisted to the event log / -# deliberation log. Keep these tight so the JSONL stays grep-able and small; -# the per-scenario *_deliberations.jsonl carries the full raw reasoning. -_ACTION_REASON_CHARS = 200 -_PLAN_SUMMARY_CHARS = 300 -_PARSE_FAILURE_RAW_CHARS = 500 -# Full LLM completion text on successful deliberation (PROVIDER_CALL events). -# 16 KB headroom: frontier models (Fable 5) emit multi-KB structured plans -# and the verbatim transcripts are first-class analysis artifacts; longer -# completions are tail-truncated rather than head-truncated so the parsed -# action JSON remains visible. -_RAW_OUTPUT_CHARS = 16384 - class TickResult(BaseModel): """Summary of a single game tick.""" @@ -59,23 +57,26 @@ class TickResult(BaseModel): plan: ActionPlan execution: ExecutionResult score: ScoreSnapshot | None = None + harness: str = "" + step_latency_s: float = 0.0 + extras: dict[str, Any] = {} class RLEGameLoop: - """Turn-based game loop: pause → read → deliberate → resolve → execute → score → unpause.""" + """Turn-based loop: pause → read → harness.step → execute → score → unpause.""" def __init__( self, config: RLEConfig, client: RimAPIClient, - agents: list[RimWorldRoleAgent], + agents: Sequence[Any] | None = None, expected_duration_days: int = 60, scorer: CompositeScorer | None = None, recorder: TimeSeriesRecorder | None = None, evaluator: ScenarioEvaluator | None = None, initial_population: int = 3, initial_wealth: float = 0.0, - visualizer: HelixVisualizer | None = None, + visualizer: Any | None = None, parallel: bool = True, sse_client: RimAPISSEClient | None = None, dashboard_export_dir: Path | None = None, @@ -88,23 +89,20 @@ def __init__( auto_dismiss_dialogs: bool = True, camera_director: CameraDirector | None = None, speed_keepalive_s: float = 10.0, + *, + harness: BaseHarness | None = None, + scenario: ScenarioConfig | None = None, ) -> None: + """``harness`` is the modern entry point. ``agents`` / ``no_agent`` / + ``parallel`` / ``visualizer`` are legacy arguments that build a Felix + or baseline harness for you (see ``rle.harness.compat``).""" + if harness is not None and agents: + raise ValueError("Pass either harness= or the legacy agents= argument, not both") self._config = config self._client = client - self._agents = agents - # Separate MapAnalyst (runs first) from role agents (run in parallel) - self._map_analyst: RimWorldRoleAgent | None = None - self._role_agents: list[RimWorldRoleAgent] = [] - for agent in agents: - if agent.ROLE_NAME == "map_analyst": - self._map_analyst = agent - else: - self._role_agents.append(agent) - self._no_agent = no_agent self._no_pause = no_pause self._state_manager = GameStateManager(client, expected_duration_days, sse_client) self._executor = ActionExecutor(client) - self._resolver = ActionResolver() self._scorer = scorer self._recorder = recorder self._evaluator = evaluator @@ -115,16 +113,7 @@ def __init__( initial_population=initial_population, initial_wealth=initial_wealth, ) - self._parse_successes = 0 - self._parse_failures = 0 - self._role_timeout_s = config.role_timeout_s - self._log_dir: Path | None = None - self._deliberation_log: list[dict[str, object]] = [] - self._parallel = parallel - self._last_phase: str = "" self._dashboard_export_dir = dashboard_export_dir - - self._visualizer = visualizer self._event_log = event_log self._cost_tracker = cost_tracker self._triggered_incidents = triggered_incidents or [] @@ -133,12 +122,28 @@ def __init__( self._camera_director = camera_director self._speed_keepalive_s = speed_keepalive_s - # Hub-spoke communication — agents read messages from their spokes - self._hub = CentralPost(max_agents=len(agents)) - self._spoke_manager = SpokeManager(self._hub) - for agent in agents: - spoke = self._spoke_manager.create_spoke(agent.agent_id, agent=agent) - agent.attach_spoke(spoke) + self._harness: BaseHarness = harness or build_legacy_harness( + agents, + no_agent=no_agent, + parallel=parallel, + visualizer=visualizer, + role_timeout_s=config.role_timeout_s, + ) + self._harness_ctx = HarnessContext( + config=config, + client=client, + expected_duration_days=expected_duration_days, + initial_population=initial_population, + scenario=scenario, + event_log=event_log, + cost_tracker=cost_tracker, + tick_timeout_s=config.tick_timeout_s, + ) + self._setup_done = False + + # ------------------------------------------------------------------ + # Plumbing + # ------------------------------------------------------------------ def _emit( self, event_type: EventType, tick: int, @@ -148,10 +153,23 @@ def _emit( if self._event_log is not None: self._event_log.emit(event_type, tick, agent=agent, **data) + @property + def harness(self) -> BaseHarness: + return self._harness + + @property + def harness_context(self) -> HarnessContext: + return self._harness_ctx + @property def cost_tracker(self) -> CostTracker | None: return self._cost_tracker + async def _ensure_setup(self) -> None: + if not self._setup_done: + await self._harness.setup(self._harness_ctx) + self._setup_done = True + async def _dismiss_blocking_dialogs(self, tick_num: int) -> None: """Close force-pause popups that stall unattended runs (issue #33). @@ -172,207 +190,69 @@ async def _dismiss_blocking_dialogs(self, tick_num: int) -> None: EventType.TICK_START, tick_num, dismissed_windows=list(closed), ) - def _update_visualizer_agent( - self, agent: RimWorldRoleAgent, plan: ActionPlan, macro_time: float, - ) -> None: - """Push one agent's post-deliberation state to the visualizer.""" - if not self._visualizer: - return - self._visualizer.update( - agent.agent_id, - progress=macro_time, - confidence=plan.confidence, - phase=agent.position.phase, - status=f"{len(plan.actions)} actions", - ) - - def _render_visualizer( - self, tick: int, day: int, exec_result: ExecutionResult, - snapshot: ScoreSnapshot | None, - ) -> None: - """Render the helix visualization for this tick.""" - if not self._visualizer: - return - extra: dict[str, str] = { - "actions": f"{exec_result.executed}/{exec_result.total}", - } - if snapshot: - extra["score"] = f"{snapshot.composite:.3f}" - self._visualizer.render(tick=tick, day=day, extra_info=extra) - - async def _deliberate_agent_with_timeout( - self, agent: RimWorldRoleAgent, state: object, - current_time: float, tick_num: int, - ) -> tuple[RimWorldRoleAgent, ActionPlan | None]: - """Run one agent's deliberation with a hard timeout. - - Deliberation is natively async (felix 0.3.0 ``acomplete()``), so a - timeout cancels the in-flight provider request instead of leaving an - orphaned worker thread. On timeout: emits a deliberation_timeout - ERROR event, records it in _deliberation_log, and returns - (agent, None) so the tick can continue. - """ - try: - return await asyncio.wait_for( - self._deliberate_agent(agent, state, current_time, tick_num), - timeout=self._role_timeout_s, - ) - except asyncio.TimeoutError: - logger.warning( - "Agent %s deliberation timed out after %.1fs (tick %d)", - agent.ROLE_NAME, self._role_timeout_s, tick_num, - ) - self._parse_failures += 1 - self._deliberation_log.append({ - "tick": tick_num, "agent": agent.ROLE_NAME, - "status": "deliberation_timeout", - "reason": f"timed out after {self._role_timeout_s}s", - }) - self._emit( - EventType.ERROR, tick_num, agent=agent.ROLE_NAME, - error_type="deliberation_timeout", - reason=f"timed out after {self._role_timeout_s}s", - timeout_s=self._role_timeout_s, - ) - return agent, None - - async def _deliberate_parallel( - self, state: object, current_time: float, tick_num: int, - ) -> list[tuple[RimWorldRoleAgent, ActionPlan | None]]: - """Run role agents concurrently on the event loop with per-task timeout.""" - return list(await asyncio.gather(*[ - self._deliberate_agent_with_timeout(a, state, current_time, tick_num) - for a in self._role_agents - ])) - - async def _deliberate_sequential( - self, state: object, current_time: float, tick_num: int, - ) -> list[tuple[RimWorldRoleAgent, ActionPlan | None]]: - """Run role agents one at a time. Agents read context from their spokes.""" - results: list[tuple[RimWorldRoleAgent, ActionPlan | None]] = [] - for agent in self._role_agents: - agent_result, plan = await self._deliberate_agent_with_timeout( - agent, state, current_time, tick_num, - ) - results.append((agent_result, plan)) - return results - - async def _deliberate_agent( - self, agent: RimWorldRoleAgent, state: object, - current_time: float, tick_num: int, - ) -> tuple[RimWorldRoleAgent, ActionPlan | None]: - """Run one agent's deliberation. + async def _step_harness( + self, state: GameState, tick_num: int, macro_time: float, + ) -> tuple[StepResult, float]: + """Run the harness for one tick under the loop-level timeout. - Agents read inter-agent context from their CentralPost spoke internally. + ``HarnessStepError`` and a timeout degrade to an empty step plus an + ERROR event so a flaky harness produces a scored (bad) tick instead of + aborting the run. Any other exception is a bug and propagates. """ + events = self._state_manager.pending_events + empty = StepResult( + plan=ActionPlan( + role=self._harness.name, tick=state.colony.tick, actions=[], + summary="harness produced no step", + ), + ) t0 = _time.monotonic() + timeout = self._harness_ctx.tick_timeout_s try: - plan = await agent.adeliberate(state, current_time) # type: ignore[arg-type] - except ActionPlanParseError as e: - latency_ms = round((_time.monotonic() - t0) * 1000, 1) + coro = self._harness.step(state, tick_num, macro_time, events) + step = await (asyncio.wait_for(coro, timeout=timeout) if timeout else coro) + except asyncio.TimeoutError: logger.warning( - "Agent %s parse failure (tick %d): %s", - agent.ROLE_NAME, tick_num, e.reason, - ) - self._parse_failures += 1 - raw_truncated = ( - e.raw_content[:_PARSE_FAILURE_RAW_CHARS] if e.raw_content else None + "Harness %s step timed out after %.1fs (tick %d)", + self._harness.name, timeout or 0.0, tick_num, ) - self._deliberation_log.append({ - "tick": tick_num, "agent": agent.ROLE_NAME, - "status": "parse_failure", "reason": e.reason, - "raw": raw_truncated, - }) self._emit( - EventType.ERROR, tick_num, agent=agent.ROLE_NAME, - error_type="parse_failure", reason=e.reason, latency_ms=latency_ms, - raw=raw_truncated, + EventType.ERROR, tick_num, agent=self._harness.name, + error_type="harness_timeout", timeout_s=timeout, ) - return agent, None - except ProviderError as e: - latency_ms = round((_time.monotonic() - t0) * 1000, 1) + step = empty + except HarnessStepError as exc: logger.warning( - "Agent %s provider error (tick %d): %s", - agent.ROLE_NAME, tick_num, e, + "Harness %s step failed (tick %d): %s", self._harness.name, tick_num, exc, ) - self._parse_failures += 1 - self._deliberation_log.append({ - "tick": tick_num, "agent": agent.ROLE_NAME, - "status": "provider_error", "reason": str(e), - }) self._emit( - EventType.ERROR, tick_num, agent=agent.ROLE_NAME, - error_type="provider_error", reason=str(e), latency_ms=latency_ms, + EventType.ERROR, tick_num, agent=self._harness.name, + error_type="harness_error", reason=str(exc), ) - return agent, None - - latency_ms = round((_time.monotonic() - t0) * 1000, 1) - self._parse_successes += 1 - actions_payload = [ - {"type": a.action_type, "target": a.target_colonist_id, - "priority": a.priority, "reason": a.reason[:_ACTION_REASON_CHARS]} - for a in plan.actions - ] - summary_truncated = plan.summary[:_PLAN_SUMMARY_CHARS] - self._deliberation_log.append({ - "tick": tick_num, "agent": plan.role, - "status": "success", "confidence": plan.confidence, - "num_actions": len(plan.actions), - "actions": actions_payload, - "summary": summary_truncated, - }) - self._emit( - EventType.DELIBERATION, tick_num, agent=plan.role, - latency_ms=latency_ms, confidence=plan.confidence, - num_actions=len(plan.actions), - actions=actions_payload, - summary=summary_truncated, - ) - - # Record token usage for cost tracking and event log - usage = agent._last_usage - if usage and isinstance(usage, dict): - pt = usage.get("prompt_tokens", 0) - ct = usage.get("completion_tokens", 0) - rt = usage.get("reasoning_tokens", 0) - if not isinstance(rt, int): - rt = 0 - if isinstance(pt, int) and isinstance(ct, int): - if self._cost_tracker: - self._cost_tracker.record_raw(pt, ct, rt) - raw_output = agent._last_raw_output - raw_output_truncated = ( - raw_output[:_RAW_OUTPUT_CHARS] if raw_output else None - ) - was_truncated = ( - raw_output is not None - and len(raw_output) > _RAW_OUTPUT_CHARS - ) - self._emit( - EventType.PROVIDER_CALL, tick_num, agent=plan.role, - prompt_tokens=pt, completion_tokens=ct, - reasoning_tokens=rt, - raw_output=raw_output_truncated, - raw_output_truncated=was_truncated, - ) - - return agent, plan + step = empty + return step, _time.monotonic() - t0 def _export_tick_json( - self, plans: list[ActionPlan], resolved: ActionPlan, - exec_result: ExecutionResult, snapshot: ScoreSnapshot | None, + self, step: StepResult, exec_result: ExecutionResult, snapshot: ScoreSnapshot | None, tick: int, day: int, macro_time: float, screenshot_data_uri: str | None = None, ) -> None: - """Write tick data as JSON for the rimapi-dashboard to consume.""" + """Write tick data as JSON for the rimapi-dashboard to consume. + + Harness-neutral schema: ``agents`` lists whatever sub-plans the harness + reported (seven for Felix, one for a single agent, none for baseline); + harness-specific telemetry rides in ``extras``. + """ if not self._dashboard_export_dir: return self._dashboard_export_dir.mkdir(parents=True, exist_ok=True) + resolved = step.plan data = { "tick": tick, "day": day, "macro_time": macro_time, - "phase": self._last_phase, + "harness": self._harness.name, + "phase": str(step.extras.get("phase", "")), "agents": [ { "role": p.role, @@ -389,7 +269,7 @@ def _export_tick_json( for a in p.actions ], } - for p in plans + for p in step.proposals ], "resolved": { "role": resolved.role, @@ -413,9 +293,10 @@ def _export_tick_json( "metrics": snapshot.metrics, } if snapshot else None, "screenshot_data_uri": screenshot_data_uri, + "extras": step.extras, } (self._dashboard_export_dir / "latest_tick.json").write_text( - json.dumps(data, indent=2), + json.dumps(data, indent=2, default=str), ) def _update_metric_context( @@ -459,22 +340,6 @@ def _record_draft_response( threat.threat_id, max(0, tick_num - seen), ) - def _broadcast_phase_if_changed(self, current_time: float) -> None: - """Broadcast PHASE_ANNOUNCE when macro_time crosses a phase boundary.""" - if current_time < 0.4: - phase = "exploration" - elif current_time < 0.7: - phase = "analysis" - else: - phase = "synthesis" - if phase != self._last_phase: - self._spoke_manager.broadcast_message( - MessageType.PHASE_ANNOUNCE, - {"phase": phase, "depth_ratio": current_time}, - sender_id="hub", - ) - self._last_phase = phase - async def _fire_scheduled_incidents(self, tick_num: int) -> None: """Fire any triggered_incidents whose tick_offset matches.""" for incident in self._triggered_incidents: @@ -500,12 +365,17 @@ async def _fire_scheduled_incidents(self, tick_num: int) -> None: incident.name, exc_info=True, ) + # ------------------------------------------------------------------ + # Tick + # ------------------------------------------------------------------ + async def run_tick(self) -> TickResult: """Execute one turn. - In pause mode (default): pause → read → deliberate → execute → unpause. - In no-pause mode: read → fire deliberation + sleep concurrently → execute. + In pause mode (default): pause → read → step → execute → unpause. + In no-pause mode: read → step → execute while the game keeps running. """ + await self._ensure_setup() tick_num = len(self._tick_results) # 1. Pause (skip in no-pause mode — game keeps running) @@ -538,199 +408,56 @@ async def run_tick(self) -> TickResult: tick_num, state, self._state_manager.pending_events, _time.time(), ) - # 3. Route previous tick's messages to agent spokes + broadcast phase changes - messages_before = self._hub.total_messages_processed - self._spoke_manager.process_all_messages() - self._broadcast_phase_if_changed(current_time) - - # 4-6. Agent deliberation + conflict resolution (skipped in no-agent mode) - plans: list[ActionPlan] = [] - if self._no_agent: - # Baseline mode: no deliberation, no actions. Colony runs unmanaged. - resolved = ActionPlan( - role="baseline", tick=state.colony.tick, actions=[], summary="No agents", - ) - else: - # Inject SSE events into all agents (including MapAnalyst) - pending_events = self._state_manager.pending_events - for agent in self._agents: - agent.set_pending_events(pending_events) - for evt in pending_events: - self._emit( - EventType.SSE_EVENT, tick_num, - sse_type=evt.event_type, sse_data=str(evt.data)[:200], - ) + # 3. Harness decides (and, for tool-using harnesses, may already act) + step, step_latency = await self._step_harness(state, tick_num, current_time) - # 4a. MapAnalyst deliberates FIRST (sequential, timeout-wrapped) - if self._map_analyst: - ma_agent, ma_plan = await self._deliberate_agent_with_timeout( - self._map_analyst, state, current_time, tick_num, + # 4. Execute — unless the harness already applied its writes + if step.execution is None: + exec_result = await self._executor.execute(step.plan) + for outcome in exec_result.outcomes: + self._emit( + EventType.ACTION_EXEC, tick_num, + action_type=outcome.action_type, + target=outcome.target_colonist_id, + success=outcome.success, + error=outcome.error, + parameters=outcome.parameters, ) - if ma_plan is not None: - plans.append(ma_plan) - self._update_visualizer_agent(ma_agent, ma_plan, current_time) - spoke = self._spoke_manager.get_spoke(ma_agent.agent_id) - if spoke and spoke.is_connected: - spoke.send_message( - MessageType.TASK_COMPLETE, - { - "role": ma_plan.role, - "summary": ma_plan.summary, - "confidence": ma_plan.confidence, - "num_actions": len(ma_plan.actions), - "action_types": [ - a.action_type for a in ma_plan.actions - ], - }, - ) - # Route MapAnalyst output to role agent spokes immediately - self._spoke_manager.process_all_messages() - - # 4b. Role agents deliberate (parallel or sequential) - # Snapshot which agents have pending spoke messages (for messages_acted_on) - agents_with_messages: set[str] = set() - for ra in self._role_agents: - spoke = self._spoke_manager.get_spoke(ra.agent_id) - if spoke and spoke.has_pending_messages(): - agents_with_messages.add(ra.agent_id) - - if self._parallel: - results = await self._deliberate_parallel(state, current_time, tick_num) - else: - results = await self._deliberate_sequential(state, current_time, tick_num) - - # Collect plans, update visualizer, send via CentralPost - agents_acted_with_messages = 0 - for agent, plan in results: - if plan is None: - continue - plans.append(plan) - if agent.agent_id in agents_with_messages: - agents_acted_with_messages += 1 - self._update_visualizer_agent(agent, plan, current_time) - spoke = self._spoke_manager.get_spoke(agent.agent_id) - if spoke and spoke.is_connected: - spoke.send_message( - MessageType.TASK_COMPLETE, - { - "role": plan.role, - "summary": plan.summary, - "confidence": plan.confidence, - "num_actions": len(plan.actions), - "action_types": [ - a.action_type for a in plan.actions - ], - }, - ) - - # Capture generation IDs from every provider call this tick — - # parse retries and failed deliberations bill tokens too — for - # billed-cost reconciliation against OpenRouter (the token-count - # estimator diverged up to 4x on the v0.3.0 spread). - if self._cost_tracker: - for any_agent in self._agents: - for gen_id in any_agent.drain_generation_ids(): - self._cost_tracker.record_generation_id(gen_id) - - # Resolve conflicts. Resolver + CentralPost counts are diagnostics - # in the event log only — they are Felix-specific and no longer - # feed the composite (#51). - resolved, resolver_stats = self._resolver.resolve(plans, state) - self._emit( - EventType.CONFLICT, tick_num, - input_plans=len(plans), - output_actions=len(resolved.actions), - conflicts_detected=resolver_stats.conflicts_total, - conflicts_resolved=resolver_stats.conflicts_resolved, - messages_routed=self._hub.total_messages_processed - messages_before, - agents_with_messages=len(agents_with_messages), - agents_acted_with_messages=agents_acted_with_messages, - ) - - # 7. Execute merged plan - exec_result = await self._executor.execute(resolved) - for outcome in exec_result.outcomes: - self._emit( - EventType.ACTION_EXEC, tick_num, - action_type=outcome.action_type, - target=outcome.target_colonist_id, - success=outcome.success, - error=outcome.error, - parameters=outcome.parameters, - ) + else: + exec_result = step.execution - # 7a. Track draft responses for the threat_response metric (issue #25) + # 4a. Track draft responses for the threat_response metric (issue #25) self._record_draft_response(exec_result, tick_num) - # 7b. Surface per-action errors back to agents so they can avoid - # re-proposing the same invalid action next tick (e.g. researching - # an already-finished project, setting priority for a disabled work type). - failed_outcomes = [o for o in exec_result.outcomes if not o.success and o.error] - if failed_outcomes: - error_summary = "; ".join( - f"{o.action_type}({o.target_colonist_id or '-'}): {o.error}" - for o in failed_outcomes[:10] - ) - self._spoke_manager.broadcast_message( - MessageType.STATUS_UPDATE, - { - "tick": state.colony.tick, - "summary": f"Last tick action errors — DO NOT REPEAT: {error_summary}", - "action_errors": [ - { - "action_type": o.action_type, - "target_colonist_id": o.target_colonist_id, - "error": o.error, - } - for o in failed_outcomes - ], - }, - sender_id="hub", - ) - - # 8. Score this tick + # 5. Score this tick snapshot: ScoreSnapshot | None = None if self._scorer: snapshot = self._scorer.score(state, self._metric_context) if self._recorder: self._recorder.record(snapshot) - if snapshot: - self._emit( - EventType.SCORE, tick_num, - composite=snapshot.composite, metrics=snapshot.metrics, - ) - - # 9. Broadcast score to all agents via CentralPost - if snapshot: - self._spoke_manager.broadcast_message( - MessageType.STATUS_UPDATE, - { - "tick": state.colony.tick, - "day": state.colony.day, - "composite_score": snapshot.composite, - "metrics": snapshot.metrics, - }, - sender_id="hub", + self._emit( + EventType.SCORE, tick_num, + composite=snapshot.composite, metrics=snapshot.metrics, ) - # 10. Render visualization - self._render_visualizer(state.colony.tick, state.colony.day, exec_result, snapshot) + # 6. Feedback to the harness (action errors, score, visualisers) + await self._harness.on_tick_end(tick_num, state, step, exec_result, snapshot) - # 11. Capture screenshot (opt-in, before export so it's in the JSON) + # 7. Capture screenshot (opt-in, before export so it's in the JSON) screenshot_uri: str | None = None if self._screenshots_enabled: ss = await self._client.take_screenshot() if ss is not None: screenshot_uri = ss.data_uri - # 12. Export tick data for dashboard + # 8. Export tick data for dashboard self._export_tick_json( - plans, resolved, exec_result, snapshot, + step, exec_result, snapshot, state.colony.tick, state.colony.day, current_time, screenshot_data_uri=screenshot_uri, ) - # 13. Unpause (skip in no-pause mode — game was never paused) + # 9. Unpause (skip in no-pause mode — game was never paused) if not self._no_pause: await self._client.unpause_game() @@ -738,13 +465,16 @@ async def run_tick(self) -> TickResult: tick=state.colony.tick, day=state.colony.day, macro_time=current_time, - plan=resolved, + plan=step.plan, execution=exec_result, score=snapshot, + harness=self._harness.name, + step_latency_s=round(step_latency, 3), + extras=step.extras, ) self._tick_results.append(result) - # 9. Update metric context and evaluate scenario + # 10. Update metric context and evaluate scenario self._update_metric_context(result, state, tick_num) if self._evaluator: eval_result = self._evaluator.evaluate( @@ -776,6 +506,7 @@ async def _speed_keepalive(self) -> None: async def run(self, max_ticks: int | None = None) -> list[TickResult]: """Run the game loop for N ticks or until stopped.""" + await self._ensure_setup() self._running = True if not await self._client.window_endpoints_available(): logger.warning( @@ -792,34 +523,30 @@ async def run(self, max_ticks: int | None = None) -> list[TickResult]: keepalive = asyncio.create_task(self._speed_keepalive()) tick_count = 0 try: - while self._running: - result = await self.run_tick() - tick_count += 1 - score_str = "" - if result.score: - score_str = f" | score={result.score.composite:.3f}" - logger.info( - "Tick %d (day %d): %d actions, %d executed%s", - tick_count, - result.day, - result.execution.total, - result.execution.executed, - score_str, - ) - if max_ticks and tick_count >= max_ticks: - break - await asyncio.sleep(self._config.tick_interval) + with self._harness.run_context(): + while self._running: + result = await self.run_tick() + tick_count += 1 + score_str = "" + if result.score: + score_str = f" | score={result.score.composite:.3f}" + logger.info( + "Tick %d (day %d): %d actions, %d executed%s", + tick_count, + result.day, + result.execution.total, + result.execution.executed, + score_str, + ) + if max_ticks and tick_count >= max_ticks: + break + await asyncio.sleep(self._config.tick_interval) finally: if keepalive is not None: keepalive.cancel() with contextlib.suppress(asyncio.CancelledError): await keepalive - # Final drain: a deliberation that timed out on the last tick may - # have completed in its worker thread after the per-tick drain. - if self._cost_tracker: - for agent in self._agents: - for gen_id in agent.drain_generation_ids(): - self._cost_tracker.record_generation_id(gen_id) + await self._harness.teardown() return self._tick_results def stop(self) -> None: @@ -840,10 +567,18 @@ def metric_context(self) -> MetricContext: @property def deliberation_log(self) -> list[dict[str, object]]: - """Per-tick agent deliberation records (status, actions, reasons, summary). + """Per-tick harness deliberation records (status, actions, reasons, summary). Returns a shallow copy. Each entry has keys: tick, agent, status, plus status-specific fields (actions, summary, confidence for success; raw, reason for parse_failure; reason for provider_error). """ - return list(self._deliberation_log) + return list(self._harness.deliberation_log) + + @property + def parse_successes(self) -> int: + return self._harness.parse_successes + + @property + def parse_failures(self) -> int: + return self._harness.parse_failures diff --git a/tests/integration/test_game_loop.py b/tests/integration/test_game_loop.py index 3a76054..23e39d4 100644 --- a/tests/integration/test_game_loop.py +++ b/tests/integration/test_game_loop.py @@ -593,8 +593,8 @@ async def test_parallel_all_agents_produce_plans(self) -> None: assert provider.acomplete.call_count == 7 # 1 MapAnalyst + 6 role agents assert result.plan.role == "orchestrator" - assert loop._parse_successes == 7 - assert loop._parse_failures == 0 + assert loop.parse_successes == 7 + assert loop.parse_failures == 0 async def test_spoke_messages_routed_after_tick(self) -> None: """After tick 1, agents should have TASK_COMPLETE messages from other agents.""" @@ -615,7 +615,7 @@ async def test_spoke_messages_routed_after_tick(self) -> None: await loop.run_tick() # Verify hub processed messages (6 TASK_COMPLETE from tick 1 + 6 from tick 2) - assert loop._hub.total_messages_processed >= 6 + assert loop.harness.hub.total_messages_processed >= 6 async def test_sequential_mode_still_works(self) -> None: provider = _make_mock_provider() @@ -631,7 +631,7 @@ async def test_sequential_mode_still_works(self) -> None: assert provider.acomplete.call_count == 7 # 1 MapAnalyst + 6 role agents assert result.plan.role == "orchestrator" - assert loop._parse_successes == 7 + assert loop.parse_successes == 7 async def test_sequential_3_ticks(self) -> None: provider = _make_mock_provider() @@ -684,8 +684,8 @@ async def test_task_complete_messages_sent(self) -> None: await loop.run_tick() # 7 agents sent TASK_COMPLETE messages - assert loop._hub.message_queue_size >= 0 # may have been processed - assert loop._hub.total_messages_processed >= 0 + assert loop.harness.hub.message_queue_size >= 0 # may have been processed + assert loop.harness.hub.total_messages_processed >= 0 # Each agent's spoke should have sent at least 1 message for agent in agents: assert agent._spoke.messages_sent >= 1 @@ -704,7 +704,7 @@ async def test_phase_announce_broadcast(self) -> None: await loop.run_tick() # First tick should have broadcast an initial phase - assert loop._last_phase != "" + assert loop.harness.last_phase != "" async def test_score_broadcast(self) -> None: """STATUS_UPDATE with scores should broadcast after scoring.""" diff --git a/tests/integration/test_harness_loop.py b/tests/integration/test_harness_loop.py new file mode 100644 index 0000000..8928362 --- /dev/null +++ b/tests/integration/test_harness_loop.py @@ -0,0 +1,211 @@ +"""RLEGameLoop driven by non-Felix harnesses (the swappable-harness contract).""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import ClassVar + +import httpx + +from rle.agents.actions import Action, ActionPlan +from rle.config import RLEConfig +from rle.harness import BaseHarness, HarnessStepError, StepResult +from rle.harness.baseline import BaselineHarness +from rle.orchestration.action_executor import ActionOutcome, ExecutionResult +from rle.orchestration.game_loop import RLEGameLoop +from rle.rimapi.client import RimAPIClient +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.scoring.composite import CompositeScorer +from rle.scoring.metrics import NEUTRAL +from rle.scoring.recorder import TimeSeriesRecorder +from rle.tracking.event_log import EventLog, EventType +from tests.integration.test_game_loop import _make_transport + + +@asynccontextmanager +async def _client() -> AsyncIterator[RimAPIClient]: + async with RimAPIClient("http://test") as client: + client._client = httpx.AsyncClient( + transport=_make_transport(), base_url="http://test", + ) + yield client + + +class _ScriptedHarness(BaseHarness): + """Proposes one work_priority write per tick; the loop executes it.""" + + name: ClassVar[str] = "scripted" + + def __init__(self) -> None: + super().__init__() + self.seen_events: list[list[RimAPIEvent]] = [] + self.tick_ends: list[float | None] = [] + + async def step( + self, state: GameState, tick: int, macro_time: float, events: list[RimAPIEvent], + ) -> StepResult: + self.seen_events.append(events) + plan = ActionPlan( + role="scripted", tick=state.colony.tick, + actions=[Action( + action_type="work_priority", target_colonist_id="col_01", + parameters={"Growing": 1}, reason="script", + )], + ) + self.parse_successes += 1 + return StepResult(plan=plan, proposals=(plan,), extras={"phase": "scripted"}) + + async def on_tick_end(self, tick, state, step, execution, score) -> None: # type: ignore[no-untyped-def] + self.tick_ends.append(score.composite if score else None) + + +class _PreExecutedHarness(BaseHarness): + """Applies its own writes (MCP-style) and reports the execution back.""" + + name: ClassVar[str] = "pre-executed" + + async def step( + self, state: GameState, tick: int, macro_time: float, events: list[RimAPIEvent], + ) -> StepResult: + outcomes = ( + ActionOutcome( + action_type="draft", endpoint="draft", target_colonist_id="col_01", + success=True, parameters={"is_drafted": True}, + ), + ActionOutcome( + action_type="draft", endpoint="draft", target_colonist_id="col_01", + success=True, parameters={"is_drafted": False}, + ), + ) + plan = ActionPlan(role="pre-executed", tick=state.colony.tick, actions=[]) + return StepResult( + plan=plan, + execution=ExecutionResult(executed=2, failed=0, total=2, outcomes=outcomes), + ) + + +class _FlakyHarness(BaseHarness): + name: ClassVar[str] = "flaky" + + async def step(self, state, tick, macro_time, events): # type: ignore[no-untyped-def] + raise HarnessStepError("upstream agent crashed") + + +class _SlowHarness(BaseHarness): + name: ClassVar[str] = "slow" + + async def step(self, state, tick, macro_time, events): # type: ignore[no-untyped-def] + await asyncio.sleep(5) + raise AssertionError("should have timed out") + + +class TestScriptedHarness: + async def test_loop_executes_harness_plan(self) -> None: + harness = _ScriptedHarness() + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=harness, + scorer=CompositeScorer(), recorder=TimeSeriesRecorder(), + ) + results = await loop.run(max_ticks=2) + + assert len(results) == 2 + assert all(r.harness == "scripted" for r in results) + assert results[0].execution.executed == 1 + assert results[0].extras == {"phase": "scripted"} + assert harness.tick_ends and harness.tick_ends[0] is not None + assert loop.parse_successes == 2 + assert len(harness.seen_events) == 2 + + async def test_dashboard_export_is_harness_neutral(self, tmp_path: Path) -> None: + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=_ScriptedHarness(), + dashboard_export_dir=tmp_path, + ) + await loop.run_tick() + data = json.loads((tmp_path / "latest_tick.json").read_text()) + assert data["harness"] == "scripted" + assert data["phase"] == "scripted" + assert len(data["agents"]) == 1 + assert data["extras"] == {"phase": "scripted"} + + +class TestPreExecutedHarness: + async def test_loop_skips_executor_and_scores_coherence(self) -> None: + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=_PreExecutedHarness(), + scorer=CompositeScorer(), + ) + await loop.run_tick() + second = await loop.run_tick() + + # Execution came from the harness untouched + assert second.execution.executed == 2 + # Tick 2 scores tick 1's contradictory draft/undraft: coherence 0.0 + assert second.score is not None + assert second.score.metrics["plan_coherence"] == 0.0 + assert second.score.metrics["efficiency"] == 1.0 + + +class TestBaselineThroughLoop: + async def test_baseline_neutral_process_metrics(self) -> None: + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=BaselineHarness(), + scorer=CompositeScorer(), + ) + await loop.run_tick() + second = await loop.run_tick() + assert second.score is not None + assert second.score.metrics["efficiency"] == NEUTRAL + assert second.score.metrics["plan_coherence"] == NEUTRAL + + async def test_legacy_no_agent_flag_still_works(self) -> None: + async with _client() as client: + loop = RLEGameLoop(RLEConfig(tick_interval=0.0), client, no_agent=True) + result = await loop.run_tick() + assert loop.harness.name == "baseline" + assert result.plan.actions == [] + + +class TestHarnessFailures: + async def test_step_error_degrades_to_empty_tick(self, tmp_path: Path) -> None: + log = EventLog(tmp_path / "events.jsonl") + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=_FlakyHarness(), event_log=log, + ) + result = await loop.run_tick() + assert result.plan.actions == [] + errors = [e for e in log.events if e.event_type == EventType.ERROR] + assert errors and errors[0].data["error_type"] == "harness_error" + + async def test_step_timeout_degrades_to_empty_tick(self, tmp_path: Path) -> None: + log = EventLog(tmp_path / "events.jsonl") + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0, tick_timeout_s=0.05), client, + harness=_SlowHarness(), event_log=log, + ) + result = await loop.run_tick() + assert result.plan.actions == [] + errors = [e for e in log.events if e.event_type == EventType.ERROR] + assert errors and errors[0].data["error_type"] == "harness_timeout" + + async def test_harness_and_agents_are_exclusive(self) -> None: + async with _client() as client: + try: + RLEGameLoop( + RLEConfig(tick_interval=0.0), client, [object()], harness=BaselineHarness(), + ) + except ValueError as exc: + assert "not both" in str(exc) + else: # pragma: no cover + raise AssertionError("expected ValueError") diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index fd54e9e..82ded9b 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -7,6 +7,8 @@ import pytest from rle.config import RLEConfig, bridge_anthropic_key, bridge_openrouter_key +from rle.harness.felix.provider_factory import build_provider +from rle.providers.claude_code import ClaudeCodeProvider class TestBridgeAnthropicKey: @@ -35,13 +37,26 @@ def test_noop_when_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: class TestProviderRegistry: def test_claude_code_provider_registered(self) -> None: - from rle.providers.claude_code import ClaudeCodeProvider - - config = RLEConfig(provider="claude-code", model="claude-fable-5") - provider = config.get_provider() + provider = build_provider("claude-code", "claude-fable-5") assert isinstance(provider, ClaudeCodeProvider) assert provider.model == "claude-fable-5" + def test_unknown_provider_lists_choices(self) -> None: + with pytest.raises(ValueError, match="anthropic"): + build_provider("nope", "x") + + +class TestConfigIsFrameworkFree: + def test_harness_defaults(self) -> None: + config = RLEConfig() + assert config.harness == "felix" + assert config.harness_options == {} + assert config.tick_timeout_s is None + + def test_no_felix_symbols_on_config(self) -> None: + assert not hasattr(RLEConfig(), "get_provider") + assert not hasattr(RLEConfig(), "get_helix_config") + class TestBridgeOpenRouterKey: def test_exports_key_to_process_env(self, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/test_harness_registry.py b/tests/unit/test_harness_registry.py new file mode 100644 index 0000000..11591f7 --- /dev/null +++ b/tests/unit/test_harness_registry.py @@ -0,0 +1,176 @@ +"""Tests for the harness plugin registry and the harness protocol surface.""" + +from __future__ import annotations + +from typing import ClassVar + +import pytest +from pydantic import BaseModel, ConfigDict + +from rle.agents.actions import ActionPlan +from rle.config import RLEConfig +from rle.harness import ( + Availability, + BaseHarness, + EmptyOptions, + HarnessContext, + HarnessNotFoundError, + HarnessOptionsError, + HarnessUnavailableError, + StepResult, + create_harness, + get_plugin, + harness_names, + list_harnesses, + parse_option_pairs, + validate_options, +) +from rle.harness.baseline import BaselineHarness +from rle.harness.compat import build_legacy_harness +from rle.rimapi.client import RimAPIClient +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent + + +def _ctx(**overrides: object) -> HarnessContext: + return HarnessContext( + config=RLEConfig(tick_interval=0.0), + client=RimAPIClient("http://test"), + **overrides, # type: ignore[arg-type] + ) + + +class TestBuiltinRegistration: + def test_builtins_discoverable(self) -> None: + names = harness_names() + assert "baseline" in names + assert "felix" in names + + def test_list_reports_package_and_availability(self) -> None: + infos = {i.name: i for i in list_harnesses()} + assert infos["baseline"].availability.ok + assert infos["baseline"].package == "rimworld-learning-environment" + assert infos["felix"].availability.ok # felix extra installed in dev + assert "Felix" in infos["felix"].description + + def test_unknown_name_lists_installed(self) -> None: + with pytest.raises(HarnessNotFoundError, match="baseline"): + get_plugin("does-not-exist") + + def test_create_baseline(self) -> None: + harness = create_harness("baseline", _ctx()) + assert isinstance(harness, BaselineHarness) + assert harness.name == "baseline" + + def test_create_felix_smoke_builds_seven_agents(self) -> None: + harness = create_harness("felix", _ctx(), {"no_think": True}, smoke=True) + assert harness.name == "felix" + assert len(harness.agents) == 7 # type: ignore[attr-defined] + + def test_felix_options_validated(self) -> None: + with pytest.raises(HarnessOptionsError, match="bogus"): + create_harness("felix", _ctx(), {"bogus": 1}, smoke=True) + + def test_baseline_rejects_options(self) -> None: + with pytest.raises(HarnessOptionsError): + create_harness("baseline", _ctx(), {"anything": True}) + + +class TestOptionParsing: + def test_json_coercion(self) -> None: + parsed = parse_option_pairs(["parallel=false", "role_timeout_s=12.5", "name=abc"]) + assert parsed == {"parallel": False, "role_timeout_s": 12.5, "name": "abc"} + + def test_bad_pair(self) -> None: + with pytest.raises(HarnessOptionsError): + parse_option_pairs(["novalue"]) + + def test_validate_passthrough_model(self) -> None: + plugin = get_plugin("baseline") + opts = EmptyOptions() + assert validate_options(plugin, opts) is opts + + +class _Opts(BaseModel): + model_config = ConfigDict(extra="forbid") + shout: bool = False + + +class _EchoHarness(BaseHarness): + name: ClassVar[str] = "echo" + + def __init__(self, shout: bool) -> None: + super().__init__() + self.shout = shout + self.steps = 0 + self.ended = 0 + self.torn_down = False + + async def step( + self, state: GameState, tick: int, macro_time: float, events: list[RimAPIEvent], + ) -> StepResult: + self.steps += 1 + return StepResult( + plan=ActionPlan(role="echo", tick=state.colony.tick, actions=[]), + extras={"shout": self.shout}, + ) + + async def on_tick_end(self, tick, state, step, execution, score) -> None: # type: ignore[no-untyped-def] + self.ended += 1 + await super().on_tick_end(tick, state, step, execution, score) + + async def teardown(self) -> None: + self.torn_down = True + + +class _UnavailablePlugin: + name = "ghost" + description = "never runs" + + def available(self) -> Availability: + return Availability.missing("ghost binary not on PATH") + + def option_schema(self) -> type[BaseModel]: + return EmptyOptions + + def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + raise AssertionError("must not be called") + + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + raise AssertionError("must not be called") + + def describe(self) -> dict[str, str]: + return {} + + +class TestUnavailable: + def test_unavailable_plugin_raises_with_reason(self, monkeypatch: pytest.MonkeyPatch) -> None: + import rle.harness.registry as registry + + monkeypatch.setattr(registry, "get_plugin", lambda name: _UnavailablePlugin()) + with pytest.raises(HarnessUnavailableError, match="ghost binary"): + registry.create_harness("ghost", _ctx()) + + +class TestCompatShim: + def test_no_agent_builds_baseline(self) -> None: + assert isinstance(build_legacy_harness([], no_agent=True), BaselineHarness) + assert isinstance(build_legacy_harness(None), BaselineHarness) + + def test_agents_build_felix(self) -> None: + felix = create_harness("felix", _ctx(), smoke=True) + rebuilt = build_legacy_harness(felix.agents, parallel=False) # type: ignore[attr-defined] + assert rebuilt.name == "felix" + + +class TestBaseHarnessDefaults: + def test_ctx_before_setup_raises(self) -> None: + with pytest.raises(RuntimeError): + _ = _EchoHarness(shout=False).ctx + + async def test_setup_stores_ctx(self) -> None: + h = _EchoHarness(shout=True) + ctx = _ctx() + await h.setup(ctx) + assert h.ctx is ctx + assert h.describe() == {"harness": "echo"} From f1388da7dd8cb0334956b9226733160682feb08c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 07:06:27 +0000 Subject: [PATCH 3/8] CLI: --harness / --harness list / --harness-opt via the plugin registry; rle.testing for plugin authors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both run_scenario.py and run_benchmark.py now build their harness through rle.harness.create_harness and carry no Felix imports. --no-agent stays as a permanent alias for --harness baseline; --no-think/--sequential/--visualize fold into FelixOptions (warned and ignored for other harnesses). - rle.harness.cli: shared argparse glue + plugin table - run_benchmark.py: --harness is repeatable (harness matrix in one run); results and summaries record harness, harness_versions, mean step latency, and a harness_failed quarantine flag derived from RIMAPI null-ref / plant-def markers; quarantined runs are excluded from means - rle.orchestration.save_loader: load + settle helper shared by both CLIs (replaces the 2s sleep in the benchmark path) - rle.testing: MockRimAPI (moved out of run_benchmark.py) and run_harness_smoke(plugin) — the contract test external harness packages run in CI - collect_metadata: felix_sdk_version -> harness_versions (from BaseHarness.describe()) - docs/harness-plugins.md: authoring guide + repo boundary rule Co-authored-by: Jason --- docs/harness-plugins.md | 120 +++++ scripts/run_benchmark.py | 677 ++++++++++----------------- scripts/run_scenario.py | 188 ++++---- src/rle/harness/__init__.py | 14 + src/rle/harness/cli.py | 134 ++++++ src/rle/orchestration/game_loop.py | 42 +- src/rle/orchestration/save_loader.py | 65 +++ src/rle/testing/__init__.py | 20 + src/rle/testing/mock_rimapi.py | 138 ++++++ src/rle/testing/smoke.py | 68 +++ src/rle/tracking/metadata.py | 12 +- tests/unit/test_metadata.py | 8 +- tests/unit/test_testing_smoke.py | 31 ++ 13 files changed, 974 insertions(+), 543 deletions(-) create mode 100644 docs/harness-plugins.md create mode 100644 src/rle/harness/cli.py create mode 100644 src/rle/orchestration/save_loader.py create mode 100644 src/rle/testing/__init__.py create mode 100644 src/rle/testing/mock_rimapi.py create mode 100644 src/rle/testing/smoke.py create mode 100644 tests/unit/test_testing_smoke.py diff --git a/docs/harness-plugins.md b/docs/harness-plugins.md new file mode 100644 index 0000000..4c4c175 --- /dev/null +++ b/docs/harness-plugins.md @@ -0,0 +1,120 @@ +# Writing an RLE harness plugin + +RLE benchmarks **harnesses × models**. A *model* is whatever LLM sits behind +the decisions; a *harness* is the machinery that turns game state into +actions — one agent or many, an agent framework, or a coding agent attached +over MCP. Both are swappable from the CLI: + +```bash +python scripts/run_benchmark.py --harness felix --model gpt-4o +python scripts/run_benchmark.py --harness opencode --model gpt-4o +python scripts/run_benchmark.py --harness list +``` + +## Repo boundary rule + +RLE core ships only harnesses that are RLE-authored code: `baseline` +(unmanaged colony) and `felix` (the original 7-agent stack). **A harness that +wraps a third-party tool lives in its own repository and package** +(`rle-harness-opencode`, `rle-harness-grok-build`, ...). Nothing tool-specific +is committed to RLE core; adding a harness is `pip install`, never a core PR. + +Start from the template: . + +## The contract + +Register a module-level `PLUGIN` object under the `rle.harnesses` entry-point +group: + +```toml +# pyproject.toml of your harness package +[project] +name = "rle-harness-mytool" +dependencies = ["rimworld-learning-environment>=0.5"] + +[project.entry-points."rle.harnesses"] +mytool = "rle_harness_mytool:PLUGIN" +``` + +`PLUGIN` implements `rle.harness.HarnessPlugin`: + +| Method | Purpose | +|---|---| +| `name`, `description` | Shown by `--harness list` | +| `available() -> Availability` | Cheap probe: is the binary / extra installed? Never import heavy deps at module load. | +| `option_schema() -> type[BaseModel]` | Pydantic model validated from `--harness-opt key=value` (use `EmptyOptions` if none). | +| `create(ctx, options) -> BaseHarness` | Build the real harness. | +| `smoke(ctx, options) -> BaseHarness` | Build a variant that runs with no external tool / LLM (used by `--smoke-test` and CI). | +| `describe() -> dict[str, str]` | Versions recorded in run metadata. | + +The harness itself subclasses `rle.harness.BaseHarness`: + +```python +class MyHarness(BaseHarness): + name = "mytool" + + async def setup(self, ctx: HarnessContext) -> None: ... + async def step(self, state, tick, macro_time, events) -> StepResult: ... + async def on_tick_end(self, tick, state, step, execution, score) -> None: ... + async def teardown(self) -> None: ... +``` + +### `StepResult` + +Return `StepResult(plan=ActionPlan(...))` and the loop executes the actions +through `ActionExecutor` (same normalisation and guards as Felix). If your +harness already applied its writes during the turn — the normal case for a +coding agent calling tools through the RLE MCP server — return the recorded +`ExecutionResult` in `StepResult.execution` and the loop skips execution and +scores what you report. `proposals` (per-sub-agent plans) and `extras` +(telemetry) are optional and surface in the dashboard export. + +### Turn-based, always + +Each tick the environment pauses the game, refreshes state, calls +`step(...)` once (bounded by `--tick-timeout` when set), executes, scores, +calls `on_tick_end(...)`, and unpauses. Coding-agent harnesses get one prompt +per tick and must return when the agent is idle; free-running sessions are +not part of v1. + +### Failure semantics + +Raise `HarnessStepError` for an expected failure (agent crashed, tool +unreachable). The loop logs an `ERROR` event, scores the tick with no actions, +and continues. Any other exception is treated as a bug and propagates. + +## What core gives you + +- `rle.harness` — `BaseHarness`, `StepResult`, `HarnessContext`, + `HarnessPlugin`, `Availability`, `EmptyOptions`, `HarnessStepError`, + `TickObserver`, registry helpers. +- `rle.harness.cli_base.HeadlessCliHarness` — scaffold for CLI coding agents + (spawn/attach, MCP attach, per-tick prompt, idle/timeout/abort, ledger + drain, token + latency capture). Tool-agnostic by design. +- `rle.harness.brief` — the harness-neutral scenario brief every harness + receives (goals, filtered state, MAP_SUMMARY, action catalog). +- `rle.mcp` — the RimAPI MCP server + per-tick write ledger (`rle-mcp`). +- `rle.testing` — `MockRimAPI` and `run_harness_smoke(plugin)`; run the + latter in your CI: + +```python +from rle.testing import run_harness_smoke + +async def test_smoke() -> None: + report = await run_harness_smoke("mytool", ticks=3) + assert report.ok +``` + +## Scoring is harness-agnostic + +Process metrics (`efficiency`, `plan_coherence`) are computed from the writes +that reached RIMAPI, never from a harness's internal messaging. A harness is +free to coordinate however it likes; what is scored is whether coherent, +valid writes reached the game and how the colony fared. Prompt engineering +beyond the neutral brief is part of the harness being benchmarked. + +## Metadata + +Every run records `harness`, `harness_options`, `harness_versions` +(`describe()` output), `model`, `provider`, `scoring_version`, and per-tick +`step_latency_s`. Leaderboards key on harness × model × scenario. diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py index e844725..4755d89 100644 --- a/scripts/run_benchmark.py +++ b/scripts/run_benchmark.py @@ -1,4 +1,4 @@ -"""CLI: run all RLE scenarios and output a leaderboard.""" +"""CLI: run all RLE scenarios for one or more harnesses and output a leaderboard.""" from __future__ import annotations @@ -11,31 +11,31 @@ import time from pathlib import Path from typing import Any -from unittest.mock import MagicMock - -import httpx -from felix_agent_sdk.core import HelixConfig -from felix_agent_sdk.providers.base import BaseProvider -from felix_agent_sdk.providers.types import CompletionResult -from felix_agent_sdk.visualization import HelixVisualizer - -from rle.agents import AGENT_DISPLAY -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer -from rle.config import RLEConfig, bridge_anthropic_key, bridge_openrouter_key + +from rle.config import RLEConfig +from rle.docker import DockerGameServer +from rle.harness import ( + HarnessContext, + HarnessNotFoundError, + HarnessOptionsError, + HarnessUnavailableError, + add_harness_args, + create_harness, + exit_with_harness_error, + harness_options_for, + maybe_handle_harness_list, + selected_harnesses, +) from rle.orchestration.game_loop import RLEGameLoop +from rle.orchestration.save_loader import load_save_and_settle from rle.rimapi.client import RimAPIClient from rle.scenarios.evaluator import ScenarioEvaluator from rle.scenarios.loader import list_scenarios from rle.scenarios.schema import ScenarioConfig from rle.scoring.composite import CompositeScorer -from rle.scoring.delta import PairedResult +from rle.scoring.delta import PairedResult, print_paired_leaderboard from rle.scoring.recorder import TimeSeriesRecorder +from rle.testing.mock_rimapi import MockRimAPI from rle.tracking.cost_tracker import CostTracker, create_cost_tracker, fetch_billed_costs from rle.tracking.event_log import EventLog from rle.tracking.hf_logger import HFLogger @@ -45,321 +45,206 @@ logger = logging.getLogger(__name__) -# Seconds to wait after loading a save before starting a run. -GAME_LOAD_WAIT_SECONDS = 2 - -# Mock data for --dry-run mode -_MOCK_ACTION_PLAN = json.dumps({ - "actions": [ - { - "action_type": "no_action", - "reason": "Mock mode — no real LLM call", - }, - ], - "summary": "Mock deliberation.", - "confidence": 0.6, -}) - -_MOCK_ROUTES: dict[str, dict | list] = { - "/api/v1/colonists": [ - { - "colonist_id": "col_01", "name": "Tynan", "health": 0.95, - "mood": 0.72, "skills": {"shooting": 8, "construction": 5, - "cooking": 3, "mining": 6, "intellectual": 4}, - "traits": ["industrious"], "current_job": "mining", - "is_drafted": False, "needs": {"food": 0.6, "rest": 0.8}, - "injuries": [], "position": [42, 18], - }, - { - "colonist_id": "col_02", "name": "Cassandra", "health": 0.88, - "mood": 0.65, "skills": {"shooting": 3, "construction": 7, - "cooking": 6, "growing": 8, "intellectual": 6}, - "traits": ["kind"], "current_job": "growing", - "is_drafted": False, "needs": {"food": 0.5, "rest": 0.7}, - "injuries": [], "position": [30, 22], - }, - { - "colonist_id": "col_03", "name": "Randy", "health": 0.92, - "mood": 0.58, "skills": {"shooting": 10, "melee": 7, - "construction": 3, "cooking": 2}, - "traits": ["tough", "brawler"], "current_job": None, - "is_drafted": False, "needs": {"food": 0.4, "rest": 0.6}, - "injuries": [], "position": [50, 10], - }, - ], - "/api/v1/resources/summary?map_id=0": { - "total_items": 800, "total_market_value": 8000.0, - "critical_resources": { - "food_summary": {"food_total": 85}, - "medicine_total": 5, "weapon_count": 2, - }, - }, - "/api/v1/map/buildings?map_id=0": [ - {"id": "s_01", "def_name": "Wall", "position": {"x": 10, "y": 0, "z": 10}, - "hit_points": 300.0, "max_hit_points": 300.0}, - ], - "/api/v1/research/summary": { - "current_project": "electricity", "progress": 0.45, - "completed": ["stonecutting"], "available": ["electricity", "battery", "smithing"], - }, - "/api/v1/incidents?map_id=0": {"incidents": []}, - "/api/v1/game/state": { - "name": "New Hope", "wealth": 8000.0, "day": 5, "tick": 300000, - "population": 3, "mood_average": 0.65, "food_days": 7.0, - }, - "/api/v1/map/weather?map_id=0": { - "weather": "clear", "temperature": 22.0, - }, - "/api/v1/map/zones?map_id=0": [], - "/api/v1/map/rooms?map_id=0": [], - "/api/v1/map/ore?map_id=0": [], - "/api/v1/map/farm/summary?map_id=0": { - "total_growing_zones": 0, "planted_cells": 0, - "harvestable_cells": 0, "crops": {}, - }, - "/api/v1/map/terrain?map_id=0": { - "width": 10, "height": 10, - "palette": ["Soil", "WaterMovingShallow", "SoilRich", "Granite_Rough"], - "grid": [100, 0], - "floor_palette": [], "floor_grid": [100, 0], - }, - "/api/v1/resources/stored?map_id=0": { - "Resources": [ - {"def_name": "WoodLog", "stack_count": 200}, - {"def_name": "Steel", "stack_count": 100}, - {"def_name": "ComponentIndustrial", "stack_count": 10}, - ], - }, - "/api/v1/map/power/info?map_id=0": { - "current_power": 0.0, - "total_consumption": 0.0, - "currently_stored_power": 0.0, - "total_power_storage": 0.0, - }, - "/api/v1/factions": [], - "/api/v1/ui/alerts?map_id=0": [], -} - - -def _make_mock_transport() -> httpx.MockTransport: - _POST_OK = httpx.Response( - 200, content=b'{"success": true, "errors": [], "warnings": []}', - headers={"content-type": "application/json"}, - ) +# Felix roster ids, for --ablation (which is a Felix-only experiment). +_ALL_AGENT_IDS = [ + "map_analyst", "resource_manager", "defense_commander", + "research_director", "social_overseer", "construction_planner", + "medical_officer", +] - def handler(request: httpx.Request) -> httpx.Response: - # All POSTs succeed in mock mode (game control, actions, etc.) - if request.method == "POST": - return _POST_OK - # GET routes matched by full path including query string - raw = request.url.raw_path.decode() - if raw in _MOCK_ROUTES: - return httpx.Response( - 200, content=json.dumps(_MOCK_ROUTES[raw]).encode(), - headers={"content-type": "application/json"}, - ) - # Also try without query string for routes stored that way - path = raw.split("?")[0] - if path in _MOCK_ROUTES: - return httpx.Response( - 200, content=json.dumps(_MOCK_ROUTES[path]).encode(), - headers={"content-type": "application/json"}, - ) - return httpx.Response(404, content=b"Not found") +# Markers in per-tick action errors that indicate the *harness/RIMAPI plumbing* +# failed rather than the model deciding badly (issue #27, #33, CLAUDE.md +# "null-ref cascades"). A run that trips one is quarantined from leaderboard +# means by default (see analyze_spread.py for the post-hoc taxonomy). +HARNESS_FAILURE_MARKERS = ( + "Object reference not set", + "NullReferenceException", + "Invalid plant definition", +) - return httpx.MockTransport(handler) +class RunError(RuntimeError): + """A harness could not be constructed for this run.""" + + +async def _load_save(client: RimAPIClient, config: RLEConfig, scenario: ScenarioConfig) -> bool: + """Load + settle the scenario save. Returns False when the run must be skipped.""" + if not scenario.save_name: + return True + try: + await load_save_and_settle(client, config.rimapi_url, scenario.save_name) + except Exception as e: + logger.warning("Could not load save %s: %s", scenario.save_name, e) + return False + return True -def _make_mock_provider() -> MagicMock: - provider = MagicMock(spec=BaseProvider) - provider.complete.return_value = CompletionResult( - content=_MOCK_ACTION_PLAN, model="mock-model", - usage={"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, - ) - provider.acomplete.return_value = provider.complete.return_value - return provider - - -def _create_agents( # type: ignore[no-untyped-def] - provider, helix, *, provider_kwargs=None, no_think=False, - exclude_agent: str | None = None, -): - all_agents = [ - MapAnalyst("map_analyst", provider, helix, spawn_time=0.0, velocity=1.0), - ResourceManager("resource_manager", provider, helix, spawn_time=0.0, velocity=1.0), - DefenseCommander( - "defense_commander", provider, helix, spawn_time=0.0, velocity=1.0, - ), - ResearchDirector( - "research_director", provider, helix, spawn_time=0.0, velocity=1.0, - ), - SocialOverseer("social_overseer", provider, helix, spawn_time=0.0, velocity=1.0), - ConstructionPlanner( - "construction_planner", provider, helix, spawn_time=0.0, velocity=1.0, - ), - MedicalOfficer("medical_officer", provider, helix, spawn_time=0.0, velocity=1.0), - ] - agents = [a for a in all_agents if a.agent_id != exclude_agent] - if provider_kwargs: - for agent in agents: - agent.set_provider_kwargs(**provider_kwargs) - if no_think: - for agent in agents: - agent.set_no_think(True) - return agents - - -def _create_visualizer(helix, agents) -> HelixVisualizer: # type: ignore[no-untyped-def] - """Create a HelixVisualizer with all agents registered.""" - visualizer = HelixVisualizer(helix, title="R L E") - for agent in agents: - display = AGENT_DISPLAY[agent.agent_id] - visualizer.register_agent( - agent.agent_id, label=display["label"], color=display["color"], - ) - return visualizer +def _harness_failed(event_log: EventLog | None, start_index: int) -> bool: + """True when any action error since ``start_index`` matches a plumbing marker.""" + if event_log is None: + return False + for event in event_log.events[start_index:]: + err = event.data.get("error") or event.data.get("reason") + if isinstance(err, str) and any(m in err for m in HARNESS_FAILURE_MARKERS): + return True + return False -async def _run_scenario( + +async def _run_scenario( # noqa: PLR0913 scenario: ScenarioConfig, config: RLEConfig, client: RimAPIClient, - provider, # type: ignore[no-untyped-def] - helix, # type: ignore[no-untyped-def] + harness_name: str, + harness_options: dict[str, Any], output_dir: Path | None, + *, max_ticks_override: int | None = None, - provider_kwargs: dict | None = None, - visualize: bool = False, - no_think: bool = False, - parallel: bool = True, - no_agent: bool = False, + smoke: bool = False, no_pause: bool = False, event_log: EventLog | None = None, cost_tracker: CostTracker | None = None, weave_module: object | None = None, -) -> dict: - agents = _create_agents(provider, helix, provider_kwargs=provider_kwargs, no_think=no_think) +) -> dict[str, Any]: + ctx = HarnessContext( + config=config, + client=client, + expected_duration_days=scenario.expected_duration_days, + initial_population=scenario.initial_population, + scenario=scenario, + event_log=event_log, + cost_tracker=cost_tracker, + tick_timeout_s=config.tick_timeout_s, + smoke=smoke, + ) if weave_module is not None: - for agent in agents: - agent.enable_weave(weave_module) + ctx.extras["weave_module"] = weave_module + try: + harness = create_harness(harness_name, ctx, harness_options, smoke=smoke) + except (HarnessNotFoundError, HarnessUnavailableError, HarnessOptionsError) as exc: + raise RunError(str(exc)) from exc + scorer = CompositeScorer(scenario.scoring_weights or None) recorder = TimeSeriesRecorder() evaluator = ScenarioEvaluator(scenario) - visualizer = _create_visualizer(helix, agents) if visualize else None + events_before = len(event_log.events) if event_log else 0 loop = RLEGameLoop( - config, client, agents, + config, client, expected_duration_days=scenario.expected_duration_days, scorer=scorer, recorder=recorder, evaluator=evaluator, initial_population=scenario.initial_population, initial_wealth=8000.0, - visualizer=visualizer, - parallel=parallel, - no_agent=no_agent, no_pause=no_pause, event_log=event_log, cost_tracker=cost_tracker, + harness=harness, + harness_context=ctx, + scenario=scenario, ) max_ticks = max_ticks_override or scenario.max_ticks t0 = time.monotonic() - if visualizer: - with visualizer.live(): - await loop.run(max_ticks=max_ticks) - else: - await loop.run(max_ticks=max_ticks) + await loop.run(max_ticks=max_ticks) elapsed = time.monotonic() - t0 final_score = 0.0 if recorder.snapshots: - final = scorer.final_score(recorder.snapshots) - final_score = final.composite + final_score = scorer.final_score(recorder.snapshots).composite outcome = "timeout" if loop.evaluation_result: outcome = loop.evaluation_result.outcome ticks_run = len(loop.tick_results) - total_calls = loop._parse_successes + loop._parse_failures - parse_rate = loop._parse_successes / total_calls if total_calls else 0.0 + total_calls = loop.parse_successes + loop.parse_failures + parse_rate = loop.parse_successes / total_calls if total_calls else 0.0 + latencies = [t.step_latency_s for t in loop.tick_results] + slug = scenario.name.lower().replace(" ", "_") if output_dir and recorder.snapshots: - csv_name = scenario.name.lower().replace(" ", "_") + ".csv" - recorder.to_csv(output_dir / csv_name) + recorder.to_csv(output_dir / f"{harness.name}_{slug}.csv") deliberation_log = loop.deliberation_log if output_dir and deliberation_log: - log_name = scenario.name.lower().replace(" ", "_") + "_deliberations.jsonl" - with open(output_dir / log_name, "w") as f: + with open(output_dir / f"{harness.name}_{slug}_deliberations.jsonl", "w") as f: for entry in deliberation_log: f.write(json.dumps(entry) + "\n") return { "name": scenario.name, "difficulty": scenario.difficulty, + "harness": harness.name, + "harness_versions": harness.describe(), "outcome": outcome, "score": final_score, "ticks": ticks_run, "elapsed_s": round(elapsed, 2), "sec_per_tick": round(elapsed / ticks_run, 2) if ticks_run else 0.0, - "parse_successes": loop._parse_successes, - "parse_failures": loop._parse_failures, + "mean_step_latency_s": ( + round(sum(latencies) / len(latencies), 3) if latencies else 0.0 + ), + "parse_successes": loop.parse_successes, + "parse_failures": loop.parse_failures, "parse_rate": round(parse_rate, 3), + "harness_failed": _harness_failed(event_log, events_before), } -def _print_leaderboard(results: list[dict], model: str | None = None) -> None: - print("\n" + "=" * 88) +def _print_leaderboard(results: list[dict[str, Any]], model: str | None = None) -> None: + print("\n" + "=" * 100) title = f"RLE BENCHMARK — {model}" if model else "RLE BENCHMARK LEADERBOARD" print(title) - print("=" * 88) + print("=" * 100) header = ( - f"{'Scenario':<25} {'Diff':<7} {'Outcome':<9} " - f"{'Score':>6} {'Ticks':>5} {'Time':>7} {'s/tick':>6} " - f"{'Parse%':>7} {'Fail':>4}" + f"{'Harness':<12} {'Scenario':<22} {'Diff':<7} {'Outcome':<9} " + f"{'Score':>6} {'Ticks':>5} {'Time':>7} {'s/step':>6} " + f"{'Parse%':>7} {'Fail':>4} {'HF':>3}" ) print(header) - print("-" * 88) + print("-" * 100) for r in results: print( - f"{r['name']:<25} {r['difficulty']:<7} {r['outcome']:<9} " + f"{r['harness']:<12} {r['name']:<22} {r['difficulty']:<7} {r['outcome']:<9} " f"{r['score']:>6.3f} {r['ticks']:>5} {r['elapsed_s']:>6.1f}s " - f"{r['sec_per_tick']:>6.2f} {r['parse_rate']:>6.1%} {r['parse_failures']:>4}" + f"{r['mean_step_latency_s']:>6.2f} {r['parse_rate']:>6.1%} " + f"{r['parse_failures']:>4} {'!' if r.get('harness_failed') else '':>3}" ) - print("-" * 88) - scores = [r["score"] for r in results] - passed = sum(1 for r in results if r["outcome"] == "victory") + print("-" * 100) + clean = [r for r in results if not r.get("harness_failed")] + scores = [r["score"] for r in clean] + passed = sum(1 for r in clean if r["outcome"] == "victory") avg = sum(scores) / len(scores) if scores else 0.0 total_parse = sum(r["parse_successes"] for r in results) total_fail = sum(r["parse_failures"] for r in results) total_calls = total_parse + total_fail overall_parse_rate = total_parse / total_calls if total_calls else 0.0 total_time = sum(r["elapsed_s"] for r in results) + quarantined = len(results) - len(clean) print( - f"Avg score: {avg:.3f} | Passed: {passed}/{len(results)} | " + f"Avg score: {avg:.3f} | Passed: {passed}/{len(clean)} | " f"Parse rate: {overall_parse_rate:.1%} ({total_fail} failures) | " f"Total time: {total_time:.1f}s" + + (f" | {quarantined} run(s) quarantined (HF = harness failure)" if quarantined else "") ) - print("=" * 88) + print("=" * 100) -def _build_provider(args: argparse.Namespace) -> tuple[BaseProvider, RLEConfig]: - """Build LLM provider from CLI args. Returns (provider, config).""" - if args.dry_run and not args.provider: - return _make_mock_provider(), RLEConfig(tick_interval=0.0) - - overrides: dict[str, str] = {} +def _build_config(args: argparse.Namespace, smoke: bool) -> RLEConfig: + overrides: dict[str, Any] = {} if args.provider: overrides["provider"] = args.provider if args.model: overrides["model"] = args.model if args.base_url: overrides["provider_base_url"] = args.base_url - config = RLEConfig(**overrides) if overrides else RLEConfig() - bridge_openrouter_key(config) - bridge_anthropic_key(config) - return config.get_provider(), config + if args.tick_interval is not None and not smoke: + overrides["tick_interval"] = args.tick_interval + if args.tick_timeout is not None: + overrides["tick_timeout_s"] = args.tick_timeout + if smoke and args.tick_interval is None: + overrides["tick_interval"] = 0.0 + return RLEConfig(**overrides) if overrides else RLEConfig() def _resolve_ticks(args: argparse.Namespace, use_mock_rimapi: bool) -> int | None: @@ -371,35 +256,23 @@ def _resolve_ticks(args: argparse.Namespace, use_mock_rimapi: bool) -> int | Non return None -_ALL_AGENT_IDS = [ - "map_analyst", "resource_manager", "defense_commander", - "research_director", "social_overseer", "construction_planner", - "medical_officer", -] - - -async def _run_ablation( +async def _run_ablation( # noqa: PLR0913 args: argparse.Namespace, config: RLEConfig, - provider: object, - helix: object, + harness_options: dict[str, Any], scenarios: list[ScenarioConfig], use_mock_rimapi: bool, num_runs: int, ticks_override: int | None, - provider_kwargs: dict[str, Any] | None, ) -> None: - """Run ablation study: full benchmark + 7 single-agent-removed benchmarks.""" + """Ablation study (Felix only): full roster + 7 single-agent-removed passes.""" output_dir = Path(args.output) if args.output else get_run_dir(args.model) output_dir.mkdir(parents=True, exist_ok=True) async with RimAPIClient(config.rimapi_url) as client: if use_mock_rimapi: - client._client = httpx.AsyncClient( - transport=_make_mock_transport(), base_url="http://mock", - ) + MockRimAPI().attach(client) - # Pass 0: full benchmark (all agents) passes: list[tuple[str, list[dict[str, Any]]]] = [] labels = ["all_agents", *_ALL_AGENT_IDS] @@ -414,44 +287,20 @@ async def _run_ablation( for scenario in scenarios: for run_id in range(num_runs): run_label = f" (run {run_id + 1}/{num_runs})" if num_runs > 1 else "" - if scenario.save_name and not use_mock_rimapi: - try: - await client.load_game(scenario.save_name) - await asyncio.sleep(GAME_LOAD_WAIT_SECONDS) - except Exception as e: - logger.warning("Could not load save %s: %s", scenario.save_name, e) - print(f" SKIP {scenario.name}{run_label} ({tag}): save load failed") - continue + if not use_mock_rimapi and not await _load_save(client, config, scenario): + print(f" SKIP {scenario.name}{run_label} ({tag}): save load failed") + continue print(f" {scenario.name}{run_label} ({tag})...") - agents = _create_agents( - provider, helix, - provider_kwargs=provider_kwargs, - no_think=args.no_think, - exclude_agent=exclude, - ) - scorer = CompositeScorer(scenario.scoring_weights or None) - recorder = TimeSeriesRecorder() - evaluator = ScenarioEvaluator(scenario) - loop = RLEGameLoop( - config, client, agents, - expected_duration_days=scenario.expected_duration_days, - scorer=scorer, recorder=recorder, evaluator=evaluator, - initial_population=scenario.initial_population, - initial_wealth=8000.0, - parallel=not args.sequential, + options = {**harness_options, "exclude_agent": exclude} + result = await _run_scenario( + scenario, config, client, "felix", options, None, + max_ticks_override=ticks_override, + smoke=use_mock_rimapi, no_pause=args.no_pause, ) - await loop.run(max_ticks=ticks_override or scenario.max_ticks) - - final_score = 0.0 - if recorder.snapshots: - final_score = scorer.final_score(recorder.snapshots).composite - pass_results.append({ - "scenario": scenario.name, - "score": final_score, - }) - print(f" score={final_score:.3f}") + pass_results.append({"scenario": scenario.name, "score": result["score"]}) + print(f" score={result['score']:.3f}") passes.append((tag, pass_results)) @@ -474,7 +323,6 @@ async def _run_ablation( ) matrix[agent_name][scenario_name] = round(full_avg - rem_avg, 4) - # Print ablation table scenario_names = list(full_scores.keys()) print(f"\n{'=' * 88}") print("ABLATION MATRIX (score delta: positive = agent helps)") @@ -493,7 +341,6 @@ async def _run_ablation( print(row) print(f"{'=' * 88}") - # Save results ablation_data = { "num_runs": num_runs, "ticks_per_scenario": ticks_override, @@ -505,43 +352,44 @@ async def _run_ablation( print(f"\nAblation results saved to {ablation_path}") -async def main(args: argparse.Namespace) -> None: +async def main(args: argparse.Namespace) -> None: # noqa: PLR0912, PLR0915 logging.basicConfig( level=getattr(logging, args.log_level.upper(), logging.INFO), format="%(asctime)s %(name)s %(levelname)s %(message)s", ) + if maybe_handle_harness_list(args): + return + # Seed RLE-side stochasticity (resolver tiebreaks, json_repair fallbacks). # RimWorld's RNG is unaffected — see metadata.collect_metadata docstring. if args.seed is not None: random.seed(args.seed) - helix = HelixConfig.default().to_geometry() scenarios = list_scenarios() - provider, config = _build_provider(args) is_smoke_test = args.smoke_test or args.dry_run if args.dry_run: logger.warning("--dry-run is deprecated, use --smoke-test") - if args.tick_interval is not None and not is_smoke_test: - config = RLEConfig(**{**config.model_dump(), "tick_interval": args.tick_interval}) - use_mock_rimapi = (is_smoke_test or args.provider is not None) and not args.docker + config = _build_config(args, is_smoke_test) + # Smoke test = mock game AND mock model. --smoke-test with --provider used + # to mean "real LLM against the fake game"; that is now + # --harness-opt smoke_llm=false territory for harnesses that support it. + use_mock_rimapi = is_smoke_test and not args.docker num_runs = getattr(args, "runs", 1) or 1 - # extra_body is an OpenAI-compatible escape hatch; the Anthropic API - # rejects unknown body fields with 400. - effective_provider = args.provider or config.provider + harness_names = selected_harnesses(args, config) + harness_option_sets = { + name: harness_options_for(name, args, config) for name in harness_names + } if args.ablation: + if harness_names != ["felix"]: + raise SystemExit("--ablation is a Felix-only experiment (use --harness felix)") ticks_override = _resolve_ticks(args, use_mock_rimapi) - provider_kwargs_abl: dict[str, Any] = {} - if args.no_think and effective_provider != "anthropic": - provider_kwargs_abl["extra_body"] = { - "chat_template_kwargs": {"enable_thinking": False}, - } await _run_ablation( - args, config, provider, helix, scenarios, use_mock_rimapi, - num_runs, ticks_override, provider_kwargs_abl or None, + args, config, harness_option_sets["felix"], scenarios, use_mock_rimapi, + num_runs, ticks_override, ) return @@ -561,31 +409,24 @@ async def main(args: argparse.Namespace) -> None: output_dir = Path(args.output) output_dir.mkdir(parents=True, exist_ok=True) - # Build provider kwargs (e.g. no-think for Qwen3.5) - provider_kwargs: dict[str, Any] = {} - if args.no_think and effective_provider != "anthropic": - provider_kwargs["extra_body"] = { - "chat_template_kwargs": {"enable_thinking": False}, - } - # Initialize W&B logger (no-op if --wandb not passed or wandb not installed) wandb_logger = WandBLogger( enabled=args.wandb, - run_name=f"{args.model or config.model}_{ticks_override or 'full'}ticks", + run_name=f"{'+'.join(harness_names)}_{config.model}_{ticks_override or 'full'}ticks", ) if wandb_logger.enabled: wandb_logger.log_config({ **collect_metadata(random_seed=args.seed), - "model": args.model or config.model, - "provider": args.provider or config.provider, - "no_think": args.no_think, - "parallel": not args.sequential, + "harnesses": harness_names, + "harness_options": harness_option_sets, + "model": config.model, + "provider": config.provider, "ticks_per_scenario": ticks_override, }) # Initialize cost tracker (fetches OpenRouter pricing; CLI may override) cost_tracker = await create_cost_tracker( - args.model or config.model, + config.model, prompt_price_override=( args.prompt_price_per_mtok / 1_000_000 if args.prompt_price_per_mtok is not None else None @@ -596,110 +437,105 @@ async def main(args: argparse.Namespace) -> None: ), ) - # Initialize event log event_log: EventLog | None = None if args.output: event_log = EventLog(Path(args.output) / "events.jsonl") - no_baseline = getattr(args, "no_baseline", False) + no_baseline = getattr(args, "no_baseline", False) or harness_names == ["baseline"] is_paired = not use_mock_rimapi and not no_baseline - results = [] + results: list[dict[str, Any]] = [] paired_results: list[PairedResult] = [] - # Docker lifecycle (optional) - docker_server = None + docker_server: DockerGameServer | None = None if args.docker: - from rle.docker import DockerGameServer docker_server = DockerGameServer( image=config.docker_image, port=config.docker_port, ) await docker_server.start() - config = RLEConfig(**{ - **config.model_dump(), - "rimapi_url": docker_server.url, - }) + config = RLEConfig(**{**config.model_dump(), "rimapi_url": docker_server.url}) + history_path: Path | None = None try: async with RimAPIClient(config.rimapi_url) as client: if use_mock_rimapi: - client._client = httpx.AsyncClient( - transport=_make_mock_transport(), base_url="http://mock", - ) - - for scenario in scenarios: - if docker_server: - await docker_server.restart() - paired = PairedResult(scenario=scenario.name) if is_paired else None + MockRimAPI().attach(client) + + for harness_name in harness_names: + harness_options = harness_option_sets[harness_name] + if len(harness_names) > 1: + print(f"\n{'#' * 60}\n# HARNESS: {harness_name}\n{'#' * 60}") + + for scenario in scenarios: + if docker_server: + await docker_server.restart() + paired = ( + PairedResult(scenario=f"{harness_name}/{scenario.name}") + if is_paired else None + ) - for run_id in range(num_runs): - run_label = f" (run {run_id + 1}/{num_runs})" if num_runs > 1 else "" + for run_id in range(num_runs): + run_label = f" (run {run_id + 1}/{num_runs})" if num_runs > 1 else "" - # Load save if available (for reproducible initial conditions) - if scenario.save_name and not use_mock_rimapi: - try: - await client.load_game(scenario.save_name) - await asyncio.sleep(GAME_LOAD_WAIT_SECONDS) - except Exception as e: - logger.warning("Could not load save %s: %s", scenario.save_name, e) + if not use_mock_rimapi and not await _load_save(client, config, scenario): print(f" SKIP {scenario.name}{run_label}: save load failed") continue - # Agent run - print(f"\nRunning: {scenario.name} ({scenario.difficulty}){run_label}...") - result = await _run_scenario( - scenario, config, client, provider, helix, output_dir, - max_ticks_override=ticks_override, - provider_kwargs=provider_kwargs or None, - visualize=args.visualize, - no_think=args.no_think, - parallel=not args.sequential, - no_pause=args.no_pause, - event_log=event_log, - cost_tracker=cost_tracker, - weave_module=wandb_logger.weave, - ) - results.append(result) - if paired: - paired.agent_scores.append(result["score"]) - print( - f" -> agent: {result['outcome']} | score={result['score']:.3f} " - f"| {result['ticks']} ticks | {result['elapsed_s']}s " - f"| parse {result['parse_rate']:.0%} ({result['parse_failures']} fail)" - ) - - # Baseline run (reload same save, no agents) - if is_paired: - if scenario.save_name: - try: - await client.load_game(scenario.save_name) - await asyncio.sleep(GAME_LOAD_WAIT_SECONDS) - except Exception as e: - logger.warning("Could not reload save: %s", e) - - print(f" baseline{run_label}...") - baseline = await _run_scenario( - scenario, config, client, provider, helix, output_dir, - max_ticks_override=ticks_override, - no_agent=True, + print( + f"\nRunning: {scenario.name} ({scenario.difficulty}) " + f"[{harness_name}]{run_label}...", + ) + try: + result = await _run_scenario( + scenario, config, client, harness_name, harness_options, + output_dir, + max_ticks_override=ticks_override, + smoke=use_mock_rimapi, + no_pause=args.no_pause, + event_log=event_log, + cost_tracker=cost_tracker, + weave_module=wandb_logger.weave, + ) + except RunError as exc: + exit_with_harness_error(exc) + return + results.append(result) + if paired: + paired.agent_scores.append(result["score"]) + print( + f" -> {harness_name}: {result['outcome']} " + f"| score={result['score']:.3f} | {result['ticks']} ticks " + f"| {result['elapsed_s']}s | parse {result['parse_rate']:.0%} " + f"({result['parse_failures']} fail)" + + (" | HARNESS FAILURE" if result["harness_failed"] else "") ) - paired.baseline_scores.append(baseline["score"]) - print(f" -> baseline: score={baseline['score']:.3f}") - if paired: - paired_results.append(paired) + # Baseline run (reload same save, unmanaged colony) + if paired is not None: + if not await _load_save(client, config, scenario): + logger.warning("Could not reload save for baseline") + print(f" baseline{run_label}...") + baseline = await _run_scenario( + scenario, config, client, "baseline", {}, output_dir, + max_ticks_override=ticks_override, + no_pause=args.no_pause, + ) + paired.baseline_scores.append(baseline["score"]) + print(f" -> baseline: score={baseline['score']:.3f}") + + if paired: + paired_results.append(paired) # Print results if is_paired and paired_results: - from rle.scoring.delta import print_paired_leaderboard - print_paired_leaderboard(paired_results, model=args.model, num_runs=num_runs) + print_paired_leaderboard(paired_results, model=config.model, num_runs=num_runs) else: - _print_leaderboard(results, model=args.model) + _print_leaderboard(results, model=config.model) # Reconcile estimates against OpenRouter's billed ground truth # (token-count estimates diverged up to 4x on the v0.3.0 spread). billed_report = None - effective_base_url = args.base_url or config.provider_base_url or "" + effective_base_url = config.provider_base_url or "" openai_key = os.environ.get("OPENAI_API_KEY", "") generation_ids = cost_tracker.generation_ids if "openrouter.ai" in effective_base_url and openai_key and generation_ids: @@ -709,20 +545,25 @@ async def main(args: argparse.Namespace) -> None: ) billed_report = await fetch_billed_costs(generation_ids, openai_key) - # Build enriched summary with metadata + clean_results = [r for r in results if not r.get("harness_failed")] metadata = collect_metadata(random_seed=args.seed) summary: dict[str, Any] = { **metadata, - "model": args.model or config.model, - "provider": args.provider or config.provider, - "base_url": args.base_url or None, - "no_think": args.no_think, - "parallel": not args.sequential, + "harness": harness_names[0] if len(harness_names) == 1 else "matrix", + "harnesses": harness_names, + "harness_options": harness_option_sets, + "harness_versions": { + r["harness"]: r["harness_versions"] for r in results + }, + "model": config.model, + "provider": config.provider, + "base_url": config.provider_base_url, "tick_interval": config.tick_interval, "ticks_per_scenario": ticks_override, "num_runs": num_runs, "paired": is_paired, "scenarios": results, + "quarantined_runs": len(results) - len(clean_results), "cost_snapshot": cost_tracker.snapshot().model_dump(), } if billed_report: @@ -732,15 +573,14 @@ async def main(args: argparse.Namespace) -> None: if is_paired and paired_results: summary["paired_results"] = [p.to_dict() for p in paired_results] - # Auto-generate run directory if --output not specified - output_dir = Path(args.output) if args.output else get_run_dir(args.model) + output_dir = Path(args.output) if args.output else get_run_dir(config.model) output_dir.mkdir(parents=True, exist_ok=True) summary_path = output_dir / "benchmark_summary.json" summary_path.write_text(json.dumps(summary, indent=2, default=str)) print(f"\nResults exported to {output_dir}/") - # Only track real benchmark runs (not mock/dry-run JSON compliance tests) - scores = [r.get("score", 0) for r in results] + # Only track real benchmark runs (not smoke tests) + scores = [r.get("score", 0) for r in clean_results] avg = sum(scores) / len(scores) if scores else 0 if not use_mock_rimapi: history_path = append_history(summary) @@ -753,9 +593,8 @@ async def main(args: argparse.Namespace) -> None: elif prev_score is not None: print(f"Baseline: {prev_score:.3f} (this run: {avg:.3f})") else: - print("(dry-run: skipping history/baseline tracking)") + print("(smoke test: skipping history/baseline tracking)") - # W&B logging (optional) if wandb_logger.enabled: wandb_logger.log_final_summary( avg_score=avg, @@ -768,8 +607,7 @@ async def main(args: argparse.Namespace) -> None: wandb_logger.finish() print("W&B run logged") - # HuggingFace Hub push (optional) - if args.push_hf: + if args.push_hf and history_path is not None: hf = HFLogger( repo_id=config.hf_dataset_repo, token=config.hf_token, ) @@ -793,7 +631,7 @@ async def main(args: argparse.Namespace) -> None: parser.add_argument("--output", help="Output directory for CSV results") parser.add_argument( "--smoke-test", action="store_true", - help="Use mock RIMAPI (combine with --provider for real LLM + fake game)", + help="Mock RIMAPI + each harness's smoke variant (no game, no LLM)", ) parser.add_argument( "--dry-run", action="store_true", @@ -805,24 +643,23 @@ async def main(args: argparse.Namespace) -> None: ) parser.add_argument( "--ablation", action="store_true", - help="(WIP) Run ablation study: full benchmark + 7 single-agent-removed runs", + help="(WIP, felix only) Run ablation study: full roster + 7 single-agent-removed runs", ) parser.add_argument( - "--provider", choices=["anthropic", "openai", "local", "claude-code"], - help="LLM provider (default: from config)", + "--provider", + help="LLM provider name passed to the harness (felix: anthropic|openai|local|claude-code)", ) parser.add_argument("--model", help="Model name (e.g. qwen/qwen3.5-9b)") parser.add_argument("--base-url", help="Provider API base URL (e.g. http://localhost:1234/v1)") + add_harness_args(parser, repeatable=True) parser.add_argument("--ticks", type=int, help="Override max ticks per scenario") parser.add_argument( "--tick-interval", type=float, help="Seconds between ticks (default: 1.0, use 30-60 for live game)", ) - parser.add_argument("--no-think", action="store_true", help="Disable thinking mode (Qwen3.5)") - parser.add_argument("--visualize", action="store_true", help="Show live helix visualization") parser.add_argument( - "--sequential", action="store_true", - help="Run agents sequentially (default: parallel)", + "--tick-timeout", type=float, default=None, + help="Loop-level cap in seconds on a whole harness step (default: none).", ) parser.add_argument( "--runs", type=int, default=1, @@ -830,7 +667,7 @@ async def main(args: argparse.Namespace) -> None: ) parser.add_argument( "--no-baseline", action="store_true", - help="Skip baseline (no-agent) runs — agent-only, no paired comparison", + help="Skip baseline (unmanaged) runs — harness-only, no paired comparison", ) parser.add_argument( "--no-pause", action="store_true", diff --git a/scripts/run_scenario.py b/scripts/run_scenario.py index 5d8b370..a88d32a 100644 --- a/scripts/run_scenario.py +++ b/scripts/run_scenario.py @@ -11,21 +11,22 @@ import sys from pathlib import Path -from felix_agent_sdk.core import HelixConfig -from felix_agent_sdk.visualization import HelixVisualizer - -from rle.agents import AGENT_DISPLAY -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer -from rle.config import RLEConfig, bridge_anthropic_key, bridge_openrouter_key -from rle.docker import wait_for_rimapi +from rle.config import RLEConfig +from rle.harness import ( + HarnessContext, + HarnessNotFoundError, + HarnessOptionsError, + HarnessUnavailableError, + add_harness_args, + create_harness, + exit_with_harness_error, + harness_options_for, + maybe_handle_harness_list, + selected_harnesses, +) from rle.orchestration.camera_director import CameraDirector from rle.orchestration.game_loop import RLEGameLoop +from rle.orchestration.save_loader import load_save_and_settle from rle.rimapi.client import RimAPIClient from rle.rimapi.sse_client import RimAPISSEClient from rle.scenarios.evaluator import ScenarioEvaluator @@ -68,36 +69,38 @@ def _write_deliberations_jsonl( def _build_run_summary( # noqa: PLR0913 args: argparse.Namespace, - config_model: str, - config_provider: str, - config_tick_interval: float, + config: RLEConfig, + harness_name: str, + harness_options: dict[str, object], + harness_describe: dict[str, str], scenario_name: str, scenario_save_name: str, max_ticks: int | None, outcome: str, final_score: float | None, ticks_run: int, + mean_step_latency_s: float | None, cost_snapshot_dict: dict[str, object], event_summary_dict: dict[str, object] | None, billed_cost_dict: dict[str, object] | None = None, ) -> dict[str, object]: """Compose the per-scenario summary JSON (metadata + config + result).""" summary: dict[str, object] = { - **collect_metadata(random_seed=args.seed), + **collect_metadata(random_seed=args.seed, harness_describe=harness_describe), "scenario": scenario_name, "scenario_save_name": scenario_save_name, - "model": args.model or config_model, - "provider": args.provider or config_provider, - "base_url": args.base_url or None, - "no_think": args.no_think, - "parallel": not args.sequential, - "no_agent": args.no_agent, + "harness": harness_name, + "harness_options": harness_options, + "model": config.model, + "provider": config.provider, + "base_url": config.provider_base_url, "no_pause": args.no_pause, - "tick_interval": config_tick_interval, + "tick_interval": config.tick_interval, "max_ticks": max_ticks, "outcome": outcome, "final_score": final_score, "ticks_run": ticks_run, + "mean_step_latency_s": mean_step_latency_s, "cost_snapshot": cost_snapshot_dict, } if billed_cost_dict is not None: @@ -107,21 +110,6 @@ def _build_run_summary( # noqa: PLR0913 return summary -def _create_agents(provider, helix): # type: ignore[no-untyped-def] - """Create all 7 role agents (MapAnalyst + 6 domain agents).""" - return [ - MapAnalyst("map_analyst", provider, helix, spawn_time=0.0, velocity=1.0), - ResourceManager("resource_manager", provider, helix, spawn_time=0.0, velocity=1.0), - DefenseCommander("defense_commander", provider, helix, spawn_time=0.0, velocity=1.0), - ResearchDirector("research_director", provider, helix, spawn_time=0.0, velocity=1.0), - SocialOverseer("social_overseer", provider, helix, spawn_time=0.0, velocity=1.0), - ConstructionPlanner( - "construction_planner", provider, helix, spawn_time=0.0, velocity=1.0, - ), - MedicalOfficer("medical_officer", provider, helix, spawn_time=0.0, velocity=1.0), - ] - - def _print_results(loop: RLEGameLoop, recorder: TimeSeriesRecorder) -> None: """Print final score summary.""" if not recorder.snapshots: @@ -147,6 +135,9 @@ async def main(args: argparse.Namespace) -> None: format="%(asctime)s %(name)s %(levelname)s %(message)s", ) + if maybe_handle_harness_list(args): + return + # Seed RLE-side stochasticity (resolver tiebreaks, json_repair fallbacks). # RimWorld's RNG is unaffected — see metadata.collect_metadata docstring. if args.seed is not None: @@ -173,7 +164,7 @@ async def main(args: argparse.Namespace) -> None: print(f"Duration: {scenario.expected_duration_days} days, max {scenario.max_ticks} ticks") # Setup - overrides: dict[str, str] = {} + overrides: dict[str, object] = {} if args.provider: overrides["provider"] = args.provider if args.model: @@ -181,30 +172,19 @@ async def main(args: argparse.Namespace) -> None: if args.base_url: overrides["provider_base_url"] = args.base_url if args.tick_interval is not None: - overrides["tick_interval"] = str(args.tick_interval) - config = RLEConfig(**overrides) if overrides else RLEConfig() - bridge_openrouter_key(config) - bridge_anthropic_key(config) - provider = config.get_provider() - helix = HelixConfig.default().to_geometry() - agents = _create_agents(provider, helix) - if args.no_think: - for agent in agents: - agent.set_no_think(True) + overrides["tick_interval"] = args.tick_interval + if args.tick_timeout is not None: + overrides["tick_timeout_s"] = args.tick_timeout + config = RLEConfig(**overrides) if overrides else RLEConfig() # type: ignore[arg-type] + + harness_name = selected_harnesses(args, config)[0] + harness_options = harness_options_for(harness_name, args, config) + print(f"Harness: {harness_name} {harness_options or ''}".rstrip()) scorer = CompositeScorer(scenario.scoring_weights or None) recorder = TimeSeriesRecorder() evaluator = ScenarioEvaluator(scenario) - visualizer = None - if args.visualize: - visualizer = HelixVisualizer(helix, title="R L E") - for agent in agents: - display = AGENT_DISPLAY[agent.agent_id] - visualizer.register_agent( - agent.agent_id, label=display["label"], color=display["color"], - ) - if args.until_death: # Natural-conclusion mode (Phase B): no scenario tick cap — the run # ends when the evaluator hits a terminal condition (all colonists @@ -221,7 +201,7 @@ async def main(args: argparse.Namespace) -> None: Path(args.output).mkdir(parents=True, exist_ok=True) event_log = EventLog(Path(args.output) / "events.jsonl") cost_tracker = await create_cost_tracker( - args.model or config.model, + config.model, prompt_price_override=_per_mtok_to_per_token(args.prompt_price_per_mtok), completion_price_override=_per_mtok_to_per_token(args.completion_price_per_mtok), ) @@ -235,29 +215,9 @@ async def main(args: argparse.Namespace) -> None: if scenario.save_name: print(f"Loading save: {scenario.save_name}") try: - await client.load_game(scenario.save_name) - # Wait for RIMAPI to respond, then poll until colonists are loaded. - # Then wait ~10s of stable state before any writes: RIMAPI returns - # HTTP 200 before Unity's main thread has finished applying the load, - # and writes that race the settle window get 500'd. - await wait_for_rimapi(config.rimapi_url, timeout=30.0) - stable_count = 0 - last_population = -1 - for _ in range(30): - await asyncio.sleep(2) - try: - colony = await client.get_colony() - if colony.population > 0 and colony.population == last_population: - stable_count += 1 - if stable_count >= 5: - break - else: - stable_count = 0 - last_population = colony.population - except Exception: - stable_count = 0 - # Unforbid all starting items so colonists can use them - unforbid_count = await client.unforbid_all_items() + unforbid_count = await load_save_and_settle( + client, config.rimapi_url, scenario.save_name, + ) if unforbid_count: print(f"Unforbid {unforbid_count} items.") print("Save loaded, game ready.") @@ -288,31 +248,45 @@ async def main(args: argparse.Namespace) -> None: output_dir=Path(args.output) if args.output else None, ) + harness_ctx = HarnessContext( + config=config, + client=client, + expected_duration_days=scenario.expected_duration_days, + initial_population=scenario.initial_population, + scenario=scenario, + event_log=event_log, + cost_tracker=cost_tracker, + tick_timeout_s=config.tick_timeout_s, + ) + try: + harness = create_harness(harness_name, harness_ctx, harness_options) + except (HarnessNotFoundError, HarnessUnavailableError, HarnessOptionsError) as exc: + sse.stop() + sse_task.cancel() + exit_with_harness_error(exc) + return + loop = RLEGameLoop( - config, client, agents, + config, client, expected_duration_days=scenario.expected_duration_days, scorer=scorer, recorder=recorder, evaluator=evaluator, initial_population=scenario.initial_population, - visualizer=visualizer, - parallel=not args.sequential, sse_client=sse, dashboard_export_dir=Path(args.output) if args.output else None, - no_agent=args.no_agent, no_pause=args.no_pause, event_log=event_log, cost_tracker=cost_tracker, triggered_incidents=scenario.triggered_incidents, auto_dismiss_dialogs=not args.no_dismiss_dialogs, camera_director=camera_director, + harness=harness, + harness_context=harness_ctx, + scenario=scenario, ) try: - if visualizer: - with visualizer.live(): - await loop.run(max_ticks=max_ticks) - else: - await loop.run(max_ticks=max_ticks) + await loop.run(max_ticks=max_ticks) finally: sse.stop() sse_task.cancel() @@ -324,7 +298,7 @@ async def main(args: argparse.Namespace) -> None: # token-count estimator diverged up to 4x in both directions on the # v0.3.0 spread (thinking-model usage shapes, caching discounts). billed_report = None - effective_base_url = args.base_url or config.provider_base_url or "" + effective_base_url = config.provider_base_url or "" openai_key = os.environ.get("OPENAI_API_KEY", "") generation_ids = cost_tracker.generation_ids if "openrouter.ai" in effective_base_url and openai_key and generation_ids: @@ -347,12 +321,14 @@ async def main(args: argparse.Namespace) -> None: await asyncio.to_thread(_write_deliberations_jsonl, log_path, deliberation_log) print(f"Deliberations exported to {log_path}") + latencies = [t.step_latency_s for t in loop.tick_results] # Replay-grade scenario summary with full metadata + cost + score. summary = _build_run_summary( args=args, - config_model=config.model, - config_provider=config.provider, - config_tick_interval=config.tick_interval, + config=config, + harness_name=harness.name, + harness_options=harness_options, + harness_describe=harness.describe(), scenario_name=scenario.name, scenario_save_name=scenario.save_name, max_ticks=max_ticks, @@ -364,6 +340,9 @@ async def main(args: argparse.Namespace) -> None: recorder.snapshots[-1].composite if recorder.snapshots else None ), ticks_run=len(loop.tick_results), + mean_step_latency_s=( + round(sum(latencies) / len(latencies), 3) if latencies else None + ), cost_snapshot_dict=cost_tracker.snapshot().model_dump(), event_summary_dict=( event_log.summary().model_dump() if event_log else None @@ -423,27 +402,22 @@ async def main(args: argparse.Namespace) -> None: parser.add_argument("scenario", nargs="?", help="Scenario name or number prefix") parser.add_argument("--list", action="store_true", help="List available scenarios") parser.add_argument( - "--provider", choices=["anthropic", "openai", "local", "claude-code"], - help="LLM provider (default: from config)", + "--provider", + help="LLM provider name passed to the harness (felix: anthropic|openai|local|claude-code)", ) parser.add_argument("--model", help="Model name (e.g. unsloth/nvidia-nemotron-3-nano-4b)") parser.add_argument("--base-url", help="Provider API base URL") + add_harness_args(parser) parser.add_argument("--ticks", type=int, help="Override max ticks") parser.add_argument( "--tick-interval", type=float, help="Seconds between ticks (default: 1.0, use 30-60 for live game)", ) - parser.add_argument("--output", help="Output directory for CSV results") - parser.add_argument("--visualize", action="store_true", help="Show live helix visualization") parser.add_argument( - "--sequential", action="store_true", - help="Run agents sequentially (default: parallel)", - ) - parser.add_argument("--no-think", action="store_true", help="Skip reasoning tokens") - parser.add_argument( - "--no-agent", action="store_true", - help="Baseline mode: no agent deliberation, colony runs unmanaged", + "--tick-timeout", type=float, default=None, + help="Loop-level cap in seconds on a whole harness step (default: none).", ) + parser.add_argument("--output", help="Output directory for CSV results") parser.add_argument( "--until-death", action="store_true", help="Ignore the scenario tick cap; run until the evaluator reaches " diff --git a/src/rle/harness/__init__.py b/src/rle/harness/__init__.py index 1da916d..4b0b25a 100644 --- a/src/rle/harness/__init__.py +++ b/src/rle/harness/__init__.py @@ -12,6 +12,14 @@ ``docs/harness-plugins.md``. """ +from rle.harness.cli import ( + add_harness_args, + exit_with_harness_error, + format_harness_table, + harness_options_for, + maybe_handle_harness_list, + selected_harnesses, +) from rle.harness.protocol import ( Availability, BaseHarness, @@ -50,10 +58,16 @@ "HarnessUnavailableError", "StepResult", "TickObserver", + "add_harness_args", "create_harness", + "exit_with_harness_error", + "format_harness_table", "get_plugin", "harness_names", + "harness_options_for", "list_harnesses", + "maybe_handle_harness_list", "parse_option_pairs", + "selected_harnesses", "validate_options", ] diff --git a/src/rle/harness/cli.py b/src/rle/harness/cli.py new file mode 100644 index 0000000..b77e6d8 --- /dev/null +++ b/src/rle/harness/cli.py @@ -0,0 +1,134 @@ +"""argparse glue shared by ``run_scenario.py`` and ``run_benchmark.py``. + +Keeps harness selection identical across CLIs: + + --harness registry name (default: RLEConfig.harness) + --harness list print discovered plugins and exit + --harness-opt key=value validated against the plugin's option schema + --no-agent permanent alias for --harness baseline + +Legacy Felix flags (``--no-think``, ``--sequential``, ``--visualize``) are +still accepted and folded into ``FelixOptions`` when the selected harness is +``felix``; for any other harness they are ignored with a warning. +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from typing import Any + +from rle.config import RLEConfig +from rle.harness.registry import list_harnesses, parse_option_pairs + +logger = logging.getLogger(__name__) + +LIST_SENTINEL = "list" + +_LEGACY_FELIX_FLAGS: dict[str, tuple[str, Any]] = { + # argparse attribute -> (FelixOptions field, value when the flag is set) + "no_think": ("no_think", True), + "sequential": ("parallel", False), + "visualize": ("visualize", True), +} + + +def add_harness_args(parser: argparse.ArgumentParser, *, repeatable: bool = False) -> None: + kwargs: dict[str, Any] = {"action": "append"} if repeatable else {} + parser.add_argument( + "--harness", dest="harness", default=None, + help=( + "Harness that decides the colony's actions (default: RLE_HARNESS / felix). " + "Use `--harness list` to see installed plugins." + + (" Repeat to run a harness matrix." if repeatable else "") + ), + **kwargs, + ) + parser.add_argument( + "--harness-opt", dest="harness_opts", action="append", default=[], + metavar="KEY=VALUE", + help="Harness-specific option (repeatable); validated by the plugin's schema.", + ) + parser.add_argument( + "--no-agent", action="store_true", + help="Baseline mode: alias for --harness baseline (colony runs unmanaged).", + ) + parser.add_argument( + "--no-think", action="store_true", + help="[felix] Inject prefill so thinking models skip reasoning.", + ) + parser.add_argument( + "--sequential", action="store_true", + help="[felix] Deliberate role agents one at a time (default: parallel).", + ) + parser.add_argument( + "--visualize", action="store_true", + help="[felix] Show the live helix visualisation.", + ) + + +def format_harness_table() -> str: + rows = list_harnesses() + if not rows: + return "No harness plugins installed (entry-point group 'rle.harnesses')." + name_w = max(len(r.name) for r in rows) + pkg_w = max(len(f"{r.package} {r.version}") for r in rows) + lines = [f"{'HARNESS':<{name_w}} {'PACKAGE':<{pkg_w}} STATUS DESCRIPTION"] + for r in rows: + status = "available" if r.availability.ok else "unavailable" + pkg = f"{r.package} {r.version}" + desc = r.description + if not r.availability.ok: + desc = f"{desc} ({r.availability.reason})" + lines.append(f"{r.name:<{name_w}} {pkg:<{pkg_w}} {status:<11} {desc}") + return "\n".join(lines) + + +def maybe_handle_harness_list(args: argparse.Namespace) -> bool: + """Print the plugin table and return True when ``--harness list`` was given.""" + raw = getattr(args, "harness", None) + names = raw if isinstance(raw, list) else [raw] + if any(n == LIST_SENTINEL for n in names if n): + print(format_harness_table()) + return True + return False + + +def selected_harnesses(args: argparse.Namespace, config: RLEConfig) -> list[str]: + """Resolve the harness name(s) from CLI flags + config.""" + if getattr(args, "no_agent", False): + return ["baseline"] + raw = getattr(args, "harness", None) + if raw is None: + return [config.harness] + names = raw if isinstance(raw, list) else [raw] + return [n for n in names if n and n != LIST_SENTINEL] or [config.harness] + + +def harness_options_for( + name: str, args: argparse.Namespace, config: RLEConfig, +) -> dict[str, Any]: + """Merge config options, legacy Felix flags, and ``--harness-opt`` pairs.""" + options: dict[str, Any] = dict(config.harness_options) + legacy_set = { + field: value + for attr, (field, value) in _LEGACY_FELIX_FLAGS.items() + if getattr(args, attr, False) + } + if legacy_set: + if name == "felix": + options.update(legacy_set) + else: + logger.warning( + "Ignoring Felix-only flags %s for harness %r", + sorted(legacy_set), name, + ) + options.update(parse_option_pairs(getattr(args, "harness_opts", None))) + return options + + +def exit_with_harness_error(exc: Exception) -> None: + print(f"error: {exc}", file=sys.stderr) + print(format_harness_table(), file=sys.stderr) + raise SystemExit(2) diff --git a/src/rle/orchestration/game_loop.py b/src/rle/orchestration/game_loop.py index 2ba498b..ca15d54 100644 --- a/src/rle/orchestration/game_loop.py +++ b/src/rle/orchestration/game_loop.py @@ -91,11 +91,14 @@ def __init__( speed_keepalive_s: float = 10.0, *, harness: BaseHarness | None = None, + harness_context: HarnessContext | None = None, scenario: ScenarioConfig | None = None, ) -> None: - """``harness`` is the modern entry point. ``agents`` / ``no_agent`` / - ``parallel`` / ``visualizer`` are legacy arguments that build a Felix - or baseline harness for you (see ``rle.harness.compat``).""" + """``harness`` is the modern entry point (pass the ``harness_context`` + it was created with so both sides share one client/event log). + ``agents`` / ``no_agent`` / ``parallel`` / ``visualizer`` are legacy + arguments that build a Felix or baseline harness for you (see + ``rle.harness.compat``).""" if harness is not None and agents: raise ValueError("Pass either harness= or the legacy agents= argument, not both") self._config = config @@ -129,16 +132,29 @@ def __init__( visualizer=visualizer, role_timeout_s=config.role_timeout_s, ) - self._harness_ctx = HarnessContext( - config=config, - client=client, - expected_duration_days=expected_duration_days, - initial_population=initial_population, - scenario=scenario, - event_log=event_log, - cost_tracker=cost_tracker, - tick_timeout_s=config.tick_timeout_s, - ) + if harness_context is None: + harness_context = HarnessContext( + config=config, + client=client, + expected_duration_days=expected_duration_days, + initial_population=initial_population, + scenario=scenario, + event_log=event_log, + cost_tracker=cost_tracker, + tick_timeout_s=config.tick_timeout_s, + ) + else: + # The loop is authoritative for run-shaped facts the caller may + # not have known when it built the context. + harness_context.expected_duration_days = expected_duration_days + harness_context.initial_population = initial_population + if scenario is not None: + harness_context.scenario = scenario + if harness_context.event_log is None: + harness_context.event_log = event_log + if harness_context.cost_tracker is None: + harness_context.cost_tracker = cost_tracker + self._harness_ctx = harness_context self._setup_done = False # ------------------------------------------------------------------ diff --git a/src/rle/orchestration/save_loader.py b/src/rle/orchestration/save_loader.py new file mode 100644 index 0000000..2c2eb48 --- /dev/null +++ b/src/rle/orchestration/save_loader.py @@ -0,0 +1,65 @@ +"""Load a scenario save and wait until the game is actually ready. + +``load_game`` returns HTTP 200 before Unity's main thread has applied the +load. Writes that race the settle window get 500'd and, worse, can start a +null-ref cascade that poisons the rest of the session. Every entry point +(single scenario, benchmark matrix, baseline reloads) must use this helper +rather than a fixed sleep so agent and baseline runs start from the same +settled state. +""" + +from __future__ import annotations + +import asyncio +import logging + +from rle.docker import wait_for_rimapi +from rle.rimapi.client import RimAPIClient + +logger = logging.getLogger(__name__) + +# The colony population must be > 0 and unchanged for this many consecutive +# polls (2 s apart) before we consider the load settled (~10 s). +STABLE_POLLS_REQUIRED = 5 +POLL_INTERVAL_S = 2.0 +MAX_POLLS = 30 + + +async def load_save_and_settle( + client: RimAPIClient, + rimapi_url: str, + save_name: str, + *, + unforbid_items: bool = True, + rimapi_timeout_s: float = 30.0, +) -> int: + """Load ``save_name`` and block until the colony is stable. + + Returns the number of starting items unforbidden (0 when disabled). + Raises whatever ``load_game`` / ``wait_for_rimapi`` raise so callers can + decide whether to skip the run. + """ + await client.load_game(save_name) + await wait_for_rimapi(rimapi_url, timeout=rimapi_timeout_s) + stable_count = 0 + last_population = -1 + for _ in range(MAX_POLLS): + await asyncio.sleep(POLL_INTERVAL_S) + try: + colony = await client.get_colony() + except Exception: + stable_count = 0 + continue + if colony.population > 0 and colony.population == last_population: + stable_count += 1 + if stable_count >= STABLE_POLLS_REQUIRED: + break + else: + stable_count = 0 + last_population = colony.population + else: + logger.warning("Save %s never reported a stable population; continuing", save_name) + if not unforbid_items: + return 0 + count = await client.unforbid_all_items() + return int(count or 0) diff --git a/src/rle/testing/__init__.py b/src/rle/testing/__init__.py new file mode 100644 index 0000000..5dcacc7 --- /dev/null +++ b/src/rle/testing/__init__.py @@ -0,0 +1,20 @@ +"""Test utilities exported for harness plugin authors. + +External harness packages depend on ``rimworld-learning-environment`` and use +these helpers so they never copy RLE internals: + +- :class:`MockRimAPI` / :func:`make_mock_transport` — fake RIMAPI transport +- :func:`run_harness_smoke` — drive a plugin through ``RLEGameLoop`` for a + few ticks against the mock and return the tick results +""" + +from rle.testing.mock_rimapi import MOCK_ROUTES, MockRimAPI, make_mock_transport +from rle.testing.smoke import SmokeReport, run_harness_smoke + +__all__ = [ + "MOCK_ROUTES", + "MockRimAPI", + "SmokeReport", + "make_mock_transport", + "run_harness_smoke", +] diff --git a/src/rle/testing/mock_rimapi.py b/src/rle/testing/mock_rimapi.py new file mode 100644 index 0000000..585eee2 --- /dev/null +++ b/src/rle/testing/mock_rimapi.py @@ -0,0 +1,138 @@ +"""In-memory stand-in for RIMAPI so harnesses can be exercised without RimWorld. + +Used by ``--smoke-test`` in the CLIs and by external harness packages' test +suites (``rle.testing.run_harness_smoke``). State never changes between +ticks — this proves plumbing, not colony management. +""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx + +from rle.rimapi.client import RimAPIClient + +MOCK_ROUTES: dict[str, dict[str, Any] | list[Any]] = { + "/api/v1/colonists": [ + { + "colonist_id": "col_01", "name": "Tynan", "health": 0.95, + "mood": 0.72, "skills": {"shooting": 8, "construction": 5, + "cooking": 3, "mining": 6, "intellectual": 4}, + "traits": ["industrious"], "current_job": "mining", + "is_drafted": False, "needs": {"food": 0.6, "rest": 0.8}, + "injuries": [], "position": [42, 18], + }, + { + "colonist_id": "col_02", "name": "Cassandra", "health": 0.88, + "mood": 0.65, "skills": {"shooting": 3, "construction": 7, + "cooking": 6, "growing": 8, "intellectual": 6}, + "traits": ["kind"], "current_job": "growing", + "is_drafted": False, "needs": {"food": 0.5, "rest": 0.7}, + "injuries": [], "position": [30, 22], + }, + { + "colonist_id": "col_03", "name": "Randy", "health": 0.92, + "mood": 0.58, "skills": {"shooting": 10, "melee": 7, + "construction": 3, "cooking": 2}, + "traits": ["tough", "brawler"], "current_job": None, + "is_drafted": False, "needs": {"food": 0.4, "rest": 0.6}, + "injuries": [], "position": [50, 10], + }, + ], + "/api/v1/resources/summary?map_id=0": { + "total_items": 800, "total_market_value": 8000.0, + "critical_resources": { + "food_summary": {"food_total": 85}, + "medicine_total": 5, "weapon_count": 2, + }, + }, + "/api/v1/map/buildings?map_id=0": [ + {"id": "s_01", "def_name": "Wall", "position": {"x": 10, "y": 0, "z": 10}, + "hit_points": 300.0, "max_hit_points": 300.0}, + ], + "/api/v1/research/summary": { + "current_project": "electricity", "progress": 0.45, + "completed": ["stonecutting"], "available": ["electricity", "battery", "smithing"], + }, + "/api/v1/incidents?map_id=0": {"incidents": []}, + "/api/v1/game/state": { + "name": "New Hope", "wealth": 8000.0, "day": 5, "tick": 300000, + "population": 3, "mood_average": 0.65, "food_days": 7.0, + }, + "/api/v1/map/weather?map_id=0": { + "weather": "clear", "temperature": 22.0, + }, + "/api/v1/map/zones?map_id=0": [], + "/api/v1/map/rooms?map_id=0": [], + "/api/v1/map/ore?map_id=0": [], + "/api/v1/map/farm/summary?map_id=0": { + "total_growing_zones": 0, "planted_cells": 0, + "harvestable_cells": 0, "crops": {}, + }, + "/api/v1/map/terrain?map_id=0": { + "width": 10, "height": 10, + "palette": ["Soil", "WaterMovingShallow", "SoilRich", "Granite_Rough"], + "grid": [100, 0], + "floor_palette": [], "floor_grid": [100, 0], + }, + "/api/v1/resources/stored?map_id=0": { + "Resources": [ + {"def_name": "WoodLog", "stack_count": 200}, + {"def_name": "Steel", "stack_count": 100}, + {"def_name": "ComponentIndustrial", "stack_count": 10}, + ], + }, + "/api/v1/map/power/info?map_id=0": { + "current_power": 0.0, + "total_consumption": 0.0, + "currently_stored_power": 0.0, + "total_power_storage": 0.0, + }, + "/api/v1/factions": [], + "/api/v1/ui/alerts?map_id=0": [], +} + +_POST_OK = b'{"success": true, "errors": [], "warnings": []}' + + +class MockRimAPI: + """Records every POST so tests can assert what a harness wrote.""" + + def __init__(self, routes: dict[str, dict[str, Any] | list[Any]] | None = None) -> None: + self.routes = dict(MOCK_ROUTES if routes is None else routes) + self.posts: list[tuple[str, Any]] = [] + + def handler(self, request: httpx.Request) -> httpx.Response: + raw = request.url.raw_path.decode() + if request.method == "POST": + body: Any = None + if request.content: + try: + body = json.loads(request.content) + except json.JSONDecodeError: + body = request.content.decode(errors="replace") + self.posts.append((raw.split("?")[0], body)) + return httpx.Response( + 200, content=_POST_OK, headers={"content-type": "application/json"}, + ) + for key in (raw, raw.split("?")[0]): + if key in self.routes: + return httpx.Response( + 200, content=json.dumps(self.routes[key]).encode(), + headers={"content-type": "application/json"}, + ) + return httpx.Response(404, content=b"Not found") + + def transport(self) -> httpx.MockTransport: + return httpx.MockTransport(self.handler) + + def attach(self, client: RimAPIClient, base_url: str = "http://mock") -> RimAPIClient: + """Point an (entered) RimAPIClient at this mock.""" + client._client = httpx.AsyncClient(transport=self.transport(), base_url=base_url) + return client + + +def make_mock_transport() -> httpx.MockTransport: + return MockRimAPI().transport() diff --git a/src/rle/testing/smoke.py b/src/rle/testing/smoke.py new file mode 100644 index 0000000..06e0838 --- /dev/null +++ b/src/rle/testing/smoke.py @@ -0,0 +1,68 @@ +"""Run any harness plugin through the real loop against the mock RIMAPI.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from pydantic import BaseModel + +from rle.config import RLEConfig +from rle.harness.protocol import HarnessContext, HarnessPlugin +from rle.harness.registry import get_plugin, validate_options +from rle.orchestration.game_loop import RLEGameLoop, TickResult +from rle.rimapi.client import RimAPIClient +from rle.scoring.composite import CompositeScorer +from rle.scoring.recorder import TimeSeriesRecorder +from rle.testing.mock_rimapi import MockRimAPI + + +@dataclass +class SmokeReport: + harness: str + ticks: list[TickResult] + posts: list[tuple[str, Any]] + final_composite: float | None + describe: dict[str, str] = field(default_factory=dict) + + @property + def ok(self) -> bool: + return len(self.ticks) > 0 + + +async def run_harness_smoke( + plugin: HarnessPlugin | str, + *, + options: dict[str, Any] | BaseModel | None = None, + ticks: int = 3, + config: RLEConfig | None = None, + mock: MockRimAPI | None = None, +) -> SmokeReport: + """Build ``plugin.smoke(...)`` and run it for ``ticks`` ticks. + + This is the contract test every harness package should run in CI: if it + passes, the plugin loads, validates options, produces ``StepResult``s the + loop can execute/score, and tears down cleanly. + """ + if isinstance(plugin, str): + plugin = get_plugin(plugin) + cfg = config or RLEConfig(tick_interval=0.0) + mock = mock or MockRimAPI() + async with RimAPIClient("http://mock") as client: + mock.attach(client) + ctx = HarnessContext(config=cfg, client=client, smoke=True) + harness = plugin.smoke(ctx, validate_options(plugin, options)) + recorder = TimeSeriesRecorder() + loop = RLEGameLoop( + cfg, client, harness=harness, harness_context=ctx, + scorer=CompositeScorer(), recorder=recorder, + ) + results = await loop.run(max_ticks=ticks) + final = recorder.snapshots[-1].composite if recorder.snapshots else None + return SmokeReport( + harness=harness.name, + ticks=results, + posts=list(mock.posts), + final_composite=final, + describe=harness.describe(), + ) diff --git a/src/rle/tracking/metadata.py b/src/rle/tracking/metadata.py index cddea43..337f186 100644 --- a/src/rle/tracking/metadata.py +++ b/src/rle/tracking/metadata.py @@ -34,13 +34,21 @@ ) -def collect_metadata(random_seed: int | None = None) -> dict[str, object]: +def collect_metadata( + random_seed: int | None = None, + harness_describe: dict[str, str] | None = None, +) -> dict[str, object]: """Gather reproducibility metadata for a benchmark run. The random_seed argument is the seed the caller passed to ``random.seed`` (or None if no seed was set). It controls only RLE-side stochasticity (json_repair fallbacks, resolver tiebreaks); RimWorld's own RNG is unaffected — that lives inside the game and is not reproducible from here. + + ``harness_describe`` is whatever the harness reports about itself + (``BaseHarness.describe()``): SDK versions, agent roster, external tool + versions. Recorded as ``harness_versions`` so a leaderboard row can be + traced to the exact harness build, whichever framework it used. """ dll_path = _rimapi_dll_path() return { @@ -50,7 +58,7 @@ def collect_metadata(random_seed: int | None = None) -> dict[str, object]: "git_branch": _git("branch", "--show-current"), "git_dirty": _git("status", "--porcelain") != "", "rle_version": _version("rimworld-learning-environment"), - "felix_sdk_version": _version("felix-agent-sdk"), + "harness_versions": dict(harness_describe or {}), "platform": sys.platform, "python_version": platform.python_version(), "docker_mode": False, diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index 1eec201..1a19672 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -46,7 +46,7 @@ def test_collect_metadata_includes_scoring_version_and_seed() -> None: "git_branch", "git_dirty", "rle_version", - "felix_sdk_version", + "harness_versions", "platform", "python_version", "rimapi_dll_path", @@ -56,6 +56,12 @@ def test_collect_metadata_includes_scoring_version_and_seed() -> None: assert key in md, f"missing metadata field: {key}" +def test_collect_metadata_records_harness_describe() -> None: + md = collect_metadata(harness_describe={"harness": "x", "tool": "1.2"}) + assert md["harness_versions"] == {"harness": "x", "tool": "1.2"} + assert collect_metadata()["harness_versions"] == {} + + def test_collect_metadata_default_seed_is_none() -> None: md = collect_metadata() assert md["random_seed"] is None diff --git a/tests/unit/test_testing_smoke.py b/tests/unit/test_testing_smoke.py new file mode 100644 index 0000000..74134ba --- /dev/null +++ b/tests/unit/test_testing_smoke.py @@ -0,0 +1,31 @@ +"""rle.testing.run_harness_smoke — the contract test external plugins run in CI.""" + +from __future__ import annotations + +import pytest + +from rle.testing import MockRimAPI, run_harness_smoke + + +@pytest.mark.parametrize("name", ["baseline", "felix"]) +async def test_builtin_plugins_pass_smoke(name: str) -> None: + report = await run_harness_smoke(name, ticks=2) + assert report.ok + assert report.harness == name + assert len(report.ticks) == 2 + assert report.final_composite is not None + assert report.describe["harness"] == name + + +async def test_smoke_records_posts() -> None: + mock = MockRimAPI() + report = await run_harness_smoke("baseline", ticks=1, mock=mock) + # pause + unpause at minimum went through the mock + assert any(path.startswith("/api/v1/game/speed") for path, _ in report.posts) + + +async def test_smoke_validates_options() -> None: + from rle.harness import HarnessOptionsError + + with pytest.raises(HarnessOptionsError): + await run_harness_smoke("baseline", options={"nope": 1}) From c916cf35b0719beab4c99c58e8dbd73d2d258c82 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 07:18:03 +0000 Subject: [PATCH 4/8] zero-Felix core: felix-agent-sdk becomes the optional 'felix' extra With the extra uninstalled, RLE core imports, 406 tests pass, --harness list reports felix as unavailable with the fix hint, and --harness baseline runs end to end. Enforced by scripts/check_harness_boundary.py (felix_agent_sdk may only be imported under src/rle/harness/felix/; no third-party harness names in src/tests/scripts) and a new test-no-felix CI job. - move role agents, base_role and the claude-code provider under rle.harness.felix (git mv; rle.agents keeps only the neutral action vocabulary + json_repair); rle/__init__ no longer exports agents - ActionOutcome / ExecutionResult move to rle.agents.actions so the harness layer never imports rle.orchestration (broke a real cycle); action_executor re-exports them - rle.harness.brief: harness-neutral scenario brief (goals, state snapshot, MAP_SUMMARY, action catalog); base_role delegates its MAP_SUMMARY builder to it - RLEConfig loses helix_preset (FelixOptions has it); mcp optional extra - HF dataset card keyed by harness/model; run metadata harness_versions - tests: Felix-only modules are collect_ignore'd without the extra, requires_felix marker for individual cases; conftest fixtures probe lazily - CI: install .[dev,felix,mcp]; new test-no-felix and external-plugin-contract jobs; boundary check in lint - README/CLAUDE/CONTRIBUTING: harness x model framing, install extras, --harness flags Co-authored-by: Jason --- .claude/rules/code-style.md | 2 +- .github/workflows/ci.yml | 55 ++- CLAUDE.md | 7 +- CONTRIBUTING.md | 2 +- README.md | 49 +- pyproject.toml | 12 +- scripts/check_harness_boundary.py | 74 +++ src/rle/__init__.py | 27 +- src/rle/agents/__init__.py | 57 +-- src/rle/agents/actions.py | 26 ++ src/rle/harness/brief.py | 239 ++++++++++ src/rle/harness/felix/agents/__init__.py | 56 +++ .../{ => harness/felix}/agents/base_role.py | 92 +--- .../felix}/agents/construction_planner.py | 2 +- .../felix}/agents/defense_commander.py | 2 +- .../{ => harness/felix}/agents/map_analyst.py | 2 +- .../felix}/agents/medical_officer.py | 2 +- .../felix}/agents/research_director.py | 2 +- .../felix}/agents/resource_manager.py | 2 +- .../felix}/agents/social_overseer.py | 2 +- src/rle/harness/felix/build.py | 18 +- src/rle/harness/felix/harness.py | 5 +- src/rle/harness/felix/provider_factory.py | 2 +- src/rle/harness/felix/providers/__init__.py | 5 + .../felix}/providers/claude_code.py | 0 src/rle/harness/protocol.py | 3 +- src/rle/harness/registry.py | 4 +- src/rle/orchestration/action_executor.py | 38 +- src/rle/providers/__init__.py | 5 - src/rle/scoring/coherence.py | 3 +- src/rle/tracking/hf_logger.py | 31 +- tests/conftest.py | 41 +- tests/integration/test_game_loop.py | 14 +- tests/integration/test_harness_loop.py | 10 +- tests/integration/test_scenario_run.py | 12 +- tests/unit/test_base_role.py | 2 +- tests/unit/test_brief.py | 90 ++++ tests/unit/test_claude_code_provider.py | 32 +- tests/unit/test_config.py | 13 - tests/unit/test_felix_provider_factory.py | 28 ++ tests/unit/test_harness_registry.py | 8 +- tests/unit/test_hf_logger.py | 19 +- tests/unit/test_role_agents.py | 18 +- tests/unit/test_testing_smoke.py | 5 +- tests/unit/test_visualizer_integration.py | 2 +- uv.lock | 422 +++++++++++++++++- 46 files changed, 1208 insertions(+), 334 deletions(-) create mode 100644 scripts/check_harness_boundary.py create mode 100644 src/rle/harness/brief.py create mode 100644 src/rle/harness/felix/agents/__init__.py rename src/rle/{ => harness/felix}/agents/base_role.py (90%) rename src/rle/{ => harness/felix}/agents/construction_planner.py (97%) rename src/rle/{ => harness/felix}/agents/defense_commander.py (97%) rename src/rle/{ => harness/felix}/agents/map_analyst.py (98%) rename src/rle/{ => harness/felix}/agents/medical_officer.py (97%) rename src/rle/{ => harness/felix}/agents/research_director.py (97%) rename src/rle/{ => harness/felix}/agents/resource_manager.py (97%) rename src/rle/{ => harness/felix}/agents/social_overseer.py (97%) create mode 100644 src/rle/harness/felix/providers/__init__.py rename src/rle/{ => harness/felix}/providers/claude_code.py (100%) delete mode 100644 src/rle/providers/__init__.py create mode 100644 tests/unit/test_brief.py create mode 100644 tests/unit/test_felix_provider_factory.py diff --git a/.claude/rules/code-style.md b/.claude/rules/code-style.md index f4fb2db..6c47ff3 100644 --- a/.claude/rules/code-style.md +++ b/.claude/rules/code-style.md @@ -1,6 +1,6 @@ # Code Style Rules -- Python 3.14+. Use `uv sync --extra dev` to install. +- Python 3.14+. Use `uv sync --extra dev --extra felix` to install. - mypy strict mode. All code must pass `mypy src/` with `strict = true`. - No `Any` type annotations for dataclass fields. Use `TYPE_CHECKING` imports to break circular dependencies. - `from __future__ import annotations` at top of every Python file. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 082b3c0..0d29fd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,11 +15,13 @@ jobs: with: python-version: "3.14" - uses: astral-sh/setup-uv@v4 - - run: uv pip install -e ".[dev]" --system + - run: uv pip install -e ".[dev,felix,mcp]" --system - name: Lint run: ruff check src/ tests/ scripts/ - name: Type check run: mypy src/ + - name: Harness boundary (felix confined, no third-party harness code in tree) + run: python scripts/check_harness_boundary.py test: runs-on: ubuntu-latest @@ -30,10 +32,33 @@ jobs: with: python-version: "3.14" - uses: astral-sh/setup-uv@v4 - - run: uv pip install -e ".[dev]" --system + - run: uv pip install -e ".[dev,felix,mcp]" --system - name: Run tests run: pytest --tb=short -q + # The environment must run any harness with the Felix SDK absent. This job + # installs core + dev only and exercises the non-Felix paths end to end. + test-no-felix: + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.14" + - uses: astral-sh/setup-uv@v4 + - run: uv pip install -e ".[dev,mcp]" --system + - name: Assert felix is absent + run: python -c "import importlib.util as u, sys; sys.exit(0 if u.find_spec('felix_agent_sdk') is None else 'felix-agent-sdk leaked into the no-felix job')" + - name: Core imports without felix + run: python -c "import rle, rle.harness, rle.orchestration.game_loop, rle.testing, rle.mcp" + - name: Run tests (Felix-only modules auto-skipped) + run: pytest --tb=short -q + - name: Harness list shows felix as unavailable, baseline available + run: python scripts/run_benchmark.py --harness list + - name: Smoke test baseline harness + run: python scripts/run_benchmark.py --smoke-test --ticks 3 --harness baseline + smoke-test: runs-on: ubuntu-latest needs: test @@ -43,6 +68,26 @@ jobs: with: python-version: "3.14" - uses: astral-sh/setup-uv@v4 - - run: uv pip install -e ".[dev]" --system - - name: Smoke test benchmark - run: python scripts/run_benchmark.py --dry-run --ticks 5 + - run: uv pip install -e ".[dev,felix,mcp]" --system + - name: Smoke test felix + baseline matrix + run: python scripts/run_benchmark.py --smoke-test --ticks 5 --harness felix --harness baseline + + # Contract test for the plugin API: an external harness package installed + # straight from GitHub must appear in --harness list and pass smoke with + # zero changes to this repo. + external-plugin-contract: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.14" + - uses: astral-sh/setup-uv@v4 + - run: uv pip install -e ".[dev,mcp]" --system + - name: Install the template harness from GitHub + run: uv pip install "git+https://github.com/AppSprout-dev/rle-harness-template" --system + - name: Template appears in the registry + run: python scripts/run_benchmark.py --harness list | tee /dev/stderr | grep -q "^template " + - name: Template passes smoke + run: python scripts/run_benchmark.py --smoke-test --ticks 3 --harness template diff --git a/CLAUDE.md b/CLAUDE.md index 4a1576d..bcc37fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # RLE — RimWorld Learning Environment -Multi-agent benchmark where 7 Felix Agent SDK role-specialized LLM agents manage a RimWorld colony. Think FLE (Factorio Learning Environment) but for multi-agent coordination under uncertainty. +A harness × model benchmark: swappable agent harnesses (the original 7-agent Felix Agent SDK stack, an unmanaged baseline, or external coding agents installed as `rle-harness-*` packages) manage a RimWorld colony and are scored identically. Think FLE (Factorio Learning Environment) but for multi-agent coordination under uncertainty, with the harness as a first-class benchmark variable. ## Prerequisites @@ -56,7 +56,7 @@ curl http://localhost:1234/v1/models ## Commands -- Install: `uv sync --extra dev` +- Install: `uv sync --extra dev --extra felix` (add `--extra mcp` for the RimAPI MCP server; core alone has no agent framework) - Test: `pytest` - Lint: `ruff check src/ tests/ scripts/` - Type check: `mypy src/` @@ -114,7 +114,8 @@ python scripts/run_scenario.py crashlanded \ **Important flags:** - `--no-think` — Required for thinking models (Nemotron, Qwen). Injects `` prefix. - `--no-pause` — Game runs continuously via SSE. Without this, game pauses each tick. -- `--no-agent` — Baseline mode: no LLM deliberation, colony runs unmanaged (for comparison). +- `--harness NAME` — Which harness decides (default `felix`; `--harness list` shows installed plugins; `--harness-opt key=value` for plugin options). +- `--no-agent` — Baseline mode: alias for `--harness baseline`; colony runs unmanaged (for comparison). - `--output results/live` — Exports `latest_tick.json` for the dashboard. - `--tick-interval 30` — Seconds between ticks. 30s gives agents time to deliberate. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2192acd..00866b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ ```bash git clone https://github.com/AppSprout-dev/RLE.git cd RLE -uv sync --extra dev +uv sync --extra dev --extra felix pytest # should pass 458+ tests ``` diff --git a/README.md b/README.md index 9ba7a0a..fb7b44a 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ # RLE — RimWorld Learning Environment -Multi-agent benchmark where 7 Felix Agent SDK role-specialized LLM agents manage a RimWorld colony. Think [FLE](https://github.com/chenhao-wang/FLE) (Factorio Learning Environment) but for **multi-agent coordination under uncertainty**. +A **harness × model** benchmark: swappable agent harnesses manage a RimWorld colony under uncertainty and are scored on the same footing against an unmanaged baseline. Think [FLE](https://github.com/chenhao-wang/FLE) (Factorio Learning Environment) but stochastic, multi-agent-capable, and with the *harness* — not just the model — as a first-class variable. ## What makes this different -- **7 agents, not 1** — MapAnalyst (spatial reasoning), ResourceManager, DefenseCommander, ResearchDirector, SocialOverseer, ConstructionPlanner, MedicalOfficer coordinate through a hub-spoke communication network -- **Spatial awareness** — deterministic terrain analysis from the game map tells agents exactly where to build, farm, and mine -- **Stochastic environment** — raids, plague, mental breaks, weather. Agents adapt, not just optimize -- **Helix-driven strategy** — agents shift from exploration (diverse strategies) to synthesis (decisive actions) as the colony progresses -- **Provider-agnostic** — runs on a free local 4B model or a cloud 30B, same architecture +- **Harnesses are swappable like models** — `--harness felix` (the original 7-agent Felix SDK stack), `--harness baseline` (unmanaged colony), or any harness package installed from PyPI/GitHub (`rle-harness-`); the core never imports an agent framework +- **Harness-agnostic scoring** — process metrics read the writes that reached the game, not any harness's internal messaging; scenarios, saves and the composite are identical for every harness +- **Spatial awareness** — deterministic terrain analysis from the game map gives every harness verified build/farm/stockpile coordinates (MAP_SUMMARY) +- **Stochastic environment** — raids, plague, mental breaks, weather. Harnesses adapt, not just optimize +- **Paired against a real baseline** — every run is compared to RimWorld's own pawn AI on the same save +- **Provider-agnostic** — runs on a free local 4B model or a cloud 30B, same environment ## Architecture @@ -17,15 +18,23 @@ RimWorld (game) ↕ Harmony patches RIMAPI mod (C# REST :8765 + SSE events) ↕ httpx async + SSE -RLE Orchestrator - ↕ CentralPost hub-spoke - MapAnalyst → spatial analysis (runs first) - 6 Role Agents (parallel deliberation) - ↕ OpenAI-compatible API -LLM (Nemotron 4B local / 30B cloud / Anthropic / OpenAI) +RLEGameLoop (environment: pause → state → harness.step → execute → score → unpause) + ↕ Harness protocol (rle.harness) — discovered via the `rle.harnesses` entry-point group + ├─ felix MapAnalyst → 6 role agents over CentralPost, merged by ActionResolver [in tree, extra `felix`] + ├─ baseline unmanaged colony [in tree] + └─ external coding agents attached over the RimAPI MCP server (rle-mcp) [own repos] + ↕ OpenAI-compatible / Anthropic / local API (provider + model are strings; the harness interprets them) +LLM ``` -## The 7 Agents +```bash +python scripts/run_benchmark.py --harness list # what is installed +python scripts/run_benchmark.py --harness felix --harness baseline --smoke-test +``` + +Writing a harness: see [docs/harness-plugins.md](docs/harness-plugins.md). Third-party harnesses (OpenCode, Grok Build, ...) live in their own `AppSprout-dev/rle-harness-*` repos and are installed with `pip`, never committed here. + +## The Felix harness: 7 agents | Agent | Domain | Key Actions | |-------|--------|-------------| @@ -63,9 +72,12 @@ curl http://localhost:1234/v1/models # LM Studio (if using local) ```bash git clone https://github.com/AppSprout-dev/RLE.git cd RLE -uv sync --extra dev +uv sync --extra dev --extra felix # core + the Felix harness +# add --extra mcp for the RimAPI MCP server used by external coding-agent harnesses ``` +Core is framework-free; `felix-agent-sdk` is an optional extra. Without it, `--harness felix` shows as unavailable in `--harness list` and everything else still runs. + ### Configure `.env` ```bash @@ -127,9 +139,12 @@ The scenario will: | `--no-pause` | Game runs continuously via SSE. Without this, game pauses each tick. | | `--output DIR` | Exports `latest_tick.json` for the dashboard. | | `--tick-interval N` | Seconds between ticks. 30s recommended for cloud models. | -| `--visualize` | Shows terminal helix visualization. | -| `--no-agent` | Baseline mode — no agents, colony runs unmanaged. | -| `--sequential` | Agents deliberate one at a time instead of in parallel. | +| `--harness NAME` | Which harness decides (default `felix`; `--harness list` shows installed plugins). | +| `--harness-opt K=V` | Harness-specific option, validated by the plugin (e.g. `role_timeout_s=90`). | +| `--no-agent` | Baseline mode — alias for `--harness baseline`, colony runs unmanaged. | +| `--visualize` | [felix] Shows terminal helix visualization. | +| `--sequential` | [felix] Agents deliberate one at a time instead of in parallel. | +| `--tick-timeout N` | Loop-level cap on a whole harness step (seconds). | ### Dashboard (optional, 3 terminals) diff --git a/pyproject.toml b/pyproject.toml index 3555d57..c137bf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,12 +5,14 @@ build-backend = "hatchling.build" [project] name = "rimworld-learning-environment" version = "0.4.1" -description = "Multi-agent benchmark where Felix Agent SDK agents play RimWorld" +description = "Harness x model benchmark: swappable agent harnesses manage a RimWorld colony" readme = "README.md" license = "MIT" requires-python = ">=3.14" +# Core is framework-free: the environment (RIMAPI client, loop, scoring, +# scenarios, tracking, harness registry) runs any harness. Agent frameworks +# arrive through extras / external harness packages. dependencies = [ - "felix-agent-sdk>=0.3.0", "httpx>=0.24", "pydantic>=2.0", "pydantic-settings>=2.0", @@ -18,10 +20,14 @@ dependencies = [ ] [project.optional-dependencies] +# The original Felix multi-agent harness (`--harness felix`). +felix = ["felix-agent-sdk>=0.3.0"] anthropic = ["felix-agent-sdk[anthropic]"] openai = ["felix-agent-sdk[openai]"] local = ["felix-agent-sdk[local]"] all = ["felix-agent-sdk[all]"] +# RimAPI MCP server for tool-using harnesses (`rle-mcp`). +mcp = ["mcp>=1.2"] dev = [ "pytest>=7.0", "pytest-asyncio>=0.24", @@ -64,7 +70,7 @@ python_version = "3.14" strict = true [[tool.mypy.overrides]] -module = ["wandb", "huggingface_hub"] +module = ["wandb", "huggingface_hub", "mcp", "mcp.*"] ignore_missing_imports = true # wandb/huggingface_hub ship untyped APIs; when the tracking extra is diff --git a/scripts/check_harness_boundary.py b/scripts/check_harness_boundary.py new file mode 100644 index 0000000..f6b0841 --- /dev/null +++ b/scripts/check_harness_boundary.py @@ -0,0 +1,74 @@ +"""Enforce the harness boundary rules (run in CI, stdlib only). + +1. ``felix_agent_sdk`` may be imported only under ``src/rle/harness/felix/``. + Everything else in core must run with the ``felix`` extra uninstalled. +2. Third-party harnesses (OpenCode, Grok Build, ...) live in their own repos. + Their names may appear in docs, never in ``src/``, ``tests/`` or ``scripts/``. + +Exit code 1 with a file:line listing on any violation. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "src" / "rle" +FELIX_DIR = SRC / "harness" / "felix" + +FELIX_IMPORT = re.compile(r"^\s*(from|import)\s+felix_agent_sdk\b", re.MULTILINE) +THIRD_PARTY = re.compile(r"\b(opencode|grok[-_ ]?build|grok_build)\b", re.IGNORECASE) +CODE_DIRS = (ROOT / "src", ROOT / "tests", ROOT / "scripts") + + +def _py_files(root: Path) -> list[Path]: + return [p for p in root.rglob("*.py") if "__pycache__" not in p.parts] + + +def check_felix_boundary() -> list[str]: + violations: list[str] = [] + for path in _py_files(SRC): + if FELIX_DIR in path.parents: + continue + text = path.read_text(encoding="utf-8") + for match in FELIX_IMPORT.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + violations.append( + f"{path.relative_to(ROOT)}:{line}: felix_agent_sdk import outside " + f"src/rle/harness/felix/", + ) + return violations + + +def check_no_third_party_harness_code() -> list[str]: + violations: list[str] = [] + for root in CODE_DIRS: + for path in _py_files(root): + if path.resolve() == Path(__file__).resolve(): + continue + text = path.read_text(encoding="utf-8") + for match in THIRD_PARTY.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + violations.append( + f"{path.relative_to(ROOT)}:{line}: third-party harness name " + f"{match.group(0)!r} in core code (belongs in its own repo)", + ) + return violations + + +def main() -> int: + violations = check_felix_boundary() + check_no_third_party_harness_code() + if violations: + print("Harness boundary violations:") + for v in violations: + print(f" {v}") + return 1 + print("Harness boundary OK: felix_agent_sdk confined to src/rle/harness/felix/; " + "no third-party harness code in tree.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/rle/__init__.py b/src/rle/__init__.py index fb93fd3..9897cfc 100644 --- a/src/rle/__init__.py +++ b/src/rle/__init__.py @@ -1,6 +1,10 @@ """RLE — RimWorld Learning Environment. -Multi-agent benchmark where Felix Agent SDK agents play RimWorld. +A harness x model benchmark: swappable harnesses (Felix multi-agent, an +unmanaged baseline, external coding agents over MCP, ...) manage a RimWorld +colony and are scored on the same footing. This top-level package is +framework-free; the Felix harness lives under ``rle.harness.felix`` behind +the optional ``felix`` extra. """ __version__ = "0.4.1" # x-release-please-version @@ -9,17 +13,10 @@ Action, ActionPlan, ActionPlanParseError, - ConstructionPlanner, - DefenseCommander, - MedicalOfficer, - ResearchDirector, - ResourceManager, - RimWorldRoleAgent, - SocialOverseer, - register_rle_agents, resolve_endpoint, ) from rle.config import RLEConfig +from rle.harness import BaseHarness, HarnessContext, StepResult, create_harness from rle.orchestration import ( ActionExecutor, ActionResolver, @@ -63,38 +60,34 @@ "ActionPlan", "ActionPlanParseError", "ActionResolver", + "BaseHarness", "resolve_endpoint", "ColonistData", "ColonyData", "CompositeScorer", - "ConstructionPlanner", "CrisisState", - "DefenseCommander", "EvaluationResult", "ExecutionResult", "GameState", "GameStateManager", + "HarnessContext", "MapData", - "MedicalOfficer", "MetricContext", "RLEConfig", "RLEGameLoop", "ResearchData", - "ResearchDirector", "ResourceData", - "ResourceManager", "RimAPIClient", - "RimWorldRoleAgent", "ScenarioConfig", "ScenarioEvaluator", "ScoreSnapshot", - "SocialOverseer", + "StepResult", "StructureData", "ThreatData", "TickResult", "TimeSeriesRecorder", "WeatherData", + "create_harness", "list_scenarios", "load_scenario", - "register_rle_agents", ] diff --git a/src/rle/agents/__init__.py b/src/rle/agents/__init__.py index 335f1b3..a90f18b 100644 --- a/src/rle/agents/__init__.py +++ b/src/rle/agents/__init__.py @@ -1,57 +1,20 @@ -"""RLE role agents and registration.""" +"""Harness-neutral action vocabulary. -from felix_agent_sdk import AgentFactory +``Action`` / ``ActionPlan`` are what every harness hands the environment; +``json_repair`` is a generic LLM-output cleaner any LLM-backed harness can +reuse. The Felix role agents that used to live here are now +``rle.harness.felix.agents`` — this package must stay importable without +``felix-agent-sdk``. +""" from rle.agents.actions import Action, ActionPlan, ActionPlanParseError, resolve_endpoint -from rle.agents.base_role import RimWorldRoleAgent -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer - -AGENT_DISPLAY: dict[str, dict[str, str]] = { - "map_analyst": {"label": "MA", "color": "blue"}, - "resource_manager": {"label": "RM", "color": "green"}, - "defense_commander": {"label": "DC", "color": "red"}, - "research_director": {"label": "RD", "color": "cyan"}, - "social_overseer": {"label": "SO", "color": "yellow"}, - "construction_planner": {"label": "CP", "color": "white"}, - "medical_officer": {"label": "MO", "color": "magenta"}, -} - -_ROLE_AGENTS: dict[str, type[RimWorldRoleAgent]] = { - "map_analyst": MapAnalyst, - "resource_manager": ResourceManager, - "defense_commander": DefenseCommander, - "research_director": ResearchDirector, - "social_overseer": SocialOverseer, - "construction_planner": ConstructionPlanner, - "medical_officer": MedicalOfficer, -} - - -def register_rle_agents() -> None: - """Register all RLE role agent types with the Felix AgentFactory.""" - for name, cls in _ROLE_AGENTS.items(): - AgentFactory.register_agent_type(name, cls) - +from rle.agents.json_repair import repair_json, try_parse_json __all__ = [ - "AGENT_DISPLAY", "Action", "ActionPlan", "ActionPlanParseError", + "repair_json", "resolve_endpoint", - "ConstructionPlanner", - "DefenseCommander", - "MapAnalyst", - "MedicalOfficer", - "ResearchDirector", - "ResourceManager", - "RimWorldRoleAgent", - "SocialOverseer", - "register_rle_agents", + "try_parse_json", ] diff --git a/src/rle/agents/actions.py b/src/rle/agents/actions.py index 5ec5790..31ab035 100644 --- a/src/rle/agents/actions.py +++ b/src/rle/agents/actions.py @@ -35,6 +35,32 @@ class ActionPlan(BaseModel): confidence: float = 0.5 +class ActionOutcome(BaseModel): + """Per-action execution result. Captures failure detail so the next + tick's deliberation context can surface it to whoever proposed it. + """ + + model_config = ConfigDict(frozen=True) + + action_type: str + endpoint: str + target_colonist_id: str | None = None + success: bool + error: str | None = None + parameters: dict[str, Any] = {} + + +class ExecutionResult(BaseModel): + """Summary of action execution for one tick.""" + + model_config = ConfigDict(frozen=True) + + executed: int + failed: int + total: int + outcomes: tuple[ActionOutcome, ...] = () + + class ActionPlanParseError(Exception): """Raised when LLM output cannot be parsed into an ActionPlan.""" diff --git a/src/rle/harness/brief.py b/src/rle/harness/brief.py new file mode 100644 index 0000000..14ccc19 --- /dev/null +++ b/src/rle/harness/brief.py @@ -0,0 +1,239 @@ +"""Harness-neutral scenario brief. + +Every harness receives the same facts: what the scenario asks for, the +current colony state, the deterministic MAP_SUMMARY (verified build / farm / +stockpile sites, water to avoid), and the action catalog it may use. Anything +beyond that — role splits, bootstrap playbooks, phase-dependent temperature, +tool-call framing — is the harness's own prompt engineering and is part of +what the benchmark measures. +""" + +from __future__ import annotations + +from typing import Any, cast + +from pydantic import BaseModel, ConfigDict + +from rle.rimapi.api_catalog import WRITE_CATALOG +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.scenarios.schema import ScenarioConfig + +# Colonist fields worth a harness's attention (full dumps blow context budgets). +_COLONIST_FIELDS = ( + "colonist_id", "name", "health", "mood", "current_job", "is_drafted", "position", +) +_MAX_EVENTS = 12 + + +def build_map_summary(state: GameState) -> str | None: + """Compact (~500 token) spatial summary from terrain + zone + room data. + + Coordinates here are verified against the terrain grid; harnesses are told + to use them verbatim rather than invent positions. + """ + terrain = state.map.terrain + if terrain is None: + return None + + lines: list[str] = [] + cx, cz = terrain.colony_center + lines.append(f"Colony center: ({cx}, {cz}).") + + if terrain.recommended_shelter: + s = terrain.recommended_shelter + lines.append( + f"SHELTER SITE (verified solid ground): " + f"place walls/doors/beds at ({s.x1},{s.z1})-({s.x2},{s.z2}). " + f"ALL blueprint actions MUST use x,z within this rectangle." + ) + if terrain.recommended_farm: + f = terrain.recommended_farm + lines.append( + f"FARM SITE (verified fertile soil): " + f"place growing_zone at x1={f.x1},z1={f.z1},x2={f.x2},z2={f.z2}. " + f"ALL growing_zone actions MUST use these exact coordinates." + ) + if terrain.recommended_stockpile: + sp = terrain.recommended_stockpile + lines.append( + f"STOCKPILE SITE (verified solid ground): " + f"place stockpile_zone at x1={sp.x1},z1={sp.z1}," + f"x2={sp.x2},z2={sp.z2}." + ) + + if terrain.water_areas: + water_strs = [f"({w.x1},{w.z1})-({w.x2},{w.z2})" for w in terrain.water_areas] + lines.append(f"WATER (do NOT build here): {', '.join(water_strs)}.") + + if state.map.zones: + zone_strs = [ + f"{z.label} ({z.zone_type}, {z.cell_count} cells)" + for z in state.map.zones[:8] + ] + lines.append(f"Zones: {'; '.join(zone_strs)}.") + else: + lines.append("Zones: NONE — create stockpile and growing zone NOW.") + + real_rooms = [r for r in state.map.rooms if r.size > 1] + if real_rooms: + room_strs = [f"{r.role} ({r.size} cells, {r.bed_count} beds)" for r in real_rooms[:6]] + lines.append(f"Rooms: {'; '.join(room_strs)}.") + else: + lines.append( + "Rooms: NONE — colonists sleeping outside. " + "Build shelter IMMEDIATELY." + ) + + if state.map.ore_deposits: + ore_strs = [ + f"{o.def_name} ({o.count} cells" + + (f", near ({o.positions[0][0]},{o.positions[0][1]})" if o.positions else "") + + ")" + for o in state.map.ore_deposits[:5] + ] + lines.append(f"Ore: {'; '.join(ore_strs)}.") + + fs = state.map.farm_summary + if fs and fs.total_growing_zones > 0: + lines.append( + f"Farms: {fs.total_growing_zones} zones, " + f"{fs.planted_cells} planted, " + f"{fs.harvestable_cells} harvestable." + ) + + return "\n".join(lines) + + +def action_catalog() -> list[dict[str, Any]]: + """Every write a harness may issue, with its parameter shape.""" + out: list[dict[str, Any]] = [ + {"action_type": "no_action", "description": "Do nothing this tick.", "params": {}}, + ] + for name, raw in sorted(WRITE_CATALOG.items()): + entry = cast(dict[str, Any], raw) + out.append({ + "action_type": name, + "description": entry.get("description", ""), + "params": entry.get("params", {}), + }) + return out + + +def scenario_goals(scenario: ScenarioConfig | None) -> dict[str, Any]: + if scenario is None: + return {} + return { + "name": scenario.name, + "description": scenario.description, + "difficulty": scenario.difficulty, + "expected_duration_days": scenario.expected_duration_days, + "victory": [f"{c.metric} {c.operator} {c.value}" for c in scenario.victory_conditions], + "failure": [f"{c.metric} {c.operator} {c.value}" for c in scenario.failure_conditions], + "scoring_weights": dict(scenario.scoring_weights), + } + + +def state_snapshot(state: GameState) -> dict[str, Any]: + """The colony as any harness should see it — no role filtering.""" + return { + "colony": state.colony.model_dump(), + "colonists": [ + {k: getattr(c, k) for k in _COLONIST_FIELDS} for c in state.colonists + ], + "resources": state.resources.model_dump(), + "research": state.research.model_dump(), + "threats": [t.model_dump() for t in state.threats], + "weather": state.weather.model_dump(), + "map": { + "size": state.map.size, + "biome": state.map.biome, + "season": state.map.season, + "temperature": state.map.temperature, + "structures": len(state.map.structures), + "zones": len(state.map.zones), + "rooms": len([r for r in state.map.rooms if r.size > 1]), + }, + } + + +class ScenarioBrief(BaseModel): + """What the environment tells a harness at the start of a tick.""" + + model_config = ConfigDict(frozen=True) + + tick: int + day: int + macro_time: float + goals: dict[str, Any] + state: dict[str, Any] + map_summary: str | None + recent_events: list[dict[str, Any]] + actions: list[dict[str, Any]] + + def to_text(self) -> str: + """Plain-text rendering for prompt-based harnesses.""" + parts = [ + f"# RimWorld colony — tick {self.tick}, day {self.day} " + f"(run progress {self.macro_time:.0%})", + ] + if self.goals: + g = self.goals + parts.append( + f"## Scenario: {g.get('name')} ({g.get('difficulty')})\n{g.get('description')}\n" + f"Victory: {'; '.join(g.get('victory', []))}\n" + f"Failure: {'; '.join(g.get('failure', []))}", + ) + if self.map_summary: + parts.append(f"## MAP_SUMMARY\n{self.map_summary}") + parts.append("## State\n" + _render(self.state)) + if self.recent_events: + parts.append( + "## Recent events\n" + + "\n".join(f"- {e['event_type']}: {e['data']}" for e in self.recent_events), + ) + parts.append( + "## Actions available\n" + + "\n".join( + f"- {a['action_type']}: {a['description']} params={a['params']}" + for a in self.actions + ), + ) + return "\n\n".join(parts) + + +def _render(data: dict[str, Any], indent: int = 0) -> str: + lines: list[str] = [] + pad = " " * indent + for key, value in data.items(): + if isinstance(value, dict): + lines.append(f"{pad}{key}:") + lines.append(_render(value, indent + 1)) + elif isinstance(value, list): + lines.append(f"{pad}{key}: {value}") + else: + lines.append(f"{pad}{key}: {value}") + return "\n".join(lines) + + +def build_brief( + state: GameState, + *, + tick: int, + macro_time: float, + scenario: ScenarioConfig | None = None, + events: list[RimAPIEvent] | None = None, +) -> ScenarioBrief: + return ScenarioBrief( + tick=tick, + day=state.colony.day, + macro_time=macro_time, + goals=scenario_goals(scenario), + state=state_snapshot(state), + map_summary=build_map_summary(state), + recent_events=[ + {"event_type": e.event_type, "data": str(e.data)[:200]} + for e in (events or [])[:_MAX_EVENTS] + ], + actions=action_catalog(), + ) diff --git a/src/rle/harness/felix/agents/__init__.py b/src/rle/harness/felix/agents/__init__.py new file mode 100644 index 0000000..721d894 --- /dev/null +++ b/src/rle/harness/felix/agents/__init__.py @@ -0,0 +1,56 @@ +"""The Felix harness's role agents (MapAnalyst + 6 domain roles). + +Everything here subclasses Felix SDK's ``LLMAgent``; nothing outside +``rle.harness.felix`` may import from this package. +""" + +from felix_agent_sdk import AgentFactory + +from rle.harness.felix.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.map_analyst import MapAnalyst +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer + +AGENT_DISPLAY: dict[str, dict[str, str]] = { + "map_analyst": {"label": "MA", "color": "blue"}, + "resource_manager": {"label": "RM", "color": "green"}, + "defense_commander": {"label": "DC", "color": "red"}, + "research_director": {"label": "RD", "color": "cyan"}, + "social_overseer": {"label": "SO", "color": "yellow"}, + "construction_planner": {"label": "CP", "color": "white"}, + "medical_officer": {"label": "MO", "color": "magenta"}, +} + +_ROLE_AGENTS: dict[str, type[RimWorldRoleAgent]] = { + "map_analyst": MapAnalyst, + "resource_manager": ResourceManager, + "defense_commander": DefenseCommander, + "research_director": ResearchDirector, + "social_overseer": SocialOverseer, + "construction_planner": ConstructionPlanner, + "medical_officer": MedicalOfficer, +} + + +def register_rle_agents() -> None: + """Register all RLE role agent types with the Felix AgentFactory.""" + for name, cls in _ROLE_AGENTS.items(): + AgentFactory.register_agent_type(name, cls) + + +__all__ = [ + "AGENT_DISPLAY", + "ConstructionPlanner", + "DefenseCommander", + "MapAnalyst", + "MedicalOfficer", + "ResearchDirector", + "ResourceManager", + "RimWorldRoleAgent", + "SocialOverseer", + "register_rle_agents", +] diff --git a/src/rle/agents/base_role.py b/src/rle/harness/felix/agents/base_role.py similarity index 90% rename from src/rle/agents/base_role.py rename to src/rle/harness/felix/agents/base_role.py index bc8f678..290dd7a 100644 --- a/src/rle/agents/base_role.py +++ b/src/rle/harness/felix/agents/base_role.py @@ -19,6 +19,7 @@ from rle.agents.actions import Action, ActionPlan, ActionPlanParseError, resolve_endpoint from rle.agents.json_repair import repair_json +from rle.harness.brief import build_map_summary from rle.rimapi.schemas import GameState from rle.rimapi.sse_client import RimAPIEvent @@ -408,95 +409,8 @@ def _get_role_description(self) -> str: @staticmethod def _build_map_summary(state: GameState) -> str | None: - """Build a compact ~500 token map summary from terrain + zone + room data. - - This text is injected into every agent's context so they share a - common spatial understanding without needing to parse raw data. - """ - terrain = state.map.terrain - if terrain is None: - return None - - lines: list[str] = [] - cx, cz = terrain.colony_center - lines.append(f"Colony center: ({cx}, {cz}).") - - # Verified build/farm/stockpile sites - if terrain.recommended_shelter: - s = terrain.recommended_shelter - lines.append( - f"SHELTER SITE (verified solid ground): " - f"place walls/doors/beds at ({s.x1},{s.z1})-({s.x2},{s.z2}). " - f"ALL blueprint actions MUST use x,z within this rectangle." - ) - if terrain.recommended_farm: - f = terrain.recommended_farm - lines.append( - f"FARM SITE (verified fertile soil): " - f"place growing_zone at x1={f.x1},z1={f.z1},x2={f.x2},z2={f.z2}. " - f"ALL growing_zone actions MUST use these exact coordinates." - ) - if terrain.recommended_stockpile: - sp = terrain.recommended_stockpile - lines.append( - f"STOCKPILE SITE (verified solid ground): " - f"place stockpile_zone at x1={sp.x1},z1={sp.z1}," - f"x2={sp.x2},z2={sp.z2}." - ) - - # Water avoidance - if terrain.water_areas: - water_strs = [ - f"({w.x1},{w.z1})-({w.x2},{w.z2})" - for w in terrain.water_areas - ] - lines.append(f"WATER (do NOT build here): {', '.join(water_strs)}.") - - # Existing zones - if state.map.zones: - zone_strs = [ - f"{z.label} ({z.zone_type}, {z.cell_count} cells)" - for z in state.map.zones[:8] - ] - lines.append(f"Zones: {'; '.join(zone_strs)}.") - else: - lines.append("Zones: NONE — create stockpile and growing zone NOW.") - - # Existing rooms - real_rooms = [r for r in state.map.rooms if r.size > 1] - if real_rooms: - room_strs = [ - f"{r.role} ({r.size} cells, {r.bed_count} beds)" - for r in real_rooms[:6] - ] - lines.append(f"Rooms: {'; '.join(room_strs)}.") - else: - lines.append( - "Rooms: NONE — colonists sleeping outside. " - "Build shelter IMMEDIATELY." - ) - - # Ore - if state.map.ore_deposits: - ore_strs = [ - f"{o.def_name} ({o.count} cells" - + (f", near ({o.positions[0][0]},{o.positions[0][1]})" - if o.positions else "") - + ")" - for o in state.map.ore_deposits[:5] - ] - lines.append(f"Ore: {'; '.join(ore_strs)}.") - - # Farm summary - fs = state.map.farm_summary - if fs and fs.total_growing_zones > 0: - lines.append( - f"Farms: {fs.total_growing_zones} zones, " - f"{fs.planted_cells} planted, " - f"{fs.harvestable_cells} harvestable." - ) - - return "\n".join(lines) + """Compact map summary shared by every agent (core builder, see rle.harness.brief).""" + return build_map_summary(state) def build_task( self, diff --git a/src/rle/agents/construction_planner.py b/src/rle/harness/felix/agents/construction_planner.py similarity index 97% rename from src/rle/agents/construction_planner.py rename to src/rle/harness/felix/agents/construction_planner.py index 0f68528..cba891b 100644 --- a/src/rle/agents/construction_planner.py +++ b/src/rle/harness/felix/agents/construction_planner.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/defense_commander.py b/src/rle/harness/felix/agents/defense_commander.py similarity index 97% rename from src/rle/agents/defense_commander.py rename to src/rle/harness/felix/agents/defense_commander.py index 28426ed..b74605d 100644 --- a/src/rle/agents/defense_commander.py +++ b/src/rle/harness/felix/agents/defense_commander.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/map_analyst.py b/src/rle/harness/felix/agents/map_analyst.py similarity index 98% rename from src/rle/agents/map_analyst.py rename to src/rle/harness/felix/agents/map_analyst.py index 0e21c07..12d0498 100644 --- a/src/rle/agents/map_analyst.py +++ b/src/rle/harness/felix/agents/map_analyst.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent if TYPE_CHECKING: from felix_agent_sdk.agents.llm_agent import LLMTask diff --git a/src/rle/agents/medical_officer.py b/src/rle/harness/felix/agents/medical_officer.py similarity index 97% rename from src/rle/agents/medical_officer.py rename to src/rle/harness/felix/agents/medical_officer.py index b7f0293..8e761df 100644 --- a/src/rle/agents/medical_officer.py +++ b/src/rle/harness/felix/agents/medical_officer.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/research_director.py b/src/rle/harness/felix/agents/research_director.py similarity index 97% rename from src/rle/agents/research_director.py rename to src/rle/harness/felix/agents/research_director.py index d9dbdaa..3603549 100644 --- a/src/rle/agents/research_director.py +++ b/src/rle/harness/felix/agents/research_director.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/resource_manager.py b/src/rle/harness/felix/agents/resource_manager.py similarity index 97% rename from src/rle/agents/resource_manager.py rename to src/rle/harness/felix/agents/resource_manager.py index a134c78..095e6d4 100644 --- a/src/rle/agents/resource_manager.py +++ b/src/rle/harness/felix/agents/resource_manager.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/social_overseer.py b/src/rle/harness/felix/agents/social_overseer.py similarity index 97% rename from src/rle/agents/social_overseer.py rename to src/rle/harness/felix/agents/social_overseer.py index 80e5a35..1bf15df 100644 --- a/src/rle/agents/social_overseer.py +++ b/src/rle/harness/felix/agents/social_overseer.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/harness/felix/build.py b/src/rle/harness/felix/build.py index 9828dad..e2467df 100644 --- a/src/rle/harness/felix/build.py +++ b/src/rle/harness/felix/build.py @@ -9,16 +9,16 @@ from felix_agent_sdk.visualization import HelixVisualizer from pydantic import BaseModel -from rle.agents import AGENT_DISPLAY -from rle.agents.base_role import RimWorldRoleAgent -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer from rle.config import bridge_anthropic_key, bridge_openrouter_key +from rle.harness.felix.agents import AGENT_DISPLAY +from rle.harness.felix.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.map_analyst import MapAnalyst +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer from rle.harness.felix.harness import FelixHarness from rle.harness.felix.options import FelixOptions from rle.harness.felix.provider_factory import build_helix, build_provider diff --git a/src/rle/harness/felix/harness.py b/src/rle/harness/felix/harness.py index 56b9569..94297e7 100644 --- a/src/rle/harness/felix/harness.py +++ b/src/rle/harness/felix/harness.py @@ -19,10 +19,9 @@ from felix_agent_sdk.communication import CentralPost, MessageType, SpokeManager from felix_agent_sdk.providers import ProviderError -from rle.agents.actions import ActionPlan, ActionPlanParseError -from rle.agents.base_role import RimWorldRoleAgent +from rle.agents.actions import ActionPlan, ActionPlanParseError, ExecutionResult +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.harness.protocol import BaseHarness, HarnessContext, StepResult -from rle.orchestration.action_executor import ExecutionResult from rle.orchestration.action_resolver import ActionResolver from rle.rimapi.schemas import GameState from rle.rimapi.sse_client import RimAPIEvent diff --git a/src/rle/harness/felix/provider_factory.py b/src/rle/harness/felix/provider_factory.py index 62a9e35..ff51497 100644 --- a/src/rle/harness/felix/provider_factory.py +++ b/src/rle/harness/felix/provider_factory.py @@ -10,7 +10,7 @@ OpenAIProvider, ) -from rle.providers.claude_code import ClaudeCodeProvider +from rle.harness.felix.providers.claude_code import ClaudeCodeProvider HELIX_PRESETS: dict[str, HelixConfig] = { "default": HelixConfig.default(), diff --git a/src/rle/harness/felix/providers/__init__.py b/src/rle/harness/felix/providers/__init__.py new file mode 100644 index 0000000..68cb756 --- /dev/null +++ b/src/rle/harness/felix/providers/__init__.py @@ -0,0 +1,5 @@ +"""Felix-SDK providers that RLE adds beyond the SDK's built-ins.""" + +from rle.harness.felix.providers.claude_code import ClaudeCodeProvider + +__all__ = ["ClaudeCodeProvider"] diff --git a/src/rle/providers/claude_code.py b/src/rle/harness/felix/providers/claude_code.py similarity index 100% rename from src/rle/providers/claude_code.py rename to src/rle/harness/felix/providers/claude_code.py diff --git a/src/rle/harness/protocol.py b/src/rle/harness/protocol.py index 3862686..778201b 100644 --- a/src/rle/harness/protocol.py +++ b/src/rle/harness/protocol.py @@ -19,8 +19,7 @@ from pydantic import BaseModel, ConfigDict -from rle.agents.actions import ActionPlan -from rle.orchestration.action_executor import ExecutionResult +from rle.agents.actions import ActionPlan, ExecutionResult from rle.rimapi.client import RimAPIClient from rle.rimapi.schemas import GameState from rle.rimapi.sse_client import RimAPIEvent diff --git a/src/rle/harness/registry.py b/src/rle/harness/registry.py index 8dd8118..62d1205 100644 --- a/src/rle/harness/registry.py +++ b/src/rle/harness/registry.py @@ -1,7 +1,7 @@ """Harness discovery via the ``rle.harnesses`` entry-point group. Built-in harnesses (``baseline``, ``felix``) and third-party packages -(``rle-harness-opencode``, ...) register the same way, so adding a harness is +(``rle-harness-``) register the same way, so adding a harness is ``pip install `` — never a change to RLE core. """ @@ -73,7 +73,7 @@ def get_plugin(name: str) -> HarnessPlugin: if ep is None: raise HarnessNotFoundError( f"Unknown harness {name!r}. Installed: {', '.join(sorted(eps)) or '(none)'}. " - "Install a harness package (e.g. rle-harness-opencode) or check the name.", + "Install a harness package (rle-harness-) or check the name.", ) return cast(HarnessPlugin, ep.load()) diff --git a/src/rle/orchestration/action_executor.py b/src/rle/orchestration/action_executor.py index add1602..cbee9b7 100644 --- a/src/rle/orchestration/action_executor.py +++ b/src/rle/orchestration/action_executor.py @@ -6,12 +6,18 @@ import logging from typing import Any, cast -from pydantic import BaseModel, ConfigDict - -from rle.agents.actions import Action, ActionPlan, resolve_endpoint +from rle.agents.actions import ( + Action, + ActionOutcome, + ActionPlan, + ExecutionResult, + resolve_endpoint, +) from rle.rimapi.api_catalog import WRITE_CATALOG from rle.rimapi.client import RimAPIClient, RimAPIResponseError +__all__ = ["ActionExecutor", "ActionOutcome", "ExecutionResult"] + logger = logging.getLogger(__name__) # Endpoints that require a valid colonist/pawn ID. @@ -27,32 +33,6 @@ }) -class ActionOutcome(BaseModel): - """Per-action execution result. Captures failure detail so the next - tick's deliberation context can surface it to the agent that proposed it. - """ - - model_config = ConfigDict(frozen=True) - - action_type: str - endpoint: str - target_colonist_id: str | None = None - success: bool - error: str | None = None - parameters: dict[str, Any] = {} - - -class ExecutionResult(BaseModel): - """Summary of action execution for one tick.""" - - model_config = ConfigDict(frozen=True) - - executed: int - failed: int - total: int - outcomes: tuple[ActionOutcome, ...] = () - - def _extract_rimapi_error(detail: str) -> str: """Pull the first error string out of a RIMAPI JSON envelope, else return raw detail.""" try: diff --git a/src/rle/providers/__init__.py b/src/rle/providers/__init__.py deleted file mode 100644 index 362afb2..0000000 --- a/src/rle/providers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""RLE-local LLM providers (beyond those shipped with felix-agent-sdk).""" - -from rle.providers.claude_code import ClaudeCodeProvider - -__all__ = ["ClaudeCodeProvider"] diff --git a/src/rle/scoring/coherence.py b/src/rle/scoring/coherence.py index e15022b..8cf74ed 100644 --- a/src/rle/scoring/coherence.py +++ b/src/rle/scoring/coherence.py @@ -12,8 +12,7 @@ from collections.abc import Iterable, Sequence from typing import Any -from rle.agents.actions import resolve_endpoint -from rle.orchestration.action_executor import ActionOutcome +from rle.agents.actions import ActionOutcome, resolve_endpoint Rect = tuple[int, int, int, int] diff --git a/src/rle/tracking/hf_logger.py b/src/rle/tracking/hf_logger.py index 1c2feac..ad54177 100644 --- a/src/rle/tracking/hf_logger.py +++ b/src/rle/tracking/hf_logger.py @@ -35,18 +35,25 @@ # RLE — RimWorld Learning Environment Benchmarks -Can 7 role-specialized LLM agents keep a RimWorld colony alive? RLE is a -multi-agent coordination benchmark: MapAnalyst + 6 domain agents -(resources, defense, research, social, construction, medical) manage a -live colony through a REST API, scored on a 10-metric weighted composite -against a no-agent baseline (RimWorld's built-in pawn AI, static -4-seed reference). +Can an LLM agent harness keep a RimWorld colony alive? RLE is a +harness x model benchmark: a swappable harness (the Felix 7-agent stack, +a coding agent attached over MCP, ...) manages a live colony through a +REST API and is scored on a weighted composite against the unmanaged +baseline (RimWorld's built-in pawn AI). Rows are keyed by harness and +model; scoring is harness-agnostic (see SCORING_VERSION in each run). - Site + featured runs: https://rle.appsprout.dev -- Harness: https://github.com/AppSprout-dev/RLE +- Environment + harness registry: https://github.com/AppSprout-dev/RLE """ +def _row_label(row: dict[str, Any]) -> str: + """``harness/model`` when the row knows its harness, else just the model.""" + harness = row.get("harness") + model = str(row.get("model", "?")) + return f"{harness}/{model}" if harness else model + + def build_dataset_card(board: dict[str, Any], date: str) -> str: """Render the dataset card README from a spread's leaderboard.json. @@ -66,14 +73,14 @@ def build_dataset_card(board: dict[str, Any], date: str) -> str: f"Baseline: no-agent, {baseline.get('n_runs', '?')} seeds, mean " f"time-to-end {baseline.get('mean_time_to_end_days', '?')} days.", "", - "| # | model | mean | final | vs baseline | ticks > base | action ok | cost |", - "|---|-------|------|-------|-------------|--------------|-----------|------|", + "| # | harness/model | mean | final | vs baseline | ticks > base | action ok | cost |", + "|---|---------------|------|-------|-------------|--------------|-----------|------|", ] for i, r in enumerate(rows, 1): real = r.get("real_cost_usd") cost = f"${real:.2f}" if real is not None else f"~${r.get('est_cost_usd', 0):.2f}" lines.append( - f"| {i} | {r['model']} | {r['mean_composite']:.3f} " + f"| {i} | {_row_label(r)} | {r['mean_composite']:.3f} " f"| {r['final_composite']:.3f} | {r['vs_baseline_mean_delta']:+.3f} " f"| {r['ticks_above_baseline']} | {r['raw_action_success']:.0%} | {cost} |" ) @@ -81,9 +88,9 @@ def build_dataset_card(board: dict[str, Any], date: str) -> str: above = [r for r in rows if r.get("vs_baseline_mean_delta", 0) > 0] lines += [ "", - f"**{len(above)} of {len(rows)} models beat the no-agent baseline.**" + f"**{len(above)} of {len(rows)} harness/model rows beat the unmanaged baseline.**" + ( - " (" + ", ".join(r["model"] for r in above) + ")" + " (" + ", ".join(_row_label(r) for r in above) + ")" if above else "" ), "", diff --git a/tests/conftest.py b/tests/conftest.py index ed76706..596f324 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,12 +3,11 @@ from __future__ import annotations import json +from importlib.util import find_spec +from typing import Any from unittest.mock import MagicMock import pytest -from felix_agent_sdk.core import HelixConfig, HelixGeometry -from felix_agent_sdk.providers.base import BaseProvider -from felix_agent_sdk.providers.types import CompletionResult from rle.config import RLEConfig from rle.rimapi.schemas import ( @@ -23,6 +22,28 @@ WeatherData, ) +# The Felix harness is an optional extra. Tests that need it either live in +# a module listed below (skipped from collection entirely) or carry the +# `requires_felix` marker; core tests must pass without it. +FELIX_AVAILABLE = find_spec("felix_agent_sdk") is not None +requires_felix = pytest.mark.skipif( + not FELIX_AVAILABLE, reason="felix-agent-sdk not installed (uv sync --extra felix)", +) + +# Modules that import the Felix SDK (or rle.harness.felix internals) at +# module level. Without the extra they cannot even be collected, so they are +# excluded wholesale; the zero-Felix CI job runs everything else. +_FELIX_ONLY_TEST_MODULES = [ + "unit/test_base_role.py", + "unit/test_claude_code_provider.py", + "unit/test_felix_provider_factory.py", + "unit/test_role_agents.py", + "unit/test_visualizer_integration.py", + "integration/test_game_loop.py", + "integration/test_scenario_run.py", +] +collect_ignore = [] if FELIX_AVAILABLE else list(_FELIX_ONLY_TEST_MODULES) + # ------------------------------------------------------------------ # Config # ------------------------------------------------------------------ @@ -35,7 +56,6 @@ def mock_config() -> RLEConfig: provider="anthropic", model="claude-sonnet-4-5", tick_interval=0.5, - helix_preset="default", max_agents=7, log_level="DEBUG", ) @@ -283,15 +303,18 @@ def sample_action_plan_json() -> str: @pytest.fixture -def helix() -> HelixGeometry: - return HelixConfig.default().to_geometry() +def helix() -> Any: + felix_core = pytest.importorskip("felix_agent_sdk.core") + return felix_core.HelixConfig.default().to_geometry() @pytest.fixture def mock_provider() -> MagicMock: - """Provider mock that returns a valid JSON action plan.""" - provider = MagicMock(spec=BaseProvider) - provider.complete.return_value = CompletionResult( + """Felix provider mock that returns a valid JSON action plan.""" + providers_base = pytest.importorskip("felix_agent_sdk.providers.base") + providers_types = pytest.importorskip("felix_agent_sdk.providers.types") + provider = MagicMock(spec=providers_base.BaseProvider) + provider.complete.return_value = providers_types.CompletionResult( content=SAMPLE_ACTION_PLAN_JSON, model="mock-model", usage={"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, diff --git a/tests/integration/test_game_loop.py b/tests/integration/test_game_loop.py index 23e39d4..3bfc541 100644 --- a/tests/integration/test_game_loop.py +++ b/tests/integration/test_game_loop.py @@ -12,14 +12,14 @@ from felix_agent_sdk.providers.base import BaseProvider from felix_agent_sdk.providers.types import CompletionResult -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer from rle.config import RLEConfig +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.map_analyst import MapAnalyst +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer from rle.orchestration.game_loop import RLEGameLoop, TickResult from rle.rimapi.client import RimAPIClient from rle.scenarios.evaluator import ScenarioEvaluator diff --git a/tests/integration/test_harness_loop.py b/tests/integration/test_harness_loop.py index 8928362..c69fd98 100644 --- a/tests/integration/test_harness_loop.py +++ b/tests/integration/test_harness_loop.py @@ -9,8 +9,6 @@ from pathlib import Path from typing import ClassVar -import httpx - from rle.agents.actions import Action, ActionPlan from rle.config import RLEConfig from rle.harness import BaseHarness, HarnessStepError, StepResult @@ -23,16 +21,14 @@ from rle.scoring.composite import CompositeScorer from rle.scoring.metrics import NEUTRAL from rle.scoring.recorder import TimeSeriesRecorder +from rle.testing import MockRimAPI from rle.tracking.event_log import EventLog, EventType -from tests.integration.test_game_loop import _make_transport @asynccontextmanager async def _client() -> AsyncIterator[RimAPIClient]: - async with RimAPIClient("http://test") as client: - client._client = httpx.AsyncClient( - transport=_make_transport(), base_url="http://test", - ) + async with RimAPIClient("http://mock") as client: + MockRimAPI().attach(client) yield client diff --git a/tests/integration/test_scenario_run.py b/tests/integration/test_scenario_run.py index 7f61b56..31dd981 100644 --- a/tests/integration/test_scenario_run.py +++ b/tests/integration/test_scenario_run.py @@ -12,13 +12,13 @@ from felix_agent_sdk.providers.base import BaseProvider from felix_agent_sdk.providers.types import CompletionResult -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer from rle.config import RLEConfig +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer from rle.orchestration.game_loop import RLEGameLoop from rle.rimapi.client import RimAPIClient from rle.scenarios.evaluator import ScenarioEvaluator diff --git a/tests/unit/test_base_role.py b/tests/unit/test_base_role.py index a1dc8de..938ca30 100644 --- a/tests/unit/test_base_role.py +++ b/tests/unit/test_base_role.py @@ -13,7 +13,7 @@ from felix_agent_sdk.providers.types import CompletionResult, MessageRole from rle.agents.actions import ActionPlan, ActionPlanParseError -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState # ------------------------------------------------------------------ diff --git a/tests/unit/test_brief.py b/tests/unit/test_brief.py new file mode 100644 index 0000000..0940fec --- /dev/null +++ b/tests/unit/test_brief.py @@ -0,0 +1,90 @@ +"""Harness-neutral scenario brief.""" + +from __future__ import annotations + +from rle.harness.brief import action_catalog, build_brief, build_map_summary +from rle.rimapi.schemas import ( + AreaRect, + ColonyData, + GameState, + MapData, + ResearchData, + ResourceData, + TerrainSummary, + WeatherData, +) +from rle.rimapi.sse_client import RimAPIEvent +from rle.scenarios.loader import list_scenarios + + +def _state(with_terrain: bool = True) -> GameState: + terrain = TerrainSummary( + colony_center=(50, 50), + recommended_shelter=AreaRect(x1=40, z1=40, x2=46, z2=46), + recommended_farm=AreaRect(x1=60, z1=40, x2=67, z2=47), + recommended_stockpile=AreaRect(x1=48, z1=52, x2=52, z2=56), + water_areas=[AreaRect(x1=0, z1=0, x2=5, z2=90)], + ) if with_terrain else None + return GameState( + colony=ColonyData( + name="T", wealth=5000.0, day=3, tick=180000, + population=3, mood_average=0.6, food_days=4.0, + ), + colonists=[], + resources=ResourceData( + food=40.0, medicine=2, steel=50, wood=120, components=3, silver=0, power_net=0.0, + ), + map=MapData( + size=(250, 250), biome="temperate_forest", season="spring", + temperature=15.0, structures=[], terrain=terrain, + ), + research=ResearchData(current_project=None, progress=0.0, completed=[], available=["a"]), + threats=[], + weather=WeatherData(condition="clear", temperature=15.0, outdoor_severity=0.0), + timestamp=0.0, + ) + + +class TestMapSummary: + def test_none_without_terrain(self) -> None: + assert build_map_summary(_state(with_terrain=False)) is None + + def test_contains_verified_sites_and_water(self) -> None: + text = build_map_summary(_state()) + assert text is not None + assert "SHELTER SITE" in text and "(40,40)-(46,46)" in text + assert "FARM SITE" in text and "x1=60,z1=40,x2=67,z2=47" in text + assert "STOCKPILE SITE" in text + assert "WATER (do NOT build here)" in text + assert "Zones: NONE" in text and "Rooms: NONE" in text + + +class TestActionCatalog: + def test_includes_no_action_and_every_write(self) -> None: + names = {a["action_type"] for a in action_catalog()} + assert "no_action" in names + assert {"work_priority", "draft", "blueprint", "growing_zone"} <= names + + +class TestBrief: + def test_brief_carries_goals_state_events_and_actions(self) -> None: + scenario = list_scenarios()[0] + events = [RimAPIEvent(event_type="raid", data={"points": 500}, timestamp=1.0)] + brief = build_brief( + _state(), tick=2, macro_time=0.1, scenario=scenario, events=events, + ) + assert brief.tick == 2 and brief.day == 3 + assert brief.goals["name"] == scenario.name + assert brief.goals["victory"] + assert brief.state["colony"]["population"] == 3 + assert brief.recent_events[0]["event_type"] == "raid" + assert brief.map_summary and "SHELTER SITE" in brief.map_summary + text = brief.to_text() + assert "## Scenario" in text and "## MAP_SUMMARY" in text + assert "## Actions available" in text and "- draft:" in text + + def test_brief_without_scenario(self) -> None: + brief = build_brief(_state(with_terrain=False), tick=0, macro_time=0.0) + assert brief.goals == {} + assert brief.map_summary is None + assert "## Scenario" not in brief.to_text() diff --git a/tests/unit/test_claude_code_provider.py b/tests/unit/test_claude_code_provider.py index 6911e6c..2348ab5 100644 --- a/tests/unit/test_claude_code_provider.py +++ b/tests/unit/test_claude_code_provider.py @@ -12,7 +12,7 @@ from felix_agent_sdk.providers.errors import ProviderError from felix_agent_sdk.providers.types import ChatMessage, MessageRole -from rle.providers.claude_code import ClaudeCodeProvider +from rle.harness.felix.providers.claude_code import ClaudeCodeProvider MESSAGES = [ ChatMessage(role=MessageRole.SYSTEM, content="You are a test agent."), @@ -52,8 +52,8 @@ def _run_complete( ) -> tuple[Any, MagicMock]: """Call provider.complete with mocked CLI resolution + subprocess.""" with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", - ), patch("rle.providers.claude_code.subprocess.run") as mock_run: + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", + ), patch("rle.harness.felix.providers.claude_code.subprocess.run") as mock_run: if side_effect is not None: mock_run.side_effect = side_effect else: @@ -131,7 +131,7 @@ def test_timeout_raises(self) -> None: def test_missing_cli_raises(self) -> None: provider = ClaudeCodeProvider() - with patch("rle.providers.claude_code.shutil.which", return_value=None): + with patch("rle.harness.felix.providers.claude_code.shutil.which", return_value=None): with pytest.raises(ProviderError, match="not found on PATH"): provider.complete(MESSAGES) @@ -142,8 +142,8 @@ def test_assistant_messages_ignored(self) -> None: ChatMessage(role=MessageRole.ASSISTANT, content=""), ] with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", - ), patch("rle.providers.claude_code.subprocess.run") as mock_run: + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", + ), patch("rle.harness.felix.providers.claude_code.subprocess.run") as mock_run: mock_run.return_value = _mock_proc(_cli_envelope()) provider.complete(messages) assert mock_run.call_args[1]["input"] == "Do the thing." @@ -167,9 +167,9 @@ async def test_parses_result_and_usage(self) -> None: provider = ClaudeCodeProvider() proc = _mock_async_proc(_cli_envelope(result="hello", input_tokens=50, output_tokens=7)) with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", ), patch( - "rle.providers.claude_code.asyncio.create_subprocess_exec", + "rle.harness.felix.providers.claude_code.asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc), ): result = await provider.acomplete(MESSAGES) @@ -180,9 +180,9 @@ async def test_nonzero_exit_raises(self) -> None: provider = ClaudeCodeProvider() proc = _mock_async_proc("", returncode=1, stderr="boom") with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", ), patch( - "rle.providers.claude_code.asyncio.create_subprocess_exec", + "rle.harness.felix.providers.claude_code.asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc), ): with pytest.raises(ProviderError, match="exited with code 1"): @@ -193,9 +193,9 @@ async def test_timeout_kills_subprocess(self) -> None: proc = _mock_async_proc("") proc.communicate = AsyncMock(side_effect=asyncio.TimeoutError) with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", ), patch( - "rle.providers.claude_code.asyncio.create_subprocess_exec", + "rle.harness.felix.providers.claude_code.asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc), ): with pytest.raises(ProviderError, match="timed out"): @@ -212,9 +212,9 @@ async def _hang(*args: Any, **kwargs: Any) -> tuple[bytes, bytes]: proc.communicate = AsyncMock(side_effect=_hang) with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", ), patch( - "rle.providers.claude_code.asyncio.create_subprocess_exec", + "rle.harness.felix.providers.claude_code.asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc), ): task = asyncio.ensure_future(provider.acomplete(MESSAGES)) @@ -229,8 +229,8 @@ class TestStreamAndTokens: def test_stream_yields_content_then_final(self) -> None: provider = ClaudeCodeProvider() with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", - ), patch("rle.providers.claude_code.subprocess.run") as mock_run: + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", + ), patch("rle.harness.felix.providers.claude_code.subprocess.run") as mock_run: mock_run.return_value = _mock_proc(_cli_envelope(result="streamed")) chunks = list(provider.stream(MESSAGES)) assert chunks[0].text == "streamed" diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 82ded9b..a437770 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -7,8 +7,6 @@ import pytest from rle.config import RLEConfig, bridge_anthropic_key, bridge_openrouter_key -from rle.harness.felix.provider_factory import build_provider -from rle.providers.claude_code import ClaudeCodeProvider class TestBridgeAnthropicKey: @@ -35,17 +33,6 @@ def test_noop_when_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: assert "ANTHROPIC_API_KEY" not in os.environ -class TestProviderRegistry: - def test_claude_code_provider_registered(self) -> None: - provider = build_provider("claude-code", "claude-fable-5") - assert isinstance(provider, ClaudeCodeProvider) - assert provider.model == "claude-fable-5" - - def test_unknown_provider_lists_choices(self) -> None: - with pytest.raises(ValueError, match="anthropic"): - build_provider("nope", "x") - - class TestConfigIsFrameworkFree: def test_harness_defaults(self) -> None: config = RLEConfig() diff --git a/tests/unit/test_felix_provider_factory.py b/tests/unit/test_felix_provider_factory.py new file mode 100644 index 0000000..862e64a --- /dev/null +++ b/tests/unit/test_felix_provider_factory.py @@ -0,0 +1,28 @@ +"""Felix provider/helix construction (moved off RLEConfig).""" + +from __future__ import annotations + +import pytest + +from rle.harness.felix.provider_factory import build_helix, build_provider +from rle.harness.felix.providers.claude_code import ClaudeCodeProvider + + +class TestProviderRegistry: + def test_claude_code_provider_registered(self) -> None: + provider = build_provider("claude-code", "claude-fable-5") + assert isinstance(provider, ClaudeCodeProvider) + assert provider.model == "claude-fable-5" + + def test_unknown_provider_lists_choices(self) -> None: + with pytest.raises(ValueError, match="anthropic"): + build_provider("nope", "x") + + +class TestHelixPresets: + def test_default(self) -> None: + assert build_helix("default") is not None + + def test_unknown_preset(self) -> None: + with pytest.raises(ValueError, match="research_heavy"): + build_helix("spiral") diff --git a/tests/unit/test_harness_registry.py b/tests/unit/test_harness_registry.py index 11591f7..a4c2dcc 100644 --- a/tests/unit/test_harness_registry.py +++ b/tests/unit/test_harness_registry.py @@ -30,6 +30,7 @@ from rle.rimapi.client import RimAPIClient from rle.rimapi.schemas import GameState from rle.rimapi.sse_client import RimAPIEvent +from tests.conftest import FELIX_AVAILABLE, requires_felix def _ctx(**overrides: object) -> HarnessContext: @@ -50,8 +51,10 @@ def test_list_reports_package_and_availability(self) -> None: infos = {i.name: i for i in list_harnesses()} assert infos["baseline"].availability.ok assert infos["baseline"].package == "rimworld-learning-environment" - assert infos["felix"].availability.ok # felix extra installed in dev + assert infos["felix"].availability.ok is FELIX_AVAILABLE assert "Felix" in infos["felix"].description + if not FELIX_AVAILABLE: + assert "extra felix" in infos["felix"].availability.reason def test_unknown_name_lists_installed(self) -> None: with pytest.raises(HarnessNotFoundError, match="baseline"): @@ -62,11 +65,13 @@ def test_create_baseline(self) -> None: assert isinstance(harness, BaselineHarness) assert harness.name == "baseline" + @requires_felix def test_create_felix_smoke_builds_seven_agents(self) -> None: harness = create_harness("felix", _ctx(), {"no_think": True}, smoke=True) assert harness.name == "felix" assert len(harness.agents) == 7 # type: ignore[attr-defined] + @requires_felix def test_felix_options_validated(self) -> None: with pytest.raises(HarnessOptionsError, match="bogus"): create_harness("felix", _ctx(), {"bogus": 1}, smoke=True) @@ -157,6 +162,7 @@ def test_no_agent_builds_baseline(self) -> None: assert isinstance(build_legacy_harness([], no_agent=True), BaselineHarness) assert isinstance(build_legacy_harness(None), BaselineHarness) + @requires_felix def test_agents_build_felix(self) -> None: felix = create_harness("felix", _ctx(), smoke=True) rebuilt = build_legacy_harness(felix.agents, parallel=False) # type: ignore[attr-defined] diff --git a/tests/unit/test_hf_logger.py b/tests/unit/test_hf_logger.py index 226927f..c720764 100644 --- a/tests/unit/test_hf_logger.py +++ b/tests/unit/test_hf_logger.py @@ -61,7 +61,7 @@ def test_real_cost_unmarked_estimate_marked(self) -> None: def test_baseline_beaters_called_out(self) -> None: card = build_dataset_card(_BOARD, "2026-06-11") - assert "1 of 3 models beat the no-agent baseline." in card + assert "1 of 3 harness/model rows beat the unmanaged baseline." in card assert "(z-ai/glm-5.1)" in card def test_date_and_baseline_framing(self) -> None: @@ -76,4 +76,19 @@ def test_no_baseline_beaters_omits_paren(self) -> None: "rows": [r for r in _BOARD["rows"] if r["model"] != "z-ai/glm-5.1"], } card = build_dataset_card(board, "2026-06-11") - assert "0 of 2 models beat the no-agent baseline." in card + assert "0 of 2 harness/model rows beat the unmanaged baseline." in card + + +class TestHarnessRows: + def test_rows_with_harness_are_labelled_harness_slash_model(self) -> None: + board = { + "baseline": _BOARD["baseline"], + "rows": [ + {**_BOARD["rows"][1], "harness": "felix"}, + {**_BOARD["rows"][0], "harness": "some-tool"}, + ], + } + card = build_dataset_card(board, "2026-09-04") + assert "| felix/z-ai/glm-5.1 |" in card + assert "| some-tool/x-ai/grok-4.3 |" in card + assert "(felix/z-ai/glm-5.1)" in card diff --git a/tests/unit/test_role_agents.py b/tests/unit/test_role_agents.py index 710015f..e9dd16d 100644 --- a/tests/unit/test_role_agents.py +++ b/tests/unit/test_role_agents.py @@ -7,16 +7,16 @@ from felix_agent_sdk import AgentFactory from felix_agent_sdk.core import HelixConfig, HelixGeometry -from rle.agents import register_rle_agents from rle.agents.actions import ActionPlan -from rle.agents.base_role import _SHARED_SYSTEM_PREFIX -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer +from rle.harness.felix.agents import register_rle_agents +from rle.harness.felix.agents.base_role import _SHARED_SYSTEM_PREFIX +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.map_analyst import MapAnalyst +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer from rle.rimapi.schemas import GameState # ================================================================== diff --git a/tests/unit/test_testing_smoke.py b/tests/unit/test_testing_smoke.py index 74134ba..f4cff29 100644 --- a/tests/unit/test_testing_smoke.py +++ b/tests/unit/test_testing_smoke.py @@ -5,9 +5,12 @@ import pytest from rle.testing import MockRimAPI, run_harness_smoke +from tests.conftest import requires_felix -@pytest.mark.parametrize("name", ["baseline", "felix"]) +@pytest.mark.parametrize( + "name", ["baseline", pytest.param("felix", marks=requires_felix)], +) async def test_builtin_plugins_pass_smoke(name: str) -> None: report = await run_harness_smoke(name, ticks=2) assert report.ok diff --git a/tests/unit/test_visualizer_integration.py b/tests/unit/test_visualizer_integration.py index 565b8a2..ba05853 100644 --- a/tests/unit/test_visualizer_integration.py +++ b/tests/unit/test_visualizer_integration.py @@ -5,7 +5,7 @@ from felix_agent_sdk.core import HelixConfig from felix_agent_sdk.visualization import HelixVisualizer -from rle.agents import AGENT_DISPLAY +from rle.harness.felix.agents import AGENT_DISPLAY class TestAgentDisplay: diff --git a/uv.lock b/uv.lock index 3232f15..d4be112 100644 --- a/uv.lock +++ b/uv.lock @@ -51,6 +51,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "certifi" version = "2026.2.25" @@ -60,6 +69,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, ] +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.7" @@ -194,6 +262,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, ] +[[package]] +name = "cryptography" +version = "50.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/ad/5d6702db60b1e40b41ef513b6967ff5848f307d50f8449baf1634f5908f1/cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20", size = 880381, upload-time = "2026-08-25T19:45:45.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/19/797e2aaac9df6a66f1550f49979dc1b1e39ecd2077501c30efa81e8d5d67/cryptography-50.0.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986", size = 4010153, upload-time = "2026-08-25T19:44:03.155Z" }, + { url = "https://files.pythonhosted.org/packages/90/34/9ce9a62ed9dc82ca9fd6a34445b6904af56e5f38b3eae2ed32e49c36053d/cryptography-50.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f", size = 4723133, upload-time = "2026-08-25T19:44:05.461Z" }, + { url = "https://files.pythonhosted.org/packages/57/26/e6d4fc8512a51a5f9ee7bfdbfb853bce1197087df40c9ad993ad370b846f/cryptography-50.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef", size = 4712478, upload-time = "2026-08-25T19:44:07.375Z" }, + { url = "https://files.pythonhosted.org/packages/e6/de/d3cdc2815697aae84126cbd6a030ca7b6b452e28a88b501b836bd3aa7a86/cryptography-50.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8", size = 4730726, upload-time = "2026-08-25T19:44:09.294Z" }, + { url = "https://files.pythonhosted.org/packages/55/32/38c0d344b98c06d34b5df8946565a9c0d6dbf32c8e0730a7f05f0a3c6cab/cryptography-50.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45", size = 5353524, upload-time = "2026-08-25T19:44:11.96Z" }, + { url = "https://files.pythonhosted.org/packages/e1/1b/82f0f0d8858d4432be1af790477edf62aef90324041aa07c57e57bef1af7/cryptography-50.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad", size = 4746720, upload-time = "2026-08-25T19:44:14.051Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/042ca458b8c64348c768284b5d23e69b92ed53d057ab779fee628564676d/cryptography-50.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49", size = 4361866, upload-time = "2026-08-25T19:44:16.167Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/e96c1ef71edef71057c7e3c3d982ce8fda554e0c52d0cc19c18845cde3eb/cryptography-50.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f", size = 4730028, upload-time = "2026-08-25T19:44:18.085Z" }, + { url = "https://files.pythonhosted.org/packages/e3/38/45abd72ef63f2e7d0754a6cacf97bd8b69512ace7f6130d24c39ece65da2/cryptography-50.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527", size = 5308405, upload-time = "2026-08-25T19:44:20.197Z" }, + { url = "https://files.pythonhosted.org/packages/85/66/6ccca4722987ddedaa7fc9c3f4708af7431f5535666c174350830888c6b7/cryptography-50.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a", size = 4746230, upload-time = "2026-08-25T19:44:22.376Z" }, + { url = "https://files.pythonhosted.org/packages/13/0e/b1f92e013228111413f2e6743948b80bc24dfd3c1b87ba98ceea16f5df89/cryptography-50.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959", size = 4862596, upload-time = "2026-08-25T19:44:24.472Z" }, + { url = "https://files.pythonhosted.org/packages/7e/22/c3654cccc856e9d682817b04ac3ee79731cb09ca6f95996a95c904de2883/cryptography-50.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b", size = 5014082, upload-time = "2026-08-25T19:44:26.709Z" }, + { url = "https://files.pythonhosted.org/packages/42/8b/cb12b1b60c91b074ca6bf0fdd59aa8f10d8bc5f73af8faece86ef0421b37/cryptography-50.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648", size = 3842826, upload-time = "2026-08-25T19:44:28.784Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f0/424cb557d99aa86ac55da5e2add02e2882e44047b6264f93ade1b975a993/cryptography-50.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f", size = 3973525, upload-time = "2026-08-25T19:44:30.7Z" }, + { url = "https://files.pythonhosted.org/packages/4d/72/3a2711d967977ab5fc80b782837c7e8d1ac7445e764c20c381a265c57ef3/cryptography-50.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a", size = 4708817, upload-time = "2026-08-25T19:44:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f2/bb1f56e10815b789df0b409a69fa4992ff3d3fef9c72747f4a6b26fed38e/cryptography-50.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367", size = 4697300, upload-time = "2026-08-25T19:44:35.144Z" }, + { url = "https://files.pythonhosted.org/packages/08/bd/ed5396be499ffcf8807a585bfe38b71a1fbdd1c342b4f9b6d0ef5162a946/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5", size = 4716039, upload-time = "2026-08-25T19:44:37.192Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6e/1cf405c5c8e8df7545378048e954792f00b7f2367af8863ce8b8f3e10607/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9", size = 5332388, upload-time = "2026-08-25T19:44:39.16Z" }, + { url = "https://files.pythonhosted.org/packages/47/92/b4317e8c32c4f47b062f5398bd79106b220a124546f42be83bf32b761e2a/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0", size = 4730293, upload-time = "2026-08-25T19:44:41.298Z" }, + { url = "https://files.pythonhosted.org/packages/39/0d/a1e7633e2c744d0f2983320a27e924ef2264c79c56e1a58d5fb0a1cfd413/cryptography-50.0.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc", size = 4346031, upload-time = "2026-08-25T19:44:43.245Z" }, + { url = "https://files.pythonhosted.org/packages/88/dd/b215616f9bab3fc18510c78a4e5c9f362d77838503c363dc747c7d4f5c6f/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17", size = 4715344, upload-time = "2026-08-25T19:44:45.291Z" }, + { url = "https://files.pythonhosted.org/packages/b1/1b/ec3ebd31741d0e963612c4fe43caa39341b9b1e031e469820e42e4c83918/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6", size = 5287201, upload-time = "2026-08-25T19:44:47.297Z" }, + { url = "https://files.pythonhosted.org/packages/1a/01/0127d11a762b31a9ee0221894f540318761783f3fdc4bc5d057698caebd5/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3", size = 4730023, upload-time = "2026-08-25T19:44:49.435Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b9/e7425ebfb599241a0c1d7000f1b466c3062da66c19d9525031315dff7213/cryptography-50.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6", size = 4847362, upload-time = "2026-08-25T19:44:51.94Z" }, + { url = "https://files.pythonhosted.org/packages/2d/fd/60d0ddf4defa12e482c9d5e0f554384d6e8ab25341fd15f060028fd92e6a/cryptography-50.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149", size = 4999247, upload-time = "2026-08-25T19:44:53.876Z" }, + { url = "https://files.pythonhosted.org/packages/4d/56/bc4f2b209e766c93372cfcd59b781a0b2b59700f62a969580415b699c2b2/cryptography-50.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf", size = 3825806, upload-time = "2026-08-25T19:44:56.209Z" }, + { url = "https://files.pythonhosted.org/packages/84/a9/ee16a903f13755e914d1eecc482fe64d1f10761c3960e5d8fa6837377aff/cryptography-50.0.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0", size = 4035307, upload-time = "2026-08-25T19:44:58.305Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a5/9ec7e81e8526c0d7a387d73386b2daed3f39e10d81a85930bd1b6bfba65c/cryptography-50.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23", size = 4751900, upload-time = "2026-08-25T19:45:00.401Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3c/0e77bd5ffcf078e9dd27d3074aad6c030d9b10d0bf69329d573c927a188c/cryptography-50.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733", size = 4738357, upload-time = "2026-08-25T19:45:02.786Z" }, + { url = "https://files.pythonhosted.org/packages/27/3a/3c5f80daa4dcd47323c7af8a2fcb90de27a33564d4fcac69846c0972691a/cryptography-50.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88", size = 4758474, upload-time = "2026-08-25T19:45:04.889Z" }, + { url = "https://files.pythonhosted.org/packages/6e/2b/214cf0cf93db9628c3c20c896b229f327f6fb1b20e4b3743d8ad3f00af8b/cryptography-50.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054", size = 5375862, upload-time = "2026-08-25T19:45:07.163Z" }, + { url = "https://files.pythonhosted.org/packages/d6/51/3f9701867a46b6c1740c9b52fc4d3bed6cbdcfedcc9b6e64305c07f39cff/cryptography-50.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5", size = 4772942, upload-time = "2026-08-25T19:45:09.396Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5c/13ea642e08e2544d0f5396122055f4820cfacb3203562197b5967125ea97/cryptography-50.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361", size = 4383347, upload-time = "2026-08-25T19:45:11.659Z" }, + { url = "https://files.pythonhosted.org/packages/84/d5/7d1fe1cb93f91c428093ff234e128c89ba8ea61a6f26aab406081f9b996e/cryptography-50.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71", size = 4758050, upload-time = "2026-08-25T19:45:13.745Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/557fc5ead96a829e0bc812a3b9dc4a52a2f27e4f7f5950da7ff27653a805/cryptography-50.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80", size = 5332955, upload-time = "2026-08-25T19:45:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/eb/5d7124083e8d8cda8f5b348f544b71ad6f707ad63193758ef4d8e569da02/cryptography-50.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239", size = 4772694, upload-time = "2026-08-25T19:45:18.315Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/f1f955e0921dd2b6d22eae7e8d24a4c4b638d10735ffbf6a71f99eb0fcb8/cryptography-50.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558", size = 4888413, upload-time = "2026-08-25T19:45:20.4Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ab/89e2b798d2c3925f82e2bb72d5979f3d2f6da2dd22ef4a8cd8b70d920039/cryptography-50.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e", size = 5044355, upload-time = "2026-08-25T19:45:22.353Z" }, + { url = "https://files.pythonhosted.org/packages/99/89/87ef49ffe383ef4e147d27b7bf2088fb0b54ea409dd87b5a89442e5828a5/cryptography-50.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2", size = 3875429, upload-time = "2026-08-25T19:45:24.418Z" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -364,6 +482,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -379,6 +510,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "huggingface-hub" version = "1.9.2" @@ -401,11 +557,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] [[package]] @@ -450,6 +606,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/8e/7def204fea9f9be8b3c21a6f2dd6c020cf56c7d5ff753e0e23ed7f9ea57e/jiter-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2c26cf47e2cad140fa23b6d58d435a7c0161f5c514284802f25e87fddfe11024", size = 187152, upload-time = "2026-02-02T12:37:22.124Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "kiwisolver" version = "1.5.0" @@ -567,6 +750,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/49/d651878698a0b67f23aa28e17f45a6d6dd3d3f933fa29087fa4ce5947b5a/matplotlib-3.10.8-cp314-cp314t-win_arm64.whl", hash = "sha256:113bb52413ea508ce954a02c10ffd0d565f9c3bc7f2eddc27dfe1731e71c7b5f", size = 8192560, upload-time = "2025-12-10T22:56:38.008Z" }, ] +[[package]] +name = "mcp" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "jsonschema" }, + { name = "mcp-types" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/6e/21fb8e5d579dbe21d96ea4d5034200d46d8bdf2261053b5bd041f3c2f612/mcp-2.1.1.tar.gz", hash = "sha256:50b7ba1ebbe117008ea7bdd288234043e69c20b403d6851d19661e6d431a75ef", size = 3984589, upload-time = "2026-08-25T16:14:02.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/af/8644cc5fa26a59afd2df2e98eeb19e72926887fa4b7441aba4ff661140db/mcp-2.1.1-py3-none-any.whl", hash = "sha256:1c6c31c5d6471c58db76af3af8af67f46d11d01f0a59077d0a308cbdb3d3e915", size = 357912, upload-time = "2026-08-25T16:13:59.024Z" }, +] + +[[package]] +name = "mcp-types" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/dd/1c4417dc0b722c23a1669032d5f044e41170fe5d4773b488a50fcce98c32/mcp_types-2.1.1.tar.gz", hash = "sha256:77dcbe48fba73cca71a673f2646a5f037a017b7a0a07ac89cec1113028890eda", size = 66674, upload-time = "2026-08-25T16:14:03.861Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/d0/242e63c510f4a17381f55b1549a3f94f5687a0595984febd2b6f87a687a0/mcp_types-2.1.1-py3-none-any.whl", hash = "sha256:26f9f7f03f2a5730717a5b98e2ab7eb640ac352d05a00cdc725c311864778295", size = 69656, upload-time = "2026-08-25T16:14:00.667Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -662,6 +883,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/bc/a8f7c3aa03452fedbb9af8be83e959adba96a6b4a35e416faffcc959c568/openai-2.31.0-py3-none-any.whl", hash = "sha256:44e1344d87e56a493d649b17e2fac519d1368cbb0745f59f1957c4c26de50a0a", size = 1153479, upload-time = "2026-04-08T21:01:39.217Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -746,6 +979,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -823,6 +1065,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyparsing" version = "3.3.2" @@ -895,6 +1151,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, +] + +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -921,6 +1199,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "requests" version = "2.33.1" @@ -954,7 +1245,6 @@ name = "rimworld-learning-environment" version = "0.4.1" source = { editable = "." } dependencies = [ - { name = "felix-agent-sdk" }, { name = "httpx" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -976,9 +1266,15 @@ dev = [ { name = "ruff" }, { name = "types-pyyaml" }, ] +felix = [ + { name = "felix-agent-sdk" }, +] local = [ { name = "felix-agent-sdk", extra = ["local"] }, ] +mcp = [ + { name = "mcp" }, +] openai = [ { name = "felix-agent-sdk", extra = ["openai"] }, ] @@ -992,7 +1288,7 @@ viz = [ [package.metadata] requires-dist = [ - { name = "felix-agent-sdk", specifier = ">=0.3.0" }, + { name = "felix-agent-sdk", marker = "extra == 'felix'", specifier = ">=0.3.0" }, { name = "felix-agent-sdk", extras = ["all"], marker = "extra == 'all'" }, { name = "felix-agent-sdk", extras = ["anthropic"], marker = "extra == 'anthropic'" }, { name = "felix-agent-sdk", extras = ["local"], marker = "extra == 'local'" }, @@ -1000,6 +1296,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.24" }, { name = "huggingface-hub", marker = "extra == 'tracking'", specifier = ">=0.20" }, { name = "matplotlib", marker = "extra == 'viz'", specifier = ">=3.5" }, + { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.2" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pydantic-settings", specifier = ">=2.0" }, @@ -1011,7 +1308,73 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0" }, { name = "wandb", marker = "extra == 'tracking'", specifier = ">=0.15" }, ] -provides-extras = ["anthropic", "openai", "local", "all", "dev", "viz", "tracking"] +provides-extras = ["felix", "anthropic", "openai", "local", "all", "mcp", "dev", "viz", "tracking"] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, +] [[package]] name = "ruff" @@ -1087,6 +1450,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sse-starlette" +version = "3.4.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/8a41e88e825ea26c44333897c7ffe35fe60153a2cfc097a5bd1d209ad281/sse_starlette-3.4.10.tar.gz", hash = "sha256:c6c87280d8feb4e55a8d79633782766b9cac6a26da5c79a145d00aa404117a86", size = 33720, upload-time = "2026-09-03T09:36:24.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/3c/96018a51c7301a64f7b0579d9ce8f9b69dd39ca8ed5aa100ba3feadee503/sse_starlette-3.4.10-py3-none-any.whl", hash = "sha256:710f5f5b0527409903a22a91699db02f76f4c2eb9204e882e4ee7cada76bdf75", size = 17120, upload-time = "2026-09-03T09:36:22.56Z" }, +] + +[[package]] +name = "starlette" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/b4/205b0d5241d934e8add0c38aa924c4f9fb7330834ff11e5444db964ec3f9/starlette-1.6.0.tar.gz", hash = "sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b", size = 2716969, upload-time = "2026-08-08T18:27:57.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl", hash = "sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c", size = 75969, upload-time = "2026-08-08T18:27:56.196Z" }, +] + [[package]] name = "tqdm" version = "4.67.3" @@ -1099,6 +1487,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typer" version = "0.24.1" @@ -1153,6 +1550,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] +[[package]] +name = "uvicorn" +version = "0.52.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/0f/3f86e61397dd33bf2ccf28188c40db6a740658aeebbbf6e7dbc101a1f487/uvicorn-0.52.4.tar.gz", hash = "sha256:73acfee47a0b133c5de13d219492d62d8a31e935f4fe6e41a232451a15379f86", size = 100627, upload-time = "2026-08-19T06:27:41.821Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/79/4a20b54ab0491485ccd8c077db2d39187c7f12b3e15485d38a7be37c81b4/uvicorn-0.52.4-py3-none-any.whl", hash = "sha256:f86e41a149d7d05a9969337e3946a9c171c06a5d42680896daaba624aeac8da1", size = 79871, upload-time = "2026-08-19T06:27:40.36Z" }, +] + [[package]] name = "wandb" version = "0.25.1" From d285b7d6d527d30904f3b51cb6935c28d2a4c101 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 07:23:24 +0000 Subject: [PATCH 5/8] rle.mcp: RimAPI as an MCP tool server with a per-tick write ledger Tool-using harnesses act during their turn, so the environment cannot execute for them. The MCP server executes each write immediately through ActionExecutor (same normalisation and guards as Felix) and records it in a TickLedger; the harness drains the ledger into StepResult(execution=...) and the loop scores what actually reached RIMAPI. - rle.mcp.ledger / session: framework-free ledger + tool logic (writes outside a tick rejected; pawn-targeted writes without an id fail loudly instead of the executor's silent skip) - rle.mcp.server: MCPServer with get_brief / get_state / list_actions / list_reads / rimapi_read / end_turn plus one tool per WRITE_CATALOG entry - rle.mcp.host: in-process streamable-HTTP host (uvicorn, free port) so an external agent's MCP client and the loop share one ledger - rle-mcp console script: stdio server for manual play against a live game - optional extra mcp>=2.1; tests cover in-memory tool calls and a real HTTP round trip via mcp.client.Client Co-authored-by: Jason --- pyproject.toml | 5 +- src/rle/mcp/__init__.py | 12 ++ src/rle/mcp/__main__.py | 51 ++++++++ src/rle/mcp/host.py | 68 ++++++++++ src/rle/mcp/ledger.py | 86 +++++++++++++ src/rle/mcp/server.py | 110 ++++++++++++++++ src/rle/mcp/session.py | 130 +++++++++++++++++++ src/rle/orchestration/action_executor.py | 4 +- tests/unit/test_mcp_server.py | 157 +++++++++++++++++++++++ uv.lock | 2 +- 10 files changed, 622 insertions(+), 3 deletions(-) create mode 100644 src/rle/mcp/__init__.py create mode 100644 src/rle/mcp/__main__.py create mode 100644 src/rle/mcp/host.py create mode 100644 src/rle/mcp/ledger.py create mode 100644 src/rle/mcp/server.py create mode 100644 src/rle/mcp/session.py create mode 100644 tests/unit/test_mcp_server.py diff --git a/pyproject.toml b/pyproject.toml index c137bf1..d35bfd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ openai = ["felix-agent-sdk[openai]"] local = ["felix-agent-sdk[local]"] all = ["felix-agent-sdk[all]"] # RimAPI MCP server for tool-using harnesses (`rle-mcp`). -mcp = ["mcp>=1.2"] +mcp = ["mcp>=2.1"] dev = [ "pytest>=7.0", "pytest-asyncio>=0.24", @@ -51,6 +51,9 @@ tracking = [ baseline = "rle.harness.baseline:PLUGIN" felix = "rle.harness.felix:PLUGIN" +[project.scripts] +rle-mcp = "rle.mcp.__main__:main" + [tool.hatch.build.targets.wheel] packages = ["src/rle"] diff --git a/src/rle/mcp/__init__.py b/src/rle/mcp/__init__.py new file mode 100644 index 0000000..0eb39cc --- /dev/null +++ b/src/rle/mcp/__init__.py @@ -0,0 +1,12 @@ +"""RimAPI as an MCP tool surface for external coding-agent harnesses. + +``ledger`` and ``session`` have no ``mcp`` dependency; ``server`` / ``host`` +need the ``mcp`` extra. Run standalone with ``rle-mcp`` (stdio) for manual +play against a live game, or let a harness host it in-process (streamable +HTTP) so the ledger and the loop share memory. +""" + +from rle.mcp.ledger import NoActiveTickError, TickLedger +from rle.mcp.session import McpSession + +__all__ = ["McpSession", "NoActiveTickError", "TickLedger"] diff --git a/src/rle/mcp/__main__.py b/src/rle/mcp/__main__.py new file mode 100644 index 0000000..f892bf6 --- /dev/null +++ b/src/rle/mcp/__main__.py @@ -0,0 +1,51 @@ +"""``rle-mcp`` — stdio MCP server against a live RIMAPI, for manual play. + +Standalone mode has no environment loop: every call is "in a tick", writes go +straight to the game, and the ledger is only informational. Benchmark runs +use ``rle.mcp.host.McpHost`` inside a harness instead. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging + +from rle.config import RLEConfig +from rle.harness.brief import build_brief +from rle.mcp.ledger import TickLedger +from rle.mcp.server import build_server +from rle.mcp.session import McpSession +from rle.orchestration.action_executor import ActionExecutor +from rle.orchestration.state_manager import GameStateManager +from rle.rimapi.client import RimAPIClient + + +async def _serve(rimapi_url: str) -> None: + async with RimAPIClient(rimapi_url) as client: + ledger = TickLedger(harness_name="manual") + session = McpSession(client=client, executor=ActionExecutor(client), ledger=ledger) + manager = GameStateManager(client, expected_duration_days=30) + try: + state = await manager.refresh() + session.begin_tick(0, state, build_brief(state, tick=0, macro_time=0.0)) + except Exception: + logging.getLogger(__name__).warning( + "RIMAPI not reachable at %s — serving with an empty brief", rimapi_url, + ) + ledger.active = True + await build_server(session).run_stdio_async() + + +def main() -> None: + parser = argparse.ArgumentParser(description="RLE RimAPI MCP server (stdio)") + parser.add_argument("--rimapi-url", default=None, help="Default: RIMAPI_URL / config") + parser.add_argument("--log-level", default="WARNING") + args = parser.parse_args() + logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING)) + config = RLEConfig() + asyncio.run(_serve(args.rimapi_url or config.rimapi_url)) + + +if __name__ == "__main__": + main() diff --git a/src/rle/mcp/host.py b/src/rle/mcp/host.py new file mode 100644 index 0000000..288cce8 --- /dev/null +++ b/src/rle/mcp/host.py @@ -0,0 +1,68 @@ +"""Host the RLE MCP server in-process over streamable HTTP. + +Harnesses that drive an external coding agent start one of these so the +agent's MCP client and the environment share a single ledger in memory. +""" + +from __future__ import annotations + +import asyncio +import socket + +import uvicorn +from mcp.server.mcpserver import MCPServer + +MCP_PATH = "/mcp" + + +def free_port(host: str = "127.0.0.1") -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind((host, 0)) + return int(s.getsockname()[1]) + + +class McpHost: + def __init__(self, server: MCPServer, *, host: str = "127.0.0.1", port: int = 0) -> None: + self._server = server + self._host = host + self._port = port or free_port(host) + self._uvicorn: uvicorn.Server | None = None + self._task: asyncio.Task[None] | None = None + + @property + def url(self) -> str: + return f"http://{self._host}:{self._port}{MCP_PATH}" + + @property + def running(self) -> bool: + return self._task is not None and not self._task.done() + + async def start(self, *, timeout_s: float = 10.0) -> str: + if self.running: + return self.url + app = self._server.streamable_http_app( + streamable_http_path=MCP_PATH, host=self._host, stateless_http=True, + ) + config = uvicorn.Config(app, host=self._host, port=self._port, log_level="warning") + self._uvicorn = uvicorn.Server(config) + self._task = asyncio.create_task(self._uvicorn.serve()) + deadline = asyncio.get_running_loop().time() + timeout_s + while not self._uvicorn.started: + if self._task.done(): + self._task.result() # re-raise startup failure + raise RuntimeError("MCP host exited before starting") + if asyncio.get_running_loop().time() > deadline: + raise TimeoutError("MCP host did not start in time") + await asyncio.sleep(0.02) + return self.url + + async def stop(self) -> None: + if self._uvicorn is not None: + self._uvicorn.should_exit = True + if self._task is not None: + try: + await asyncio.wait_for(self._task, timeout=10.0) + except (asyncio.TimeoutError, asyncio.CancelledError): + self._task.cancel() + self._task = None + self._uvicorn = None diff --git a/src/rle/mcp/ledger.py b/src/rle/mcp/ledger.py new file mode 100644 index 0000000..ab28bea --- /dev/null +++ b/src/rle/mcp/ledger.py @@ -0,0 +1,86 @@ +"""Per-tick write ledger shared between the MCP tools and the harness. + +Tool-using harnesses act *during* their turn (they need tool results to +decide the next call), so nothing is left for the loop to execute. The ledger +records every write attempted in the current tick and turns it into the +``StepResult`` the environment scores. No ``mcp`` import here — the harness +base and the scorer depend on this module even when the MCP extra is absent. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any + +from rle.agents.actions import Action, ActionOutcome, ActionPlan, ExecutionResult +from rle.harness.protocol import StepResult + + +class NoActiveTickError(RuntimeError): + """A write arrived while the environment was not inside a harness step.""" + + +@dataclass +class TickLedger: + harness_name: str = "mcp" + active: bool = False + tick: int = -1 + game_tick: int = 0 + actions: list[Action] = field(default_factory=list) + outcomes: list[ActionOutcome] = field(default_factory=list) + summary: str = "" + extras: dict[str, Any] = field(default_factory=dict) + turn_done: asyncio.Event = field(default_factory=asyncio.Event) + + def begin(self, tick: int, game_tick: int) -> None: + self.active = True + self.tick = tick + self.game_tick = game_tick + self.actions = [] + self.outcomes = [] + self.summary = "" + self.extras = {} + self.turn_done = asyncio.Event() + + def require_active(self) -> None: + if not self.active: + raise NoActiveTickError( + "No tick is in progress — the environment is between turns. " + "Wait for the next prompt before acting.", + ) + + def record(self, action: Action, outcome: ActionOutcome | None) -> None: + self.require_active() + self.actions.append(action) + if outcome is not None: + self.outcomes.append(outcome) + + def end_turn(self, summary: str = "") -> None: + if summary: + self.summary = summary + self.turn_done.set() + + def finish(self) -> StepResult: + """Close the tick and package what happened as a StepResult.""" + self.active = False + executed = sum(1 for o in self.outcomes if o.success) + failed = len(self.outcomes) - executed + plan = ActionPlan( + role=self.harness_name, + tick=self.game_tick, + actions=list(self.actions), + summary=self.summary or f"{len(self.actions)} tool call(s)", + ) + execution = ExecutionResult( + executed=executed, + failed=failed, + total=len(self.outcomes), + outcomes=tuple(self.outcomes), + ) + return StepResult( + plan=plan, + execution=execution, + proposals=(plan,), + extras={"turn_ended": self.turn_done.is_set(), **self.extras}, + ) diff --git a/src/rle/mcp/server.py b/src/rle/mcp/server.py new file mode 100644 index 0000000..b730540 --- /dev/null +++ b/src/rle/mcp/server.py @@ -0,0 +1,110 @@ +"""RimAPI as an MCP tool server (requires the ``mcp`` extra). + +One tool per ``WRITE_CATALOG`` entry (executed immediately through +``ActionExecutor`` and recorded in the tick ledger), a generic read tool over +``READ_CATALOG``, the harness-neutral brief, and ``end_turn``. Any MCP-capable +coding agent can attach and play; the harness packages that do so live in +their own repositories. +""" + +from __future__ import annotations + +import json +from typing import Any, cast + +from mcp.server.mcpserver import MCPServer + +from rle.mcp.session import McpSession +from rle.rimapi.api_catalog import READ_CATALOG, WRITE_CATALOG + +SERVER_NAME = "rle" +INSTRUCTIONS = ( + "You are managing a RimWorld colony through the RLE benchmark environment. " + "Each turn: call get_brief to see the scenario, colony state, MAP_SUMMARY and " + "available actions; issue writes with the action tools (use MAP_SUMMARY " + "coordinates verbatim, never invent positions; pawn ids are integers); then " + "call end_turn with a one-line summary. Writes outside a turn are rejected." +) + + +def _dumps(value: Any) -> str: + return json.dumps(value, indent=2, default=str) + + +def build_server(session: McpSession) -> MCPServer: + server = MCPServer(SERVER_NAME, instructions=INSTRUCTIONS) + + @server.tool( + description="Scenario goals, colony state, MAP_SUMMARY and action catalog for this turn.", + ) + def get_brief() -> str: + return session.brief_text() + + @server.tool( + description="Current colony state as JSON (same data as get_brief, machine-readable).", + ) + def get_state() -> str: + return _dumps(session.state_json()) + + @server.tool(description="List every write action with its parameter shape.") + def list_actions() -> str: + return _dumps(session.actions()) + + @server.tool(description="List RIMAPI read endpoints available to rimapi_read.") + def list_reads() -> str: + return _dumps({ + name: cast(dict[str, Any], entry).get("description", "") + for name, entry in sorted(READ_CATALOG.items()) + }) + + @server.tool( + description=( + "Read a RIMAPI endpoint by catalog name (see list_reads). " + "params become query-string parameters, e.g. {\"map_id\": 0}." + ), + ) + async def rimapi_read(endpoint: str, params: dict[str, Any] | None = None) -> str: + try: + return _dumps(await session.read(endpoint, params)) + except Exception as exc: # surfaced to the agent, never crashes the server + return _dumps({"error": str(exc)}) + + @server.tool( + description=( + "Finish this turn. Call once you have issued all writes for the tick; " + "the environment then advances the game and scores the tick." + ), + ) + def end_turn(summary: str = "") -> str: + session.ledger.end_turn(summary) + n = len(session.ledger.actions) + return f"Turn ended after {n} action(s)." + + for name, raw in sorted(WRITE_CATALOG.items()): + entry = cast(dict[str, Any], raw) + server.add_tool( + _make_action_tool(session, name), + name=name, + description=( + f"{entry.get('description', name)}. " + f"parameters shape: {json.dumps(entry.get('params', {}), default=str)}. " + "Executes immediately and returns {ok, error}." + ), + ) + return server + + +def _make_action_tool(session: McpSession, action_type: str) -> Any: + async def tool( + parameters: dict[str, Any] | None = None, + target_colonist_id: str | None = None, + reason: str = "", + ) -> str: + try: + result = await session.act(action_type, parameters, target_colonist_id, reason) + except Exception as exc: + result = {"ok": False, "action_type": action_type, "error": str(exc)} + return _dumps(result) + + tool.__name__ = action_type + return tool diff --git a/src/rle/mcp/session.py b/src/rle/mcp/session.py new file mode 100644 index 0000000..c4a76e9 --- /dev/null +++ b/src/rle/mcp/session.py @@ -0,0 +1,130 @@ +"""Shared state between the environment and the MCP tool handlers.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any, cast + +from rle.agents.actions import Action, ActionOutcome, ActionPlan, resolve_endpoint +from rle.harness.brief import ScenarioBrief, action_catalog +from rle.mcp.ledger import TickLedger +from rle.orchestration.action_executor import NEEDS_PAWN, ActionExecutor +from rle.rimapi.api_catalog import READ_CATALOG +from rle.rimapi.client import RimAPIClient +from rle.rimapi.schemas import GameState +from rle.tracking.event_log import EventType + +EmitFn = Callable[..., None] + + +def _noop_emit(*_args: Any, **_kwargs: Any) -> None: + return None + + +@dataclass +class McpSession: + """What the tools need: a RIMAPI client, an executor, the current tick's + brief/state, and the ledger to record into.""" + + client: RimAPIClient + executor: ActionExecutor + ledger: TickLedger + brief: ScenarioBrief | None = None + state: GameState | None = None + emit: EmitFn = _noop_emit + extras: dict[str, Any] = field(default_factory=dict) + + def begin_tick( + self, tick: int, state: GameState, brief: ScenarioBrief | None, + ) -> None: + self.state = state + self.brief = brief + self.ledger.begin(tick, state.colony.tick) + + async def act( + self, + action_type: str, + parameters: dict[str, Any] | None = None, + target_colonist_id: str | None = None, + reason: str = "", + ) -> dict[str, Any]: + """Execute one write immediately, record it, and return the outcome.""" + self.ledger.require_active() + action = Action( + action_type=action_type, + target_colonist_id=target_colonist_id, + parameters=dict(parameters or {}), + reason=reason, + ) + endpoint = resolve_endpoint(action_type) + if endpoint == "no_action": + self.ledger.record(action, None) + return {"ok": True, "action_type": action_type, "note": "no-op recorded"} + outcome: ActionOutcome + if endpoint in NEEDS_PAWN and not target_colonist_id: + # The executor would silently skip this; tell the agent instead. + outcome = ActionOutcome( + action_type=action_type, endpoint=endpoint, target_colonist_id=None, + success=False, error="target_colonist_id is required for this action", + parameters=action.parameters, + ) + self.ledger.record(action, outcome) + return {"ok": False, "action_type": action_type, "endpoint": endpoint, + "error": outcome.error} + result = await self.executor.execute( + ActionPlan(role=self.ledger.harness_name, tick=self.ledger.game_tick, actions=[action]), + ) + if result.outcomes: + outcome = result.outcomes[0] + else: + # The executor skipped it (unknown endpoint). + outcome = ActionOutcome( + action_type=action_type, endpoint=endpoint, + target_colonist_id=target_colonist_id, success=False, + error="skipped by executor (unknown action)", + parameters=action.parameters, + ) + self.ledger.record(action, outcome) + self.emit( + EventType.ACTION_EXEC, self.ledger.tick, + action_type=outcome.action_type, target=outcome.target_colonist_id, + success=outcome.success, error=outcome.error, parameters=outcome.parameters, + via="mcp", + ) + return { + "ok": outcome.success, + "action_type": action_type, + "endpoint": endpoint, + "error": outcome.error, + } + + async def read(self, endpoint: str, params: dict[str, Any] | None = None) -> Any: + """GET a READ_CATALOG endpoint by name.""" + raw = READ_CATALOG.get(endpoint) + if raw is None: + known = ", ".join(sorted(READ_CATALOG)) + raise KeyError(f"Unknown read endpoint {endpoint!r}. Known: {known}") + entry = cast(dict[str, Any], raw) + path = str(entry["path"]) + if params: + query = "&".join(f"{k}={v}" for k, v in params.items()) + path = f"{path}?{query}" + return await self.client.call(str(entry.get("method", "GET")), path) + + def brief_text(self) -> str: + if not self.ledger.active or self.brief is None: + return ( + "No tick is in progress. The environment will prompt you when the " + "next turn starts." + ) + return self.brief.to_text() + + def state_json(self) -> dict[str, Any]: + if self.brief is None: + return {} + return dict(self.brief.state) + + @staticmethod + def actions() -> list[dict[str, Any]]: + return action_catalog() diff --git a/src/rle/orchestration/action_executor.py b/src/rle/orchestration/action_executor.py index cbee9b7..b1478eb 100644 --- a/src/rle/orchestration/action_executor.py +++ b/src/rle/orchestration/action_executor.py @@ -16,7 +16,7 @@ from rle.rimapi.api_catalog import WRITE_CATALOG from rle.rimapi.client import RimAPIClient, RimAPIResponseError -__all__ = ["ActionExecutor", "ActionOutcome", "ExecutionResult"] +__all__ = ["NEEDS_PAWN", "ActionExecutor", "ActionOutcome", "ExecutionResult"] logger = logging.getLogger(__name__) @@ -31,6 +31,8 @@ "bed_rest", "tend", }) +# Public alias for callers that validate before dispatch (MCP tools). +NEEDS_PAWN = _NEEDS_PAWN def _extract_rimapi_error(detail: str) -> str: diff --git a/tests/unit/test_mcp_server.py b/tests/unit/test_mcp_server.py new file mode 100644 index 0000000..8fa9689 --- /dev/null +++ b/tests/unit/test_mcp_server.py @@ -0,0 +1,157 @@ +"""RimAPI MCP server: tools execute immediately and land in the tick ledger.""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import pytest + +from rle.agents.actions import Action, ActionOutcome +from rle.harness.brief import build_brief +from rle.mcp import McpSession, NoActiveTickError, TickLedger +from rle.orchestration.action_executor import ActionExecutor +from rle.orchestration.state_manager import GameStateManager +from rle.rimapi.client import RimAPIClient +from rle.scenarios.loader import list_scenarios +from rle.testing import MockRimAPI + +mcp_server = pytest.importorskip("rle.mcp.server") +mcp_host = pytest.importorskip("rle.mcp.host") +mcp_client = pytest.importorskip("mcp.client.client") + + +@asynccontextmanager +async def _session() -> AsyncIterator[tuple[McpSession, MockRimAPI]]: + mock = MockRimAPI() + async with RimAPIClient("http://mock") as client: + mock.attach(client) + ledger = TickLedger(harness_name="test-tool") + session = McpSession(client=client, executor=ActionExecutor(client), ledger=ledger) + state = await GameStateManager(client, 30).refresh() + brief = build_brief(state, tick=0, macro_time=0.0, scenario=list_scenarios()[0]) + session.begin_tick(0, state, brief) + yield session, mock + + +def _text(result: object) -> str: + content = getattr(result, "content", None) + assert content, f"no content in {result!r}" + return str(content[0].text) + + +class TestLedger: + def test_writes_outside_tick_rejected(self) -> None: + ledger = TickLedger() + with pytest.raises(NoActiveTickError): + ledger.require_active() + + def test_finish_packages_step_result(self) -> None: + ledger = TickLedger(harness_name="x") + ledger.begin(3, 1800) + ledger.record( + Action(action_type="draft", target_colonist_id="1", parameters={"is_drafted": True}), + ActionOutcome(action_type="draft", endpoint="draft", target_colonist_id="1", + success=True, parameters={"is_drafted": True}), + ) + ledger.record( + Action(action_type="blueprint", parameters={}), + ActionOutcome(action_type="blueprint", endpoint="blueprint", success=False, + error="no coords"), + ) + ledger.end_turn("done") + step = ledger.finish() + assert not ledger.active + assert step.plan.role == "x" and step.plan.tick == 1800 + assert step.plan.summary == "done" + assert step.execution is not None + assert (step.execution.executed, step.execution.failed, step.execution.total) == (1, 1, 2) + assert step.extras["turn_ended"] is True + + +class TestToolsInMemory: + async def test_brief_state_actions_and_reads(self) -> None: + async with _session() as (session, _mock): + server = mcp_server.build_server(session) + names = {t.name for t in await server.list_tools()} + assert {"get_brief", "get_state", "list_actions", "rimapi_read", "end_turn"} <= names + assert {"work_priority", "draft", "blueprint", "growing_zone"} <= names + + brief = _text(await server.call_tool("get_brief", {})) + assert "## Scenario" in brief and "## Actions available" in brief + state = json.loads(_text(await server.call_tool("get_state", {}))) + assert state["colony"]["population"] == 3 + reads = json.loads(_text(await server.call_tool("list_reads", {}))) + assert "colonists" in reads + colonists = json.loads(_text(await server.call_tool( + "rimapi_read", {"endpoint": "colonists"}, + ))) + assert isinstance(colonists, list) and colonists[0]["name"] == "Tynan" + bad = json.loads(_text(await server.call_tool( + "rimapi_read", {"endpoint": "nope"}, + ))) + assert "Unknown read endpoint" in bad["error"] + + async def test_write_tool_executes_and_records(self) -> None: + async with _session() as (session, mock): + server = mcp_server.build_server(session) + out = json.loads(_text(await server.call_tool( + "work_priority", + {"parameters": {"Growing": 1}, "target_colonist_id": "col_01", "reason": "food"}, + ))) + assert out["ok"] is True + assert any(p.endswith("/colonist/work-priority") for p, _ in mock.posts) + assert len(session.ledger.actions) == 1 + assert session.ledger.outcomes[0].success + + ended = _text(await server.call_tool("end_turn", {"summary": "ok"})) + assert "Turn ended after 1 action(s)" in ended + step = session.ledger.finish() + assert step.execution is not None and step.execution.executed == 1 + assert step.plan.summary == "ok" + + async def test_write_after_turn_closed_is_rejected(self) -> None: + async with _session() as (session, _mock): + server = mcp_server.build_server(session) + session.ledger.finish() + out = json.loads(_text(await server.call_tool( + "draft", {"parameters": {"is_drafted": True}, "target_colonist_id": "col_01"}, + ))) + assert out["ok"] is False and "No tick is in progress" in out["error"] + assert "No tick is in progress" in _text(await server.call_tool("get_brief", {})) + + async def test_no_action_is_recorded_without_execution(self) -> None: + async with _session() as (session, mock): + before = len(mock.posts) + result = await session.act("no_action", reason="nothing to do") + assert result["ok"] is True + assert len(mock.posts) == before + assert len(session.ledger.actions) == 1 and session.ledger.outcomes == [] + + async def test_missing_pawn_id_reports_failure(self) -> None: + async with _session() as (session, _mock): + result = await session.act("draft", {"is_drafted": True}) + assert result["ok"] is False + assert "target_colonist_id is required" in (result["error"] or "") + assert session.ledger.outcomes[0].success is False + + +class TestHttpHost: + async def test_client_over_streamable_http(self) -> None: + async with _session() as (session, _mock): + host = mcp_host.McpHost(mcp_server.build_server(session)) + url = await host.start() + try: + assert url.endswith("/mcp") + async with mcp_client.Client(url) as client: + tools = await client.list_tools() + assert any(t.name == "end_turn" for t in tools.tools) + result = await client.call_tool( + "research_target", {"parameters": {"project": "Electricity"}}, + ) + assert json.loads(_text(result))["ok"] is True + assert len(session.ledger.actions) == 1 + finally: + await host.stop() + assert not host.running diff --git a/uv.lock b/uv.lock index d4be112..d9053fe 100644 --- a/uv.lock +++ b/uv.lock @@ -1296,7 +1296,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.24" }, { name = "huggingface-hub", marker = "extra == 'tracking'", specifier = ">=0.20" }, { name = "matplotlib", marker = "extra == 'viz'", specifier = ">=3.5" }, - { name = "mcp", marker = "extra == 'mcp'", specifier = ">=1.2" }, + { name = "mcp", marker = "extra == 'mcp'", specifier = ">=2.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pydantic-settings", specifier = ">=2.0" }, From 9ad3dc754e547e03f873870120fe43d3fa61bc7f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 07:26:36 +0000 Subject: [PATCH 6/8] HeadlessCliHarness scaffold + ScriptedMcpHarness for external coding-agent harnesses rle.harness.cli_base.HeadlessCliHarness is the tool-agnostic base that harness packages wrapping a CLI coding agent subclass. It hosts the RLE MCP server in-process, builds the neutral brief and turn prompt, runs the turn protocol (prompt -> agent acts through tools -> end_turn or idle grace), applies turn_timeout_s, drains the ledger into StepResult(execution=...), and records latency, tokens and the deliberation log. Subclasses implement start_agent / send_turn / stop_agent only. rle.testing.scripted_agent.ScriptedMcpHarness plays a fixed tool script over a real MCP client connection; external plugins return it from smoke() so CI exercises the full round trip without their binary. Integration tests run it through RLEGameLoop against the mock RIMAPI and cover turn timeout / agent error degradation. Co-authored-by: Jason --- src/rle/harness/cli_base.py | 270 +++++++++++++++++++++ src/rle/testing/__init__.py | 3 + src/rle/testing/scripted_agent.py | 67 +++++ tests/integration/test_cli_base_harness.py | 110 +++++++++ 4 files changed, 450 insertions(+) create mode 100644 src/rle/harness/cli_base.py create mode 100644 src/rle/testing/scripted_agent.py create mode 100644 tests/integration/test_cli_base_harness.py diff --git a/src/rle/harness/cli_base.py b/src/rle/harness/cli_base.py new file mode 100644 index 0000000..15c71d2 --- /dev/null +++ b/src/rle/harness/cli_base.py @@ -0,0 +1,270 @@ +"""Scaffold for harnesses that drive an external coding agent over MCP. + +Tool-agnostic by design: a concrete harness (in its own package) implements +how to start the agent, hand it a prompt, and stop it. This base owns +everything else — hosting the RLE MCP server in-process, the per-tick brief, +the turn protocol (prompt → agent acts through tools → ``end_turn`` or idle +→ ledger drained into ``StepResult``), timeouts, cost/latency accounting, +and the deliberation log — so every CLI harness is scored identically. + +Requires the ``mcp`` extra. +""" + +from __future__ import annotations + +import asyncio +import logging +import time as _time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any, ClassVar + +from pydantic import BaseModel, ConfigDict, Field + +from rle.harness.brief import ScenarioBrief, build_brief +from rle.harness.protocol import BaseHarness, HarnessContext, HarnessStepError, StepResult +from rle.mcp.host import McpHost +from rle.mcp.ledger import TickLedger +from rle.mcp.server import build_server +from rle.mcp.session import McpSession +from rle.orchestration.action_executor import ActionExecutor +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.tracking.event_log import EventType + +logger = logging.getLogger(__name__) + +_RAW_OUTPUT_CHARS = 16384 + +TURN_RULES = ( + "Rules for this turn:\n" + "1. Call get_brief first. Use MAP_SUMMARY coordinates verbatim — never invent positions.\n" + "2. Act only through the RLE tools (never shell into the game). Pawn ids are integers.\n" + "3. Each tool call executes immediately and returns {ok, error}; do not repeat a failed call " + "with the same arguments.\n" + "4. When you have issued this tick's writes, call end_turn with a one-line summary. " + "Doing nothing is allowed — still call end_turn.\n" +) + + +class HeadlessCliOptions(BaseModel): + """Options common to every CLI-agent harness; subclasses extend.""" + + model_config = ConfigDict(extra="forbid") + + model: str | None = Field( + default=None, + description="Model identifier handed to the agent (defaults to RLEConfig.model).", + ) + turn_timeout_s: float = Field( + default=180.0, + description="Hard cap on one turn. The agent is aborted and the tick scored as-is.", + ) + idle_grace_s: float = Field( + default=3.0, + description=( + "If the agent finishes responding without calling end_turn, wait this long " + "for late tool calls before closing the tick." + ), + ) + extra_instructions: str = Field( + default="", + description="Appended to every turn prompt (harness-side prompt engineering).", + ) + + +@dataclass +class TurnResult: + """What the agent produced for one prompt.""" + + text: str = "" + prompt_tokens: int = 0 + completion_tokens: int = 0 + reasoning_tokens: int = 0 + extras: dict[str, Any] = field(default_factory=dict) + + +class HeadlessCliHarness(BaseHarness, ABC): + name: ClassVar[str] = "headless-cli" + + def __init__(self, options: HeadlessCliOptions) -> None: + super().__init__() + self.options = options + self._session: McpSession | None = None + self._host: McpHost | None = None + self._agent_started = False + + # ------------------------------------------------------------------ + # Tool-specific hooks + # ------------------------------------------------------------------ + + @abstractmethod + async def start_agent(self, mcp_url: str) -> None: + """Launch or attach to the agent and register the RLE MCP server.""" + + @abstractmethod + async def send_turn(self, prompt: str) -> TurnResult: + """Deliver one turn prompt and return when the agent has finished responding.""" + + @abstractmethod + async def stop_agent(self) -> None: + """Abort any in-flight turn and shut the agent down.""" + + async def abort_turn(self) -> None: + """Called on timeout; default just stops and restarts nothing.""" + return None + + def agent_versions(self) -> dict[str, str]: + """Version info for run metadata (binary --version etc.).""" + return {} + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + @property + def session(self) -> McpSession: + if self._session is None: + raise RuntimeError("setup() has not been called") + return self._session + + @property + def mcp_url(self) -> str: + if self._host is None: + raise RuntimeError("setup() has not been called") + return self._host.url + + async def setup(self, ctx: HarnessContext) -> None: + await super().setup(ctx) + ledger = TickLedger(harness_name=self.name) + self._session = McpSession( + client=ctx.client, executor=ActionExecutor(ctx.client), ledger=ledger, emit=ctx.emit, + ) + self._host = McpHost(build_server(self._session)) + url = await self._host.start() + await self.start_agent(url) + self._agent_started = True + + async def teardown(self) -> None: + if self._agent_started: + try: + await self.stop_agent() + except Exception: + logger.debug("stop_agent failed", exc_info=True) + self._agent_started = False + if self._host is not None: + await self._host.stop() + + def describe(self) -> dict[str, str]: + info = {"harness": self.name, "model": self.options.model or self.ctx.config.model} + info.update(self.agent_versions()) + return info + + # ------------------------------------------------------------------ + # Turn protocol + # ------------------------------------------------------------------ + + def render_prompt(self, brief: ScenarioBrief) -> str: + parts = [ + f"RLE turn — tick {brief.tick}, day {brief.day}. " + "You manage this RimWorld colony through the `rle` MCP tools.", + TURN_RULES, + ] + if self.options.extra_instructions: + parts.append(self.options.extra_instructions) + parts.append( + "Brief preview (call get_brief for the full version):\n" + + brief.to_text()[:4000], + ) + return "\n\n".join(parts) + + async def step( + self, state: GameState, tick: int, macro_time: float, events: list[RimAPIEvent], + ) -> StepResult: + session = self.session + brief = build_brief( + state, tick=tick, macro_time=macro_time, scenario=self.ctx.scenario, events=events, + ) + session.begin_tick(tick, state, brief) + prompt = self.render_prompt(brief) + + t0 = _time.monotonic() + status = "success" + turn = TurnResult() + try: + turn = await asyncio.wait_for( + self._run_turn(prompt, session.ledger), timeout=self.options.turn_timeout_s, + ) + except asyncio.TimeoutError: + status = "turn_timeout" + logger.warning("%s turn timed out after %.0fs (tick %d)", + self.name, self.options.turn_timeout_s, tick) + await self.abort_turn() + except HarnessStepError as exc: + status = "agent_error" + logger.warning("%s agent error (tick %d): %s", self.name, tick, exc) + turn = TurnResult(text=str(exc)) + latency_ms = round((_time.monotonic() - t0) * 1000, 1) + + step = session.ledger.finish() + if status == "success": + self.parse_successes += 1 + else: + self.parse_failures += 1 + + n_actions = len(step.plan.actions) + self.deliberation_log.append({ + "tick": tick, "agent": self.name, "status": status, + "num_actions": n_actions, "summary": step.plan.summary, + "latency_ms": latency_ms, + }) + self.ctx.emit( + EventType.DELIBERATION if status == "success" else EventType.ERROR, tick, + agent=self.name, latency_ms=latency_ms, num_actions=n_actions, + summary=step.plan.summary, error_type=None if status == "success" else status, + ) + if turn.prompt_tokens or turn.completion_tokens: + if self.ctx.cost_tracker: + self.ctx.cost_tracker.record_raw( + turn.prompt_tokens, turn.completion_tokens, turn.reasoning_tokens, + ) + self.ctx.emit( + EventType.PROVIDER_CALL, tick, agent=self.name, + prompt_tokens=turn.prompt_tokens, completion_tokens=turn.completion_tokens, + reasoning_tokens=turn.reasoning_tokens, + raw_output=turn.text[:_RAW_OUTPUT_CHARS], + raw_output_truncated=len(turn.text) > _RAW_OUTPUT_CHARS, + ) + + extras = {**step.extras, "status": status, "latency_ms": latency_ms, **turn.extras} + return StepResult( + plan=step.plan, execution=step.execution, proposals=step.proposals, extras=extras, + ) + + async def _run_turn(self, prompt: str, ledger: TickLedger) -> TurnResult: + """Prompt the agent, then wait for end_turn (or a short idle grace).""" + send = asyncio.create_task(self.send_turn(prompt)) + done_wait = asyncio.create_task(ledger.turn_done.wait()) + try: + await asyncio.wait({send, done_wait}, return_when=asyncio.FIRST_COMPLETED) + if send.done(): + result = send.result() + if not ledger.turn_done.is_set(): + # Agent replied without end_turn: allow trailing tool calls. + try: + await asyncio.wait_for( + ledger.turn_done.wait(), timeout=self.options.idle_grace_s, + ) + except asyncio.TimeoutError: + pass + return result + # end_turn arrived first; let the agent finish its reply briefly. + try: + return await asyncio.wait_for(send, timeout=self.options.idle_grace_s) + except asyncio.TimeoutError: + send.cancel() + return TurnResult(text="(agent still responding after end_turn)") + finally: + for task in (send, done_wait): + if not task.done(): + task.cancel() diff --git a/src/rle/testing/__init__.py b/src/rle/testing/__init__.py index 5dcacc7..6ae77d3 100644 --- a/src/rle/testing/__init__.py +++ b/src/rle/testing/__init__.py @@ -6,6 +6,9 @@ - :class:`MockRimAPI` / :func:`make_mock_transport` — fake RIMAPI transport - :func:`run_harness_smoke` — drive a plugin through ``RLEGameLoop`` for a few ticks against the mock and return the tick results +- ``rle.testing.scripted_agent.ScriptedMcpHarness`` — a fake coding agent + that plays a fixed tool script through the RLE MCP server (needs the + ``mcp`` extra; imported separately so this package stays extra-free) """ from rle.testing.mock_rimapi import MOCK_ROUTES, MockRimAPI, make_mock_transport diff --git a/src/rle/testing/scripted_agent.py b/src/rle/testing/scripted_agent.py new file mode 100644 index 0000000..5394611 --- /dev/null +++ b/src/rle/testing/scripted_agent.py @@ -0,0 +1,67 @@ +"""A scripted stand-in for an external coding agent (smoke tests). + +Connects to the RLE MCP server like a real agent would, reads the brief, +issues a fixed script of tool calls, and ends the turn. External harness +packages use :class:`ScriptedMcpHarness` for their ``plugin.smoke`` so the +full MCP round trip is exercised in CI without the real binary. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any, ClassVar + +from mcp.client.client import Client + +from rle.harness.cli_base import HeadlessCliHarness, HeadlessCliOptions, TurnResult + +DEFAULT_SCRIPT: tuple[tuple[str, dict[str, Any]], ...] = ( + ("get_brief", {}), + ("research_target", {"parameters": {"project": "Electricity"}, "reason": "smoke"}), + ("end_turn", {"summary": "scripted smoke turn"}), +) + + +class ScriptedMcpHarness(HeadlessCliHarness): + """Plays a fixed tool-call script through the MCP server each turn.""" + + name: ClassVar[str] = "scripted-mcp" + + def __init__( + self, + options: HeadlessCliOptions | None = None, + script: Sequence[tuple[str, dict[str, Any]]] = DEFAULT_SCRIPT, + *, + name: str | None = None, + ) -> None: + super().__init__(options or HeadlessCliOptions()) + self.script = list(script) + self.turns: list[str] = [] + self.calls: list[tuple[str, dict[str, Any]]] = [] + if name: + self.name = name # type: ignore[misc] # per-instance override for plugins + self._url: str | None = None + + async def start_agent(self, mcp_url: str) -> None: + self._url = mcp_url + + async def send_turn(self, prompt: str) -> TurnResult: + assert self._url is not None + self.turns.append(prompt) + texts: list[str] = [] + async with Client(self._url) as client: + for tool, args in self.script: + result = await client.call_tool(tool, args) + self.calls.append((tool, args)) + content = getattr(result, "content", None) or [] + if content: + texts.append(str(getattr(content[0], "text", ""))) + return TurnResult( + text="\n".join(texts), prompt_tokens=len(prompt) // 4, completion_tokens=32, + ) + + async def stop_agent(self) -> None: + self._url = None + + def agent_versions(self) -> dict[str, str]: + return {"scripted_agent": "1"} diff --git a/tests/integration/test_cli_base_harness.py b/tests/integration/test_cli_base_harness.py new file mode 100644 index 0000000..7a8a7bf --- /dev/null +++ b/tests/integration/test_cli_base_harness.py @@ -0,0 +1,110 @@ +"""HeadlessCliHarness driven by the scripted MCP agent through the real loop.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any, ClassVar + +import pytest + +from rle.config import RLEConfig +from rle.harness import HarnessContext, HarnessStepError +from rle.orchestration.game_loop import RLEGameLoop +from rle.rimapi.client import RimAPIClient +from rle.scoring.composite import CompositeScorer +from rle.testing import MockRimAPI +from rle.tracking.event_log import EventLog, EventType + +cli_base = pytest.importorskip("rle.harness.cli_base") +scripted = pytest.importorskip("rle.testing.scripted_agent") + + +@asynccontextmanager +async def _env() -> AsyncIterator[tuple[RimAPIClient, MockRimAPI]]: + mock = MockRimAPI() + async with RimAPIClient("http://mock") as client: + mock.attach(client) + yield client, mock + + +class TestScriptedMcpHarness: + async def test_full_round_trip_through_loop(self, tmp_path: Path) -> None: + log = EventLog(tmp_path / "events.jsonl") + harness = scripted.ScriptedMcpHarness() + async with _env() as (client, mock): + config = RLEConfig(tick_interval=0.0) + ctx = HarnessContext(config=config, client=client, event_log=log) + loop = RLEGameLoop( + config, client, harness=harness, harness_context=ctx, + scorer=CompositeScorer(), event_log=log, + ) + results = await loop.run(max_ticks=2) + + assert len(results) == 2 + # research_target went to RIMAPI via the MCP tool, once per tick + research_posts = [p for p, _ in mock.posts if "research" in p] + assert len(research_posts) == 2 + # Ledger reported the execution; the loop did not re-execute + assert results[0].execution.executed == 1 + assert results[0].plan.role == "scripted-mcp" + assert results[0].plan.summary == "scripted smoke turn" + assert results[0].extras["status"] == "success" + assert results[0].extras["turn_ended"] is True + assert harness.parse_successes == 2 + assert len(harness.turns) == 2 and "get_brief" in harness.turns[0] + kinds = {e.event_type for e in log.events} + assert {EventType.DELIBERATION, EventType.ACTION_EXEC, EventType.PROVIDER_CALL} <= kinds + # MCP host torn down with the loop + assert not harness._host.running # type: ignore[union-attr] + + async def test_describe_includes_agent_versions(self) -> None: + harness = scripted.ScriptedMcpHarness( + cli_base.HeadlessCliOptions(model="some/model"), name="my-tool", + ) + async with _env() as (client, _mock): + ctx = HarnessContext(config=RLEConfig(tick_interval=0.0), client=client) + await harness.setup(ctx) + try: + info = harness.describe() + finally: + await harness.teardown() + assert info == {"harness": "my-tool", "model": "some/model", "scripted_agent": "1"} + + +class _NeverEndsTurn(scripted.ScriptedMcpHarness): # type: ignore[misc] + name: ClassVar[str] = "never-ends" + + async def send_turn(self, prompt: str) -> Any: + await asyncio.sleep(10) + return cli_base.TurnResult() + + +class _Crashes(scripted.ScriptedMcpHarness): # type: ignore[misc] + name: ClassVar[str] = "crashes" + + async def send_turn(self, prompt: str) -> Any: + raise HarnessStepError("binary exited 1") + + +class TestTurnFailures: + async def test_turn_timeout_scores_empty_tick(self) -> None: + harness = _NeverEndsTurn( + cli_base.HeadlessCliOptions(turn_timeout_s=0.2, idle_grace_s=0.05), + ) + async with _env() as (client, _mock): + loop = RLEGameLoop(RLEConfig(tick_interval=0.0), client, harness=harness) + result = await loop.run(max_ticks=1) + assert result[0].extras["status"] == "turn_timeout" + assert result[0].execution.total == 0 + assert harness.parse_failures == 1 + + async def test_agent_error_scores_empty_tick(self) -> None: + harness = _Crashes(cli_base.HeadlessCliOptions(idle_grace_s=0.05)) + async with _env() as (client, _mock): + loop = RLEGameLoop(RLEConfig(tick_interval=0.0), client, harness=harness) + result = await loop.run(max_ticks=1) + assert result[0].extras["status"] == "agent_error" + assert harness.parse_failures == 1 From 6aac546ad93e1b30e5b5f37c9022f3ae53dfa1d1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 07:36:13 +0000 Subject: [PATCH 7/8] registry: smoke variants run without the external tool; ship py.typed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_harness(smoke=True) no longer gates on plugin.available() — smoke harnesses exist so CI can exercise a plugin without its binary. A smoke that still needs a missing dependency (felix without the SDK) surfaces as HarnessUnavailableError with the plugin's own reason. src/rle/py.typed lets external harness packages type-check against RLE under mypy --strict. Co-authored-by: Jason --- docs/harness-plugins.md | 17 ++++++++++++++--- src/rle/harness/registry.py | 15 ++++++++++++--- src/rle/py.typed | 0 tests/unit/test_harness_registry.py | 20 +++++++++++++++++--- 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 src/rle/py.typed diff --git a/docs/harness-plugins.md b/docs/harness-plugins.md index 4c4c175..1931b26 100644 --- a/docs/harness-plugins.md +++ b/docs/harness-plugins.md @@ -88,9 +88,20 @@ and continues. Any other exception is treated as a bug and propagates. - `rle.harness` — `BaseHarness`, `StepResult`, `HarnessContext`, `HarnessPlugin`, `Availability`, `EmptyOptions`, `HarnessStepError`, `TickObserver`, registry helpers. -- `rle.harness.cli_base.HeadlessCliHarness` — scaffold for CLI coding agents - (spawn/attach, MCP attach, per-tick prompt, idle/timeout/abort, ledger - drain, token + latency capture). Tool-agnostic by design. +- `rle.harness.cli_base.HeadlessCliHarness` — scaffold for CLI coding agents. + Subclass and implement three hooks: `start_agent(mcp_url)` (launch/attach + the tool and register the RLE MCP server), `send_turn(prompt) -> TurnResult` + (deliver one prompt, return when the agent has finished responding, with + token counts if you have them), `stop_agent()`. The base hosts the MCP + server in-process, builds the brief and prompt, waits for `end_turn` (or a + short idle grace), drains the ledger into `StepResult`, applies + `turn_timeout_s`, and records latency/cost/deliberation log. Options extend + `HeadlessCliOptions` (`model`, `turn_timeout_s`, `idle_grace_s`, + `extra_instructions`). Needs the `mcp` extra. +- `rle.testing.scripted_agent.ScriptedMcpHarness` — a fake coding agent that + plays a fixed tool script through the MCP server. Return it from + `plugin.smoke()` so your package's CI exercises the full round trip + without the real binary. - `rle.harness.brief` — the harness-neutral scenario brief every harness receives (goals, filtered state, MAP_SUMMARY, action catalog). - `rle.mcp` — the RimAPI MCP server + per-tick write ledger (`rle-mcp`). diff --git a/src/rle/harness/registry.py b/src/rle/harness/registry.py index 62d1205..d1e2317 100644 --- a/src/rle/harness/registry.py +++ b/src/rle/harness/registry.py @@ -139,12 +139,21 @@ def create_harness( ) -> BaseHarness: """Resolve ``name`` through the registry and build a ready-to-setup harness.""" plugin = get_plugin(name) + opts = validate_options(plugin, options) + if smoke or ctx.smoke: + # Smoke variants must run without the external tool, so availability + # is not a gate here; a plugin whose smoke still needs a missing + # dependency surfaces that as unavailable. + try: + return plugin.smoke(ctx, opts) + except ImportError as exc: + reason = plugin.available().reason or str(exc) + raise HarnessUnavailableError( + f"Harness {name!r} cannot run its smoke variant: {reason}", + ) from exc availability = plugin.available() if not availability.ok: raise HarnessUnavailableError( f"Harness {name!r} is installed but unavailable: {availability.reason}", ) - opts = validate_options(plugin, options) - if smoke or ctx.smoke: - return plugin.smoke(ctx, opts) return plugin.create(ctx, opts) diff --git a/src/rle/py.typed b/src/rle/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_harness_registry.py b/tests/unit/test_harness_registry.py index a4c2dcc..c3673c1 100644 --- a/tests/unit/test_harness_registry.py +++ b/tests/unit/test_harness_registry.py @@ -7,6 +7,7 @@ import pytest from pydantic import BaseModel, ConfigDict +import rle.harness.registry as registry from rle.agents.actions import ActionPlan from rle.config import RLEConfig from rle.harness import ( @@ -142,20 +143,33 @@ def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: raise AssertionError("must not be called") def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: - raise AssertionError("must not be called") + return BaselineHarness() def describe(self) -> dict[str, str]: return {} +class _SmokeNeedsDep(_UnavailablePlugin): + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + raise ImportError("No module named 'ghosttool'") + + class TestUnavailable: def test_unavailable_plugin_raises_with_reason(self, monkeypatch: pytest.MonkeyPatch) -> None: - import rle.harness.registry as registry - monkeypatch.setattr(registry, "get_plugin", lambda name: _UnavailablePlugin()) with pytest.raises(HarnessUnavailableError, match="ghost binary"): registry.create_harness("ghost", _ctx()) + def test_smoke_does_not_require_the_tool(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Smoke variants exist precisely so CI can run without the binary.""" + monkeypatch.setattr(registry, "get_plugin", lambda name: _UnavailablePlugin()) + assert isinstance(registry.create_harness("ghost", _ctx(), smoke=True), BaselineHarness) + + def test_smoke_missing_dependency_is_unavailable(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(registry, "get_plugin", lambda name: _SmokeNeedsDep()) + with pytest.raises(HarnessUnavailableError, match="ghost binary"): + registry.create_harness("ghost", _ctx(), smoke=True) + class TestCompatShim: def test_no_agent_builds_baseline(self) -> None: From ffebcc51cae9c99b6fc6a052b26197515c5468c2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 4 Sep 2026 07:48:13 +0000 Subject: [PATCH 8/8] hygiene: harness x model leaderboard, quarantine-aware aggregation, ADR-004, docs - Leaderboard rows keyed by (harness, model); legacy history defaults to felix. Scenarios flagged harness_failed are excluded from means and counted; mean step latency and cost are Pareto columns; reads the cost_snapshot block the CLIs actually write. - update_baseline keys per harness x model and ignores quarantined scenarios. - ADR-004 records the swappable-harness decision, the repo boundary rule and the alternatives rejected. - CLAUDE.md / CONTRIBUTING.md architecture and package structure reflect the harness layer, MCP server, testing exports and the external harness repos. - CI contract job no longer relies on tee /dev/stderr. Co-authored-by: Jason --- .github/workflows/ci.yml | 4 +- CLAUDE.md | 105 ++++++++++++-------- CONTRIBUTING.md | 38 +++++--- docs/adr/004-swappable-harnesses.md | 142 ++++++++++++++++++++++++++++ src/rle/tracking/history.py | 14 ++- src/rle/tracking/leaderboard.py | 107 ++++++++++++++++----- tests/unit/test_leaderboard.py | 37 +++++++- 7 files changed, 366 insertions(+), 81 deletions(-) create mode 100644 docs/adr/004-swappable-harnesses.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d29fd9..a7fbc87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,6 +88,8 @@ jobs: - name: Install the template harness from GitHub run: uv pip install "git+https://github.com/AppSprout-dev/rle-harness-template" --system - name: Template appears in the registry - run: python scripts/run_benchmark.py --harness list | tee /dev/stderr | grep -q "^template " + run: | + python scripts/run_benchmark.py --harness list + python scripts/run_benchmark.py --harness list | grep -q "^template " - name: Template passes smoke run: python scripts/run_benchmark.py --smoke-test --ticks 3 --harness template diff --git a/CLAUDE.md b/CLAUDE.md index bcc37fb..a7f73f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,7 +61,9 @@ curl http://localhost:1234/v1/models - Lint: `ruff check src/ tests/ scripts/` - Type check: `mypy src/` - List scenarios: `python scripts/run_scenario.py --list` -- Smoke test: `python scripts/run_benchmark.py --smoke-test --ticks 5` +- Smoke test: `python scripts/run_benchmark.py --smoke-test --ticks 5 --harness felix --harness baseline` +- Harness boundary: `python scripts/check_harness_boundary.py` +- List harnesses: `python scripts/run_benchmark.py --harness list` - Compare runs: `python scripts/compare_benchmarks.py results/run1 results/run2` ### Configure `.env` @@ -170,26 +172,33 @@ RIMAPI mod (REST :8765 + SSE /api/v1/events) ↕ RimAPIClient (httpx async) + RimAPISSEClient (event stream) ↕ -RLEGameLoop - unpause → read state → drain SSE → inject events → route spoke messages - → MapAnalyst deliberates FIRST (spatial analysis) - → broadcast MapAnalyst output via CentralPost - → 6 role agents deliberate (parallel) → resolve conflicts → execute actions - → score → broadcast score → export tick JSON → render helix +RLEGameLoop (environment — no agent framework imports) + pause → read state → drain SSE → harness.step(state, tick, macro_time, events) + → execute StepResult.plan (unless the harness already applied its writes) + → score → harness.on_tick_end → export tick JSON → unpause → evaluate + ↕ Harness protocol (rle.harness.BaseHarness), discovered via `rle.harnesses` entry points + ├── felix (in tree, extra `felix`) CentralPost hub-spoke → MapAnalyst FIRST → + │ 6 role agents (parallel) → ActionResolver → merged ActionPlan → helix viz + ├── baseline (in tree) unmanaged colony — the paired control + └── (own repos: rle-harness-template / -opencode / -grok-build) + HeadlessCliHarness → RLE MCP server (rle.mcp, in-process HTTP) → + coding agent acts through tools during its turn → TickLedger → StepResult.execution ↕ -CentralPost hub-spoke (TASK_COMPLETE, STATUS_UPDATE, PHASE_ANNOUNCE) - ↕ ↕ ↕ ↕ ↕ ↕ ↕ -7 Agents (MapAnalyst + 6 Role Agents) +ActionExecutor → RIMAPI write calls (shared by every harness; MCP tools call it too) ↕ -ActionResolver → merged ActionPlan - ↕ -ActionExecutor → RIMAPI write calls - ↕ -CompositeScorer → ScoreSnapshot per tick +CompositeScorer (scoring 1.2: outcomes + efficiency + plan_coherence) → ScoreSnapshot per tick ↕ ScenarioEvaluator → victory/defeat/timeout ↕ -HelixVisualizer (terminal) + Dashboard (React :3000 via latest_tick.json :9000) +Dashboard (React :3000 via latest_tick.json :9000; `harness` + `extras` fields are harness-neutral) +``` + +**Repo boundary rule:** only RLE-authored harnesses (`baseline`, `felix`) live here. Harnesses wrapping third-party tools ship as their own `AppSprout-dev/rle-harness-*` packages and register under the same entry-point group. `scripts/check_harness_boundary.py` (run in CI) fails on any `felix_agent_sdk` import outside `src/rle/harness/felix/` and on any third-party harness name in `src/`, `tests/`, `scripts/`. Writing a harness: `docs/harness-plugins.md`; design rationale: ADR-004. + +```bash +python scripts/run_benchmark.py --harness list # installed plugins + availability +python scripts/run_benchmark.py --harness felix --harness baseline --smoke-test +python scripts/run_scenario.py crashlanded --harness felix --harness-opt no_think=true --harness-opt parallel=false ``` ## Agents (map to roles, not colonists) @@ -337,29 +346,47 @@ GitHub Actions workflows in `.github/workflows/`: ``` src/rle/ -├── config.py # RLEConfig (pydantic-settings) +├── config.py # RLEConfig (pydantic-settings; framework-free: provider/model/harness are strings) +├── py.typed # external harness packages type-check against RLE ├── rimapi/ # RIMAPI async HTTP client + SSE + Pydantic schemas │ ├── client.py # RimAPIClient (REST read/write + state adapters + terrain analysis) │ ├── schemas.py # GameState, MapData, TerrainSummary, ZoneData, etc. │ └── sse_client.py # RimAPISSEClient (real-time event stream) -├── agents/ # 7 agents (MapAnalyst + 6 role agents) + base class -│ ├── base_role.py # RimWorldRoleAgent (spoke context, SSE events, MAP_SUMMARY, bootstrap) -│ ├── actions.py # Action, ActionPlan, resolve_endpoint() -│ ├── json_repair.py # Strip think tags, trailing commas, extract JSON -│ ├── map_analyst.py # MapAnalyst (spatial analysis, runs first) -│ ├── resource_manager.py -│ ├── defense_commander.py -│ ├── research_director.py -│ ├── social_overseer.py -│ ├── construction_planner.py -│ └── medical_officer.py -├── orchestration/ # Game loop, state manager, action executor/resolver -│ ├── game_loop.py # RLEGameLoop (MapAnalyst-first, parallel deliberation, CentralPost) +├── agents/ # Harness-neutral action vocabulary (NO agent framework here) +│ ├── actions.py # Action, ActionPlan, ActionOutcome, ExecutionResult, resolve_endpoint() +│ └── json_repair.py # Strip think tags, trailing commas, extract JSON +├── harness/ # Swappable harnesses +│ ├── protocol.py # BaseHarness, StepResult, HarnessContext, HarnessPlugin, Availability +│ ├── registry.py # entry-point discovery (`rle.harnesses`), option validation, create_harness() +│ ├── cli.py # --harness / --harness list / --harness-opt argparse glue +│ ├── brief.py # harness-neutral scenario brief (goals, state, MAP_SUMMARY, action catalog) +│ ├── baseline.py # BaselineHarness (unmanaged colony) + PLUGIN +│ ├── compat.py # RLEGameLoop(agents=..., no_agent=...) legacy shim +│ ├── cli_base.py # HeadlessCliHarness: scaffold for CLI coding agents over MCP (extra `mcp`) +│ └── felix/ # The Felix multi-agent harness (extra `felix`; only place felix_agent_sdk is imported) +│ ├── plugin.py # PLUGIN (lazy SDK imports), harness.py (FelixHarness), build.py, options.py +│ ├── provider_factory.py # Felix providers + helix presets (moved off RLEConfig) +│ ├── agents/ # RimWorldRoleAgent + MapAnalyst + 6 role agents +│ └── providers/ # ClaudeCodeProvider (claude -p) +├── mcp/ # RimAPI as an MCP tool server (extra `mcp`) +│ ├── ledger.py # TickLedger: writes made during a turn → StepResult +│ ├── session.py # tool logic: act()/read()/brief (framework-free) +│ ├── server.py # MCPServer: one tool per WRITE_CATALOG entry + get_brief/end_turn/... +│ ├── host.py # in-process streamable-HTTP host (shared ledger with the loop) +│ └── __main__.py # `rle-mcp` stdio server for manual play +├── testing/ # Exported for plugin authors +│ ├── mock_rimapi.py # MockRimAPI transport (records POSTs) +│ ├── smoke.py # run_harness_smoke(plugin) — the plugin contract test +│ └── scripted_agent.py # ScriptedMcpHarness: fake coding agent over a real MCP client +├── orchestration/ # Environment: game loop, state manager, executor/resolver +│ ├── game_loop.py # RLEGameLoop (harness-agnostic; pause/state/step/execute/score) +│ ├── save_loader.py # load_save_and_settle() shared by both CLIs │ ├── state_manager.py # GameStateManager (SSE drain, macro time, history) │ ├── action_executor.py # Routes actions to RIMAPI write endpoints -│ └── action_resolver.py # 4-rule conflict resolution -├── scoring/ # 10 metrics, composite scorer, bootstrap CIs, CSV recorder -│ ├── metrics.py # 10 individual metric functions (8 colony + 2 process) +│ └── action_resolver.py # 4-rule conflict resolution (used by FelixHarness) +├── scoring/ # 9 metrics, composite scorer, bootstrap CIs, CSV recorder +│ ├── metrics.py # 9 metric functions (7 colony + efficiency + plan_coherence); NEUTRAL = 0.5 +│ ├── coherence.py # contradiction detection over a tick's executed writes │ ├── composite.py # CompositeScorer (weighted aggregation) │ ├── bootstrap.py # BootstrapCI, bootstrap_ci(), bootstrap_paired_delta() │ ├── delta.py # PairedResult (agent vs baseline stats, Welch's t-test) @@ -367,16 +394,17 @@ src/rle/ ├── tracking/ # Benchmark history, cost tracking, observability │ ├── cost_tracker.py # CostTracker + OpenRouter pricing API │ ├── event_log.py # Structured JSONL event log (deliberations, actions, errors) -│ ├── leaderboard.py # Model×scenario matrix, Pareto frontier -│ ├── history.py # JSONL run history + per-model baselines +│ ├── leaderboard.py # Harness×model×scenario matrix (quarantine-aware), Pareto frontier +│ ├── history.py # JSONL run history + per-harness×model baselines │ ├── metadata.py # Git commit, versions, reproducibility metadata │ ├── wandb_logger.py # Weights & Biases integration (optional) │ └── hf_logger.py # HuggingFace Hub export (optional) ├── docker.py # DockerGameServer lifecycle + wait_for_rimapi() └── scenarios/ # YAML schema, loader, evaluator, 6 definitions scripts/ -├── run_scenario.py # Single scenario CLI (auto-loads save, unforbids items) -├── run_benchmark.py # Full benchmark suite CLI (--docker, --smoke-test, --runs) +├── run_scenario.py # Single scenario CLI (auto-loads save, unforbids items, --harness) +├── run_benchmark.py # Full benchmark suite CLI (--docker, --smoke-test, --runs, repeatable --harness) +├── check_harness_boundary.py # CI guard: felix confined to harness/felix; no third-party harness code in tree ├── run_spread_n1.sh # N=1 multi-model spread runner (sequential, continue-on-error) ├── compare_benchmarks.py # Paired statistical comparison of benchmark runs ├── analyze_spread.py # Cross-model leaderboard vs baseline + failure taxonomy @@ -398,6 +426,9 @@ docker/ - [felix-agent-sdk](https://github.com/AppSprout-dev/felix-agent-sdk) — Agent framework (LLMAgent, CentralPost, HelixGeometry, providers) - [RIMAPI](https://github.com/IlyaChichkov/RIMAPI) — C# RimWorld mod (REST API + SSE). [Our fork](https://github.com/AppSprout-dev/RIMAPI) has the `rle-testing` branch with extra endpoints pending upstream merge. - [rimapi-dashboard](https://github.com/AppSprout-dev/rimapi-dashboard) — React dashboard with 5 RLE widgets. Runs on :3000, reads from :9000. +- [rle-harness-template](https://github.com/AppSprout-dev/rle-harness-template) — Template repo for a harness plugin (RLE CI installs it as the plugin-API contract test). +- [rle-harness-opencode](https://github.com/AppSprout-dev/rle-harness-opencode) — OpenCode (`opencode serve` + HTTP API) as a harness over the RLE MCP server. +- [rle-harness-grok-build](https://github.com/AppSprout-dev/rle-harness-grok-build) — Grok Build (headless `grok -p`, session resumed per tick) as a harness over the RLE MCP server. ## RIMAPI Fork Status diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00866b8..7a73416 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,37 +105,51 @@ OPENAI_API_KEY= python scripts/run_benchmark.py \ - **Pydantic v2** — frozen models for all data structures - **No `Any` types** in metric contexts — use `TYPE_CHECKING` imports to break circular deps - **No scipy/numpy** — stdlib only for statistics (see ADR-003) -- **Parallel by default** — agents deliberate concurrently via `asyncio.to_thread` +- **Core is framework-free** — `felix_agent_sdk` is imported only under `src/rle/harness/felix/`; everything else must run with the `felix` extra uninstalled (the `test-no-felix` CI job checks this) +- **Harness-agnostic scoring** — metrics read the executed write stream, never a harness's internal messaging +- **Parallel by default (felix)** — role agents deliberate concurrently - **JSON repair** — LLM output goes through `json_repair.py` before parsing -- **CentralPost for inter-agent context** — not orchestrator-passed lists -- **SSE events in agent context** — each role agent gets relevant events in `filter_game_state()` +- **CentralPost for inter-agent context (felix)** — not orchestrator-passed lists +- **SSE events in agent context** — the loop passes each tick's events to `harness.step()`; Felix role agents get relevant ones in `filter_game_state()` -## Adding a new agent +## Adding a new harness -1. Create `src/rle/agents/your_agent.py` subclassing `RimWorldRoleAgent` +Harnesses are plugins discovered through the `rle.harnesses` entry-point group. Only +RLE-authored harnesses (`baseline`, `felix`) live in this repo; a harness that wraps a +third-party tool gets its own repo — start from +[rle-harness-template](https://github.com/AppSprout-dev/rle-harness-template) and read +`docs/harness-plugins.md`. `scripts/check_harness_boundary.py` (CI) rejects third-party +harness code and stray `felix_agent_sdk` imports in this tree. + +## Adding a new Felix role agent + +1. Create `src/rle/harness/felix/agents/your_agent.py` subclassing `RimWorldRoleAgent` 2. Set `ROLE_NAME`, `ALLOWED_ACTIONS`, `TEMPERATURE_RANGE` class vars 3. Implement `filter_game_state()`, `_get_task_description()`, `_get_role_description()` 4. Add `"recent_events": self._format_events("relevant_event_type")` to `filter_game_state()` -5. Register in `src/rle/agents/__init__.py` — add to `_ROLE_AGENTS` and `AGENT_DISPLAY` +5. Register in `src/rle/harness/felix/agents/__init__.py` (`_ROLE_AGENTS`, `AGENT_DISPLAY`) and the roster in `src/rle/harness/felix/build.py` 6. Add tests in `tests/unit/test_role_agents.py` ## Adding a new scenario 1. Create `src/rle/scenarios/definitions/NN_your_scenario.yaml` -2. Follow the schema: name, description, difficulty, expected_duration_days, initial_population, victory_conditions, failure_conditions, max_ticks, scoring_weights (include all 10 metrics) +2. Follow the schema: name, description, difficulty, expected_duration_days, initial_population, victory_conditions, failure_conditions, max_ticks, scoring_weights (all 9 metrics, summing to 1.0) 3. The loader auto-discovers YAML files — no registration needed ## Project structure ``` src/rle/ -├── config.py # RLEConfig (env vars, provider, helix preset) +├── config.py # RLEConfig (env vars; provider/model/harness as strings) ├── docker.py # Docker container lifecycle + RIMAPI health checks ├── rimapi/ # RIMAPI client + SSE + schemas -├── agents/ # 7 agents (MapAnalyst + 6 role) + base class + JSON repair -├── orchestration/ # Game loop, state manager, executor, resolver -├── scoring/ # 10 metrics, composite scorer, bootstrap CIs, CSV recorder -├── tracking/ # Cost tracking, event log, leaderboard, W&B/HF loggers +├── agents/ # Harness-neutral action vocabulary + JSON repair +├── harness/ # Harness protocol, registry, CLI glue, brief, baseline, felix/ (extra), cli_base +├── mcp/ # RimAPI as an MCP tool server + per-tick ledger (extra `mcp`) +├── testing/ # MockRimAPI, run_harness_smoke, ScriptedMcpHarness (for plugin authors) +├── orchestration/ # Game loop, save loader, state manager, executor, resolver +├── scoring/ # 9 metrics, coherence, composite scorer, bootstrap CIs, CSV recorder +├── tracking/ # Cost tracking, event log, leaderboard (harness×model), W&B/HF loggers └── scenarios/ # YAML schema, loader, evaluator, 6 definitions docker/ # HeadlessRim Dockerfile, compose, entrypoint .github/workflows/ # CI (lint+test+smoke) and benchmark (Docker) workflows diff --git a/docs/adr/004-swappable-harnesses.md b/docs/adr/004-swappable-harnesses.md new file mode 100644 index 0000000..afefeef --- /dev/null +++ b/docs/adr/004-swappable-harnesses.md @@ -0,0 +1,142 @@ +# ADR-004: Swappable harnesses (harness x model benchmark) + +**Date:** 2026-09-04 +**Status:** Accepted +**Deciders:** @jkbennitt +**Tracks:** #51 (scoring 1.2), #6, #8, #46 + +## Decision + +Make the *harness* — the machinery that turns colony state into actions each +tick — a first-class benchmark variable, swappable from the CLI exactly like +the model. RLE core becomes a framework-free environment; the original Felix +7-agent stack becomes one harness among several, behind an optional extra. +Harnesses are discovered through the `rle.harnesses` entry-point group; +harnesses that wrap third-party tools live in their own repositories. + +## Context + +RLE was built around one harness before "harness" was a common term: seven +Felix SDK role agents over a CentralPost hub, merged by a conflict resolver. +Only the model behind that stack could be swapped. Two consequences: + +1. **Scientific:** the benchmark could not answer whether the multi-agent + design itself helps. There was no way to run the same model under a + different decision architecture on the same scenarios and saves. +2. **Structural:** `felix-agent-sdk` was a core dependency imported by the + game loop, config, scripts and tracking. Two of the ten composite metrics + (`coordination`, `communication_efficiency`) read CentralPost / resolver + counters — Felix-shaped by construction and, it turned out, ≈1.0 in every + run (issue #51). + +Meanwhile, capable open-source coding agents (OpenCode, Grok Build, ...) ship +headless modes and native MCP support. They are harnesses in the same sense, +and the interesting benchmark question is *harness x model*. + +## What changes + +### Harness protocol (`rle.harness`) + +`RLEGameLoop` owns only the environment: pause, state refresh, execution, +scoring, evaluation, export. Each tick it calls `harness.step(state, tick, +macro_time, events) -> StepResult`. `StepResult.plan` is executed by the loop; +a harness that already applied its writes during its turn (tool-using coding +agents) returns `StepResult.execution` and the loop scores that instead. +`on_tick_end` delivers execution results and the score back to the harness. +`HarnessStepError` and a loop-level `tick_timeout_s` degrade to an empty, +scored tick rather than aborting the run. + +`FelixHarness` receives everything Felix-specific verbatim from the old loop +(hub/spoke wiring, MapAnalyst-first deliberation, per-agent timeouts, phase +and score broadcasts, helix visualiser, generation-id accounting). +`BaselineHarness` is the unmanaged colony. Legacy `RLEGameLoop(agents=..., +no_agent=...)` calls still work through `rle.harness.compat`. + +### Registry = entry points + +```toml +[project.entry-points."rle.harnesses"] +baseline = "rle.harness.baseline:PLUGIN" +felix = "rle.harness.felix:PLUGIN" +``` + +Built-ins and third-party packages register identically. A plugin exposes +`available()`, `option_schema()` (pydantic, validated from `--harness-opt +key=value`), `create()`, `smoke()` (runs with no external tool or LLM) and +`describe()` (versions for run metadata). CLI: `--harness NAME`, `--harness +list`, `--harness-opt K=V`; `--no-agent` remains an alias for `--harness +baseline`; `run_benchmark.py` accepts `--harness` repeatedly for a matrix. + +### Zero-Felix core + +`felix-agent-sdk` is the optional `felix` extra. Role agents, `base_role` and +the claude-code provider moved under `rle.harness.felix/`; `rle.agents` keeps +only the neutral action vocabulary. `scripts/check_harness_boundary.py` and a +`test-no-felix` CI job enforce that `felix_agent_sdk` is imported nowhere else +and that no third-party harness name appears in `src/`, `tests/` or +`scripts/`. The harness-neutral scenario brief (`rle.harness.brief`: goals, +state, MAP_SUMMARY, action catalog) is what every harness receives; prompt +engineering beyond it is the harness under test. + +### RimAPI as MCP + +`rle.mcp` exposes one tool per `WRITE_CATALOG` entry (executed immediately +through `ActionExecutor`, recorded in a per-tick ledger), `get_brief`, +`get_state`, `rimapi_read`, `end_turn`. The harness hosts it in-process over +streamable HTTP so the agent's MCP client and the loop share one ledger. +`rle.harness.cli_base.HeadlessCliHarness` is the tool-agnostic scaffold for +CLI coding agents (turn protocol, timeouts, ledger drain, cost/latency); +`rle.testing.scripted_agent.ScriptedMcpHarness` plays a fixed tool script so +plugin CI exercises the full round trip without the binary. + +### Repo boundary rule + +RLE core ships only RLE-authored harnesses (`baseline`, `felix`). Harnesses +wrapping third-party tools are separate `AppSprout-dev/rle-harness-*` +packages (template, OpenCode, Grok Build). RLE CI installs the template from +GitHub as the plugin-API contract test. + +### Scoring 1.2 (#51) + +`coordination` and `communication_efficiency` removed; `plan_coherence` +(1 − contradictory executed writes / executed writes per tick) added. +`efficiency` and `plan_coherence` return a neutral 0.5 for ticks with no +writes so the baseline earns no free process points. Both are computed from +the writes that reached RIMAPI — the only surface every harness shares. + +### Tracking + +Runs record `harness`, `harness_options`, `harness_versions` +(`describe()`), per-tick `step_latency_s`, and a `harness_failed` +quarantine flag (RIMAPI null-ref / plant-def markers) that excludes a run +from leaderboard means. Leaderboard rows are keyed by harness x model with +cost and mean step latency as Pareto axes. Both CLIs use one load-and-settle +helper (`rle.orchestration.save_loader`) instead of a fixed sleep. + +## Alternatives rejected + +1. **Keep Felix as core, add adapters inside it.** Would keep the SDK as a + hard dependency and leave process metrics Felix-shaped; other harnesses + would be benchmarked through Felix's own abstractions. +2. **In-tree adapters for OpenCode / Grok Build.** Ties RLE's release cadence + to third-party tools and grows core with every harness; the entry-point + registry makes a package the natural unit. +3. **Buffer MCP writes and execute after the turn.** Coding agents need tool + results inside their turn to decide the next call; immediate execution + with a ledger is the only shape that works, so `StepResult.execution` + exists. +4. **Fix `coordination` by counting unresolved conflicts.** Still measures a + Felix-internal process a single-agent harness cannot have; scoring on + executed writes is the fair common surface. + +## Consequences + +**Positive:** the benchmark can now attribute results to the harness, the +model, or both; adding a harness is `pip install`; core is lighter and +importable without any agent framework; process metrics discriminate. + +**Negative / follow-ups:** the pinned Crashlanded baseline sidecar is stale +until recalibrated against a live game (scoring 1.2); the coding-agent +harnesses are verified against mocks and documented CLI/API surfaces, not yet +against a live colony; `helix_preset` and other Felix knobs moved from +`RLEConfig` to `--harness-opt`, which is a CLI change for existing scripts. diff --git a/src/rle/tracking/history.py b/src/rle/tracking/history.py index c164948..787988f 100644 --- a/src/rle/tracking/history.py +++ b/src/rle/tracking/history.py @@ -43,16 +43,21 @@ def load_history() -> list[dict[str, object]]: def update_baseline(summary: dict[str, Any]) -> tuple[bool, float | None]: - """Update baseline if this run's avg score is a new best for the model. + """Update baseline if this run's avg score is a new best for the + harness x model pair. Quarantined (harness-failure) scenarios are ignored. Returns (is_new_best, previous_score_or_None). """ BASELINES_DIR.mkdir(parents=True, exist_ok=True) - model = summary.get("model", "unknown") - slug = re.sub(r"[^\w\-]", "_", model) + model = str(summary.get("model", "unknown")) + harness = str(summary.get("harness") or "felix") + slug = re.sub(r"[^\w\-]", "_", f"{harness}__{model}" if harness != "felix" else model) baseline_path = BASELINES_DIR / f"{slug}.json" - scenarios = summary.get("scenarios", []) + scenarios = [ + s for s in summary.get("scenarios", []) + if isinstance(s, dict) and not s.get("harness_failed") + ] if not scenarios: return False, None avg_score = sum(s.get("score", 0) for s in scenarios) / len(scenarios) @@ -66,6 +71,7 @@ def update_baseline(summary: dict[str, Any]) -> tuple[bool, float | None]: baseline = { "model": model, + "harness": harness, "avg_score": round(avg_score, 4), "timestamp": summary.get("timestamp", ""), "git_commit": summary.get("git_commit", ""), diff --git a/src/rle/tracking/leaderboard.py b/src/rle/tracking/leaderboard.py index dc7f8c7..98519f1 100644 --- a/src/rle/tracking/leaderboard.py +++ b/src/rle/tracking/leaderboard.py @@ -1,7 +1,9 @@ """Leaderboard generation from RLE benchmark history. -Builds model x scenario results matrix with significance markers, -cost-normalized rankings, and Pareto frontier computation. +Rows are keyed by **harness x model**: the same model under two harnesses is +two rows, because the harness is a benchmark variable. Scenario scores are +averaged across runs (quarantined harness-failure runs excluded), with +bootstrap CIs when N >= 2, plus cost and mean step latency as Pareto axes. """ from __future__ import annotations @@ -12,33 +14,54 @@ from rle.scoring.bootstrap import bootstrap_ci +# History entries written before the harness layer existed were all Felix runs. +LEGACY_HARNESS = "felix" + class LeaderboardEntry(BaseModel): - """One model's benchmark results.""" + """One harness x model's benchmark results.""" model_config = ConfigDict(frozen=True) model: str + harness: str = LEGACY_HARNESS composite_score: float composite_ci: tuple[float, float] | None = None total_cost_usd: float = 0.0 cost_per_scenario: float = 0.0 total_tokens: int = 0 total_wall_time_s: float = 0.0 + mean_step_latency_s: float = 0.0 n_runs: int = 1 + n_quarantined: int = 0 scenarios: dict[str, float] = {} significance_vs_baseline: dict[str, str] = {} timestamp: str = "" git_commit: str = "" + @property + def label(self) -> str: + return f"{self.harness}/{self.model}" + + +def _run_harness(entry: dict[str, Any]) -> str: + harness = entry.get("harness") + return str(harness) if harness else LEGACY_HARNESS + -def _collect_scenario_scores( - runs: list[dict[str, Any]], -) -> dict[str, list[float]]: - """Group scenario scores across multiple runs for one model.""" +def _clean_scenarios(run: dict[str, Any]) -> list[dict[str, Any]]: + """Scenario results minus those flagged as harness/plumbing failures.""" + return [ + sc for sc in run.get("scenarios", []) + if isinstance(sc, dict) and not sc.get("harness_failed") + ] + + +def _collect_scenario_scores(runs: list[dict[str, Any]]) -> dict[str, list[float]]: + """Group scenario scores across multiple runs for one harness x model.""" scores: dict[str, list[float]] = {} for run in runs: - for sc in run.get("scenarios", []): + for sc in _clean_scenarios(run): name = sc.get("name", "") score = sc.get("score") if name and isinstance(score, (int, float)): @@ -51,8 +74,7 @@ def _per_run_composites(runs: list[dict[str, Any]]) -> list[float]: composites: list[float] = [] for run in runs: scenario_scores = [ - sc["score"] - for sc in run.get("scenarios", []) + sc["score"] for sc in _clean_scenarios(run) if isinstance(sc.get("score"), (int, float)) ] if scenario_scores: @@ -60,18 +82,32 @@ def _per_run_composites(runs: list[dict[str, Any]]) -> list[float]: return composites +def _mean_latency(runs: list[dict[str, Any]]) -> float: + latencies = [ + float(sc["mean_step_latency_s"]) + for run in runs for sc in _clean_scenarios(run) + if isinstance(sc.get("mean_step_latency_s"), (int, float)) + ] + return round(sum(latencies) / len(latencies), 3) if latencies else 0.0 + + +def _cost_block(run: dict[str, Any]) -> dict[str, Any]: + block = run.get("cost_snapshot") or run.get("cost") or {} + return block if isinstance(block, dict) else {} + + class Leaderboard: """Manages the RLE benchmark leaderboard.""" def from_history(self, history: list[dict[str, Any]]) -> list[LeaderboardEntry]: """Build sorted leaderboard from benchmark_history.jsonl entries.""" - by_model: dict[str, list[dict[str, Any]]] = {} + by_key: dict[tuple[str, str], list[dict[str, Any]]] = {} for entry in history: - model = entry.get("model", "unknown") - by_model.setdefault(model, []).append(entry) + key = (_run_harness(entry), str(entry.get("model", "unknown"))) + by_key.setdefault(key, []).append(entry) entries: list[LeaderboardEntry] = [] - for model, runs in by_model.items(): + for (harness, model), runs in by_key.items(): latest = runs[-1] scenario_scores = _collect_scenario_scores(runs) scenario_means = {k: sum(v) / len(v) for k, v in scenario_scores.items()} @@ -79,26 +115,34 @@ def from_history(self, history: list[dict[str, Any]]) -> list[LeaderboardEntry]: composites = _per_run_composites(runs) composite = sum(composites) / len(composites) if composites else 0.0 n_runs = len(runs) + n_quarantined = sum( + 1 for run in runs for sc in run.get("scenarios", []) + if isinstance(sc, dict) and sc.get("harness_failed") + ) ci: tuple[float, float] | None = None if len(composites) >= 2: bci = bootstrap_ci(composites) ci = (bci.ci_lower, bci.ci_upper) - cost = float(latest.get("cost", {}).get("estimated_cost_usd", 0.0)) - tokens = int(latest.get("cost", {}).get("total_tokens", 0)) - wall = float(latest.get("cost", {}).get("wall_time_s", 0.0)) + cost_block = _cost_block(latest) + cost = float(cost_block.get("estimated_cost_usd", 0.0)) + tokens = int(cost_block.get("total_tokens", 0)) + wall = float(cost_block.get("wall_time_s", 0.0)) n_scenarios = max(len(scenario_means), 1) entries.append(LeaderboardEntry( model=model, + harness=harness, composite_score=round(composite, 4), composite_ci=ci, total_cost_usd=cost, cost_per_scenario=round(cost / n_scenarios, 4), total_tokens=tokens, total_wall_time_s=wall, + mean_step_latency_s=_mean_latency(runs), n_runs=n_runs, + n_quarantined=n_quarantined, scenarios=scenario_means, timestamp=str(latest.get("timestamp", "")), git_commit=str(latest.get("git_commit", "")), @@ -108,27 +152,33 @@ def from_history(self, history: list[dict[str, Any]]) -> list[LeaderboardEntry]: return entries def to_markdown(self, entries: list[LeaderboardEntry]) -> str: - """Render model x scenario matrix as Markdown table.""" + """Render harness x model x scenario matrix as Markdown table.""" if not entries: return "" - all_scenarios = sorted( - {s for e in entries for s in e.scenarios} - ) + all_scenarios = sorted({s for e in entries for s in e.scenarios}) short_names = [s.split()[0] if " " in s else s[:12] for s in all_scenarios] - header = "| Model | " + " | ".join(short_names) + " | Avg | Cost |" - sep = "|" + "|".join("---" for _ in range(len(short_names) + 3)) + "|" + header = ( + "| Harness | Model | " + " | ".join(short_names) + + " | Avg | Cost | s/step | N |" + ) + sep = "|" + "|".join("---" for _ in range(len(short_names) + 6)) + "|" rows = [header, sep] for entry in entries: - cells = [entry.model] + cells = [entry.harness, entry.model] for scenario in all_scenarios: score = entry.scenarios.get(scenario) sig = entry.significance_vs_baseline.get(scenario, "") cells.append(f"{score:.2f}{sig}" if score is not None else "—") cells.append(f"{entry.composite_score:.2f}") cells.append(f"${entry.total_cost_usd:.2f}") + cells.append(f"{entry.mean_step_latency_s:.1f}") + n = str(entry.n_runs) + if entry.n_quarantined: + n += f" ({entry.n_quarantined} quarantined)" + cells.append(n) rows.append("| " + " | ".join(cells) + " |") return "\n".join(rows) @@ -139,17 +189,22 @@ def to_csv(self, entries: list[LeaderboardEntry], path: str) -> None: return all_scenarios = sorted({s for e in entries for s in e.scenarios}) - header = ["model"] + all_scenarios + ["avg", "cost_usd", "n_runs"] + header = ( + ["harness", "model"] + all_scenarios + + ["avg", "cost_usd", "mean_step_latency_s", "n_runs", "n_quarantined"] + ) lines = [",".join(header)] for entry in entries: - row = [entry.model] + row = [entry.harness, entry.model] for s in all_scenarios: score = entry.scenarios.get(s) row.append(f"{score:.4f}" if score is not None else "") row.append(f"{entry.composite_score:.4f}") row.append(f"{entry.total_cost_usd:.4f}") + row.append(f"{entry.mean_step_latency_s:.3f}") row.append(str(entry.n_runs)) + row.append(str(entry.n_quarantined)) lines.append(",".join(row)) with open(path, "w", encoding="utf-8") as f: diff --git a/tests/unit/test_leaderboard.py b/tests/unit/test_leaderboard.py index ee30a02..0dbdd81 100644 --- a/tests/unit/test_leaderboard.py +++ b/tests/unit/test_leaderboard.py @@ -72,7 +72,7 @@ def test_produces_valid_markdown(self) -> None: md = lb.to_markdown(entries) lines = md.strip().split("\n") assert len(lines) >= 3 # header + separator + at least 1 row - assert "| Model |" in lines[0] + assert "| Harness | Model |" in lines[0] assert "---" in lines[1] def test_includes_cost_column(self) -> None: @@ -130,3 +130,38 @@ def test_single_entry(self) -> None: def test_empty(self) -> None: lb = Leaderboard() assert lb.pareto_frontier([]) == [] + + +class TestHarnessKeys: + def test_same_model_two_harnesses_are_two_rows(self) -> None: + history = [ + {**_history_entry("gpt-4o", [_scenario("S1", 0.8)]), "harness": "felix"}, + {**_history_entry("gpt-4o", [_scenario("S1", 0.6)]), "harness": "some-tool"}, + ] + entries = Leaderboard().from_history(history) + keys = {(e.harness, e.model) for e in entries} + assert keys == {("felix", "gpt-4o"), ("some-tool", "gpt-4o")} + assert entries[0].label == "felix/gpt-4o" + + def test_legacy_entries_default_to_felix(self) -> None: + entries = Leaderboard().from_history(HISTORY) + assert all(e.harness == "felix" for e in entries) + + def test_quarantined_scenarios_excluded_and_counted(self) -> None: + run = _history_entry("m", [ + {"name": "S1", "score": 0.9, "mean_step_latency_s": 2.0}, + {"name": "S2", "score": 0.1, "harness_failed": True, "mean_step_latency_s": 50.0}, + ]) + entry = Leaderboard().from_history([run])[0] + assert entry.scenarios == {"S1": 0.9} + assert entry.composite_score == 0.9 + assert entry.n_quarantined == 1 + assert entry.mean_step_latency_s == 2.0 + md = Leaderboard().to_markdown([entry]) + assert "1 (1 quarantined)" in md and "s/step" in md + + def test_reads_cost_snapshot_block(self) -> None: + run = {**_history_entry("m", [_scenario("S1", 0.5)])} + run["cost_snapshot"] = run.pop("cost") + run["cost_snapshot"]["estimated_cost_usd"] = 3.5 + assert Leaderboard().from_history([run])[0].total_cost_usd == 3.5