diff --git a/.claude/rules/code-style.md b/.claude/rules/code-style.md index f4fb2db..6c47ff3 100644 --- a/.claude/rules/code-style.md +++ b/.claude/rules/code-style.md @@ -1,6 +1,6 @@ # Code Style Rules -- Python 3.14+. Use `uv sync --extra dev` to install. +- Python 3.14+. Use `uv sync --extra dev --extra felix` to install. - mypy strict mode. All code must pass `mypy src/` with `strict = true`. - No `Any` type annotations for dataclass fields. Use `TYPE_CHECKING` imports to break circular dependencies. - `from __future__ import annotations` at top of every Python file. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 082b3c0..a7fbc87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,11 +15,13 @@ jobs: with: python-version: "3.14" - uses: astral-sh/setup-uv@v4 - - run: uv pip install -e ".[dev]" --system + - run: uv pip install -e ".[dev,felix,mcp]" --system - name: Lint run: ruff check src/ tests/ scripts/ - name: Type check run: mypy src/ + - name: Harness boundary (felix confined, no third-party harness code in tree) + run: python scripts/check_harness_boundary.py test: runs-on: ubuntu-latest @@ -30,10 +32,33 @@ jobs: with: python-version: "3.14" - uses: astral-sh/setup-uv@v4 - - run: uv pip install -e ".[dev]" --system + - run: uv pip install -e ".[dev,felix,mcp]" --system - name: Run tests run: pytest --tb=short -q + # The environment must run any harness with the Felix SDK absent. This job + # installs core + dev only and exercises the non-Felix paths end to end. + test-no-felix: + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.14" + - uses: astral-sh/setup-uv@v4 + - run: uv pip install -e ".[dev,mcp]" --system + - name: Assert felix is absent + run: python -c "import importlib.util as u, sys; sys.exit(0 if u.find_spec('felix_agent_sdk') is None else 'felix-agent-sdk leaked into the no-felix job')" + - name: Core imports without felix + run: python -c "import rle, rle.harness, rle.orchestration.game_loop, rle.testing, rle.mcp" + - name: Run tests (Felix-only modules auto-skipped) + run: pytest --tb=short -q + - name: Harness list shows felix as unavailable, baseline available + run: python scripts/run_benchmark.py --harness list + - name: Smoke test baseline harness + run: python scripts/run_benchmark.py --smoke-test --ticks 3 --harness baseline + smoke-test: runs-on: ubuntu-latest needs: test @@ -43,6 +68,28 @@ jobs: with: python-version: "3.14" - uses: astral-sh/setup-uv@v4 - - run: uv pip install -e ".[dev]" --system - - name: Smoke test benchmark - run: python scripts/run_benchmark.py --dry-run --ticks 5 + - run: uv pip install -e ".[dev,felix,mcp]" --system + - name: Smoke test felix + baseline matrix + run: python scripts/run_benchmark.py --smoke-test --ticks 5 --harness felix --harness baseline + + # Contract test for the plugin API: an external harness package installed + # straight from GitHub must appear in --harness list and pass smoke with + # zero changes to this repo. + external-plugin-contract: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.14" + - uses: astral-sh/setup-uv@v4 + - run: uv pip install -e ".[dev,mcp]" --system + - name: Install the template harness from GitHub + run: uv pip install "git+https://github.com/AppSprout-dev/rle-harness-template" --system + - name: Template appears in the registry + run: | + python scripts/run_benchmark.py --harness list + python scripts/run_benchmark.py --harness list | grep -q "^template " + - name: Template passes smoke + run: python scripts/run_benchmark.py --smoke-test --ticks 3 --harness template diff --git a/CLAUDE.md b/CLAUDE.md index b1006b8..a7f73f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # RLE — RimWorld Learning Environment -Multi-agent benchmark where 7 Felix Agent SDK role-specialized LLM agents manage a RimWorld colony. Think FLE (Factorio Learning Environment) but for multi-agent coordination under uncertainty. +A harness × model benchmark: swappable agent harnesses (the original 7-agent Felix Agent SDK stack, an unmanaged baseline, or external coding agents installed as `rle-harness-*` packages) manage a RimWorld colony and are scored identically. Think FLE (Factorio Learning Environment) but for multi-agent coordination under uncertainty, with the harness as a first-class benchmark variable. ## Prerequisites @@ -56,12 +56,14 @@ curl http://localhost:1234/v1/models ## Commands -- Install: `uv sync --extra dev` +- Install: `uv sync --extra dev --extra felix` (add `--extra mcp` for the RimAPI MCP server; core alone has no agent framework) - Test: `pytest` - Lint: `ruff check src/ tests/ scripts/` - Type check: `mypy src/` - List scenarios: `python scripts/run_scenario.py --list` -- Smoke test: `python scripts/run_benchmark.py --smoke-test --ticks 5` +- Smoke test: `python scripts/run_benchmark.py --smoke-test --ticks 5 --harness felix --harness baseline` +- Harness boundary: `python scripts/check_harness_boundary.py` +- List harnesses: `python scripts/run_benchmark.py --harness list` - Compare runs: `python scripts/compare_benchmarks.py results/run1 results/run2` ### Configure `.env` @@ -114,7 +116,8 @@ python scripts/run_scenario.py crashlanded \ **Important flags:** - `--no-think` — Required for thinking models (Nemotron, Qwen). Injects `` prefix. - `--no-pause` — Game runs continuously via SSE. Without this, game pauses each tick. -- `--no-agent` — Baseline mode: no LLM deliberation, colony runs unmanaged (for comparison). +- `--harness NAME` — Which harness decides (default `felix`; `--harness list` shows installed plugins; `--harness-opt key=value` for plugin options). +- `--no-agent` — Baseline mode: alias for `--harness baseline`; colony runs unmanaged (for comparison). - `--output results/live` — Exports `latest_tick.json` for the dashboard. - `--tick-interval 30` — Seconds between ticks. 30s gives agents time to deliberate. @@ -169,26 +172,33 @@ RIMAPI mod (REST :8765 + SSE /api/v1/events) ↕ RimAPIClient (httpx async) + RimAPISSEClient (event stream) ↕ -RLEGameLoop - unpause → read state → drain SSE → inject events → route spoke messages - → MapAnalyst deliberates FIRST (spatial analysis) - → broadcast MapAnalyst output via CentralPost - → 6 role agents deliberate (parallel) → resolve conflicts → execute actions - → score → broadcast score → export tick JSON → render helix +RLEGameLoop (environment — no agent framework imports) + pause → read state → drain SSE → harness.step(state, tick, macro_time, events) + → execute StepResult.plan (unless the harness already applied its writes) + → score → harness.on_tick_end → export tick JSON → unpause → evaluate + ↕ Harness protocol (rle.harness.BaseHarness), discovered via `rle.harnesses` entry points + ├── felix (in tree, extra `felix`) CentralPost hub-spoke → MapAnalyst FIRST → + │ 6 role agents (parallel) → ActionResolver → merged ActionPlan → helix viz + ├── baseline (in tree) unmanaged colony — the paired control + └── (own repos: rle-harness-template / -opencode / -grok-build) + HeadlessCliHarness → RLE MCP server (rle.mcp, in-process HTTP) → + coding agent acts through tools during its turn → TickLedger → StepResult.execution ↕ -CentralPost hub-spoke (TASK_COMPLETE, STATUS_UPDATE, PHASE_ANNOUNCE) - ↕ ↕ ↕ ↕ ↕ ↕ ↕ -7 Agents (MapAnalyst + 6 Role Agents) +ActionExecutor → RIMAPI write calls (shared by every harness; MCP tools call it too) ↕ -ActionResolver → merged ActionPlan - ↕ -ActionExecutor → RIMAPI write calls - ↕ -CompositeScorer → ScoreSnapshot per tick +CompositeScorer (scoring 1.2: outcomes + efficiency + plan_coherence) → ScoreSnapshot per tick ↕ ScenarioEvaluator → victory/defeat/timeout ↕ -HelixVisualizer (terminal) + Dashboard (React :3000 via latest_tick.json :9000) +Dashboard (React :3000 via latest_tick.json :9000; `harness` + `extras` fields are harness-neutral) +``` + +**Repo boundary rule:** only RLE-authored harnesses (`baseline`, `felix`) live here. Harnesses wrapping third-party tools ship as their own `AppSprout-dev/rle-harness-*` packages and register under the same entry-point group. `scripts/check_harness_boundary.py` (run in CI) fails on any `felix_agent_sdk` import outside `src/rle/harness/felix/` and on any third-party harness name in `src/`, `tests/`, `scripts/`. Writing a harness: `docs/harness-plugins.md`; design rationale: ADR-004. + +```bash +python scripts/run_benchmark.py --harness list # installed plugins + availability +python scripts/run_benchmark.py --harness felix --harness baseline --smoke-test +python scripts/run_scenario.py crashlanded --harness felix --harness-opt no_think=true --harness-opt parallel=false ``` ## Agents (map to roles, not colonists) @@ -268,22 +278,21 @@ Macro helix: `t = min(1.0, game_day / expected_duration_days)` drives agent beha - **Analysis** (0.4 <= t < 0.7): Medium temp, evaluate trade-offs - **Synthesis** (t >= 0.7): Low temperature, decisive actions -## Scoring (10 metrics, weighted composite) +## Scoring (9 metrics, weighted composite, SCORING_VERSION 1.2) | Metric | Default Weight | Source | |--------|---------------|--------| -| survival | 0.25 | alive/started colonists | -| threat_response | 0.15 | draft response speed | -| mood | 0.15 | avg colonist mood (from real RIMAPI data) | +| survival | 0.24 | alive/started colonists | +| threat_response | 0.14 | draft response speed | +| mood | 0.12 | avg colonist mood (from real RIMAPI data) | | food_security | 0.10 | food count / 10 (from /api/v1/resources/summary) | -| wealth | 0.10 | wealth growth ratio | -| research | 0.10 | % research tree completed | +| wealth | 0.08 | wealth growth ratio | +| research | 0.08 | % research tree completed | | self_sufficiency | 0.10 | power + food + population stability | -| efficiency | 0.05 | action execution rate | -| coordination | 0.00* | conflicts resolved / total conflicts | -| communication_efficiency | 0.00* | messages acted on / total messages | +| efficiency | 0.06 | executed / proposed writes per tick (neutral 0.5 when no writes) | +| plan_coherence | 0.08 | 1 − contradictory executed writes / executed writes per tick (neutral 0.5 when no writes); see `scoring/coherence.py` | -*Process metrics have 0.0 weight until game loop wires MetricContext counters. Target: coordination=0.12, communication_efficiency=0.08. +Process metrics are harness-agnostic: they read the executed write stream (`ExecutionResult.outcomes`), never CentralPost or resolver counters. Those Felix-specific counts are still emitted on the `CONFLICT` event as diagnostics. `coordination` / `communication_efficiency` were removed in 1.2 (issue #51) because they were ≈1.0 by construction. Bump `SCORING_VERSION` in `tracking/metadata.py` whenever weights or metric implementations change; pinned `.baseline.json` sidecars are rejected on mismatch until recalibrated with `scripts/calibrate_baseline.py`. Scenarios can override weights. TimeSeriesRecorder exports per-tick CSV. @@ -337,29 +346,47 @@ GitHub Actions workflows in `.github/workflows/`: ``` src/rle/ -├── config.py # RLEConfig (pydantic-settings) +├── config.py # RLEConfig (pydantic-settings; framework-free: provider/model/harness are strings) +├── py.typed # external harness packages type-check against RLE ├── rimapi/ # RIMAPI async HTTP client + SSE + Pydantic schemas │ ├── client.py # RimAPIClient (REST read/write + state adapters + terrain analysis) │ ├── schemas.py # GameState, MapData, TerrainSummary, ZoneData, etc. │ └── sse_client.py # RimAPISSEClient (real-time event stream) -├── agents/ # 7 agents (MapAnalyst + 6 role agents) + base class -│ ├── base_role.py # RimWorldRoleAgent (spoke context, SSE events, MAP_SUMMARY, bootstrap) -│ ├── actions.py # Action, ActionPlan, resolve_endpoint() -│ ├── json_repair.py # Strip think tags, trailing commas, extract JSON -│ ├── map_analyst.py # MapAnalyst (spatial analysis, runs first) -│ ├── resource_manager.py -│ ├── defense_commander.py -│ ├── research_director.py -│ ├── social_overseer.py -│ ├── construction_planner.py -│ └── medical_officer.py -├── orchestration/ # Game loop, state manager, action executor/resolver -│ ├── game_loop.py # RLEGameLoop (MapAnalyst-first, parallel deliberation, CentralPost) +├── agents/ # Harness-neutral action vocabulary (NO agent framework here) +│ ├── actions.py # Action, ActionPlan, ActionOutcome, ExecutionResult, resolve_endpoint() +│ └── json_repair.py # Strip think tags, trailing commas, extract JSON +├── harness/ # Swappable harnesses +│ ├── protocol.py # BaseHarness, StepResult, HarnessContext, HarnessPlugin, Availability +│ ├── registry.py # entry-point discovery (`rle.harnesses`), option validation, create_harness() +│ ├── cli.py # --harness / --harness list / --harness-opt argparse glue +│ ├── brief.py # harness-neutral scenario brief (goals, state, MAP_SUMMARY, action catalog) +│ ├── baseline.py # BaselineHarness (unmanaged colony) + PLUGIN +│ ├── compat.py # RLEGameLoop(agents=..., no_agent=...) legacy shim +│ ├── cli_base.py # HeadlessCliHarness: scaffold for CLI coding agents over MCP (extra `mcp`) +│ └── felix/ # The Felix multi-agent harness (extra `felix`; only place felix_agent_sdk is imported) +│ ├── plugin.py # PLUGIN (lazy SDK imports), harness.py (FelixHarness), build.py, options.py +│ ├── provider_factory.py # Felix providers + helix presets (moved off RLEConfig) +│ ├── agents/ # RimWorldRoleAgent + MapAnalyst + 6 role agents +│ └── providers/ # ClaudeCodeProvider (claude -p) +├── mcp/ # RimAPI as an MCP tool server (extra `mcp`) +│ ├── ledger.py # TickLedger: writes made during a turn → StepResult +│ ├── session.py # tool logic: act()/read()/brief (framework-free) +│ ├── server.py # MCPServer: one tool per WRITE_CATALOG entry + get_brief/end_turn/... +│ ├── host.py # in-process streamable-HTTP host (shared ledger with the loop) +│ └── __main__.py # `rle-mcp` stdio server for manual play +├── testing/ # Exported for plugin authors +│ ├── mock_rimapi.py # MockRimAPI transport (records POSTs) +│ ├── smoke.py # run_harness_smoke(plugin) — the plugin contract test +│ └── scripted_agent.py # ScriptedMcpHarness: fake coding agent over a real MCP client +├── orchestration/ # Environment: game loop, state manager, executor/resolver +│ ├── game_loop.py # RLEGameLoop (harness-agnostic; pause/state/step/execute/score) +│ ├── save_loader.py # load_save_and_settle() shared by both CLIs │ ├── state_manager.py # GameStateManager (SSE drain, macro time, history) │ ├── action_executor.py # Routes actions to RIMAPI write endpoints -│ └── action_resolver.py # 4-rule conflict resolution -├── scoring/ # 10 metrics, composite scorer, bootstrap CIs, CSV recorder -│ ├── metrics.py # 10 individual metric functions (8 colony + 2 process) +│ └── action_resolver.py # 4-rule conflict resolution (used by FelixHarness) +├── scoring/ # 9 metrics, composite scorer, bootstrap CIs, CSV recorder +│ ├── metrics.py # 9 metric functions (7 colony + efficiency + plan_coherence); NEUTRAL = 0.5 +│ ├── coherence.py # contradiction detection over a tick's executed writes │ ├── composite.py # CompositeScorer (weighted aggregation) │ ├── bootstrap.py # BootstrapCI, bootstrap_ci(), bootstrap_paired_delta() │ ├── delta.py # PairedResult (agent vs baseline stats, Welch's t-test) @@ -367,16 +394,17 @@ src/rle/ ├── tracking/ # Benchmark history, cost tracking, observability │ ├── cost_tracker.py # CostTracker + OpenRouter pricing API │ ├── event_log.py # Structured JSONL event log (deliberations, actions, errors) -│ ├── leaderboard.py # Model×scenario matrix, Pareto frontier -│ ├── history.py # JSONL run history + per-model baselines +│ ├── leaderboard.py # Harness×model×scenario matrix (quarantine-aware), Pareto frontier +│ ├── history.py # JSONL run history + per-harness×model baselines │ ├── metadata.py # Git commit, versions, reproducibility metadata │ ├── wandb_logger.py # Weights & Biases integration (optional) │ └── hf_logger.py # HuggingFace Hub export (optional) ├── docker.py # DockerGameServer lifecycle + wait_for_rimapi() └── scenarios/ # YAML schema, loader, evaluator, 6 definitions scripts/ -├── run_scenario.py # Single scenario CLI (auto-loads save, unforbids items) -├── run_benchmark.py # Full benchmark suite CLI (--docker, --smoke-test, --runs) +├── run_scenario.py # Single scenario CLI (auto-loads save, unforbids items, --harness) +├── run_benchmark.py # Full benchmark suite CLI (--docker, --smoke-test, --runs, repeatable --harness) +├── check_harness_boundary.py # CI guard: felix confined to harness/felix; no third-party harness code in tree ├── run_spread_n1.sh # N=1 multi-model spread runner (sequential, continue-on-error) ├── compare_benchmarks.py # Paired statistical comparison of benchmark runs ├── analyze_spread.py # Cross-model leaderboard vs baseline + failure taxonomy @@ -398,6 +426,9 @@ docker/ - [felix-agent-sdk](https://github.com/AppSprout-dev/felix-agent-sdk) — Agent framework (LLMAgent, CentralPost, HelixGeometry, providers) - [RIMAPI](https://github.com/IlyaChichkov/RIMAPI) — C# RimWorld mod (REST API + SSE). [Our fork](https://github.com/AppSprout-dev/RIMAPI) has the `rle-testing` branch with extra endpoints pending upstream merge. - [rimapi-dashboard](https://github.com/AppSprout-dev/rimapi-dashboard) — React dashboard with 5 RLE widgets. Runs on :3000, reads from :9000. +- [rle-harness-template](https://github.com/AppSprout-dev/rle-harness-template) — Template repo for a harness plugin (RLE CI installs it as the plugin-API contract test). +- [rle-harness-opencode](https://github.com/AppSprout-dev/rle-harness-opencode) — OpenCode (`opencode serve` + HTTP API) as a harness over the RLE MCP server. +- [rle-harness-grok-build](https://github.com/AppSprout-dev/rle-harness-grok-build) — Grok Build (headless `grok -p`, session resumed per tick) as a harness over the RLE MCP server. ## RIMAPI Fork Status diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2192acd..7a73416 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ ```bash git clone https://github.com/AppSprout-dev/RLE.git cd RLE -uv sync --extra dev +uv sync --extra dev --extra felix pytest # should pass 458+ tests ``` @@ -105,37 +105,51 @@ OPENAI_API_KEY= python scripts/run_benchmark.py \ - **Pydantic v2** — frozen models for all data structures - **No `Any` types** in metric contexts — use `TYPE_CHECKING` imports to break circular deps - **No scipy/numpy** — stdlib only for statistics (see ADR-003) -- **Parallel by default** — agents deliberate concurrently via `asyncio.to_thread` +- **Core is framework-free** — `felix_agent_sdk` is imported only under `src/rle/harness/felix/`; everything else must run with the `felix` extra uninstalled (the `test-no-felix` CI job checks this) +- **Harness-agnostic scoring** — metrics read the executed write stream, never a harness's internal messaging +- **Parallel by default (felix)** — role agents deliberate concurrently - **JSON repair** — LLM output goes through `json_repair.py` before parsing -- **CentralPost for inter-agent context** — not orchestrator-passed lists -- **SSE events in agent context** — each role agent gets relevant events in `filter_game_state()` +- **CentralPost for inter-agent context (felix)** — not orchestrator-passed lists +- **SSE events in agent context** — the loop passes each tick's events to `harness.step()`; Felix role agents get relevant ones in `filter_game_state()` -## Adding a new agent +## Adding a new harness -1. Create `src/rle/agents/your_agent.py` subclassing `RimWorldRoleAgent` +Harnesses are plugins discovered through the `rle.harnesses` entry-point group. Only +RLE-authored harnesses (`baseline`, `felix`) live in this repo; a harness that wraps a +third-party tool gets its own repo — start from +[rle-harness-template](https://github.com/AppSprout-dev/rle-harness-template) and read +`docs/harness-plugins.md`. `scripts/check_harness_boundary.py` (CI) rejects third-party +harness code and stray `felix_agent_sdk` imports in this tree. + +## Adding a new Felix role agent + +1. Create `src/rle/harness/felix/agents/your_agent.py` subclassing `RimWorldRoleAgent` 2. Set `ROLE_NAME`, `ALLOWED_ACTIONS`, `TEMPERATURE_RANGE` class vars 3. Implement `filter_game_state()`, `_get_task_description()`, `_get_role_description()` 4. Add `"recent_events": self._format_events("relevant_event_type")` to `filter_game_state()` -5. Register in `src/rle/agents/__init__.py` — add to `_ROLE_AGENTS` and `AGENT_DISPLAY` +5. Register in `src/rle/harness/felix/agents/__init__.py` (`_ROLE_AGENTS`, `AGENT_DISPLAY`) and the roster in `src/rle/harness/felix/build.py` 6. Add tests in `tests/unit/test_role_agents.py` ## Adding a new scenario 1. Create `src/rle/scenarios/definitions/NN_your_scenario.yaml` -2. Follow the schema: name, description, difficulty, expected_duration_days, initial_population, victory_conditions, failure_conditions, max_ticks, scoring_weights (include all 10 metrics) +2. Follow the schema: name, description, difficulty, expected_duration_days, initial_population, victory_conditions, failure_conditions, max_ticks, scoring_weights (all 9 metrics, summing to 1.0) 3. The loader auto-discovers YAML files — no registration needed ## Project structure ``` src/rle/ -├── config.py # RLEConfig (env vars, provider, helix preset) +├── config.py # RLEConfig (env vars; provider/model/harness as strings) ├── docker.py # Docker container lifecycle + RIMAPI health checks ├── rimapi/ # RIMAPI client + SSE + schemas -├── agents/ # 7 agents (MapAnalyst + 6 role) + base class + JSON repair -├── orchestration/ # Game loop, state manager, executor, resolver -├── scoring/ # 10 metrics, composite scorer, bootstrap CIs, CSV recorder -├── tracking/ # Cost tracking, event log, leaderboard, W&B/HF loggers +├── agents/ # Harness-neutral action vocabulary + JSON repair +├── harness/ # Harness protocol, registry, CLI glue, brief, baseline, felix/ (extra), cli_base +├── mcp/ # RimAPI as an MCP tool server + per-tick ledger (extra `mcp`) +├── testing/ # MockRimAPI, run_harness_smoke, ScriptedMcpHarness (for plugin authors) +├── orchestration/ # Game loop, save loader, state manager, executor, resolver +├── scoring/ # 9 metrics, coherence, composite scorer, bootstrap CIs, CSV recorder +├── tracking/ # Cost tracking, event log, leaderboard (harness×model), W&B/HF loggers └── scenarios/ # YAML schema, loader, evaluator, 6 definitions docker/ # HeadlessRim Dockerfile, compose, entrypoint .github/workflows/ # CI (lint+test+smoke) and benchmark (Docker) workflows diff --git a/README.md b/README.md index b8a9f7b..fb7b44a 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ # RLE — RimWorld Learning Environment -Multi-agent benchmark where 7 Felix Agent SDK role-specialized LLM agents manage a RimWorld colony. Think [FLE](https://github.com/chenhao-wang/FLE) (Factorio Learning Environment) but for **multi-agent coordination under uncertainty**. +A **harness × model** benchmark: swappable agent harnesses manage a RimWorld colony under uncertainty and are scored on the same footing against an unmanaged baseline. Think [FLE](https://github.com/chenhao-wang/FLE) (Factorio Learning Environment) but stochastic, multi-agent-capable, and with the *harness* — not just the model — as a first-class variable. ## What makes this different -- **7 agents, not 1** — MapAnalyst (spatial reasoning), ResourceManager, DefenseCommander, ResearchDirector, SocialOverseer, ConstructionPlanner, MedicalOfficer coordinate through a hub-spoke communication network -- **Spatial awareness** — deterministic terrain analysis from the game map tells agents exactly where to build, farm, and mine -- **Stochastic environment** — raids, plague, mental breaks, weather. Agents adapt, not just optimize -- **Helix-driven strategy** — agents shift from exploration (diverse strategies) to synthesis (decisive actions) as the colony progresses -- **Provider-agnostic** — runs on a free local 4B model or a cloud 30B, same architecture +- **Harnesses are swappable like models** — `--harness felix` (the original 7-agent Felix SDK stack), `--harness baseline` (unmanaged colony), or any harness package installed from PyPI/GitHub (`rle-harness-`); the core never imports an agent framework +- **Harness-agnostic scoring** — process metrics read the writes that reached the game, not any harness's internal messaging; scenarios, saves and the composite are identical for every harness +- **Spatial awareness** — deterministic terrain analysis from the game map gives every harness verified build/farm/stockpile coordinates (MAP_SUMMARY) +- **Stochastic environment** — raids, plague, mental breaks, weather. Harnesses adapt, not just optimize +- **Paired against a real baseline** — every run is compared to RimWorld's own pawn AI on the same save +- **Provider-agnostic** — runs on a free local 4B model or a cloud 30B, same environment ## Architecture @@ -17,15 +18,23 @@ RimWorld (game) ↕ Harmony patches RIMAPI mod (C# REST :8765 + SSE events) ↕ httpx async + SSE -RLE Orchestrator - ↕ CentralPost hub-spoke - MapAnalyst → spatial analysis (runs first) - 6 Role Agents (parallel deliberation) - ↕ OpenAI-compatible API -LLM (Nemotron 4B local / 30B cloud / Anthropic / OpenAI) +RLEGameLoop (environment: pause → state → harness.step → execute → score → unpause) + ↕ Harness protocol (rle.harness) — discovered via the `rle.harnesses` entry-point group + ├─ felix MapAnalyst → 6 role agents over CentralPost, merged by ActionResolver [in tree, extra `felix`] + ├─ baseline unmanaged colony [in tree] + └─ external coding agents attached over the RimAPI MCP server (rle-mcp) [own repos] + ↕ OpenAI-compatible / Anthropic / local API (provider + model are strings; the harness interprets them) +LLM ``` -## The 7 Agents +```bash +python scripts/run_benchmark.py --harness list # what is installed +python scripts/run_benchmark.py --harness felix --harness baseline --smoke-test +``` + +Writing a harness: see [docs/harness-plugins.md](docs/harness-plugins.md). Third-party harnesses (OpenCode, Grok Build, ...) live in their own `AppSprout-dev/rle-harness-*` repos and are installed with `pip`, never committed here. + +## The Felix harness: 7 agents | Agent | Domain | Key Actions | |-------|--------|-------------| @@ -63,9 +72,12 @@ curl http://localhost:1234/v1/models # LM Studio (if using local) ```bash git clone https://github.com/AppSprout-dev/RLE.git cd RLE -uv sync --extra dev +uv sync --extra dev --extra felix # core + the Felix harness +# add --extra mcp for the RimAPI MCP server used by external coding-agent harnesses ``` +Core is framework-free; `felix-agent-sdk` is an optional extra. Without it, `--harness felix` shows as unavailable in `--harness list` and everything else still runs. + ### Configure `.env` ```bash @@ -127,9 +139,12 @@ The scenario will: | `--no-pause` | Game runs continuously via SSE. Without this, game pauses each tick. | | `--output DIR` | Exports `latest_tick.json` for the dashboard. | | `--tick-interval N` | Seconds between ticks. 30s recommended for cloud models. | -| `--visualize` | Shows terminal helix visualization. | -| `--no-agent` | Baseline mode — no agents, colony runs unmanaged. | -| `--sequential` | Agents deliberate one at a time instead of in parallel. | +| `--harness NAME` | Which harness decides (default `felix`; `--harness list` shows installed plugins). | +| `--harness-opt K=V` | Harness-specific option, validated by the plugin (e.g. `role_timeout_s=90`). | +| `--no-agent` | Baseline mode — alias for `--harness baseline`, colony runs unmanaged. | +| `--visualize` | [felix] Shows terminal helix visualization. | +| `--sequential` | [felix] Agents deliberate one at a time instead of in parallel. | +| `--tick-timeout N` | Loop-level cap on a whole harness step (seconds). | ### Dashboard (optional, 3 terminals) @@ -194,22 +209,21 @@ Measured against a pinned no-agent baseline (4 seeds, mean time-to-end 8.0 days) ## Scoring -10 metrics, weighted composite (scenarios can override weights): +9 metrics, weighted composite (`SCORING_VERSION = "1.2"`; scenarios can override weights): | Metric | Default Weight | What it measures | |--------|---------------|------------------| -| survival | 0.25 | alive / started colonists | -| threat_response | 0.15 | draft response speed | -| mood | 0.15 | avg colonist mood | +| survival | 0.24 | alive / started colonists | +| threat_response | 0.14 | draft response speed | +| mood | 0.12 | avg colonist mood | | food_security | 0.10 | days of food (10+ = 1.0) | -| wealth | 0.10 | wealth growth ratio | -| research | 0.10 | % research tree completed | +| wealth | 0.08 | wealth growth ratio | +| research | 0.08 | % research tree completed | | self_sufficiency | 0.10 | power + food + population stability | -| efficiency | 0.05 | action execution rate | -| coordination | 0.00* | conflicts resolved / total conflicts | -| communication_efficiency | 0.00* | messages acted on / total messages | +| efficiency | 0.06 | executed / proposed writes per tick | +| plan_coherence | 0.08 | 1 − contradictory executed writes / executed writes per tick | -*Process metrics weighted 0.0 until game loop wires MetricContext counters. +Both process metrics (`efficiency`, `plan_coherence`) are computed from the writes that actually reached RIMAPI, so any harness is scored the same way, and both return a neutral 0.5 for ticks with no writes so the unmanaged baseline earns no free points. The pre-1.2 `coordination` / `communication_efficiency` metrics were removed because they were ≈1.0 by construction (issue #51). ## Development diff --git a/docs/adr/003-automated-benchmark-infrastructure.md b/docs/adr/003-automated-benchmark-infrastructure.md index aa476b6..2b345d9 100644 --- a/docs/adr/003-automated-benchmark-infrastructure.md +++ b/docs/adr/003-automated-benchmark-infrastructure.md @@ -44,6 +44,8 @@ Two new metrics folded into the composite score: Process metrics get 20% combined weight. No historical data to protect (pre-release). All 6 scenario YAMLs updated. +> **Superseded (scoring 1.2, issue #51, ADR-004).** Both metrics turned out to be ≈1.0 by construction (the resolver always "resolves" every conflict it detects; empty message traffic scored 1.0) and were Felix-specific. They were replaced by a single harness-agnostic `plan_coherence` metric computed from the executed write stream, with neutral 0.5 defaults for ticks that issue no writes. + ### 3. Bootstrap confidence intervals (stdlib-only) We evaluated `resample` (scikit-hep, best modern option) but it pulls in scipy (~150MB) + numpy (~50MB). For percentile bootstrap CIs and Welch's t-test at N≥4, stdlib `random.choices()` + `math` is mathematically correct and keeps the install lightweight. The existing hand-rolled Welch's t-test in `delta.py` uses a normal CDF approximation (Abramowitz & Stegun) documented as "very accurate for df > 30, approximate for smaller df." Our N≥4 minimum guarantees sufficient accuracy. diff --git a/docs/adr/004-swappable-harnesses.md b/docs/adr/004-swappable-harnesses.md new file mode 100644 index 0000000..afefeef --- /dev/null +++ b/docs/adr/004-swappable-harnesses.md @@ -0,0 +1,142 @@ +# ADR-004: Swappable harnesses (harness x model benchmark) + +**Date:** 2026-09-04 +**Status:** Accepted +**Deciders:** @jkbennitt +**Tracks:** #51 (scoring 1.2), #6, #8, #46 + +## Decision + +Make the *harness* — the machinery that turns colony state into actions each +tick — a first-class benchmark variable, swappable from the CLI exactly like +the model. RLE core becomes a framework-free environment; the original Felix +7-agent stack becomes one harness among several, behind an optional extra. +Harnesses are discovered through the `rle.harnesses` entry-point group; +harnesses that wrap third-party tools live in their own repositories. + +## Context + +RLE was built around one harness before "harness" was a common term: seven +Felix SDK role agents over a CentralPost hub, merged by a conflict resolver. +Only the model behind that stack could be swapped. Two consequences: + +1. **Scientific:** the benchmark could not answer whether the multi-agent + design itself helps. There was no way to run the same model under a + different decision architecture on the same scenarios and saves. +2. **Structural:** `felix-agent-sdk` was a core dependency imported by the + game loop, config, scripts and tracking. Two of the ten composite metrics + (`coordination`, `communication_efficiency`) read CentralPost / resolver + counters — Felix-shaped by construction and, it turned out, ≈1.0 in every + run (issue #51). + +Meanwhile, capable open-source coding agents (OpenCode, Grok Build, ...) ship +headless modes and native MCP support. They are harnesses in the same sense, +and the interesting benchmark question is *harness x model*. + +## What changes + +### Harness protocol (`rle.harness`) + +`RLEGameLoop` owns only the environment: pause, state refresh, execution, +scoring, evaluation, export. Each tick it calls `harness.step(state, tick, +macro_time, events) -> StepResult`. `StepResult.plan` is executed by the loop; +a harness that already applied its writes during its turn (tool-using coding +agents) returns `StepResult.execution` and the loop scores that instead. +`on_tick_end` delivers execution results and the score back to the harness. +`HarnessStepError` and a loop-level `tick_timeout_s` degrade to an empty, +scored tick rather than aborting the run. + +`FelixHarness` receives everything Felix-specific verbatim from the old loop +(hub/spoke wiring, MapAnalyst-first deliberation, per-agent timeouts, phase +and score broadcasts, helix visualiser, generation-id accounting). +`BaselineHarness` is the unmanaged colony. Legacy `RLEGameLoop(agents=..., +no_agent=...)` calls still work through `rle.harness.compat`. + +### Registry = entry points + +```toml +[project.entry-points."rle.harnesses"] +baseline = "rle.harness.baseline:PLUGIN" +felix = "rle.harness.felix:PLUGIN" +``` + +Built-ins and third-party packages register identically. A plugin exposes +`available()`, `option_schema()` (pydantic, validated from `--harness-opt +key=value`), `create()`, `smoke()` (runs with no external tool or LLM) and +`describe()` (versions for run metadata). CLI: `--harness NAME`, `--harness +list`, `--harness-opt K=V`; `--no-agent` remains an alias for `--harness +baseline`; `run_benchmark.py` accepts `--harness` repeatedly for a matrix. + +### Zero-Felix core + +`felix-agent-sdk` is the optional `felix` extra. Role agents, `base_role` and +the claude-code provider moved under `rle.harness.felix/`; `rle.agents` keeps +only the neutral action vocabulary. `scripts/check_harness_boundary.py` and a +`test-no-felix` CI job enforce that `felix_agent_sdk` is imported nowhere else +and that no third-party harness name appears in `src/`, `tests/` or +`scripts/`. The harness-neutral scenario brief (`rle.harness.brief`: goals, +state, MAP_SUMMARY, action catalog) is what every harness receives; prompt +engineering beyond it is the harness under test. + +### RimAPI as MCP + +`rle.mcp` exposes one tool per `WRITE_CATALOG` entry (executed immediately +through `ActionExecutor`, recorded in a per-tick ledger), `get_brief`, +`get_state`, `rimapi_read`, `end_turn`. The harness hosts it in-process over +streamable HTTP so the agent's MCP client and the loop share one ledger. +`rle.harness.cli_base.HeadlessCliHarness` is the tool-agnostic scaffold for +CLI coding agents (turn protocol, timeouts, ledger drain, cost/latency); +`rle.testing.scripted_agent.ScriptedMcpHarness` plays a fixed tool script so +plugin CI exercises the full round trip without the binary. + +### Repo boundary rule + +RLE core ships only RLE-authored harnesses (`baseline`, `felix`). Harnesses +wrapping third-party tools are separate `AppSprout-dev/rle-harness-*` +packages (template, OpenCode, Grok Build). RLE CI installs the template from +GitHub as the plugin-API contract test. + +### Scoring 1.2 (#51) + +`coordination` and `communication_efficiency` removed; `plan_coherence` +(1 − contradictory executed writes / executed writes per tick) added. +`efficiency` and `plan_coherence` return a neutral 0.5 for ticks with no +writes so the baseline earns no free process points. Both are computed from +the writes that reached RIMAPI — the only surface every harness shares. + +### Tracking + +Runs record `harness`, `harness_options`, `harness_versions` +(`describe()`), per-tick `step_latency_s`, and a `harness_failed` +quarantine flag (RIMAPI null-ref / plant-def markers) that excludes a run +from leaderboard means. Leaderboard rows are keyed by harness x model with +cost and mean step latency as Pareto axes. Both CLIs use one load-and-settle +helper (`rle.orchestration.save_loader`) instead of a fixed sleep. + +## Alternatives rejected + +1. **Keep Felix as core, add adapters inside it.** Would keep the SDK as a + hard dependency and leave process metrics Felix-shaped; other harnesses + would be benchmarked through Felix's own abstractions. +2. **In-tree adapters for OpenCode / Grok Build.** Ties RLE's release cadence + to third-party tools and grows core with every harness; the entry-point + registry makes a package the natural unit. +3. **Buffer MCP writes and execute after the turn.** Coding agents need tool + results inside their turn to decide the next call; immediate execution + with a ledger is the only shape that works, so `StepResult.execution` + exists. +4. **Fix `coordination` by counting unresolved conflicts.** Still measures a + Felix-internal process a single-agent harness cannot have; scoring on + executed writes is the fair common surface. + +## Consequences + +**Positive:** the benchmark can now attribute results to the harness, the +model, or both; adding a harness is `pip install`; core is lighter and +importable without any agent framework; process metrics discriminate. + +**Negative / follow-ups:** the pinned Crashlanded baseline sidecar is stale +until recalibrated against a live game (scoring 1.2); the coding-agent +harnesses are verified against mocks and documented CLI/API surfaces, not yet +against a live colony; `helix_preset` and other Felix knobs moved from +`RLEConfig` to `--harness-opt`, which is a CLI change for existing scripts. diff --git a/docs/harness-plugins.md b/docs/harness-plugins.md new file mode 100644 index 0000000..1931b26 --- /dev/null +++ b/docs/harness-plugins.md @@ -0,0 +1,131 @@ +# Writing an RLE harness plugin + +RLE benchmarks **harnesses × models**. A *model* is whatever LLM sits behind +the decisions; a *harness* is the machinery that turns game state into +actions — one agent or many, an agent framework, or a coding agent attached +over MCP. Both are swappable from the CLI: + +```bash +python scripts/run_benchmark.py --harness felix --model gpt-4o +python scripts/run_benchmark.py --harness opencode --model gpt-4o +python scripts/run_benchmark.py --harness list +``` + +## Repo boundary rule + +RLE core ships only harnesses that are RLE-authored code: `baseline` +(unmanaged colony) and `felix` (the original 7-agent stack). **A harness that +wraps a third-party tool lives in its own repository and package** +(`rle-harness-opencode`, `rle-harness-grok-build`, ...). Nothing tool-specific +is committed to RLE core; adding a harness is `pip install`, never a core PR. + +Start from the template: . + +## The contract + +Register a module-level `PLUGIN` object under the `rle.harnesses` entry-point +group: + +```toml +# pyproject.toml of your harness package +[project] +name = "rle-harness-mytool" +dependencies = ["rimworld-learning-environment>=0.5"] + +[project.entry-points."rle.harnesses"] +mytool = "rle_harness_mytool:PLUGIN" +``` + +`PLUGIN` implements `rle.harness.HarnessPlugin`: + +| Method | Purpose | +|---|---| +| `name`, `description` | Shown by `--harness list` | +| `available() -> Availability` | Cheap probe: is the binary / extra installed? Never import heavy deps at module load. | +| `option_schema() -> type[BaseModel]` | Pydantic model validated from `--harness-opt key=value` (use `EmptyOptions` if none). | +| `create(ctx, options) -> BaseHarness` | Build the real harness. | +| `smoke(ctx, options) -> BaseHarness` | Build a variant that runs with no external tool / LLM (used by `--smoke-test` and CI). | +| `describe() -> dict[str, str]` | Versions recorded in run metadata. | + +The harness itself subclasses `rle.harness.BaseHarness`: + +```python +class MyHarness(BaseHarness): + name = "mytool" + + async def setup(self, ctx: HarnessContext) -> None: ... + async def step(self, state, tick, macro_time, events) -> StepResult: ... + async def on_tick_end(self, tick, state, step, execution, score) -> None: ... + async def teardown(self) -> None: ... +``` + +### `StepResult` + +Return `StepResult(plan=ActionPlan(...))` and the loop executes the actions +through `ActionExecutor` (same normalisation and guards as Felix). If your +harness already applied its writes during the turn — the normal case for a +coding agent calling tools through the RLE MCP server — return the recorded +`ExecutionResult` in `StepResult.execution` and the loop skips execution and +scores what you report. `proposals` (per-sub-agent plans) and `extras` +(telemetry) are optional and surface in the dashboard export. + +### Turn-based, always + +Each tick the environment pauses the game, refreshes state, calls +`step(...)` once (bounded by `--tick-timeout` when set), executes, scores, +calls `on_tick_end(...)`, and unpauses. Coding-agent harnesses get one prompt +per tick and must return when the agent is idle; free-running sessions are +not part of v1. + +### Failure semantics + +Raise `HarnessStepError` for an expected failure (agent crashed, tool +unreachable). The loop logs an `ERROR` event, scores the tick with no actions, +and continues. Any other exception is treated as a bug and propagates. + +## What core gives you + +- `rle.harness` — `BaseHarness`, `StepResult`, `HarnessContext`, + `HarnessPlugin`, `Availability`, `EmptyOptions`, `HarnessStepError`, + `TickObserver`, registry helpers. +- `rle.harness.cli_base.HeadlessCliHarness` — scaffold for CLI coding agents. + Subclass and implement three hooks: `start_agent(mcp_url)` (launch/attach + the tool and register the RLE MCP server), `send_turn(prompt) -> TurnResult` + (deliver one prompt, return when the agent has finished responding, with + token counts if you have them), `stop_agent()`. The base hosts the MCP + server in-process, builds the brief and prompt, waits for `end_turn` (or a + short idle grace), drains the ledger into `StepResult`, applies + `turn_timeout_s`, and records latency/cost/deliberation log. Options extend + `HeadlessCliOptions` (`model`, `turn_timeout_s`, `idle_grace_s`, + `extra_instructions`). Needs the `mcp` extra. +- `rle.testing.scripted_agent.ScriptedMcpHarness` — a fake coding agent that + plays a fixed tool script through the MCP server. Return it from + `plugin.smoke()` so your package's CI exercises the full round trip + without the real binary. +- `rle.harness.brief` — the harness-neutral scenario brief every harness + receives (goals, filtered state, MAP_SUMMARY, action catalog). +- `rle.mcp` — the RimAPI MCP server + per-tick write ledger (`rle-mcp`). +- `rle.testing` — `MockRimAPI` and `run_harness_smoke(plugin)`; run the + latter in your CI: + +```python +from rle.testing import run_harness_smoke + +async def test_smoke() -> None: + report = await run_harness_smoke("mytool", ticks=3) + assert report.ok +``` + +## Scoring is harness-agnostic + +Process metrics (`efficiency`, `plan_coherence`) are computed from the writes +that reached RIMAPI, never from a harness's internal messaging. A harness is +free to coordinate however it likes; what is scored is whether coherent, +valid writes reached the game and how the colony fared. Prompt engineering +beyond the neutral brief is part of the harness being benchmarked. + +## Metadata + +Every run records `harness`, `harness_options`, `harness_versions` +(`describe()` output), `model`, `provider`, `scoring_version`, and per-tick +`step_latency_s`. Leaderboards key on harness × model × scenario. diff --git a/pyproject.toml b/pyproject.toml index a11941b..d35bfd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,12 +5,14 @@ build-backend = "hatchling.build" [project] name = "rimworld-learning-environment" version = "0.4.1" -description = "Multi-agent benchmark where Felix Agent SDK agents play RimWorld" +description = "Harness x model benchmark: swappable agent harnesses manage a RimWorld colony" readme = "README.md" license = "MIT" requires-python = ">=3.14" +# Core is framework-free: the environment (RIMAPI client, loop, scoring, +# scenarios, tracking, harness registry) runs any harness. Agent frameworks +# arrive through extras / external harness packages. dependencies = [ - "felix-agent-sdk>=0.3.0", "httpx>=0.24", "pydantic>=2.0", "pydantic-settings>=2.0", @@ -18,10 +20,14 @@ dependencies = [ ] [project.optional-dependencies] +# The original Felix multi-agent harness (`--harness felix`). +felix = ["felix-agent-sdk>=0.3.0"] anthropic = ["felix-agent-sdk[anthropic]"] openai = ["felix-agent-sdk[openai]"] local = ["felix-agent-sdk[local]"] all = ["felix-agent-sdk[all]"] +# RimAPI MCP server for tool-using harnesses (`rle-mcp`). +mcp = ["mcp>=2.1"] dev = [ "pytest>=7.0", "pytest-asyncio>=0.24", @@ -38,6 +44,16 @@ tracking = [ "huggingface-hub>=0.20", ] +# Harness registry. Only RLE-authored harnesses live in this tree; harnesses +# wrapping third-party tools (OpenCode, Grok Build, ...) ship as their own +# packages and register under the same group. +[project.entry-points."rle.harnesses"] +baseline = "rle.harness.baseline:PLUGIN" +felix = "rle.harness.felix:PLUGIN" + +[project.scripts] +rle-mcp = "rle.mcp.__main__:main" + [tool.hatch.build.targets.wheel] packages = ["src/rle"] @@ -57,7 +73,7 @@ python_version = "3.14" strict = true [[tool.mypy.overrides]] -module = ["wandb", "huggingface_hub"] +module = ["wandb", "huggingface_hub", "mcp", "mcp.*"] ignore_missing_imports = true # wandb/huggingface_hub ship untyped APIs; when the tracking extra is diff --git a/scripts/check_harness_boundary.py b/scripts/check_harness_boundary.py new file mode 100644 index 0000000..f6b0841 --- /dev/null +++ b/scripts/check_harness_boundary.py @@ -0,0 +1,74 @@ +"""Enforce the harness boundary rules (run in CI, stdlib only). + +1. ``felix_agent_sdk`` may be imported only under ``src/rle/harness/felix/``. + Everything else in core must run with the ``felix`` extra uninstalled. +2. Third-party harnesses (OpenCode, Grok Build, ...) live in their own repos. + Their names may appear in docs, never in ``src/``, ``tests/`` or ``scripts/``. + +Exit code 1 with a file:line listing on any violation. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +SRC = ROOT / "src" / "rle" +FELIX_DIR = SRC / "harness" / "felix" + +FELIX_IMPORT = re.compile(r"^\s*(from|import)\s+felix_agent_sdk\b", re.MULTILINE) +THIRD_PARTY = re.compile(r"\b(opencode|grok[-_ ]?build|grok_build)\b", re.IGNORECASE) +CODE_DIRS = (ROOT / "src", ROOT / "tests", ROOT / "scripts") + + +def _py_files(root: Path) -> list[Path]: + return [p for p in root.rglob("*.py") if "__pycache__" not in p.parts] + + +def check_felix_boundary() -> list[str]: + violations: list[str] = [] + for path in _py_files(SRC): + if FELIX_DIR in path.parents: + continue + text = path.read_text(encoding="utf-8") + for match in FELIX_IMPORT.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + violations.append( + f"{path.relative_to(ROOT)}:{line}: felix_agent_sdk import outside " + f"src/rle/harness/felix/", + ) + return violations + + +def check_no_third_party_harness_code() -> list[str]: + violations: list[str] = [] + for root in CODE_DIRS: + for path in _py_files(root): + if path.resolve() == Path(__file__).resolve(): + continue + text = path.read_text(encoding="utf-8") + for match in THIRD_PARTY.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + violations.append( + f"{path.relative_to(ROOT)}:{line}: third-party harness name " + f"{match.group(0)!r} in core code (belongs in its own repo)", + ) + return violations + + +def main() -> int: + violations = check_felix_boundary() + check_no_third_party_harness_code() + if violations: + print("Harness boundary violations:") + for v in violations: + print(f" {v}") + return 1 + print("Harness boundary OK: felix_agent_sdk confined to src/rle/harness/felix/; " + "no third-party harness code in tree.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_benchmark.py b/scripts/run_benchmark.py index e844725..4755d89 100644 --- a/scripts/run_benchmark.py +++ b/scripts/run_benchmark.py @@ -1,4 +1,4 @@ -"""CLI: run all RLE scenarios and output a leaderboard.""" +"""CLI: run all RLE scenarios for one or more harnesses and output a leaderboard.""" from __future__ import annotations @@ -11,31 +11,31 @@ import time from pathlib import Path from typing import Any -from unittest.mock import MagicMock - -import httpx -from felix_agent_sdk.core import HelixConfig -from felix_agent_sdk.providers.base import BaseProvider -from felix_agent_sdk.providers.types import CompletionResult -from felix_agent_sdk.visualization import HelixVisualizer - -from rle.agents import AGENT_DISPLAY -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer -from rle.config import RLEConfig, bridge_anthropic_key, bridge_openrouter_key + +from rle.config import RLEConfig +from rle.docker import DockerGameServer +from rle.harness import ( + HarnessContext, + HarnessNotFoundError, + HarnessOptionsError, + HarnessUnavailableError, + add_harness_args, + create_harness, + exit_with_harness_error, + harness_options_for, + maybe_handle_harness_list, + selected_harnesses, +) from rle.orchestration.game_loop import RLEGameLoop +from rle.orchestration.save_loader import load_save_and_settle from rle.rimapi.client import RimAPIClient from rle.scenarios.evaluator import ScenarioEvaluator from rle.scenarios.loader import list_scenarios from rle.scenarios.schema import ScenarioConfig from rle.scoring.composite import CompositeScorer -from rle.scoring.delta import PairedResult +from rle.scoring.delta import PairedResult, print_paired_leaderboard from rle.scoring.recorder import TimeSeriesRecorder +from rle.testing.mock_rimapi import MockRimAPI from rle.tracking.cost_tracker import CostTracker, create_cost_tracker, fetch_billed_costs from rle.tracking.event_log import EventLog from rle.tracking.hf_logger import HFLogger @@ -45,321 +45,206 @@ logger = logging.getLogger(__name__) -# Seconds to wait after loading a save before starting a run. -GAME_LOAD_WAIT_SECONDS = 2 - -# Mock data for --dry-run mode -_MOCK_ACTION_PLAN = json.dumps({ - "actions": [ - { - "action_type": "no_action", - "reason": "Mock mode — no real LLM call", - }, - ], - "summary": "Mock deliberation.", - "confidence": 0.6, -}) - -_MOCK_ROUTES: dict[str, dict | list] = { - "/api/v1/colonists": [ - { - "colonist_id": "col_01", "name": "Tynan", "health": 0.95, - "mood": 0.72, "skills": {"shooting": 8, "construction": 5, - "cooking": 3, "mining": 6, "intellectual": 4}, - "traits": ["industrious"], "current_job": "mining", - "is_drafted": False, "needs": {"food": 0.6, "rest": 0.8}, - "injuries": [], "position": [42, 18], - }, - { - "colonist_id": "col_02", "name": "Cassandra", "health": 0.88, - "mood": 0.65, "skills": {"shooting": 3, "construction": 7, - "cooking": 6, "growing": 8, "intellectual": 6}, - "traits": ["kind"], "current_job": "growing", - "is_drafted": False, "needs": {"food": 0.5, "rest": 0.7}, - "injuries": [], "position": [30, 22], - }, - { - "colonist_id": "col_03", "name": "Randy", "health": 0.92, - "mood": 0.58, "skills": {"shooting": 10, "melee": 7, - "construction": 3, "cooking": 2}, - "traits": ["tough", "brawler"], "current_job": None, - "is_drafted": False, "needs": {"food": 0.4, "rest": 0.6}, - "injuries": [], "position": [50, 10], - }, - ], - "/api/v1/resources/summary?map_id=0": { - "total_items": 800, "total_market_value": 8000.0, - "critical_resources": { - "food_summary": {"food_total": 85}, - "medicine_total": 5, "weapon_count": 2, - }, - }, - "/api/v1/map/buildings?map_id=0": [ - {"id": "s_01", "def_name": "Wall", "position": {"x": 10, "y": 0, "z": 10}, - "hit_points": 300.0, "max_hit_points": 300.0}, - ], - "/api/v1/research/summary": { - "current_project": "electricity", "progress": 0.45, - "completed": ["stonecutting"], "available": ["electricity", "battery", "smithing"], - }, - "/api/v1/incidents?map_id=0": {"incidents": []}, - "/api/v1/game/state": { - "name": "New Hope", "wealth": 8000.0, "day": 5, "tick": 300000, - "population": 3, "mood_average": 0.65, "food_days": 7.0, - }, - "/api/v1/map/weather?map_id=0": { - "weather": "clear", "temperature": 22.0, - }, - "/api/v1/map/zones?map_id=0": [], - "/api/v1/map/rooms?map_id=0": [], - "/api/v1/map/ore?map_id=0": [], - "/api/v1/map/farm/summary?map_id=0": { - "total_growing_zones": 0, "planted_cells": 0, - "harvestable_cells": 0, "crops": {}, - }, - "/api/v1/map/terrain?map_id=0": { - "width": 10, "height": 10, - "palette": ["Soil", "WaterMovingShallow", "SoilRich", "Granite_Rough"], - "grid": [100, 0], - "floor_palette": [], "floor_grid": [100, 0], - }, - "/api/v1/resources/stored?map_id=0": { - "Resources": [ - {"def_name": "WoodLog", "stack_count": 200}, - {"def_name": "Steel", "stack_count": 100}, - {"def_name": "ComponentIndustrial", "stack_count": 10}, - ], - }, - "/api/v1/map/power/info?map_id=0": { - "current_power": 0.0, - "total_consumption": 0.0, - "currently_stored_power": 0.0, - "total_power_storage": 0.0, - }, - "/api/v1/factions": [], - "/api/v1/ui/alerts?map_id=0": [], -} - - -def _make_mock_transport() -> httpx.MockTransport: - _POST_OK = httpx.Response( - 200, content=b'{"success": true, "errors": [], "warnings": []}', - headers={"content-type": "application/json"}, - ) +# Felix roster ids, for --ablation (which is a Felix-only experiment). +_ALL_AGENT_IDS = [ + "map_analyst", "resource_manager", "defense_commander", + "research_director", "social_overseer", "construction_planner", + "medical_officer", +] - def handler(request: httpx.Request) -> httpx.Response: - # All POSTs succeed in mock mode (game control, actions, etc.) - if request.method == "POST": - return _POST_OK - # GET routes matched by full path including query string - raw = request.url.raw_path.decode() - if raw in _MOCK_ROUTES: - return httpx.Response( - 200, content=json.dumps(_MOCK_ROUTES[raw]).encode(), - headers={"content-type": "application/json"}, - ) - # Also try without query string for routes stored that way - path = raw.split("?")[0] - if path in _MOCK_ROUTES: - return httpx.Response( - 200, content=json.dumps(_MOCK_ROUTES[path]).encode(), - headers={"content-type": "application/json"}, - ) - return httpx.Response(404, content=b"Not found") +# Markers in per-tick action errors that indicate the *harness/RIMAPI plumbing* +# failed rather than the model deciding badly (issue #27, #33, CLAUDE.md +# "null-ref cascades"). A run that trips one is quarantined from leaderboard +# means by default (see analyze_spread.py for the post-hoc taxonomy). +HARNESS_FAILURE_MARKERS = ( + "Object reference not set", + "NullReferenceException", + "Invalid plant definition", +) - return httpx.MockTransport(handler) +class RunError(RuntimeError): + """A harness could not be constructed for this run.""" + + +async def _load_save(client: RimAPIClient, config: RLEConfig, scenario: ScenarioConfig) -> bool: + """Load + settle the scenario save. Returns False when the run must be skipped.""" + if not scenario.save_name: + return True + try: + await load_save_and_settle(client, config.rimapi_url, scenario.save_name) + except Exception as e: + logger.warning("Could not load save %s: %s", scenario.save_name, e) + return False + return True -def _make_mock_provider() -> MagicMock: - provider = MagicMock(spec=BaseProvider) - provider.complete.return_value = CompletionResult( - content=_MOCK_ACTION_PLAN, model="mock-model", - usage={"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, - ) - provider.acomplete.return_value = provider.complete.return_value - return provider - - -def _create_agents( # type: ignore[no-untyped-def] - provider, helix, *, provider_kwargs=None, no_think=False, - exclude_agent: str | None = None, -): - all_agents = [ - MapAnalyst("map_analyst", provider, helix, spawn_time=0.0, velocity=1.0), - ResourceManager("resource_manager", provider, helix, spawn_time=0.0, velocity=1.0), - DefenseCommander( - "defense_commander", provider, helix, spawn_time=0.0, velocity=1.0, - ), - ResearchDirector( - "research_director", provider, helix, spawn_time=0.0, velocity=1.0, - ), - SocialOverseer("social_overseer", provider, helix, spawn_time=0.0, velocity=1.0), - ConstructionPlanner( - "construction_planner", provider, helix, spawn_time=0.0, velocity=1.0, - ), - MedicalOfficer("medical_officer", provider, helix, spawn_time=0.0, velocity=1.0), - ] - agents = [a for a in all_agents if a.agent_id != exclude_agent] - if provider_kwargs: - for agent in agents: - agent.set_provider_kwargs(**provider_kwargs) - if no_think: - for agent in agents: - agent.set_no_think(True) - return agents - - -def _create_visualizer(helix, agents) -> HelixVisualizer: # type: ignore[no-untyped-def] - """Create a HelixVisualizer with all agents registered.""" - visualizer = HelixVisualizer(helix, title="R L E") - for agent in agents: - display = AGENT_DISPLAY[agent.agent_id] - visualizer.register_agent( - agent.agent_id, label=display["label"], color=display["color"], - ) - return visualizer +def _harness_failed(event_log: EventLog | None, start_index: int) -> bool: + """True when any action error since ``start_index`` matches a plumbing marker.""" + if event_log is None: + return False + for event in event_log.events[start_index:]: + err = event.data.get("error") or event.data.get("reason") + if isinstance(err, str) and any(m in err for m in HARNESS_FAILURE_MARKERS): + return True + return False -async def _run_scenario( + +async def _run_scenario( # noqa: PLR0913 scenario: ScenarioConfig, config: RLEConfig, client: RimAPIClient, - provider, # type: ignore[no-untyped-def] - helix, # type: ignore[no-untyped-def] + harness_name: str, + harness_options: dict[str, Any], output_dir: Path | None, + *, max_ticks_override: int | None = None, - provider_kwargs: dict | None = None, - visualize: bool = False, - no_think: bool = False, - parallel: bool = True, - no_agent: bool = False, + smoke: bool = False, no_pause: bool = False, event_log: EventLog | None = None, cost_tracker: CostTracker | None = None, weave_module: object | None = None, -) -> dict: - agents = _create_agents(provider, helix, provider_kwargs=provider_kwargs, no_think=no_think) +) -> dict[str, Any]: + ctx = HarnessContext( + config=config, + client=client, + expected_duration_days=scenario.expected_duration_days, + initial_population=scenario.initial_population, + scenario=scenario, + event_log=event_log, + cost_tracker=cost_tracker, + tick_timeout_s=config.tick_timeout_s, + smoke=smoke, + ) if weave_module is not None: - for agent in agents: - agent.enable_weave(weave_module) + ctx.extras["weave_module"] = weave_module + try: + harness = create_harness(harness_name, ctx, harness_options, smoke=smoke) + except (HarnessNotFoundError, HarnessUnavailableError, HarnessOptionsError) as exc: + raise RunError(str(exc)) from exc + scorer = CompositeScorer(scenario.scoring_weights or None) recorder = TimeSeriesRecorder() evaluator = ScenarioEvaluator(scenario) - visualizer = _create_visualizer(helix, agents) if visualize else None + events_before = len(event_log.events) if event_log else 0 loop = RLEGameLoop( - config, client, agents, + config, client, expected_duration_days=scenario.expected_duration_days, scorer=scorer, recorder=recorder, evaluator=evaluator, initial_population=scenario.initial_population, initial_wealth=8000.0, - visualizer=visualizer, - parallel=parallel, - no_agent=no_agent, no_pause=no_pause, event_log=event_log, cost_tracker=cost_tracker, + harness=harness, + harness_context=ctx, + scenario=scenario, ) max_ticks = max_ticks_override or scenario.max_ticks t0 = time.monotonic() - if visualizer: - with visualizer.live(): - await loop.run(max_ticks=max_ticks) - else: - await loop.run(max_ticks=max_ticks) + await loop.run(max_ticks=max_ticks) elapsed = time.monotonic() - t0 final_score = 0.0 if recorder.snapshots: - final = scorer.final_score(recorder.snapshots) - final_score = final.composite + final_score = scorer.final_score(recorder.snapshots).composite outcome = "timeout" if loop.evaluation_result: outcome = loop.evaluation_result.outcome ticks_run = len(loop.tick_results) - total_calls = loop._parse_successes + loop._parse_failures - parse_rate = loop._parse_successes / total_calls if total_calls else 0.0 + total_calls = loop.parse_successes + loop.parse_failures + parse_rate = loop.parse_successes / total_calls if total_calls else 0.0 + latencies = [t.step_latency_s for t in loop.tick_results] + slug = scenario.name.lower().replace(" ", "_") if output_dir and recorder.snapshots: - csv_name = scenario.name.lower().replace(" ", "_") + ".csv" - recorder.to_csv(output_dir / csv_name) + recorder.to_csv(output_dir / f"{harness.name}_{slug}.csv") deliberation_log = loop.deliberation_log if output_dir and deliberation_log: - log_name = scenario.name.lower().replace(" ", "_") + "_deliberations.jsonl" - with open(output_dir / log_name, "w") as f: + with open(output_dir / f"{harness.name}_{slug}_deliberations.jsonl", "w") as f: for entry in deliberation_log: f.write(json.dumps(entry) + "\n") return { "name": scenario.name, "difficulty": scenario.difficulty, + "harness": harness.name, + "harness_versions": harness.describe(), "outcome": outcome, "score": final_score, "ticks": ticks_run, "elapsed_s": round(elapsed, 2), "sec_per_tick": round(elapsed / ticks_run, 2) if ticks_run else 0.0, - "parse_successes": loop._parse_successes, - "parse_failures": loop._parse_failures, + "mean_step_latency_s": ( + round(sum(latencies) / len(latencies), 3) if latencies else 0.0 + ), + "parse_successes": loop.parse_successes, + "parse_failures": loop.parse_failures, "parse_rate": round(parse_rate, 3), + "harness_failed": _harness_failed(event_log, events_before), } -def _print_leaderboard(results: list[dict], model: str | None = None) -> None: - print("\n" + "=" * 88) +def _print_leaderboard(results: list[dict[str, Any]], model: str | None = None) -> None: + print("\n" + "=" * 100) title = f"RLE BENCHMARK — {model}" if model else "RLE BENCHMARK LEADERBOARD" print(title) - print("=" * 88) + print("=" * 100) header = ( - f"{'Scenario':<25} {'Diff':<7} {'Outcome':<9} " - f"{'Score':>6} {'Ticks':>5} {'Time':>7} {'s/tick':>6} " - f"{'Parse%':>7} {'Fail':>4}" + f"{'Harness':<12} {'Scenario':<22} {'Diff':<7} {'Outcome':<9} " + f"{'Score':>6} {'Ticks':>5} {'Time':>7} {'s/step':>6} " + f"{'Parse%':>7} {'Fail':>4} {'HF':>3}" ) print(header) - print("-" * 88) + print("-" * 100) for r in results: print( - f"{r['name']:<25} {r['difficulty']:<7} {r['outcome']:<9} " + f"{r['harness']:<12} {r['name']:<22} {r['difficulty']:<7} {r['outcome']:<9} " f"{r['score']:>6.3f} {r['ticks']:>5} {r['elapsed_s']:>6.1f}s " - f"{r['sec_per_tick']:>6.2f} {r['parse_rate']:>6.1%} {r['parse_failures']:>4}" + f"{r['mean_step_latency_s']:>6.2f} {r['parse_rate']:>6.1%} " + f"{r['parse_failures']:>4} {'!' if r.get('harness_failed') else '':>3}" ) - print("-" * 88) - scores = [r["score"] for r in results] - passed = sum(1 for r in results if r["outcome"] == "victory") + print("-" * 100) + clean = [r for r in results if not r.get("harness_failed")] + scores = [r["score"] for r in clean] + passed = sum(1 for r in clean if r["outcome"] == "victory") avg = sum(scores) / len(scores) if scores else 0.0 total_parse = sum(r["parse_successes"] for r in results) total_fail = sum(r["parse_failures"] for r in results) total_calls = total_parse + total_fail overall_parse_rate = total_parse / total_calls if total_calls else 0.0 total_time = sum(r["elapsed_s"] for r in results) + quarantined = len(results) - len(clean) print( - f"Avg score: {avg:.3f} | Passed: {passed}/{len(results)} | " + f"Avg score: {avg:.3f} | Passed: {passed}/{len(clean)} | " f"Parse rate: {overall_parse_rate:.1%} ({total_fail} failures) | " f"Total time: {total_time:.1f}s" + + (f" | {quarantined} run(s) quarantined (HF = harness failure)" if quarantined else "") ) - print("=" * 88) + print("=" * 100) -def _build_provider(args: argparse.Namespace) -> tuple[BaseProvider, RLEConfig]: - """Build LLM provider from CLI args. Returns (provider, config).""" - if args.dry_run and not args.provider: - return _make_mock_provider(), RLEConfig(tick_interval=0.0) - - overrides: dict[str, str] = {} +def _build_config(args: argparse.Namespace, smoke: bool) -> RLEConfig: + overrides: dict[str, Any] = {} if args.provider: overrides["provider"] = args.provider if args.model: overrides["model"] = args.model if args.base_url: overrides["provider_base_url"] = args.base_url - config = RLEConfig(**overrides) if overrides else RLEConfig() - bridge_openrouter_key(config) - bridge_anthropic_key(config) - return config.get_provider(), config + if args.tick_interval is not None and not smoke: + overrides["tick_interval"] = args.tick_interval + if args.tick_timeout is not None: + overrides["tick_timeout_s"] = args.tick_timeout + if smoke and args.tick_interval is None: + overrides["tick_interval"] = 0.0 + return RLEConfig(**overrides) if overrides else RLEConfig() def _resolve_ticks(args: argparse.Namespace, use_mock_rimapi: bool) -> int | None: @@ -371,35 +256,23 @@ def _resolve_ticks(args: argparse.Namespace, use_mock_rimapi: bool) -> int | Non return None -_ALL_AGENT_IDS = [ - "map_analyst", "resource_manager", "defense_commander", - "research_director", "social_overseer", "construction_planner", - "medical_officer", -] - - -async def _run_ablation( +async def _run_ablation( # noqa: PLR0913 args: argparse.Namespace, config: RLEConfig, - provider: object, - helix: object, + harness_options: dict[str, Any], scenarios: list[ScenarioConfig], use_mock_rimapi: bool, num_runs: int, ticks_override: int | None, - provider_kwargs: dict[str, Any] | None, ) -> None: - """Run ablation study: full benchmark + 7 single-agent-removed benchmarks.""" + """Ablation study (Felix only): full roster + 7 single-agent-removed passes.""" output_dir = Path(args.output) if args.output else get_run_dir(args.model) output_dir.mkdir(parents=True, exist_ok=True) async with RimAPIClient(config.rimapi_url) as client: if use_mock_rimapi: - client._client = httpx.AsyncClient( - transport=_make_mock_transport(), base_url="http://mock", - ) + MockRimAPI().attach(client) - # Pass 0: full benchmark (all agents) passes: list[tuple[str, list[dict[str, Any]]]] = [] labels = ["all_agents", *_ALL_AGENT_IDS] @@ -414,44 +287,20 @@ async def _run_ablation( for scenario in scenarios: for run_id in range(num_runs): run_label = f" (run {run_id + 1}/{num_runs})" if num_runs > 1 else "" - if scenario.save_name and not use_mock_rimapi: - try: - await client.load_game(scenario.save_name) - await asyncio.sleep(GAME_LOAD_WAIT_SECONDS) - except Exception as e: - logger.warning("Could not load save %s: %s", scenario.save_name, e) - print(f" SKIP {scenario.name}{run_label} ({tag}): save load failed") - continue + if not use_mock_rimapi and not await _load_save(client, config, scenario): + print(f" SKIP {scenario.name}{run_label} ({tag}): save load failed") + continue print(f" {scenario.name}{run_label} ({tag})...") - agents = _create_agents( - provider, helix, - provider_kwargs=provider_kwargs, - no_think=args.no_think, - exclude_agent=exclude, - ) - scorer = CompositeScorer(scenario.scoring_weights or None) - recorder = TimeSeriesRecorder() - evaluator = ScenarioEvaluator(scenario) - loop = RLEGameLoop( - config, client, agents, - expected_duration_days=scenario.expected_duration_days, - scorer=scorer, recorder=recorder, evaluator=evaluator, - initial_population=scenario.initial_population, - initial_wealth=8000.0, - parallel=not args.sequential, + options = {**harness_options, "exclude_agent": exclude} + result = await _run_scenario( + scenario, config, client, "felix", options, None, + max_ticks_override=ticks_override, + smoke=use_mock_rimapi, no_pause=args.no_pause, ) - await loop.run(max_ticks=ticks_override or scenario.max_ticks) - - final_score = 0.0 - if recorder.snapshots: - final_score = scorer.final_score(recorder.snapshots).composite - pass_results.append({ - "scenario": scenario.name, - "score": final_score, - }) - print(f" score={final_score:.3f}") + pass_results.append({"scenario": scenario.name, "score": result["score"]}) + print(f" score={result['score']:.3f}") passes.append((tag, pass_results)) @@ -474,7 +323,6 @@ async def _run_ablation( ) matrix[agent_name][scenario_name] = round(full_avg - rem_avg, 4) - # Print ablation table scenario_names = list(full_scores.keys()) print(f"\n{'=' * 88}") print("ABLATION MATRIX (score delta: positive = agent helps)") @@ -493,7 +341,6 @@ async def _run_ablation( print(row) print(f"{'=' * 88}") - # Save results ablation_data = { "num_runs": num_runs, "ticks_per_scenario": ticks_override, @@ -505,43 +352,44 @@ async def _run_ablation( print(f"\nAblation results saved to {ablation_path}") -async def main(args: argparse.Namespace) -> None: +async def main(args: argparse.Namespace) -> None: # noqa: PLR0912, PLR0915 logging.basicConfig( level=getattr(logging, args.log_level.upper(), logging.INFO), format="%(asctime)s %(name)s %(levelname)s %(message)s", ) + if maybe_handle_harness_list(args): + return + # Seed RLE-side stochasticity (resolver tiebreaks, json_repair fallbacks). # RimWorld's RNG is unaffected — see metadata.collect_metadata docstring. if args.seed is not None: random.seed(args.seed) - helix = HelixConfig.default().to_geometry() scenarios = list_scenarios() - provider, config = _build_provider(args) is_smoke_test = args.smoke_test or args.dry_run if args.dry_run: logger.warning("--dry-run is deprecated, use --smoke-test") - if args.tick_interval is not None and not is_smoke_test: - config = RLEConfig(**{**config.model_dump(), "tick_interval": args.tick_interval}) - use_mock_rimapi = (is_smoke_test or args.provider is not None) and not args.docker + config = _build_config(args, is_smoke_test) + # Smoke test = mock game AND mock model. --smoke-test with --provider used + # to mean "real LLM against the fake game"; that is now + # --harness-opt smoke_llm=false territory for harnesses that support it. + use_mock_rimapi = is_smoke_test and not args.docker num_runs = getattr(args, "runs", 1) or 1 - # extra_body is an OpenAI-compatible escape hatch; the Anthropic API - # rejects unknown body fields with 400. - effective_provider = args.provider or config.provider + harness_names = selected_harnesses(args, config) + harness_option_sets = { + name: harness_options_for(name, args, config) for name in harness_names + } if args.ablation: + if harness_names != ["felix"]: + raise SystemExit("--ablation is a Felix-only experiment (use --harness felix)") ticks_override = _resolve_ticks(args, use_mock_rimapi) - provider_kwargs_abl: dict[str, Any] = {} - if args.no_think and effective_provider != "anthropic": - provider_kwargs_abl["extra_body"] = { - "chat_template_kwargs": {"enable_thinking": False}, - } await _run_ablation( - args, config, provider, helix, scenarios, use_mock_rimapi, - num_runs, ticks_override, provider_kwargs_abl or None, + args, config, harness_option_sets["felix"], scenarios, use_mock_rimapi, + num_runs, ticks_override, ) return @@ -561,31 +409,24 @@ async def main(args: argparse.Namespace) -> None: output_dir = Path(args.output) output_dir.mkdir(parents=True, exist_ok=True) - # Build provider kwargs (e.g. no-think for Qwen3.5) - provider_kwargs: dict[str, Any] = {} - if args.no_think and effective_provider != "anthropic": - provider_kwargs["extra_body"] = { - "chat_template_kwargs": {"enable_thinking": False}, - } - # Initialize W&B logger (no-op if --wandb not passed or wandb not installed) wandb_logger = WandBLogger( enabled=args.wandb, - run_name=f"{args.model or config.model}_{ticks_override or 'full'}ticks", + run_name=f"{'+'.join(harness_names)}_{config.model}_{ticks_override or 'full'}ticks", ) if wandb_logger.enabled: wandb_logger.log_config({ **collect_metadata(random_seed=args.seed), - "model": args.model or config.model, - "provider": args.provider or config.provider, - "no_think": args.no_think, - "parallel": not args.sequential, + "harnesses": harness_names, + "harness_options": harness_option_sets, + "model": config.model, + "provider": config.provider, "ticks_per_scenario": ticks_override, }) # Initialize cost tracker (fetches OpenRouter pricing; CLI may override) cost_tracker = await create_cost_tracker( - args.model or config.model, + config.model, prompt_price_override=( args.prompt_price_per_mtok / 1_000_000 if args.prompt_price_per_mtok is not None else None @@ -596,110 +437,105 @@ async def main(args: argparse.Namespace) -> None: ), ) - # Initialize event log event_log: EventLog | None = None if args.output: event_log = EventLog(Path(args.output) / "events.jsonl") - no_baseline = getattr(args, "no_baseline", False) + no_baseline = getattr(args, "no_baseline", False) or harness_names == ["baseline"] is_paired = not use_mock_rimapi and not no_baseline - results = [] + results: list[dict[str, Any]] = [] paired_results: list[PairedResult] = [] - # Docker lifecycle (optional) - docker_server = None + docker_server: DockerGameServer | None = None if args.docker: - from rle.docker import DockerGameServer docker_server = DockerGameServer( image=config.docker_image, port=config.docker_port, ) await docker_server.start() - config = RLEConfig(**{ - **config.model_dump(), - "rimapi_url": docker_server.url, - }) + config = RLEConfig(**{**config.model_dump(), "rimapi_url": docker_server.url}) + history_path: Path | None = None try: async with RimAPIClient(config.rimapi_url) as client: if use_mock_rimapi: - client._client = httpx.AsyncClient( - transport=_make_mock_transport(), base_url="http://mock", - ) - - for scenario in scenarios: - if docker_server: - await docker_server.restart() - paired = PairedResult(scenario=scenario.name) if is_paired else None + MockRimAPI().attach(client) + + for harness_name in harness_names: + harness_options = harness_option_sets[harness_name] + if len(harness_names) > 1: + print(f"\n{'#' * 60}\n# HARNESS: {harness_name}\n{'#' * 60}") + + for scenario in scenarios: + if docker_server: + await docker_server.restart() + paired = ( + PairedResult(scenario=f"{harness_name}/{scenario.name}") + if is_paired else None + ) - for run_id in range(num_runs): - run_label = f" (run {run_id + 1}/{num_runs})" if num_runs > 1 else "" + for run_id in range(num_runs): + run_label = f" (run {run_id + 1}/{num_runs})" if num_runs > 1 else "" - # Load save if available (for reproducible initial conditions) - if scenario.save_name and not use_mock_rimapi: - try: - await client.load_game(scenario.save_name) - await asyncio.sleep(GAME_LOAD_WAIT_SECONDS) - except Exception as e: - logger.warning("Could not load save %s: %s", scenario.save_name, e) + if not use_mock_rimapi and not await _load_save(client, config, scenario): print(f" SKIP {scenario.name}{run_label}: save load failed") continue - # Agent run - print(f"\nRunning: {scenario.name} ({scenario.difficulty}){run_label}...") - result = await _run_scenario( - scenario, config, client, provider, helix, output_dir, - max_ticks_override=ticks_override, - provider_kwargs=provider_kwargs or None, - visualize=args.visualize, - no_think=args.no_think, - parallel=not args.sequential, - no_pause=args.no_pause, - event_log=event_log, - cost_tracker=cost_tracker, - weave_module=wandb_logger.weave, - ) - results.append(result) - if paired: - paired.agent_scores.append(result["score"]) - print( - f" -> agent: {result['outcome']} | score={result['score']:.3f} " - f"| {result['ticks']} ticks | {result['elapsed_s']}s " - f"| parse {result['parse_rate']:.0%} ({result['parse_failures']} fail)" - ) - - # Baseline run (reload same save, no agents) - if is_paired: - if scenario.save_name: - try: - await client.load_game(scenario.save_name) - await asyncio.sleep(GAME_LOAD_WAIT_SECONDS) - except Exception as e: - logger.warning("Could not reload save: %s", e) - - print(f" baseline{run_label}...") - baseline = await _run_scenario( - scenario, config, client, provider, helix, output_dir, - max_ticks_override=ticks_override, - no_agent=True, + print( + f"\nRunning: {scenario.name} ({scenario.difficulty}) " + f"[{harness_name}]{run_label}...", + ) + try: + result = await _run_scenario( + scenario, config, client, harness_name, harness_options, + output_dir, + max_ticks_override=ticks_override, + smoke=use_mock_rimapi, + no_pause=args.no_pause, + event_log=event_log, + cost_tracker=cost_tracker, + weave_module=wandb_logger.weave, + ) + except RunError as exc: + exit_with_harness_error(exc) + return + results.append(result) + if paired: + paired.agent_scores.append(result["score"]) + print( + f" -> {harness_name}: {result['outcome']} " + f"| score={result['score']:.3f} | {result['ticks']} ticks " + f"| {result['elapsed_s']}s | parse {result['parse_rate']:.0%} " + f"({result['parse_failures']} fail)" + + (" | HARNESS FAILURE" if result["harness_failed"] else "") ) - paired.baseline_scores.append(baseline["score"]) - print(f" -> baseline: score={baseline['score']:.3f}") - if paired: - paired_results.append(paired) + # Baseline run (reload same save, unmanaged colony) + if paired is not None: + if not await _load_save(client, config, scenario): + logger.warning("Could not reload save for baseline") + print(f" baseline{run_label}...") + baseline = await _run_scenario( + scenario, config, client, "baseline", {}, output_dir, + max_ticks_override=ticks_override, + no_pause=args.no_pause, + ) + paired.baseline_scores.append(baseline["score"]) + print(f" -> baseline: score={baseline['score']:.3f}") + + if paired: + paired_results.append(paired) # Print results if is_paired and paired_results: - from rle.scoring.delta import print_paired_leaderboard - print_paired_leaderboard(paired_results, model=args.model, num_runs=num_runs) + print_paired_leaderboard(paired_results, model=config.model, num_runs=num_runs) else: - _print_leaderboard(results, model=args.model) + _print_leaderboard(results, model=config.model) # Reconcile estimates against OpenRouter's billed ground truth # (token-count estimates diverged up to 4x on the v0.3.0 spread). billed_report = None - effective_base_url = args.base_url or config.provider_base_url or "" + effective_base_url = config.provider_base_url or "" openai_key = os.environ.get("OPENAI_API_KEY", "") generation_ids = cost_tracker.generation_ids if "openrouter.ai" in effective_base_url and openai_key and generation_ids: @@ -709,20 +545,25 @@ async def main(args: argparse.Namespace) -> None: ) billed_report = await fetch_billed_costs(generation_ids, openai_key) - # Build enriched summary with metadata + clean_results = [r for r in results if not r.get("harness_failed")] metadata = collect_metadata(random_seed=args.seed) summary: dict[str, Any] = { **metadata, - "model": args.model or config.model, - "provider": args.provider or config.provider, - "base_url": args.base_url or None, - "no_think": args.no_think, - "parallel": not args.sequential, + "harness": harness_names[0] if len(harness_names) == 1 else "matrix", + "harnesses": harness_names, + "harness_options": harness_option_sets, + "harness_versions": { + r["harness"]: r["harness_versions"] for r in results + }, + "model": config.model, + "provider": config.provider, + "base_url": config.provider_base_url, "tick_interval": config.tick_interval, "ticks_per_scenario": ticks_override, "num_runs": num_runs, "paired": is_paired, "scenarios": results, + "quarantined_runs": len(results) - len(clean_results), "cost_snapshot": cost_tracker.snapshot().model_dump(), } if billed_report: @@ -732,15 +573,14 @@ async def main(args: argparse.Namespace) -> None: if is_paired and paired_results: summary["paired_results"] = [p.to_dict() for p in paired_results] - # Auto-generate run directory if --output not specified - output_dir = Path(args.output) if args.output else get_run_dir(args.model) + output_dir = Path(args.output) if args.output else get_run_dir(config.model) output_dir.mkdir(parents=True, exist_ok=True) summary_path = output_dir / "benchmark_summary.json" summary_path.write_text(json.dumps(summary, indent=2, default=str)) print(f"\nResults exported to {output_dir}/") - # Only track real benchmark runs (not mock/dry-run JSON compliance tests) - scores = [r.get("score", 0) for r in results] + # Only track real benchmark runs (not smoke tests) + scores = [r.get("score", 0) for r in clean_results] avg = sum(scores) / len(scores) if scores else 0 if not use_mock_rimapi: history_path = append_history(summary) @@ -753,9 +593,8 @@ async def main(args: argparse.Namespace) -> None: elif prev_score is not None: print(f"Baseline: {prev_score:.3f} (this run: {avg:.3f})") else: - print("(dry-run: skipping history/baseline tracking)") + print("(smoke test: skipping history/baseline tracking)") - # W&B logging (optional) if wandb_logger.enabled: wandb_logger.log_final_summary( avg_score=avg, @@ -768,8 +607,7 @@ async def main(args: argparse.Namespace) -> None: wandb_logger.finish() print("W&B run logged") - # HuggingFace Hub push (optional) - if args.push_hf: + if args.push_hf and history_path is not None: hf = HFLogger( repo_id=config.hf_dataset_repo, token=config.hf_token, ) @@ -793,7 +631,7 @@ async def main(args: argparse.Namespace) -> None: parser.add_argument("--output", help="Output directory for CSV results") parser.add_argument( "--smoke-test", action="store_true", - help="Use mock RIMAPI (combine with --provider for real LLM + fake game)", + help="Mock RIMAPI + each harness's smoke variant (no game, no LLM)", ) parser.add_argument( "--dry-run", action="store_true", @@ -805,24 +643,23 @@ async def main(args: argparse.Namespace) -> None: ) parser.add_argument( "--ablation", action="store_true", - help="(WIP) Run ablation study: full benchmark + 7 single-agent-removed runs", + help="(WIP, felix only) Run ablation study: full roster + 7 single-agent-removed runs", ) parser.add_argument( - "--provider", choices=["anthropic", "openai", "local", "claude-code"], - help="LLM provider (default: from config)", + "--provider", + help="LLM provider name passed to the harness (felix: anthropic|openai|local|claude-code)", ) parser.add_argument("--model", help="Model name (e.g. qwen/qwen3.5-9b)") parser.add_argument("--base-url", help="Provider API base URL (e.g. http://localhost:1234/v1)") + add_harness_args(parser, repeatable=True) parser.add_argument("--ticks", type=int, help="Override max ticks per scenario") parser.add_argument( "--tick-interval", type=float, help="Seconds between ticks (default: 1.0, use 30-60 for live game)", ) - parser.add_argument("--no-think", action="store_true", help="Disable thinking mode (Qwen3.5)") - parser.add_argument("--visualize", action="store_true", help="Show live helix visualization") parser.add_argument( - "--sequential", action="store_true", - help="Run agents sequentially (default: parallel)", + "--tick-timeout", type=float, default=None, + help="Loop-level cap in seconds on a whole harness step (default: none).", ) parser.add_argument( "--runs", type=int, default=1, @@ -830,7 +667,7 @@ async def main(args: argparse.Namespace) -> None: ) parser.add_argument( "--no-baseline", action="store_true", - help="Skip baseline (no-agent) runs — agent-only, no paired comparison", + help="Skip baseline (unmanaged) runs — harness-only, no paired comparison", ) parser.add_argument( "--no-pause", action="store_true", diff --git a/scripts/run_scenario.py b/scripts/run_scenario.py index 5d8b370..a88d32a 100644 --- a/scripts/run_scenario.py +++ b/scripts/run_scenario.py @@ -11,21 +11,22 @@ import sys from pathlib import Path -from felix_agent_sdk.core import HelixConfig -from felix_agent_sdk.visualization import HelixVisualizer - -from rle.agents import AGENT_DISPLAY -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer -from rle.config import RLEConfig, bridge_anthropic_key, bridge_openrouter_key -from rle.docker import wait_for_rimapi +from rle.config import RLEConfig +from rle.harness import ( + HarnessContext, + HarnessNotFoundError, + HarnessOptionsError, + HarnessUnavailableError, + add_harness_args, + create_harness, + exit_with_harness_error, + harness_options_for, + maybe_handle_harness_list, + selected_harnesses, +) from rle.orchestration.camera_director import CameraDirector from rle.orchestration.game_loop import RLEGameLoop +from rle.orchestration.save_loader import load_save_and_settle from rle.rimapi.client import RimAPIClient from rle.rimapi.sse_client import RimAPISSEClient from rle.scenarios.evaluator import ScenarioEvaluator @@ -68,36 +69,38 @@ def _write_deliberations_jsonl( def _build_run_summary( # noqa: PLR0913 args: argparse.Namespace, - config_model: str, - config_provider: str, - config_tick_interval: float, + config: RLEConfig, + harness_name: str, + harness_options: dict[str, object], + harness_describe: dict[str, str], scenario_name: str, scenario_save_name: str, max_ticks: int | None, outcome: str, final_score: float | None, ticks_run: int, + mean_step_latency_s: float | None, cost_snapshot_dict: dict[str, object], event_summary_dict: dict[str, object] | None, billed_cost_dict: dict[str, object] | None = None, ) -> dict[str, object]: """Compose the per-scenario summary JSON (metadata + config + result).""" summary: dict[str, object] = { - **collect_metadata(random_seed=args.seed), + **collect_metadata(random_seed=args.seed, harness_describe=harness_describe), "scenario": scenario_name, "scenario_save_name": scenario_save_name, - "model": args.model or config_model, - "provider": args.provider or config_provider, - "base_url": args.base_url or None, - "no_think": args.no_think, - "parallel": not args.sequential, - "no_agent": args.no_agent, + "harness": harness_name, + "harness_options": harness_options, + "model": config.model, + "provider": config.provider, + "base_url": config.provider_base_url, "no_pause": args.no_pause, - "tick_interval": config_tick_interval, + "tick_interval": config.tick_interval, "max_ticks": max_ticks, "outcome": outcome, "final_score": final_score, "ticks_run": ticks_run, + "mean_step_latency_s": mean_step_latency_s, "cost_snapshot": cost_snapshot_dict, } if billed_cost_dict is not None: @@ -107,21 +110,6 @@ def _build_run_summary( # noqa: PLR0913 return summary -def _create_agents(provider, helix): # type: ignore[no-untyped-def] - """Create all 7 role agents (MapAnalyst + 6 domain agents).""" - return [ - MapAnalyst("map_analyst", provider, helix, spawn_time=0.0, velocity=1.0), - ResourceManager("resource_manager", provider, helix, spawn_time=0.0, velocity=1.0), - DefenseCommander("defense_commander", provider, helix, spawn_time=0.0, velocity=1.0), - ResearchDirector("research_director", provider, helix, spawn_time=0.0, velocity=1.0), - SocialOverseer("social_overseer", provider, helix, spawn_time=0.0, velocity=1.0), - ConstructionPlanner( - "construction_planner", provider, helix, spawn_time=0.0, velocity=1.0, - ), - MedicalOfficer("medical_officer", provider, helix, spawn_time=0.0, velocity=1.0), - ] - - def _print_results(loop: RLEGameLoop, recorder: TimeSeriesRecorder) -> None: """Print final score summary.""" if not recorder.snapshots: @@ -147,6 +135,9 @@ async def main(args: argparse.Namespace) -> None: format="%(asctime)s %(name)s %(levelname)s %(message)s", ) + if maybe_handle_harness_list(args): + return + # Seed RLE-side stochasticity (resolver tiebreaks, json_repair fallbacks). # RimWorld's RNG is unaffected — see metadata.collect_metadata docstring. if args.seed is not None: @@ -173,7 +164,7 @@ async def main(args: argparse.Namespace) -> None: print(f"Duration: {scenario.expected_duration_days} days, max {scenario.max_ticks} ticks") # Setup - overrides: dict[str, str] = {} + overrides: dict[str, object] = {} if args.provider: overrides["provider"] = args.provider if args.model: @@ -181,30 +172,19 @@ async def main(args: argparse.Namespace) -> None: if args.base_url: overrides["provider_base_url"] = args.base_url if args.tick_interval is not None: - overrides["tick_interval"] = str(args.tick_interval) - config = RLEConfig(**overrides) if overrides else RLEConfig() - bridge_openrouter_key(config) - bridge_anthropic_key(config) - provider = config.get_provider() - helix = HelixConfig.default().to_geometry() - agents = _create_agents(provider, helix) - if args.no_think: - for agent in agents: - agent.set_no_think(True) + overrides["tick_interval"] = args.tick_interval + if args.tick_timeout is not None: + overrides["tick_timeout_s"] = args.tick_timeout + config = RLEConfig(**overrides) if overrides else RLEConfig() # type: ignore[arg-type] + + harness_name = selected_harnesses(args, config)[0] + harness_options = harness_options_for(harness_name, args, config) + print(f"Harness: {harness_name} {harness_options or ''}".rstrip()) scorer = CompositeScorer(scenario.scoring_weights or None) recorder = TimeSeriesRecorder() evaluator = ScenarioEvaluator(scenario) - visualizer = None - if args.visualize: - visualizer = HelixVisualizer(helix, title="R L E") - for agent in agents: - display = AGENT_DISPLAY[agent.agent_id] - visualizer.register_agent( - agent.agent_id, label=display["label"], color=display["color"], - ) - if args.until_death: # Natural-conclusion mode (Phase B): no scenario tick cap — the run # ends when the evaluator hits a terminal condition (all colonists @@ -221,7 +201,7 @@ async def main(args: argparse.Namespace) -> None: Path(args.output).mkdir(parents=True, exist_ok=True) event_log = EventLog(Path(args.output) / "events.jsonl") cost_tracker = await create_cost_tracker( - args.model or config.model, + config.model, prompt_price_override=_per_mtok_to_per_token(args.prompt_price_per_mtok), completion_price_override=_per_mtok_to_per_token(args.completion_price_per_mtok), ) @@ -235,29 +215,9 @@ async def main(args: argparse.Namespace) -> None: if scenario.save_name: print(f"Loading save: {scenario.save_name}") try: - await client.load_game(scenario.save_name) - # Wait for RIMAPI to respond, then poll until colonists are loaded. - # Then wait ~10s of stable state before any writes: RIMAPI returns - # HTTP 200 before Unity's main thread has finished applying the load, - # and writes that race the settle window get 500'd. - await wait_for_rimapi(config.rimapi_url, timeout=30.0) - stable_count = 0 - last_population = -1 - for _ in range(30): - await asyncio.sleep(2) - try: - colony = await client.get_colony() - if colony.population > 0 and colony.population == last_population: - stable_count += 1 - if stable_count >= 5: - break - else: - stable_count = 0 - last_population = colony.population - except Exception: - stable_count = 0 - # Unforbid all starting items so colonists can use them - unforbid_count = await client.unforbid_all_items() + unforbid_count = await load_save_and_settle( + client, config.rimapi_url, scenario.save_name, + ) if unforbid_count: print(f"Unforbid {unforbid_count} items.") print("Save loaded, game ready.") @@ -288,31 +248,45 @@ async def main(args: argparse.Namespace) -> None: output_dir=Path(args.output) if args.output else None, ) + harness_ctx = HarnessContext( + config=config, + client=client, + expected_duration_days=scenario.expected_duration_days, + initial_population=scenario.initial_population, + scenario=scenario, + event_log=event_log, + cost_tracker=cost_tracker, + tick_timeout_s=config.tick_timeout_s, + ) + try: + harness = create_harness(harness_name, harness_ctx, harness_options) + except (HarnessNotFoundError, HarnessUnavailableError, HarnessOptionsError) as exc: + sse.stop() + sse_task.cancel() + exit_with_harness_error(exc) + return + loop = RLEGameLoop( - config, client, agents, + config, client, expected_duration_days=scenario.expected_duration_days, scorer=scorer, recorder=recorder, evaluator=evaluator, initial_population=scenario.initial_population, - visualizer=visualizer, - parallel=not args.sequential, sse_client=sse, dashboard_export_dir=Path(args.output) if args.output else None, - no_agent=args.no_agent, no_pause=args.no_pause, event_log=event_log, cost_tracker=cost_tracker, triggered_incidents=scenario.triggered_incidents, auto_dismiss_dialogs=not args.no_dismiss_dialogs, camera_director=camera_director, + harness=harness, + harness_context=harness_ctx, + scenario=scenario, ) try: - if visualizer: - with visualizer.live(): - await loop.run(max_ticks=max_ticks) - else: - await loop.run(max_ticks=max_ticks) + await loop.run(max_ticks=max_ticks) finally: sse.stop() sse_task.cancel() @@ -324,7 +298,7 @@ async def main(args: argparse.Namespace) -> None: # token-count estimator diverged up to 4x in both directions on the # v0.3.0 spread (thinking-model usage shapes, caching discounts). billed_report = None - effective_base_url = args.base_url or config.provider_base_url or "" + effective_base_url = config.provider_base_url or "" openai_key = os.environ.get("OPENAI_API_KEY", "") generation_ids = cost_tracker.generation_ids if "openrouter.ai" in effective_base_url and openai_key and generation_ids: @@ -347,12 +321,14 @@ async def main(args: argparse.Namespace) -> None: await asyncio.to_thread(_write_deliberations_jsonl, log_path, deliberation_log) print(f"Deliberations exported to {log_path}") + latencies = [t.step_latency_s for t in loop.tick_results] # Replay-grade scenario summary with full metadata + cost + score. summary = _build_run_summary( args=args, - config_model=config.model, - config_provider=config.provider, - config_tick_interval=config.tick_interval, + config=config, + harness_name=harness.name, + harness_options=harness_options, + harness_describe=harness.describe(), scenario_name=scenario.name, scenario_save_name=scenario.save_name, max_ticks=max_ticks, @@ -364,6 +340,9 @@ async def main(args: argparse.Namespace) -> None: recorder.snapshots[-1].composite if recorder.snapshots else None ), ticks_run=len(loop.tick_results), + mean_step_latency_s=( + round(sum(latencies) / len(latencies), 3) if latencies else None + ), cost_snapshot_dict=cost_tracker.snapshot().model_dump(), event_summary_dict=( event_log.summary().model_dump() if event_log else None @@ -423,27 +402,22 @@ async def main(args: argparse.Namespace) -> None: parser.add_argument("scenario", nargs="?", help="Scenario name or number prefix") parser.add_argument("--list", action="store_true", help="List available scenarios") parser.add_argument( - "--provider", choices=["anthropic", "openai", "local", "claude-code"], - help="LLM provider (default: from config)", + "--provider", + help="LLM provider name passed to the harness (felix: anthropic|openai|local|claude-code)", ) parser.add_argument("--model", help="Model name (e.g. unsloth/nvidia-nemotron-3-nano-4b)") parser.add_argument("--base-url", help="Provider API base URL") + add_harness_args(parser) parser.add_argument("--ticks", type=int, help="Override max ticks") parser.add_argument( "--tick-interval", type=float, help="Seconds between ticks (default: 1.0, use 30-60 for live game)", ) - parser.add_argument("--output", help="Output directory for CSV results") - parser.add_argument("--visualize", action="store_true", help="Show live helix visualization") parser.add_argument( - "--sequential", action="store_true", - help="Run agents sequentially (default: parallel)", - ) - parser.add_argument("--no-think", action="store_true", help="Skip reasoning tokens") - parser.add_argument( - "--no-agent", action="store_true", - help="Baseline mode: no agent deliberation, colony runs unmanaged", + "--tick-timeout", type=float, default=None, + help="Loop-level cap in seconds on a whole harness step (default: none).", ) + parser.add_argument("--output", help="Output directory for CSV results") parser.add_argument( "--until-death", action="store_true", help="Ignore the scenario tick cap; run until the evaluator reaches " diff --git a/src/rle/__init__.py b/src/rle/__init__.py index fb93fd3..9897cfc 100644 --- a/src/rle/__init__.py +++ b/src/rle/__init__.py @@ -1,6 +1,10 @@ """RLE — RimWorld Learning Environment. -Multi-agent benchmark where Felix Agent SDK agents play RimWorld. +A harness x model benchmark: swappable harnesses (Felix multi-agent, an +unmanaged baseline, external coding agents over MCP, ...) manage a RimWorld +colony and are scored on the same footing. This top-level package is +framework-free; the Felix harness lives under ``rle.harness.felix`` behind +the optional ``felix`` extra. """ __version__ = "0.4.1" # x-release-please-version @@ -9,17 +13,10 @@ Action, ActionPlan, ActionPlanParseError, - ConstructionPlanner, - DefenseCommander, - MedicalOfficer, - ResearchDirector, - ResourceManager, - RimWorldRoleAgent, - SocialOverseer, - register_rle_agents, resolve_endpoint, ) from rle.config import RLEConfig +from rle.harness import BaseHarness, HarnessContext, StepResult, create_harness from rle.orchestration import ( ActionExecutor, ActionResolver, @@ -63,38 +60,34 @@ "ActionPlan", "ActionPlanParseError", "ActionResolver", + "BaseHarness", "resolve_endpoint", "ColonistData", "ColonyData", "CompositeScorer", - "ConstructionPlanner", "CrisisState", - "DefenseCommander", "EvaluationResult", "ExecutionResult", "GameState", "GameStateManager", + "HarnessContext", "MapData", - "MedicalOfficer", "MetricContext", "RLEConfig", "RLEGameLoop", "ResearchData", - "ResearchDirector", "ResourceData", - "ResourceManager", "RimAPIClient", - "RimWorldRoleAgent", "ScenarioConfig", "ScenarioEvaluator", "ScoreSnapshot", - "SocialOverseer", + "StepResult", "StructureData", "ThreatData", "TickResult", "TimeSeriesRecorder", "WeatherData", + "create_harness", "list_scenarios", "load_scenario", - "register_rle_agents", ] diff --git a/src/rle/agents/__init__.py b/src/rle/agents/__init__.py index 335f1b3..a90f18b 100644 --- a/src/rle/agents/__init__.py +++ b/src/rle/agents/__init__.py @@ -1,57 +1,20 @@ -"""RLE role agents and registration.""" +"""Harness-neutral action vocabulary. -from felix_agent_sdk import AgentFactory +``Action`` / ``ActionPlan`` are what every harness hands the environment; +``json_repair`` is a generic LLM-output cleaner any LLM-backed harness can +reuse. The Felix role agents that used to live here are now +``rle.harness.felix.agents`` — this package must stay importable without +``felix-agent-sdk``. +""" from rle.agents.actions import Action, ActionPlan, ActionPlanParseError, resolve_endpoint -from rle.agents.base_role import RimWorldRoleAgent -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer - -AGENT_DISPLAY: dict[str, dict[str, str]] = { - "map_analyst": {"label": "MA", "color": "blue"}, - "resource_manager": {"label": "RM", "color": "green"}, - "defense_commander": {"label": "DC", "color": "red"}, - "research_director": {"label": "RD", "color": "cyan"}, - "social_overseer": {"label": "SO", "color": "yellow"}, - "construction_planner": {"label": "CP", "color": "white"}, - "medical_officer": {"label": "MO", "color": "magenta"}, -} - -_ROLE_AGENTS: dict[str, type[RimWorldRoleAgent]] = { - "map_analyst": MapAnalyst, - "resource_manager": ResourceManager, - "defense_commander": DefenseCommander, - "research_director": ResearchDirector, - "social_overseer": SocialOverseer, - "construction_planner": ConstructionPlanner, - "medical_officer": MedicalOfficer, -} - - -def register_rle_agents() -> None: - """Register all RLE role agent types with the Felix AgentFactory.""" - for name, cls in _ROLE_AGENTS.items(): - AgentFactory.register_agent_type(name, cls) - +from rle.agents.json_repair import repair_json, try_parse_json __all__ = [ - "AGENT_DISPLAY", "Action", "ActionPlan", "ActionPlanParseError", + "repair_json", "resolve_endpoint", - "ConstructionPlanner", - "DefenseCommander", - "MapAnalyst", - "MedicalOfficer", - "ResearchDirector", - "ResourceManager", - "RimWorldRoleAgent", - "SocialOverseer", - "register_rle_agents", + "try_parse_json", ] diff --git a/src/rle/agents/actions.py b/src/rle/agents/actions.py index 5ec5790..31ab035 100644 --- a/src/rle/agents/actions.py +++ b/src/rle/agents/actions.py @@ -35,6 +35,32 @@ class ActionPlan(BaseModel): confidence: float = 0.5 +class ActionOutcome(BaseModel): + """Per-action execution result. Captures failure detail so the next + tick's deliberation context can surface it to whoever proposed it. + """ + + model_config = ConfigDict(frozen=True) + + action_type: str + endpoint: str + target_colonist_id: str | None = None + success: bool + error: str | None = None + parameters: dict[str, Any] = {} + + +class ExecutionResult(BaseModel): + """Summary of action execution for one tick.""" + + model_config = ConfigDict(frozen=True) + + executed: int + failed: int + total: int + outcomes: tuple[ActionOutcome, ...] = () + + class ActionPlanParseError(Exception): """Raised when LLM output cannot be parsed into an ActionPlan.""" diff --git a/src/rle/config.py b/src/rle/config.py index 89e11ef..cdc06bd 100644 --- a/src/rle/config.py +++ b/src/rle/config.py @@ -3,34 +3,19 @@ from __future__ import annotations import os +from typing import Any -from felix_agent_sdk.core import HelixConfig -from felix_agent_sdk.providers import ( - AnthropicProvider, - BaseProvider, - LocalProvider, - OpenAIProvider, -) from pydantic_settings import BaseSettings -from rle.providers.claude_code import ClaudeCodeProvider - -_HELIX_PRESETS: dict[str, HelixConfig] = { - "default": HelixConfig.default(), - "research_heavy": HelixConfig.research_heavy(), - "fast_convergence": HelixConfig.fast_convergence(), -} - -_PROVIDER_CLASSES: dict[str, type[BaseProvider]] = { - "anthropic": AnthropicProvider, - "openai": OpenAIProvider, - "local": LocalProvider, - "claude-code": ClaudeCodeProvider, -} - class RLEConfig(BaseSettings): - """Top-level configuration for the RimWorld Learning Environment.""" + """Top-level configuration for the RimWorld Learning Environment. + + Deliberately framework-free: provider/model/harness are strings, and the + harness that gets built from them (see ``rle.harness``) decides what they + mean. Felix-only knobs (helix preset, no-think, parallelism) live in + ``FelixOptions`` and arrive via ``harness_options``. + """ model_config = {"env_prefix": "", "env_file": ".env", "extra": "ignore"} @@ -41,13 +26,21 @@ class RLEConfig(BaseSettings): openrouter_api_key: str | None = None anthropic_api_key: str | None = None tick_interval: float = 1.0 + harness: str = "felix" + """Which harness decides the colony's actions. Resolved through the + ``rle.harnesses`` entry-point registry (``--harness list``).""" + harness_options: dict[str, Any] = {} + """Harness-specific options validated against the plugin's schema + (``RLE_HARNESS_OPTIONS`` as JSON, or ``--harness-opt key=value``).""" + tick_timeout_s: float | None = None + """Loop-level cap on a whole harness step. ``None`` = no cap (the Felix + harness applies its own per-agent ``role_timeout_s``).""" role_timeout_s: float = 60.0 - """Max wall-clock seconds for a single agent's deliberation. Hung LLM + """Max wall-clock seconds for a single Felix agent's deliberation. Hung LLM calls beyond this fire a deliberation_timeout ERROR event and the agent - contributes no actions for the tick. The hung thread eventually unwinds - via the provider's own timeout (Python threads can't be force-killed). - Docker-benchmark average is ~7s per deliberation; 60s leaves ~8x headroom.""" - helix_preset: str = "default" + contributes no actions for the tick. Kept on RLEConfig for the legacy + ``RLEGameLoop(agents=...)`` path; ``--harness-opt role_timeout_s=`` is the + modern spelling.""" max_agents: int = 7 log_level: str = "INFO" docker_image: str = "rle-headless:latest" @@ -57,30 +50,6 @@ class RLEConfig(BaseSettings): hf_dataset_repo: str = "AppSprout/rle-benchmarks" """Target HF dataset repo (HF_DATASET_REPO in .env to override).""" - def get_helix_config(self) -> HelixConfig: - """Return the HelixConfig preset matching ``helix_preset``.""" - try: - return _HELIX_PRESETS[self.helix_preset] - except KeyError: - raise ValueError( - f"Unknown helix preset {self.helix_preset!r}. " - f"Choose from: {list(_HELIX_PRESETS)}" - ) from None - - def get_provider(self) -> BaseProvider: - """Construct an LLM provider from the current config.""" - cls = _PROVIDER_CLASSES.get(self.provider) - if cls is None: - raise ValueError( - f"Unknown provider {self.provider!r}. " - f"Choose from: {list(_PROVIDER_CLASSES)}" - ) - kwargs: dict[str, str] = {"model": self.model} - if self.provider_base_url: - kwargs["base_url"] = self.provider_base_url - return cls(**kwargs) # type: ignore[arg-type] # subclasses accept kwargs - - def bridge_openrouter_key(config: RLEConfig) -> None: """If OPENROUTER_API_KEY is set but OPENAI_API_KEY isn't, bridge them.""" if config.openrouter_api_key and not os.environ.get("OPENAI_API_KEY"): diff --git a/src/rle/harness/__init__.py b/src/rle/harness/__init__.py new file mode 100644 index 0000000..4b0b25a --- /dev/null +++ b/src/rle/harness/__init__.py @@ -0,0 +1,73 @@ +"""Swappable harnesses — the decision-making side of an RLE run. + +Public API for harness authors (kept stable; external harness packages depend +on it): + +- :class:`BaseHarness`, :class:`StepResult`, :class:`HarnessContext` +- :class:`HarnessPlugin`, :class:`Availability`, :class:`EmptyOptions` +- :class:`TickObserver`, :class:`HarnessStepError` +- :func:`create_harness`, :func:`list_harnesses`, :func:`get_plugin` + +Register a harness under the ``rle.harnesses`` entry-point group; see +``docs/harness-plugins.md``. +""" + +from rle.harness.cli import ( + add_harness_args, + exit_with_harness_error, + format_harness_table, + harness_options_for, + maybe_handle_harness_list, + selected_harnesses, +) +from rle.harness.protocol import ( + Availability, + BaseHarness, + EmptyOptions, + HarnessContext, + HarnessPlugin, + HarnessStepError, + StepResult, + TickObserver, +) +from rle.harness.registry import ( + ENTRY_POINT_GROUP, + HarnessInfo, + HarnessNotFoundError, + HarnessOptionsError, + HarnessUnavailableError, + create_harness, + get_plugin, + harness_names, + list_harnesses, + parse_option_pairs, + validate_options, +) + +__all__ = [ + "ENTRY_POINT_GROUP", + "Availability", + "BaseHarness", + "EmptyOptions", + "HarnessContext", + "HarnessInfo", + "HarnessNotFoundError", + "HarnessOptionsError", + "HarnessPlugin", + "HarnessStepError", + "HarnessUnavailableError", + "StepResult", + "TickObserver", + "add_harness_args", + "create_harness", + "exit_with_harness_error", + "format_harness_table", + "get_plugin", + "harness_names", + "harness_options_for", + "list_harnesses", + "maybe_handle_harness_list", + "parse_option_pairs", + "selected_harnesses", + "validate_options", +] diff --git a/src/rle/harness/baseline.py b/src/rle/harness/baseline.py new file mode 100644 index 0000000..952fab3 --- /dev/null +++ b/src/rle/harness/baseline.py @@ -0,0 +1,58 @@ +"""Unmanaged baseline harness — the colony runs on RimWorld's own AI.""" + +from __future__ import annotations + +from typing import ClassVar + +from pydantic import BaseModel + +from rle.agents.actions import ActionPlan +from rle.harness.protocol import ( + Availability, + BaseHarness, + EmptyOptions, + HarnessContext, + StepResult, +) +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent + + +class BaselineHarness(BaseHarness): + """Proposes nothing every tick. Paired against every other harness.""" + + name: ClassVar[str] = "baseline" + + async def step( + self, state: GameState, tick: int, macro_time: float, + events: list[RimAPIEvent], + ) -> StepResult: + return StepResult( + plan=ActionPlan( + role="baseline", tick=state.colony.tick, actions=[], + summary="No agents", + ), + ) + + +class BaselinePlugin: + name = "baseline" + description = "Unmanaged colony (RimWorld built-in AI). The paired control for every run." + + def available(self) -> Availability: + return Availability.available() + + def option_schema(self) -> type[BaseModel]: + return EmptyOptions + + def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + return BaselineHarness() + + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + return BaselineHarness() + + def describe(self) -> dict[str, str]: + return {"harness": self.name} + + +PLUGIN = BaselinePlugin() diff --git a/src/rle/harness/brief.py b/src/rle/harness/brief.py new file mode 100644 index 0000000..14ccc19 --- /dev/null +++ b/src/rle/harness/brief.py @@ -0,0 +1,239 @@ +"""Harness-neutral scenario brief. + +Every harness receives the same facts: what the scenario asks for, the +current colony state, the deterministic MAP_SUMMARY (verified build / farm / +stockpile sites, water to avoid), and the action catalog it may use. Anything +beyond that — role splits, bootstrap playbooks, phase-dependent temperature, +tool-call framing — is the harness's own prompt engineering and is part of +what the benchmark measures. +""" + +from __future__ import annotations + +from typing import Any, cast + +from pydantic import BaseModel, ConfigDict + +from rle.rimapi.api_catalog import WRITE_CATALOG +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.scenarios.schema import ScenarioConfig + +# Colonist fields worth a harness's attention (full dumps blow context budgets). +_COLONIST_FIELDS = ( + "colonist_id", "name", "health", "mood", "current_job", "is_drafted", "position", +) +_MAX_EVENTS = 12 + + +def build_map_summary(state: GameState) -> str | None: + """Compact (~500 token) spatial summary from terrain + zone + room data. + + Coordinates here are verified against the terrain grid; harnesses are told + to use them verbatim rather than invent positions. + """ + terrain = state.map.terrain + if terrain is None: + return None + + lines: list[str] = [] + cx, cz = terrain.colony_center + lines.append(f"Colony center: ({cx}, {cz}).") + + if terrain.recommended_shelter: + s = terrain.recommended_shelter + lines.append( + f"SHELTER SITE (verified solid ground): " + f"place walls/doors/beds at ({s.x1},{s.z1})-({s.x2},{s.z2}). " + f"ALL blueprint actions MUST use x,z within this rectangle." + ) + if terrain.recommended_farm: + f = terrain.recommended_farm + lines.append( + f"FARM SITE (verified fertile soil): " + f"place growing_zone at x1={f.x1},z1={f.z1},x2={f.x2},z2={f.z2}. " + f"ALL growing_zone actions MUST use these exact coordinates." + ) + if terrain.recommended_stockpile: + sp = terrain.recommended_stockpile + lines.append( + f"STOCKPILE SITE (verified solid ground): " + f"place stockpile_zone at x1={sp.x1},z1={sp.z1}," + f"x2={sp.x2},z2={sp.z2}." + ) + + if terrain.water_areas: + water_strs = [f"({w.x1},{w.z1})-({w.x2},{w.z2})" for w in terrain.water_areas] + lines.append(f"WATER (do NOT build here): {', '.join(water_strs)}.") + + if state.map.zones: + zone_strs = [ + f"{z.label} ({z.zone_type}, {z.cell_count} cells)" + for z in state.map.zones[:8] + ] + lines.append(f"Zones: {'; '.join(zone_strs)}.") + else: + lines.append("Zones: NONE — create stockpile and growing zone NOW.") + + real_rooms = [r for r in state.map.rooms if r.size > 1] + if real_rooms: + room_strs = [f"{r.role} ({r.size} cells, {r.bed_count} beds)" for r in real_rooms[:6]] + lines.append(f"Rooms: {'; '.join(room_strs)}.") + else: + lines.append( + "Rooms: NONE — colonists sleeping outside. " + "Build shelter IMMEDIATELY." + ) + + if state.map.ore_deposits: + ore_strs = [ + f"{o.def_name} ({o.count} cells" + + (f", near ({o.positions[0][0]},{o.positions[0][1]})" if o.positions else "") + + ")" + for o in state.map.ore_deposits[:5] + ] + lines.append(f"Ore: {'; '.join(ore_strs)}.") + + fs = state.map.farm_summary + if fs and fs.total_growing_zones > 0: + lines.append( + f"Farms: {fs.total_growing_zones} zones, " + f"{fs.planted_cells} planted, " + f"{fs.harvestable_cells} harvestable." + ) + + return "\n".join(lines) + + +def action_catalog() -> list[dict[str, Any]]: + """Every write a harness may issue, with its parameter shape.""" + out: list[dict[str, Any]] = [ + {"action_type": "no_action", "description": "Do nothing this tick.", "params": {}}, + ] + for name, raw in sorted(WRITE_CATALOG.items()): + entry = cast(dict[str, Any], raw) + out.append({ + "action_type": name, + "description": entry.get("description", ""), + "params": entry.get("params", {}), + }) + return out + + +def scenario_goals(scenario: ScenarioConfig | None) -> dict[str, Any]: + if scenario is None: + return {} + return { + "name": scenario.name, + "description": scenario.description, + "difficulty": scenario.difficulty, + "expected_duration_days": scenario.expected_duration_days, + "victory": [f"{c.metric} {c.operator} {c.value}" for c in scenario.victory_conditions], + "failure": [f"{c.metric} {c.operator} {c.value}" for c in scenario.failure_conditions], + "scoring_weights": dict(scenario.scoring_weights), + } + + +def state_snapshot(state: GameState) -> dict[str, Any]: + """The colony as any harness should see it — no role filtering.""" + return { + "colony": state.colony.model_dump(), + "colonists": [ + {k: getattr(c, k) for k in _COLONIST_FIELDS} for c in state.colonists + ], + "resources": state.resources.model_dump(), + "research": state.research.model_dump(), + "threats": [t.model_dump() for t in state.threats], + "weather": state.weather.model_dump(), + "map": { + "size": state.map.size, + "biome": state.map.biome, + "season": state.map.season, + "temperature": state.map.temperature, + "structures": len(state.map.structures), + "zones": len(state.map.zones), + "rooms": len([r for r in state.map.rooms if r.size > 1]), + }, + } + + +class ScenarioBrief(BaseModel): + """What the environment tells a harness at the start of a tick.""" + + model_config = ConfigDict(frozen=True) + + tick: int + day: int + macro_time: float + goals: dict[str, Any] + state: dict[str, Any] + map_summary: str | None + recent_events: list[dict[str, Any]] + actions: list[dict[str, Any]] + + def to_text(self) -> str: + """Plain-text rendering for prompt-based harnesses.""" + parts = [ + f"# RimWorld colony — tick {self.tick}, day {self.day} " + f"(run progress {self.macro_time:.0%})", + ] + if self.goals: + g = self.goals + parts.append( + f"## Scenario: {g.get('name')} ({g.get('difficulty')})\n{g.get('description')}\n" + f"Victory: {'; '.join(g.get('victory', []))}\n" + f"Failure: {'; '.join(g.get('failure', []))}", + ) + if self.map_summary: + parts.append(f"## MAP_SUMMARY\n{self.map_summary}") + parts.append("## State\n" + _render(self.state)) + if self.recent_events: + parts.append( + "## Recent events\n" + + "\n".join(f"- {e['event_type']}: {e['data']}" for e in self.recent_events), + ) + parts.append( + "## Actions available\n" + + "\n".join( + f"- {a['action_type']}: {a['description']} params={a['params']}" + for a in self.actions + ), + ) + return "\n\n".join(parts) + + +def _render(data: dict[str, Any], indent: int = 0) -> str: + lines: list[str] = [] + pad = " " * indent + for key, value in data.items(): + if isinstance(value, dict): + lines.append(f"{pad}{key}:") + lines.append(_render(value, indent + 1)) + elif isinstance(value, list): + lines.append(f"{pad}{key}: {value}") + else: + lines.append(f"{pad}{key}: {value}") + return "\n".join(lines) + + +def build_brief( + state: GameState, + *, + tick: int, + macro_time: float, + scenario: ScenarioConfig | None = None, + events: list[RimAPIEvent] | None = None, +) -> ScenarioBrief: + return ScenarioBrief( + tick=tick, + day=state.colony.day, + macro_time=macro_time, + goals=scenario_goals(scenario), + state=state_snapshot(state), + map_summary=build_map_summary(state), + recent_events=[ + {"event_type": e.event_type, "data": str(e.data)[:200]} + for e in (events or [])[:_MAX_EVENTS] + ], + actions=action_catalog(), + ) diff --git a/src/rle/harness/cli.py b/src/rle/harness/cli.py new file mode 100644 index 0000000..b77e6d8 --- /dev/null +++ b/src/rle/harness/cli.py @@ -0,0 +1,134 @@ +"""argparse glue shared by ``run_scenario.py`` and ``run_benchmark.py``. + +Keeps harness selection identical across CLIs: + + --harness registry name (default: RLEConfig.harness) + --harness list print discovered plugins and exit + --harness-opt key=value validated against the plugin's option schema + --no-agent permanent alias for --harness baseline + +Legacy Felix flags (``--no-think``, ``--sequential``, ``--visualize``) are +still accepted and folded into ``FelixOptions`` when the selected harness is +``felix``; for any other harness they are ignored with a warning. +""" + +from __future__ import annotations + +import argparse +import logging +import sys +from typing import Any + +from rle.config import RLEConfig +from rle.harness.registry import list_harnesses, parse_option_pairs + +logger = logging.getLogger(__name__) + +LIST_SENTINEL = "list" + +_LEGACY_FELIX_FLAGS: dict[str, tuple[str, Any]] = { + # argparse attribute -> (FelixOptions field, value when the flag is set) + "no_think": ("no_think", True), + "sequential": ("parallel", False), + "visualize": ("visualize", True), +} + + +def add_harness_args(parser: argparse.ArgumentParser, *, repeatable: bool = False) -> None: + kwargs: dict[str, Any] = {"action": "append"} if repeatable else {} + parser.add_argument( + "--harness", dest="harness", default=None, + help=( + "Harness that decides the colony's actions (default: RLE_HARNESS / felix). " + "Use `--harness list` to see installed plugins." + + (" Repeat to run a harness matrix." if repeatable else "") + ), + **kwargs, + ) + parser.add_argument( + "--harness-opt", dest="harness_opts", action="append", default=[], + metavar="KEY=VALUE", + help="Harness-specific option (repeatable); validated by the plugin's schema.", + ) + parser.add_argument( + "--no-agent", action="store_true", + help="Baseline mode: alias for --harness baseline (colony runs unmanaged).", + ) + parser.add_argument( + "--no-think", action="store_true", + help="[felix] Inject prefill so thinking models skip reasoning.", + ) + parser.add_argument( + "--sequential", action="store_true", + help="[felix] Deliberate role agents one at a time (default: parallel).", + ) + parser.add_argument( + "--visualize", action="store_true", + help="[felix] Show the live helix visualisation.", + ) + + +def format_harness_table() -> str: + rows = list_harnesses() + if not rows: + return "No harness plugins installed (entry-point group 'rle.harnesses')." + name_w = max(len(r.name) for r in rows) + pkg_w = max(len(f"{r.package} {r.version}") for r in rows) + lines = [f"{'HARNESS':<{name_w}} {'PACKAGE':<{pkg_w}} STATUS DESCRIPTION"] + for r in rows: + status = "available" if r.availability.ok else "unavailable" + pkg = f"{r.package} {r.version}" + desc = r.description + if not r.availability.ok: + desc = f"{desc} ({r.availability.reason})" + lines.append(f"{r.name:<{name_w}} {pkg:<{pkg_w}} {status:<11} {desc}") + return "\n".join(lines) + + +def maybe_handle_harness_list(args: argparse.Namespace) -> bool: + """Print the plugin table and return True when ``--harness list`` was given.""" + raw = getattr(args, "harness", None) + names = raw if isinstance(raw, list) else [raw] + if any(n == LIST_SENTINEL for n in names if n): + print(format_harness_table()) + return True + return False + + +def selected_harnesses(args: argparse.Namespace, config: RLEConfig) -> list[str]: + """Resolve the harness name(s) from CLI flags + config.""" + if getattr(args, "no_agent", False): + return ["baseline"] + raw = getattr(args, "harness", None) + if raw is None: + return [config.harness] + names = raw if isinstance(raw, list) else [raw] + return [n for n in names if n and n != LIST_SENTINEL] or [config.harness] + + +def harness_options_for( + name: str, args: argparse.Namespace, config: RLEConfig, +) -> dict[str, Any]: + """Merge config options, legacy Felix flags, and ``--harness-opt`` pairs.""" + options: dict[str, Any] = dict(config.harness_options) + legacy_set = { + field: value + for attr, (field, value) in _LEGACY_FELIX_FLAGS.items() + if getattr(args, attr, False) + } + if legacy_set: + if name == "felix": + options.update(legacy_set) + else: + logger.warning( + "Ignoring Felix-only flags %s for harness %r", + sorted(legacy_set), name, + ) + options.update(parse_option_pairs(getattr(args, "harness_opts", None))) + return options + + +def exit_with_harness_error(exc: Exception) -> None: + print(f"error: {exc}", file=sys.stderr) + print(format_harness_table(), file=sys.stderr) + raise SystemExit(2) diff --git a/src/rle/harness/cli_base.py b/src/rle/harness/cli_base.py new file mode 100644 index 0000000..15c71d2 --- /dev/null +++ b/src/rle/harness/cli_base.py @@ -0,0 +1,270 @@ +"""Scaffold for harnesses that drive an external coding agent over MCP. + +Tool-agnostic by design: a concrete harness (in its own package) implements +how to start the agent, hand it a prompt, and stop it. This base owns +everything else — hosting the RLE MCP server in-process, the per-tick brief, +the turn protocol (prompt → agent acts through tools → ``end_turn`` or idle +→ ledger drained into ``StepResult``), timeouts, cost/latency accounting, +and the deliberation log — so every CLI harness is scored identically. + +Requires the ``mcp`` extra. +""" + +from __future__ import annotations + +import asyncio +import logging +import time as _time +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any, ClassVar + +from pydantic import BaseModel, ConfigDict, Field + +from rle.harness.brief import ScenarioBrief, build_brief +from rle.harness.protocol import BaseHarness, HarnessContext, HarnessStepError, StepResult +from rle.mcp.host import McpHost +from rle.mcp.ledger import TickLedger +from rle.mcp.server import build_server +from rle.mcp.session import McpSession +from rle.orchestration.action_executor import ActionExecutor +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.tracking.event_log import EventType + +logger = logging.getLogger(__name__) + +_RAW_OUTPUT_CHARS = 16384 + +TURN_RULES = ( + "Rules for this turn:\n" + "1. Call get_brief first. Use MAP_SUMMARY coordinates verbatim — never invent positions.\n" + "2. Act only through the RLE tools (never shell into the game). Pawn ids are integers.\n" + "3. Each tool call executes immediately and returns {ok, error}; do not repeat a failed call " + "with the same arguments.\n" + "4. When you have issued this tick's writes, call end_turn with a one-line summary. " + "Doing nothing is allowed — still call end_turn.\n" +) + + +class HeadlessCliOptions(BaseModel): + """Options common to every CLI-agent harness; subclasses extend.""" + + model_config = ConfigDict(extra="forbid") + + model: str | None = Field( + default=None, + description="Model identifier handed to the agent (defaults to RLEConfig.model).", + ) + turn_timeout_s: float = Field( + default=180.0, + description="Hard cap on one turn. The agent is aborted and the tick scored as-is.", + ) + idle_grace_s: float = Field( + default=3.0, + description=( + "If the agent finishes responding without calling end_turn, wait this long " + "for late tool calls before closing the tick." + ), + ) + extra_instructions: str = Field( + default="", + description="Appended to every turn prompt (harness-side prompt engineering).", + ) + + +@dataclass +class TurnResult: + """What the agent produced for one prompt.""" + + text: str = "" + prompt_tokens: int = 0 + completion_tokens: int = 0 + reasoning_tokens: int = 0 + extras: dict[str, Any] = field(default_factory=dict) + + +class HeadlessCliHarness(BaseHarness, ABC): + name: ClassVar[str] = "headless-cli" + + def __init__(self, options: HeadlessCliOptions) -> None: + super().__init__() + self.options = options + self._session: McpSession | None = None + self._host: McpHost | None = None + self._agent_started = False + + # ------------------------------------------------------------------ + # Tool-specific hooks + # ------------------------------------------------------------------ + + @abstractmethod + async def start_agent(self, mcp_url: str) -> None: + """Launch or attach to the agent and register the RLE MCP server.""" + + @abstractmethod + async def send_turn(self, prompt: str) -> TurnResult: + """Deliver one turn prompt and return when the agent has finished responding.""" + + @abstractmethod + async def stop_agent(self) -> None: + """Abort any in-flight turn and shut the agent down.""" + + async def abort_turn(self) -> None: + """Called on timeout; default just stops and restarts nothing.""" + return None + + def agent_versions(self) -> dict[str, str]: + """Version info for run metadata (binary --version etc.).""" + return {} + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + @property + def session(self) -> McpSession: + if self._session is None: + raise RuntimeError("setup() has not been called") + return self._session + + @property + def mcp_url(self) -> str: + if self._host is None: + raise RuntimeError("setup() has not been called") + return self._host.url + + async def setup(self, ctx: HarnessContext) -> None: + await super().setup(ctx) + ledger = TickLedger(harness_name=self.name) + self._session = McpSession( + client=ctx.client, executor=ActionExecutor(ctx.client), ledger=ledger, emit=ctx.emit, + ) + self._host = McpHost(build_server(self._session)) + url = await self._host.start() + await self.start_agent(url) + self._agent_started = True + + async def teardown(self) -> None: + if self._agent_started: + try: + await self.stop_agent() + except Exception: + logger.debug("stop_agent failed", exc_info=True) + self._agent_started = False + if self._host is not None: + await self._host.stop() + + def describe(self) -> dict[str, str]: + info = {"harness": self.name, "model": self.options.model or self.ctx.config.model} + info.update(self.agent_versions()) + return info + + # ------------------------------------------------------------------ + # Turn protocol + # ------------------------------------------------------------------ + + def render_prompt(self, brief: ScenarioBrief) -> str: + parts = [ + f"RLE turn — tick {brief.tick}, day {brief.day}. " + "You manage this RimWorld colony through the `rle` MCP tools.", + TURN_RULES, + ] + if self.options.extra_instructions: + parts.append(self.options.extra_instructions) + parts.append( + "Brief preview (call get_brief for the full version):\n" + + brief.to_text()[:4000], + ) + return "\n\n".join(parts) + + async def step( + self, state: GameState, tick: int, macro_time: float, events: list[RimAPIEvent], + ) -> StepResult: + session = self.session + brief = build_brief( + state, tick=tick, macro_time=macro_time, scenario=self.ctx.scenario, events=events, + ) + session.begin_tick(tick, state, brief) + prompt = self.render_prompt(brief) + + t0 = _time.monotonic() + status = "success" + turn = TurnResult() + try: + turn = await asyncio.wait_for( + self._run_turn(prompt, session.ledger), timeout=self.options.turn_timeout_s, + ) + except asyncio.TimeoutError: + status = "turn_timeout" + logger.warning("%s turn timed out after %.0fs (tick %d)", + self.name, self.options.turn_timeout_s, tick) + await self.abort_turn() + except HarnessStepError as exc: + status = "agent_error" + logger.warning("%s agent error (tick %d): %s", self.name, tick, exc) + turn = TurnResult(text=str(exc)) + latency_ms = round((_time.monotonic() - t0) * 1000, 1) + + step = session.ledger.finish() + if status == "success": + self.parse_successes += 1 + else: + self.parse_failures += 1 + + n_actions = len(step.plan.actions) + self.deliberation_log.append({ + "tick": tick, "agent": self.name, "status": status, + "num_actions": n_actions, "summary": step.plan.summary, + "latency_ms": latency_ms, + }) + self.ctx.emit( + EventType.DELIBERATION if status == "success" else EventType.ERROR, tick, + agent=self.name, latency_ms=latency_ms, num_actions=n_actions, + summary=step.plan.summary, error_type=None if status == "success" else status, + ) + if turn.prompt_tokens or turn.completion_tokens: + if self.ctx.cost_tracker: + self.ctx.cost_tracker.record_raw( + turn.prompt_tokens, turn.completion_tokens, turn.reasoning_tokens, + ) + self.ctx.emit( + EventType.PROVIDER_CALL, tick, agent=self.name, + prompt_tokens=turn.prompt_tokens, completion_tokens=turn.completion_tokens, + reasoning_tokens=turn.reasoning_tokens, + raw_output=turn.text[:_RAW_OUTPUT_CHARS], + raw_output_truncated=len(turn.text) > _RAW_OUTPUT_CHARS, + ) + + extras = {**step.extras, "status": status, "latency_ms": latency_ms, **turn.extras} + return StepResult( + plan=step.plan, execution=step.execution, proposals=step.proposals, extras=extras, + ) + + async def _run_turn(self, prompt: str, ledger: TickLedger) -> TurnResult: + """Prompt the agent, then wait for end_turn (or a short idle grace).""" + send = asyncio.create_task(self.send_turn(prompt)) + done_wait = asyncio.create_task(ledger.turn_done.wait()) + try: + await asyncio.wait({send, done_wait}, return_when=asyncio.FIRST_COMPLETED) + if send.done(): + result = send.result() + if not ledger.turn_done.is_set(): + # Agent replied without end_turn: allow trailing tool calls. + try: + await asyncio.wait_for( + ledger.turn_done.wait(), timeout=self.options.idle_grace_s, + ) + except asyncio.TimeoutError: + pass + return result + # end_turn arrived first; let the agent finish its reply briefly. + try: + return await asyncio.wait_for(send, timeout=self.options.idle_grace_s) + except asyncio.TimeoutError: + send.cancel() + return TurnResult(text="(agent still responding after end_turn)") + finally: + for task in (send, done_wait): + if not task.done(): + task.cancel() diff --git a/src/rle/harness/compat.py b/src/rle/harness/compat.py new file mode 100644 index 0000000..19c05ec --- /dev/null +++ b/src/rle/harness/compat.py @@ -0,0 +1,40 @@ +"""Backward-compatibility shims for callers that predate the harness layer. + +``RLEGameLoop(config, client, agents, no_agent=..., parallel=..., visualizer=...)`` +still works: this module turns those legacy arguments into a harness. It is +the one place in core that reaches for the Felix harness by module path — +through ``importlib`` so that importing the loop never imports the SDK +(documented exception to the no-inline-imports rule: optional dependency). +""" + +from __future__ import annotations + +from collections.abc import Sequence +from importlib import import_module +from typing import Any + +from rle.harness.baseline import BaselineHarness +from rle.harness.protocol import BaseHarness + +FELIX_HARNESS_MODULE = "rle.harness.felix.harness" + + +def build_legacy_harness( + agents: Sequence[Any] | None, + *, + no_agent: bool = False, + parallel: bool = True, + visualizer: Any | None = None, + role_timeout_s: float = 60.0, +) -> BaseHarness: + if no_agent or not agents: + return BaselineHarness() + module = import_module(FELIX_HARNESS_MODULE) + felix_cls = module.FelixHarness + harness: BaseHarness = felix_cls( + list(agents), + parallel=parallel, + role_timeout_s=role_timeout_s, + visualizer=visualizer, + ) + return harness diff --git a/src/rle/harness/felix/__init__.py b/src/rle/harness/felix/__init__.py new file mode 100644 index 0000000..73a76da --- /dev/null +++ b/src/rle/harness/felix/__init__.py @@ -0,0 +1,11 @@ +"""Felix multi-agent harness (optional extra ``felix``). + +Importing this package must stay cheap and Felix-free: the ``felix`` entry +point resolves to :data:`PLUGIN`, whose methods import the SDK-dependent +modules (``harness``, ``build``, ``provider_factory``) only when a Felix +harness is actually requested. +""" + +from rle.harness.felix.plugin import PLUGIN, FelixPlugin + +__all__ = ["PLUGIN", "FelixPlugin"] diff --git a/src/rle/harness/felix/agents/__init__.py b/src/rle/harness/felix/agents/__init__.py new file mode 100644 index 0000000..721d894 --- /dev/null +++ b/src/rle/harness/felix/agents/__init__.py @@ -0,0 +1,56 @@ +"""The Felix harness's role agents (MapAnalyst + 6 domain roles). + +Everything here subclasses Felix SDK's ``LLMAgent``; nothing outside +``rle.harness.felix`` may import from this package. +""" + +from felix_agent_sdk import AgentFactory + +from rle.harness.felix.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.map_analyst import MapAnalyst +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer + +AGENT_DISPLAY: dict[str, dict[str, str]] = { + "map_analyst": {"label": "MA", "color": "blue"}, + "resource_manager": {"label": "RM", "color": "green"}, + "defense_commander": {"label": "DC", "color": "red"}, + "research_director": {"label": "RD", "color": "cyan"}, + "social_overseer": {"label": "SO", "color": "yellow"}, + "construction_planner": {"label": "CP", "color": "white"}, + "medical_officer": {"label": "MO", "color": "magenta"}, +} + +_ROLE_AGENTS: dict[str, type[RimWorldRoleAgent]] = { + "map_analyst": MapAnalyst, + "resource_manager": ResourceManager, + "defense_commander": DefenseCommander, + "research_director": ResearchDirector, + "social_overseer": SocialOverseer, + "construction_planner": ConstructionPlanner, + "medical_officer": MedicalOfficer, +} + + +def register_rle_agents() -> None: + """Register all RLE role agent types with the Felix AgentFactory.""" + for name, cls in _ROLE_AGENTS.items(): + AgentFactory.register_agent_type(name, cls) + + +__all__ = [ + "AGENT_DISPLAY", + "ConstructionPlanner", + "DefenseCommander", + "MapAnalyst", + "MedicalOfficer", + "ResearchDirector", + "ResourceManager", + "RimWorldRoleAgent", + "SocialOverseer", + "register_rle_agents", +] diff --git a/src/rle/agents/base_role.py b/src/rle/harness/felix/agents/base_role.py similarity index 90% rename from src/rle/agents/base_role.py rename to src/rle/harness/felix/agents/base_role.py index bc8f678..290dd7a 100644 --- a/src/rle/agents/base_role.py +++ b/src/rle/harness/felix/agents/base_role.py @@ -19,6 +19,7 @@ from rle.agents.actions import Action, ActionPlan, ActionPlanParseError, resolve_endpoint from rle.agents.json_repair import repair_json +from rle.harness.brief import build_map_summary from rle.rimapi.schemas import GameState from rle.rimapi.sse_client import RimAPIEvent @@ -408,95 +409,8 @@ def _get_role_description(self) -> str: @staticmethod def _build_map_summary(state: GameState) -> str | None: - """Build a compact ~500 token map summary from terrain + zone + room data. - - This text is injected into every agent's context so they share a - common spatial understanding without needing to parse raw data. - """ - terrain = state.map.terrain - if terrain is None: - return None - - lines: list[str] = [] - cx, cz = terrain.colony_center - lines.append(f"Colony center: ({cx}, {cz}).") - - # Verified build/farm/stockpile sites - if terrain.recommended_shelter: - s = terrain.recommended_shelter - lines.append( - f"SHELTER SITE (verified solid ground): " - f"place walls/doors/beds at ({s.x1},{s.z1})-({s.x2},{s.z2}). " - f"ALL blueprint actions MUST use x,z within this rectangle." - ) - if terrain.recommended_farm: - f = terrain.recommended_farm - lines.append( - f"FARM SITE (verified fertile soil): " - f"place growing_zone at x1={f.x1},z1={f.z1},x2={f.x2},z2={f.z2}. " - f"ALL growing_zone actions MUST use these exact coordinates." - ) - if terrain.recommended_stockpile: - sp = terrain.recommended_stockpile - lines.append( - f"STOCKPILE SITE (verified solid ground): " - f"place stockpile_zone at x1={sp.x1},z1={sp.z1}," - f"x2={sp.x2},z2={sp.z2}." - ) - - # Water avoidance - if terrain.water_areas: - water_strs = [ - f"({w.x1},{w.z1})-({w.x2},{w.z2})" - for w in terrain.water_areas - ] - lines.append(f"WATER (do NOT build here): {', '.join(water_strs)}.") - - # Existing zones - if state.map.zones: - zone_strs = [ - f"{z.label} ({z.zone_type}, {z.cell_count} cells)" - for z in state.map.zones[:8] - ] - lines.append(f"Zones: {'; '.join(zone_strs)}.") - else: - lines.append("Zones: NONE — create stockpile and growing zone NOW.") - - # Existing rooms - real_rooms = [r for r in state.map.rooms if r.size > 1] - if real_rooms: - room_strs = [ - f"{r.role} ({r.size} cells, {r.bed_count} beds)" - for r in real_rooms[:6] - ] - lines.append(f"Rooms: {'; '.join(room_strs)}.") - else: - lines.append( - "Rooms: NONE — colonists sleeping outside. " - "Build shelter IMMEDIATELY." - ) - - # Ore - if state.map.ore_deposits: - ore_strs = [ - f"{o.def_name} ({o.count} cells" - + (f", near ({o.positions[0][0]},{o.positions[0][1]})" - if o.positions else "") - + ")" - for o in state.map.ore_deposits[:5] - ] - lines.append(f"Ore: {'; '.join(ore_strs)}.") - - # Farm summary - fs = state.map.farm_summary - if fs and fs.total_growing_zones > 0: - lines.append( - f"Farms: {fs.total_growing_zones} zones, " - f"{fs.planted_cells} planted, " - f"{fs.harvestable_cells} harvestable." - ) - - return "\n".join(lines) + """Compact map summary shared by every agent (core builder, see rle.harness.brief).""" + return build_map_summary(state) def build_task( self, diff --git a/src/rle/agents/construction_planner.py b/src/rle/harness/felix/agents/construction_planner.py similarity index 97% rename from src/rle/agents/construction_planner.py rename to src/rle/harness/felix/agents/construction_planner.py index 0f68528..cba891b 100644 --- a/src/rle/agents/construction_planner.py +++ b/src/rle/harness/felix/agents/construction_planner.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/defense_commander.py b/src/rle/harness/felix/agents/defense_commander.py similarity index 97% rename from src/rle/agents/defense_commander.py rename to src/rle/harness/felix/agents/defense_commander.py index 28426ed..b74605d 100644 --- a/src/rle/agents/defense_commander.py +++ b/src/rle/harness/felix/agents/defense_commander.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/map_analyst.py b/src/rle/harness/felix/agents/map_analyst.py similarity index 98% rename from src/rle/agents/map_analyst.py rename to src/rle/harness/felix/agents/map_analyst.py index 0e21c07..12d0498 100644 --- a/src/rle/agents/map_analyst.py +++ b/src/rle/harness/felix/agents/map_analyst.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent if TYPE_CHECKING: from felix_agent_sdk.agents.llm_agent import LLMTask diff --git a/src/rle/agents/medical_officer.py b/src/rle/harness/felix/agents/medical_officer.py similarity index 97% rename from src/rle/agents/medical_officer.py rename to src/rle/harness/felix/agents/medical_officer.py index b7f0293..8e761df 100644 --- a/src/rle/agents/medical_officer.py +++ b/src/rle/harness/felix/agents/medical_officer.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/research_director.py b/src/rle/harness/felix/agents/research_director.py similarity index 97% rename from src/rle/agents/research_director.py rename to src/rle/harness/felix/agents/research_director.py index d9dbdaa..3603549 100644 --- a/src/rle/agents/research_director.py +++ b/src/rle/harness/felix/agents/research_director.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/resource_manager.py b/src/rle/harness/felix/agents/resource_manager.py similarity index 97% rename from src/rle/agents/resource_manager.py rename to src/rle/harness/felix/agents/resource_manager.py index a134c78..095e6d4 100644 --- a/src/rle/agents/resource_manager.py +++ b/src/rle/harness/felix/agents/resource_manager.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/agents/social_overseer.py b/src/rle/harness/felix/agents/social_overseer.py similarity index 97% rename from src/rle/agents/social_overseer.py rename to src/rle/harness/felix/agents/social_overseer.py index 80e5a35..1bf15df 100644 --- a/src/rle/agents/social_overseer.py +++ b/src/rle/harness/felix/agents/social_overseer.py @@ -4,7 +4,7 @@ from typing import Any, ClassVar -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState diff --git a/src/rle/harness/felix/build.py b/src/rle/harness/felix/build.py new file mode 100644 index 0000000..e2467df --- /dev/null +++ b/src/rle/harness/felix/build.py @@ -0,0 +1,105 @@ +"""Construct a fully wired FelixHarness from HarnessContext + FelixOptions.""" + +from __future__ import annotations + +from typing import Any + +from felix_agent_sdk.core import HelixGeometry +from felix_agent_sdk.providers.base import BaseProvider +from felix_agent_sdk.visualization import HelixVisualizer +from pydantic import BaseModel + +from rle.config import bridge_anthropic_key, bridge_openrouter_key +from rle.harness.felix.agents import AGENT_DISPLAY +from rle.harness.felix.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.map_analyst import MapAnalyst +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer +from rle.harness.felix.harness import FelixHarness +from rle.harness.felix.options import FelixOptions +from rle.harness.felix.provider_factory import build_helix, build_provider +from rle.harness.felix.smoke import SmokeProvider +from rle.harness.protocol import HarnessContext + +# Context extras understood by this builder. +WEAVE_MODULE_EXTRA = "weave_module" + + +def create_agents( + provider: BaseProvider, + helix: HelixGeometry, + *, + exclude_agent: str | None = None, + provider_kwargs: dict[str, Any] | None = None, + no_think: bool = False, +) -> list[RimWorldRoleAgent]: + """The canonical 7-agent roster (MapAnalyst + 6 domain roles).""" + roster: list[RimWorldRoleAgent] = [ + MapAnalyst("map_analyst", provider, helix, spawn_time=0.0, velocity=1.0), + ResourceManager("resource_manager", provider, helix, spawn_time=0.0, velocity=1.0), + DefenseCommander("defense_commander", provider, helix, spawn_time=0.0, velocity=1.0), + ResearchDirector("research_director", provider, helix, spawn_time=0.0, velocity=1.0), + SocialOverseer("social_overseer", provider, helix, spawn_time=0.0, velocity=1.0), + ConstructionPlanner( + "construction_planner", provider, helix, spawn_time=0.0, velocity=1.0, + ), + MedicalOfficer("medical_officer", provider, helix, spawn_time=0.0, velocity=1.0), + ] + agents = [a for a in roster if a.agent_id != exclude_agent] + for agent in agents: + if provider_kwargs: + agent.set_provider_kwargs(**provider_kwargs) + if no_think: + agent.set_no_think(True) + return agents + + +def create_visualizer( + helix: HelixGeometry, agents: list[RimWorldRoleAgent], title: str = "R L E", +) -> HelixVisualizer: + visualizer = HelixVisualizer(helix, title=title) + for agent in agents: + display = AGENT_DISPLAY[agent.agent_id] + visualizer.register_agent( + agent.agent_id, label=display["label"], color=display["color"], + ) + return visualizer + + +def build_felix_harness( + ctx: HarnessContext, options: BaseModel, *, smoke: bool = False, +) -> FelixHarness: + opts = options if isinstance(options, FelixOptions) else FelixOptions.model_validate( + options.model_dump(), + ) + provider: BaseProvider + if smoke or ctx.smoke: + provider = SmokeProvider() + else: + bridge_openrouter_key(ctx.config) + bridge_anthropic_key(ctx.config) + provider = build_provider( + ctx.config.provider, ctx.config.model, ctx.config.provider_base_url, + ) + helix = build_helix(opts.helix_preset) + agents = create_agents( + provider, helix, + exclude_agent=opts.exclude_agent, + provider_kwargs=opts.provider_kwargs or None, + no_think=opts.no_think, + ) + weave_module = ctx.extras.get(WEAVE_MODULE_EXTRA) + if weave_module is not None: + for agent in agents: + agent.enable_weave(weave_module) + visualizer = create_visualizer(helix, agents) if opts.visualize else None + return FelixHarness( + agents, + parallel=opts.parallel, + role_timeout_s=opts.role_timeout_s, + visualizer=visualizer, + ) diff --git a/src/rle/harness/felix/harness.py b/src/rle/harness/felix/harness.py new file mode 100644 index 0000000..94297e7 --- /dev/null +++ b/src/rle/harness/felix/harness.py @@ -0,0 +1,465 @@ +"""FelixHarness — the original RLE harness: MapAnalyst-first, six role agents +deliberating in parallel over Felix SDK's CentralPost hub-spoke bus, merged by +``ActionResolver``. + +Everything Felix-specific that used to live in ``RLEGameLoop`` lives here: +hub/spoke wiring, per-agent timeouts, phase broadcasts, action-error and score +feedback, helix visualisation, generation-id / token accounting. +""" + +from __future__ import annotations + +import asyncio +import logging +import time as _time +from contextlib import AbstractContextManager, nullcontext +from importlib.metadata import PackageNotFoundError, version +from typing import Any, ClassVar + +from felix_agent_sdk.communication import CentralPost, MessageType, SpokeManager +from felix_agent_sdk.providers import ProviderError + +from rle.agents.actions import ActionPlan, ActionPlanParseError, ExecutionResult +from rle.harness.felix.agents.base_role import RimWorldRoleAgent +from rle.harness.protocol import BaseHarness, HarnessContext, StepResult +from rle.orchestration.action_resolver import ActionResolver +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.scoring.composite import ScoreSnapshot +from rle.tracking.event_log import EventType + +logger = logging.getLogger(__name__) + +# Truncation limits for human-readable text persisted to the event log / +# deliberation log. Keep these tight so the JSONL stays grep-able and small; +# the per-scenario *_deliberations.jsonl carries the full raw reasoning. +_ACTION_REASON_CHARS = 200 +_PLAN_SUMMARY_CHARS = 300 +_PARSE_FAILURE_RAW_CHARS = 500 +# Full LLM completion text on successful deliberation (PROVIDER_CALL events). +# 16 KB headroom: frontier models emit multi-KB structured plans and the +# verbatim transcripts are first-class analysis artifacts; longer completions +# are tail-truncated so the parsed action JSON remains visible. +_RAW_OUTPUT_CHARS = 16384 + + +def felix_sdk_version() -> str: + try: + return version("felix-agent-sdk") + except PackageNotFoundError: + return "unknown" + + +def phase_for(macro_time: float) -> str: + """Helix macro phase for a normalised run position (0..1).""" + if macro_time < 0.4: + return "exploration" + if macro_time < 0.7: + return "analysis" + return "synthesis" + + +class FelixHarness(BaseHarness): + name: ClassVar[str] = "felix" + + def __init__( + self, + agents: list[RimWorldRoleAgent], + *, + parallel: bool = True, + role_timeout_s: float = 60.0, + visualizer: Any | None = None, + ) -> None: + super().__init__() + self._agents = agents + self._map_analyst: RimWorldRoleAgent | None = None + self._role_agents: list[RimWorldRoleAgent] = [] + for agent in agents: + if agent.ROLE_NAME == "map_analyst": + self._map_analyst = agent + else: + self._role_agents.append(agent) + self._parallel = parallel + self._role_timeout_s = role_timeout_s + self._visualizer = visualizer + self._resolver = ActionResolver() + self.last_phase: str = "" + + # Hub-spoke communication — agents read messages from their spokes. + # Wired at construction so callers can inspect spokes before setup(). + self.hub = CentralPost(max_agents=max(1, len(agents))) + self.spoke_manager = SpokeManager(self.hub) + for agent in agents: + spoke = self.spoke_manager.create_spoke(agent.agent_id, agent=agent) + agent.attach_spoke(spoke) + + @property + def agents(self) -> list[RimWorldRoleAgent]: + return list(self._agents) + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + async def setup(self, ctx: HarnessContext) -> None: + await super().setup(ctx) + if self._visualizer is not None: + self.observers.append(_HelixRenderObserver(self._visualizer)) + + async def teardown(self) -> None: + # Final drain: a deliberation that timed out on the last tick may have + # completed after the per-tick drain. + self._drain_generation_ids() + + def run_context(self) -> AbstractContextManager[Any]: + if self._visualizer is not None: + return self._visualizer.live() # type: ignore[no-any-return] + return nullcontext() + + @property + def visualizer(self) -> Any | None: + return self._visualizer + + def describe(self) -> dict[str, str]: + return { + "harness": self.name, + "agents": ",".join(a.ROLE_NAME for a in self._agents), + "felix_agent_sdk": felix_sdk_version(), + } + + # ------------------------------------------------------------------ + # Per-tick step + # ------------------------------------------------------------------ + + async def step( + self, state: GameState, tick: int, macro_time: float, + events: list[RimAPIEvent], + ) -> StepResult: + # Route previous tick's messages to agent spokes + broadcast phase changes + messages_before = self.hub.total_messages_processed + self.spoke_manager.process_all_messages() + self._broadcast_phase_if_changed(macro_time) + + for agent in self._agents: + agent.set_pending_events(events) + for evt in events: + self.ctx.emit( + EventType.SSE_EVENT, tick, + sse_type=evt.event_type, sse_data=str(evt.data)[:200], + ) + + plans: list[ActionPlan] = [] + + # MapAnalyst deliberates FIRST (sequential, timeout-wrapped) + if self._map_analyst: + ma_agent, ma_plan = await self._deliberate_agent_with_timeout( + self._map_analyst, state, macro_time, tick, + ) + if ma_plan is not None: + plans.append(ma_plan) + self._update_visualizer_agent(ma_agent, ma_plan, macro_time) + self._send_task_complete(ma_agent, ma_plan) + # Route MapAnalyst output to role agent spokes immediately + self.spoke_manager.process_all_messages() + + # Snapshot which agents have pending spoke messages (diagnostics) + agents_with_messages: set[str] = set() + for ra in self._role_agents: + spoke = self.spoke_manager.get_spoke(ra.agent_id) + if spoke and spoke.has_pending_messages(): + agents_with_messages.add(ra.agent_id) + + if self._parallel: + results = await self._deliberate_parallel(state, macro_time, tick) + else: + results = await self._deliberate_sequential(state, macro_time, tick) + + agents_acted_with_messages = 0 + for agent, plan in results: + if plan is None: + continue + plans.append(plan) + if agent.agent_id in agents_with_messages: + agents_acted_with_messages += 1 + self._update_visualizer_agent(agent, plan, macro_time) + self._send_task_complete(agent, plan) + + # Capture generation IDs from every provider call this tick — parse + # retries and failed deliberations bill tokens too. + self._drain_generation_ids() + + # Resolve conflicts. Resolver + CentralPost counts are diagnostics in + # the event log only — they no longer feed the composite (#51). + resolved, resolver_stats = self._resolver.resolve(plans, state) + self.ctx.emit( + EventType.CONFLICT, tick, + input_plans=len(plans), + output_actions=len(resolved.actions), + conflicts_detected=resolver_stats.conflicts_total, + conflicts_resolved=resolver_stats.conflicts_resolved, + messages_routed=self.hub.total_messages_processed - messages_before, + agents_with_messages=len(agents_with_messages), + agents_acted_with_messages=agents_acted_with_messages, + ) + + return StepResult( + plan=resolved, + proposals=tuple(plans), + extras={ + "phase": self.last_phase, + "conflicts_detected": resolver_stats.conflicts_total, + "conflicts_resolved": resolver_stats.conflicts_resolved, + }, + ) + + async def on_tick_end( + self, tick: int, state: GameState, step: StepResult, + execution: ExecutionResult, score: ScoreSnapshot | None, + ) -> None: + # Surface per-action errors back to agents so they can avoid + # re-proposing the same invalid action next tick. + failed_outcomes = [o for o in execution.outcomes if not o.success and o.error] + if failed_outcomes: + error_summary = "; ".join( + f"{o.action_type}({o.target_colonist_id or '-'}): {o.error}" + for o in failed_outcomes[:10] + ) + self.spoke_manager.broadcast_message( + MessageType.STATUS_UPDATE, + { + "tick": state.colony.tick, + "summary": f"Last tick action errors — DO NOT REPEAT: {error_summary}", + "action_errors": [ + { + "action_type": o.action_type, + "target_colonist_id": o.target_colonist_id, + "error": o.error, + } + for o in failed_outcomes + ], + }, + sender_id="hub", + ) + + if score: + self.spoke_manager.broadcast_message( + MessageType.STATUS_UPDATE, + { + "tick": state.colony.tick, + "day": state.colony.day, + "composite_score": score.composite, + "metrics": score.metrics, + }, + sender_id="hub", + ) + + await super().on_tick_end(tick, state, step, execution, score) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _send_task_complete(self, agent: RimWorldRoleAgent, plan: ActionPlan) -> None: + spoke = self.spoke_manager.get_spoke(agent.agent_id) + if spoke and spoke.is_connected: + spoke.send_message( + MessageType.TASK_COMPLETE, + { + "role": plan.role, + "summary": plan.summary, + "confidence": plan.confidence, + "num_actions": len(plan.actions), + "action_types": [a.action_type for a in plan.actions], + }, + ) + + def _drain_generation_ids(self) -> None: + tracker = self._ctx.cost_tracker if self._ctx else None + if tracker is None: + return + for agent in self._agents: + for gen_id in agent.drain_generation_ids(): + tracker.record_generation_id(gen_id) + + def _broadcast_phase_if_changed(self, macro_time: float) -> None: + phase = phase_for(macro_time) + if phase != self.last_phase: + self.spoke_manager.broadcast_message( + MessageType.PHASE_ANNOUNCE, + {"phase": phase, "depth_ratio": macro_time}, + sender_id="hub", + ) + self.last_phase = phase + + def _update_visualizer_agent( + self, agent: RimWorldRoleAgent, plan: ActionPlan, macro_time: float, + ) -> None: + if self._visualizer is None: + return + self._visualizer.update( + agent.agent_id, + progress=macro_time, + confidence=plan.confidence, + phase=agent.position.phase, + status=f"{len(plan.actions)} actions", + ) + + async def _deliberate_parallel( + self, state: GameState, macro_time: float, tick: int, + ) -> list[tuple[RimWorldRoleAgent, ActionPlan | None]]: + return list(await asyncio.gather(*[ + self._deliberate_agent_with_timeout(a, state, macro_time, tick) + for a in self._role_agents + ])) + + async def _deliberate_sequential( + self, state: GameState, macro_time: float, tick: int, + ) -> list[tuple[RimWorldRoleAgent, ActionPlan | None]]: + results: list[tuple[RimWorldRoleAgent, ActionPlan | None]] = [] + for agent in self._role_agents: + results.append( + await self._deliberate_agent_with_timeout(agent, state, macro_time, tick), + ) + return results + + async def _deliberate_agent_with_timeout( + self, agent: RimWorldRoleAgent, state: GameState, macro_time: float, tick: int, + ) -> tuple[RimWorldRoleAgent, ActionPlan | None]: + """Run one agent's deliberation with a hard timeout. + + On timeout: emits a deliberation_timeout ERROR event, records it in the + deliberation log, and returns ``(agent, None)`` so the tick continues. + """ + try: + return await asyncio.wait_for( + self._deliberate_agent(agent, state, macro_time, tick), + timeout=self._role_timeout_s, + ) + except asyncio.TimeoutError: + logger.warning( + "Agent %s deliberation timed out after %.1fs (tick %d)", + agent.ROLE_NAME, self._role_timeout_s, tick, + ) + self.parse_failures += 1 + self.deliberation_log.append({ + "tick": tick, "agent": agent.ROLE_NAME, + "status": "deliberation_timeout", + "reason": f"timed out after {self._role_timeout_s}s", + }) + self.ctx.emit( + EventType.ERROR, tick, agent=agent.ROLE_NAME, + error_type="deliberation_timeout", + reason=f"timed out after {self._role_timeout_s}s", + timeout_s=self._role_timeout_s, + ) + return agent, None + + async def _deliberate_agent( + self, agent: RimWorldRoleAgent, state: GameState, macro_time: float, tick: int, + ) -> tuple[RimWorldRoleAgent, ActionPlan | None]: + t0 = _time.monotonic() + try: + plan = await agent.adeliberate(state, macro_time) + except ActionPlanParseError as e: + latency_ms = round((_time.monotonic() - t0) * 1000, 1) + logger.warning( + "Agent %s parse failure (tick %d): %s", agent.ROLE_NAME, tick, e.reason, + ) + self.parse_failures += 1 + raw_truncated = ( + e.raw_content[:_PARSE_FAILURE_RAW_CHARS] if e.raw_content else None + ) + self.deliberation_log.append({ + "tick": tick, "agent": agent.ROLE_NAME, + "status": "parse_failure", "reason": e.reason, + "raw": raw_truncated, + }) + self.ctx.emit( + EventType.ERROR, tick, agent=agent.ROLE_NAME, + error_type="parse_failure", reason=e.reason, latency_ms=latency_ms, + raw=raw_truncated, + ) + return agent, None + except ProviderError as e: + latency_ms = round((_time.monotonic() - t0) * 1000, 1) + logger.warning( + "Agent %s provider error (tick %d): %s", agent.ROLE_NAME, tick, e, + ) + self.parse_failures += 1 + self.deliberation_log.append({ + "tick": tick, "agent": agent.ROLE_NAME, + "status": "provider_error", "reason": str(e), + }) + self.ctx.emit( + EventType.ERROR, tick, agent=agent.ROLE_NAME, + error_type="provider_error", reason=str(e), latency_ms=latency_ms, + ) + return agent, None + + latency_ms = round((_time.monotonic() - t0) * 1000, 1) + self.parse_successes += 1 + actions_payload = [ + {"type": a.action_type, "target": a.target_colonist_id, + "priority": a.priority, "reason": a.reason[:_ACTION_REASON_CHARS]} + for a in plan.actions + ] + summary_truncated = plan.summary[:_PLAN_SUMMARY_CHARS] + self.deliberation_log.append({ + "tick": tick, "agent": plan.role, + "status": "success", "confidence": plan.confidence, + "num_actions": len(plan.actions), + "actions": actions_payload, + "summary": summary_truncated, + }) + self.ctx.emit( + EventType.DELIBERATION, tick, agent=plan.role, + latency_ms=latency_ms, confidence=plan.confidence, + num_actions=len(plan.actions), + actions=actions_payload, + summary=summary_truncated, + ) + + usage = agent._last_usage + if usage and isinstance(usage, dict): + pt = usage.get("prompt_tokens", 0) + ct = usage.get("completion_tokens", 0) + rt = usage.get("reasoning_tokens", 0) + if not isinstance(rt, int): + rt = 0 + if isinstance(pt, int) and isinstance(ct, int): + if self.ctx.cost_tracker: + self.ctx.cost_tracker.record_raw(pt, ct, rt) + raw_output = agent._last_raw_output + raw_output_truncated = ( + raw_output[:_RAW_OUTPUT_CHARS] if raw_output else None + ) + was_truncated = ( + raw_output is not None and len(raw_output) > _RAW_OUTPUT_CHARS + ) + self.ctx.emit( + EventType.PROVIDER_CALL, tick, agent=plan.role, + prompt_tokens=pt, completion_tokens=ct, + reasoning_tokens=rt, + raw_output=raw_output_truncated, + raw_output_truncated=was_truncated, + ) + + return agent, plan + + +class _HelixRenderObserver: + """Renders the terminal helix at the end of each tick.""" + + def __init__(self, visualizer: Any) -> None: + self._visualizer = visualizer + + def on_tick_end( + self, tick: int, day: int, step: StepResult, + execution: ExecutionResult, score: ScoreSnapshot | None, + ) -> None: + extra: dict[str, str] = { + "actions": f"{execution.executed}/{execution.total}", + } + if score: + extra["score"] = f"{score.composite:.3f}" + self._visualizer.render(tick=tick, day=day, extra_info=extra) diff --git a/src/rle/harness/felix/options.py b/src/rle/harness/felix/options.py new file mode 100644 index 0000000..f41c452 --- /dev/null +++ b/src/rle/harness/felix/options.py @@ -0,0 +1,50 @@ +"""Options accepted by the Felix harness (``--harness-opt key=value``).""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class FelixOptions(BaseModel): + """Knobs that only make sense for the Felix multi-agent harness. + + These used to be top-level CLI flags / ``RLEConfig`` fields. They live + here now so the environment and other harnesses never see them. + """ + + model_config = ConfigDict(extra="forbid") + + parallel: bool = Field( + default=True, + description="Deliberate the six role agents concurrently (MapAnalyst always first).", + ) + no_think: bool = Field( + default=False, + description="Inject a assistant prefill so thinking models skip reasoning.", + ) + helix_preset: str = Field( + default="default", + description="HelixConfig preset: default | research_heavy | fast_convergence.", + ) + role_timeout_s: float = Field( + default=60.0, + description=( + "Max wall-clock seconds for a single agent's deliberation. Hung LLM calls " + "beyond this fire a deliberation_timeout ERROR event and the agent " + "contributes no actions for the tick." + ), + ) + exclude_agent: str | None = Field( + default=None, + description="Drop one role agent by id (ablation runs).", + ) + provider_kwargs: dict[str, Any] = Field( + default_factory=dict, + description="Extra kwargs forwarded to provider.complete() (e.g. extra_body).", + ) + visualize: bool = Field( + default=False, + description="Render the terminal helix visualiser.", + ) diff --git a/src/rle/harness/felix/plugin.py b/src/rle/harness/felix/plugin.py new file mode 100644 index 0000000..224b24e --- /dev/null +++ b/src/rle/harness/felix/plugin.py @@ -0,0 +1,60 @@ +"""Entry-point plugin for the Felix multi-agent harness. + +This module is the optional-dependency boundary for ``felix-agent-sdk``: it +is imported by the plugin registry on every ``--harness list``, so it must +not import the SDK at module load. The SDK-dependent modules are imported +inside the methods that need them (documented exception to the +no-inline-imports rule). +""" + +from __future__ import annotations + +from importlib.util import find_spec +from typing import Any + +from pydantic import BaseModel + +from rle.harness.protocol import Availability, BaseHarness, HarnessContext + +FELIX_DESCRIPTION = ( + "MapAnalyst + 6 role agents over Felix SDK CentralPost, merged by ActionResolver " + "(the original RLE harness)." +) + + +class FelixPlugin: + name = "felix" + description = FELIX_DESCRIPTION + + def available(self) -> Availability: + if find_spec("felix_agent_sdk") is None: + return Availability.missing( + "felix-agent-sdk is not installed — `uv sync --extra felix`", + ) + return Availability.available() + + def option_schema(self) -> type[BaseModel]: + from rle.harness.felix.options import FelixOptions # noqa: PLC0415 - optional dep + + return FelixOptions + + def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + from rle.harness.felix.build import build_felix_harness # noqa: PLC0415 - optional dep + + return build_felix_harness(ctx, options) + + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + from rle.harness.felix.build import build_felix_harness # noqa: PLC0415 - optional dep + + return build_felix_harness(ctx, options, smoke=True) + + def describe(self) -> dict[str, str]: + info: dict[str, Any] = {"harness": self.name} + if self.available().ok: + from rle.harness.felix.harness import felix_sdk_version # noqa: PLC0415 + + info["felix_agent_sdk"] = felix_sdk_version() + return info + + +PLUGIN = FelixPlugin() diff --git a/src/rle/harness/felix/provider_factory.py b/src/rle/harness/felix/provider_factory.py new file mode 100644 index 0000000..ff51497 --- /dev/null +++ b/src/rle/harness/felix/provider_factory.py @@ -0,0 +1,48 @@ +"""Build Felix SDK providers and helix geometry from RLE's string config.""" + +from __future__ import annotations + +from felix_agent_sdk.core import HelixConfig, HelixGeometry +from felix_agent_sdk.providers import ( + AnthropicProvider, + BaseProvider, + LocalProvider, + OpenAIProvider, +) + +from rle.harness.felix.providers.claude_code import ClaudeCodeProvider + +HELIX_PRESETS: dict[str, HelixConfig] = { + "default": HelixConfig.default(), + "research_heavy": HelixConfig.research_heavy(), + "fast_convergence": HelixConfig.fast_convergence(), +} + +PROVIDER_CLASSES: dict[str, type[BaseProvider]] = { + "anthropic": AnthropicProvider, + "openai": OpenAIProvider, + "local": LocalProvider, + "claude-code": ClaudeCodeProvider, +} + + +def build_provider(provider: str, model: str, base_url: str | None = None) -> BaseProvider: + """Construct a Felix provider from provider name + model (+ optional base URL).""" + cls = PROVIDER_CLASSES.get(provider) + if cls is None: + raise ValueError( + f"Unknown provider {provider!r}. Choose from: {list(PROVIDER_CLASSES)}" + ) + kwargs: dict[str, str] = {"model": model} + if base_url: + kwargs["base_url"] = base_url + return cls(**kwargs) # type: ignore[arg-type] # subclasses accept kwargs + + +def build_helix(preset: str = "default") -> HelixGeometry: + try: + return HELIX_PRESETS[preset].to_geometry() + except KeyError: + raise ValueError( + f"Unknown helix preset {preset!r}. Choose from: {list(HELIX_PRESETS)}" + ) from None diff --git a/src/rle/harness/felix/providers/__init__.py b/src/rle/harness/felix/providers/__init__.py new file mode 100644 index 0000000..68cb756 --- /dev/null +++ b/src/rle/harness/felix/providers/__init__.py @@ -0,0 +1,5 @@ +"""Felix-SDK providers that RLE adds beyond the SDK's built-ins.""" + +from rle.harness.felix.providers.claude_code import ClaudeCodeProvider + +__all__ = ["ClaudeCodeProvider"] diff --git a/src/rle/providers/claude_code.py b/src/rle/harness/felix/providers/claude_code.py similarity index 100% rename from src/rle/providers/claude_code.py rename to src/rle/harness/felix/providers/claude_code.py diff --git a/src/rle/harness/felix/smoke.py b/src/rle/harness/felix/smoke.py new file mode 100644 index 0000000..7361d57 --- /dev/null +++ b/src/rle/harness/felix/smoke.py @@ -0,0 +1,70 @@ +"""Deterministic stand-in provider for ``--smoke-test`` runs (no LLM calls).""" + +from __future__ import annotations + +import json +from collections.abc import Iterator, Sequence +from typing import Any + +from felix_agent_sdk.providers.base import BaseProvider +from felix_agent_sdk.providers.types import ( + ChatMessage, + CompletionResult, + ProviderConfig, + StreamChunk, +) + +SMOKE_ACTION_PLAN = json.dumps({ + "actions": [ + {"action_type": "no_action", "reason": "Smoke test — no real LLM call"}, + ], + "summary": "Smoke deliberation.", + "confidence": 0.6, +}) + + +class SmokeProvider(BaseProvider): + """Returns a fixed, always-parseable action plan.""" + + def __init__(self, content: str = SMOKE_ACTION_PLAN, model: str = "smoke") -> None: + super().__init__(ProviderConfig(model=model, api_key=None)) + self._content = content + self.calls = 0 + + @property + def provider_name(self) -> str: + return "smoke" + + def _result(self) -> CompletionResult: + self.calls += 1 + return CompletionResult( + content=self._content, + model=self.config.model, + usage={"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, + ) + + def complete( + self, messages: Sequence[ChatMessage], *, temperature: float | None = None, + max_tokens: int | None = None, stop_sequences: list[str] | None = None, + **kwargs: Any, + ) -> CompletionResult: + return self._result() + + async def acomplete( + self, messages: Sequence[ChatMessage], *, temperature: float | None = None, + max_tokens: int | None = None, stop_sequences: list[str] | None = None, + **kwargs: Any, + ) -> CompletionResult: + return self._result() + + def stream( + self, messages: Sequence[ChatMessage], *, temperature: float | None = None, + max_tokens: int | None = None, stop_sequences: list[str] | None = None, + **kwargs: Any, + ) -> Iterator[StreamChunk]: + result = self._result() + yield StreamChunk(text=result.content) + yield StreamChunk(text="", is_final=True, usage=result.usage) + + def count_tokens(self, messages: Sequence[ChatMessage]) -> int: + return sum(len(m.content) // 4 for m in messages) diff --git a/src/rle/harness/protocol.py b/src/rle/harness/protocol.py new file mode 100644 index 0000000..778201b --- /dev/null +++ b/src/rle/harness/protocol.py @@ -0,0 +1,194 @@ +"""Harness protocol — the seam between the RLE environment and whatever +decides what the colony does each tick. + +The environment (``RLEGameLoop``) owns the game: pause/unpause, state +refresh, action execution, scoring, evaluation, export. A *harness* owns the +decision-making: one or many LLM agents, a coding agent attached over MCP, a +scripted policy, or nothing at all (the unmanaged baseline). Harnesses are +benchmarked side by side with models, so nothing in this module may depend on +any particular agent framework. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable +from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict + +from rle.agents.actions import ActionPlan, ExecutionResult +from rle.rimapi.client import RimAPIClient +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.scoring.composite import ScoreSnapshot +from rle.tracking.cost_tracker import CostTracker +from rle.tracking.event_log import EventLog, EventType + +if TYPE_CHECKING: + from rle.config import RLEConfig + from rle.scenarios.schema import ScenarioConfig + + +class HarnessStepError(Exception): + """A harness failed to produce a step. The loop records an ERROR event and + treats the tick as having no actions; the run continues.""" + + +class StepResult(BaseModel): + """What a harness hands back for one tick. + + ``plan`` is the merged set of actions to apply. ``execution`` is set when + the harness has *already* applied its writes (coding agents acting through + the RLE MCP server see tool results inside their turn, so nothing is left + for the loop to execute); when it is ``None`` the loop runs + ``ActionExecutor`` itself. ``proposals`` are optional per-sub-agent plans + kept for dashboards and post-hoc analysis; ``extras`` is a free-form bag + for harness-specific telemetry (helix phase, session ids, ...). + """ + + model_config = ConfigDict(frozen=True) + + plan: ActionPlan + execution: ExecutionResult | None = None + proposals: tuple[ActionPlan, ...] = () + extras: dict[str, Any] = {} + + +@dataclass +class HarnessContext: + """Everything the environment lends a harness for the duration of a run.""" + + config: RLEConfig + client: RimAPIClient + expected_duration_days: int = 60 + initial_population: int = 3 + scenario: ScenarioConfig | None = None + event_log: EventLog | None = None + cost_tracker: CostTracker | None = None + tick_timeout_s: float | None = None + smoke: bool = False + extras: dict[str, Any] = field(default_factory=dict) + + def emit( + self, event_type: EventType, tick: int, + agent: str | None = None, **data: object, + ) -> None: + """Emit to the run's event log if one is configured.""" + if self.event_log is not None: + self.event_log.emit(event_type, tick, agent=agent, **data) + + +@runtime_checkable +class TickObserver(Protocol): + """Receives the end-of-tick summary (visualisers, recorders, cameras).""" + + def on_tick_end( + self, tick: int, day: int, step: StepResult, + execution: ExecutionResult, score: ScoreSnapshot | None, + ) -> None: ... + + +class BaseHarness(ABC): + """Base class every harness derives from. + + Lifecycle per run: ``setup`` once, then per tick ``step`` followed by + ``on_tick_end`` (after execution + scoring), then ``teardown``. Subclasses + should keep ``deliberation_log`` / ``parse_successes`` / ``parse_failures`` + updated so run reports stay uniform across harnesses. + """ + + name: ClassVar[str] = "base" + + def __init__(self) -> None: + self.deliberation_log: list[dict[str, object]] = [] + self.parse_successes = 0 + self.parse_failures = 0 + self.observers: list[TickObserver] = [] + self._ctx: HarnessContext | None = None + + @property + def ctx(self) -> HarnessContext: + if self._ctx is None: + raise RuntimeError(f"{type(self).__name__}.setup() has not been called") + return self._ctx + + async def setup(self, ctx: HarnessContext) -> None: + self._ctx = ctx + + @abstractmethod + async def step( + self, state: GameState, tick: int, macro_time: float, + events: list[RimAPIEvent], + ) -> StepResult: + """Decide (and optionally apply) this tick's actions.""" + + async def on_tick_end( + self, tick: int, state: GameState, step: StepResult, + execution: ExecutionResult, score: ScoreSnapshot | None, + ) -> None: + """Feedback hook after execution and scoring. Default: notify observers.""" + for observer in self.observers: + observer.on_tick_end(state.colony.tick, state.colony.day, step, execution, score) + + async def teardown(self) -> None: + return None + + def run_context(self) -> AbstractContextManager[Any]: + """Context the loop enters around ``run()`` (e.g. a live terminal UI).""" + return nullcontext() + + def describe(self) -> dict[str, str]: + """Version / identity info recorded in run metadata.""" + return {"harness": self.name} + + +class Availability(BaseModel): + """Whether a plugin can run here, and if not, why.""" + + model_config = ConfigDict(frozen=True) + + ok: bool + reason: str = "" + + @classmethod + def available(cls) -> Availability: + return cls(ok=True) + + @classmethod + def missing(cls, reason: str) -> Availability: + return cls(ok=False, reason=reason) + + +class HarnessPlugin(Protocol): + """Entry-point contract under the ``rle.harnesses`` group. + + A plugin is a module-level object (conventionally ``PLUGIN``) whose + ``create`` builds a harness from validated options. ``available`` must be + cheap and must not import optional dependencies at module load — probe + for them inside the method. + """ + + name: str + description: str + + def available(self) -> Availability: ... + + def option_schema(self) -> type[BaseModel]: ... + + def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: ... + + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: ... + + def describe(self) -> dict[str, str]: ... + + +class EmptyOptions(BaseModel): + """Option schema for harnesses that take no options.""" + + model_config = ConfigDict(extra="forbid") + + +OptionsFactory = Callable[[], type[BaseModel]] diff --git a/src/rle/harness/registry.py b/src/rle/harness/registry.py new file mode 100644 index 0000000..d1e2317 --- /dev/null +++ b/src/rle/harness/registry.py @@ -0,0 +1,159 @@ +"""Harness discovery via the ``rle.harnesses`` entry-point group. + +Built-in harnesses (``baseline``, ``felix``) and third-party packages +(``rle-harness-``) register the same way, so adding a harness is +``pip install `` — never a change to RLE core. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from importlib.metadata import EntryPoint, entry_points +from typing import Any, cast + +from pydantic import BaseModel, ValidationError + +from rle.harness.protocol import Availability, BaseHarness, HarnessContext, HarnessPlugin + +ENTRY_POINT_GROUP = "rle.harnesses" + + +class HarnessNotFoundError(LookupError): + """No plugin registered under the requested name.""" + + +class HarnessUnavailableError(RuntimeError): + """The plugin exists but cannot run here (missing extra, binary, ...).""" + + +class HarnessOptionsError(ValueError): + """Harness options failed validation against the plugin's schema.""" + + +@dataclass(frozen=True) +class HarnessInfo: + name: str + description: str + availability: Availability + package: str + version: str + + +def _entry_points() -> dict[str, EntryPoint]: + found: dict[str, EntryPoint] = {} + for ep in entry_points(group=ENTRY_POINT_GROUP): + if ep.name in found: + other = found[ep.name] + raise RuntimeError( + f"Two packages register harness {ep.name!r}: " + f"{_dist_name(other)} and {_dist_name(ep)}. Uninstall one.", + ) + found[ep.name] = ep + return found + + +def _dist_name(ep: EntryPoint) -> str: + dist = ep.dist + return dist.metadata["Name"] if dist is not None else "?" + + +def _dist_version(ep: EntryPoint) -> str: + dist = ep.dist + return dist.version if dist is not None else "?" + + +def harness_names() -> list[str]: + return sorted(_entry_points()) + + +def get_plugin(name: str) -> HarnessPlugin: + eps = _entry_points() + ep = eps.get(name) + if ep is None: + raise HarnessNotFoundError( + f"Unknown harness {name!r}. Installed: {', '.join(sorted(eps)) or '(none)'}. " + "Install a harness package (rle-harness-) or check the name.", + ) + return cast(HarnessPlugin, ep.load()) + + +def list_harnesses() -> list[HarnessInfo]: + infos: list[HarnessInfo] = [] + for name, ep in sorted(_entry_points().items()): + try: + plugin = cast(HarnessPlugin, ep.load()) + availability = plugin.available() + description = plugin.description + except Exception as exc: # a broken plugin must not break the CLI + availability = Availability.missing(f"plugin failed to load: {exc}") + description = "" + infos.append(HarnessInfo( + name=name, + description=description, + availability=availability, + package=_dist_name(ep), + version=_dist_version(ep), + )) + return infos + + +def parse_option_pairs(pairs: list[str] | None) -> dict[str, Any]: + """Turn ``["key=value", ...]`` into a dict, decoding JSON-looking values. + + ``true``/``false``/numbers/quoted strings/objects parse as JSON; anything + else is kept as the raw string. + """ + out: dict[str, Any] = {} + for pair in pairs or []: + key, sep, raw = pair.partition("=") + if not sep or not key: + raise HarnessOptionsError(f"--harness-opt expects key=value, got {pair!r}") + try: + out[key.strip()] = json.loads(raw) + except json.JSONDecodeError: + out[key.strip()] = raw + return out + + +def validate_options(plugin: HarnessPlugin, raw: dict[str, Any] | BaseModel | None) -> BaseModel: + schema = plugin.option_schema() + if isinstance(raw, BaseModel): + if isinstance(raw, schema): + return raw + raw = raw.model_dump() + try: + return schema.model_validate(raw or {}) + except ValidationError as exc: + raise HarnessOptionsError( + f"Invalid options for harness {plugin.name!r}:\n{exc}", + ) from exc + + +def create_harness( + name: str, + ctx: HarnessContext, + options: dict[str, Any] | BaseModel | None = None, + *, + smoke: bool = False, +) -> BaseHarness: + """Resolve ``name`` through the registry and build a ready-to-setup harness.""" + plugin = get_plugin(name) + opts = validate_options(plugin, options) + if smoke or ctx.smoke: + # Smoke variants must run without the external tool, so availability + # is not a gate here; a plugin whose smoke still needs a missing + # dependency surfaces that as unavailable. + try: + return plugin.smoke(ctx, opts) + except ImportError as exc: + reason = plugin.available().reason or str(exc) + raise HarnessUnavailableError( + f"Harness {name!r} cannot run its smoke variant: {reason}", + ) from exc + availability = plugin.available() + if not availability.ok: + raise HarnessUnavailableError( + f"Harness {name!r} is installed but unavailable: {availability.reason}", + ) + return plugin.create(ctx, opts) diff --git a/src/rle/mcp/__init__.py b/src/rle/mcp/__init__.py new file mode 100644 index 0000000..0eb39cc --- /dev/null +++ b/src/rle/mcp/__init__.py @@ -0,0 +1,12 @@ +"""RimAPI as an MCP tool surface for external coding-agent harnesses. + +``ledger`` and ``session`` have no ``mcp`` dependency; ``server`` / ``host`` +need the ``mcp`` extra. Run standalone with ``rle-mcp`` (stdio) for manual +play against a live game, or let a harness host it in-process (streamable +HTTP) so the ledger and the loop share memory. +""" + +from rle.mcp.ledger import NoActiveTickError, TickLedger +from rle.mcp.session import McpSession + +__all__ = ["McpSession", "NoActiveTickError", "TickLedger"] diff --git a/src/rle/mcp/__main__.py b/src/rle/mcp/__main__.py new file mode 100644 index 0000000..f892bf6 --- /dev/null +++ b/src/rle/mcp/__main__.py @@ -0,0 +1,51 @@ +"""``rle-mcp`` — stdio MCP server against a live RIMAPI, for manual play. + +Standalone mode has no environment loop: every call is "in a tick", writes go +straight to the game, and the ledger is only informational. Benchmark runs +use ``rle.mcp.host.McpHost`` inside a harness instead. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging + +from rle.config import RLEConfig +from rle.harness.brief import build_brief +from rle.mcp.ledger import TickLedger +from rle.mcp.server import build_server +from rle.mcp.session import McpSession +from rle.orchestration.action_executor import ActionExecutor +from rle.orchestration.state_manager import GameStateManager +from rle.rimapi.client import RimAPIClient + + +async def _serve(rimapi_url: str) -> None: + async with RimAPIClient(rimapi_url) as client: + ledger = TickLedger(harness_name="manual") + session = McpSession(client=client, executor=ActionExecutor(client), ledger=ledger) + manager = GameStateManager(client, expected_duration_days=30) + try: + state = await manager.refresh() + session.begin_tick(0, state, build_brief(state, tick=0, macro_time=0.0)) + except Exception: + logging.getLogger(__name__).warning( + "RIMAPI not reachable at %s — serving with an empty brief", rimapi_url, + ) + ledger.active = True + await build_server(session).run_stdio_async() + + +def main() -> None: + parser = argparse.ArgumentParser(description="RLE RimAPI MCP server (stdio)") + parser.add_argument("--rimapi-url", default=None, help="Default: RIMAPI_URL / config") + parser.add_argument("--log-level", default="WARNING") + args = parser.parse_args() + logging.basicConfig(level=getattr(logging, args.log_level.upper(), logging.WARNING)) + config = RLEConfig() + asyncio.run(_serve(args.rimapi_url or config.rimapi_url)) + + +if __name__ == "__main__": + main() diff --git a/src/rle/mcp/host.py b/src/rle/mcp/host.py new file mode 100644 index 0000000..288cce8 --- /dev/null +++ b/src/rle/mcp/host.py @@ -0,0 +1,68 @@ +"""Host the RLE MCP server in-process over streamable HTTP. + +Harnesses that drive an external coding agent start one of these so the +agent's MCP client and the environment share a single ledger in memory. +""" + +from __future__ import annotations + +import asyncio +import socket + +import uvicorn +from mcp.server.mcpserver import MCPServer + +MCP_PATH = "/mcp" + + +def free_port(host: str = "127.0.0.1") -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind((host, 0)) + return int(s.getsockname()[1]) + + +class McpHost: + def __init__(self, server: MCPServer, *, host: str = "127.0.0.1", port: int = 0) -> None: + self._server = server + self._host = host + self._port = port or free_port(host) + self._uvicorn: uvicorn.Server | None = None + self._task: asyncio.Task[None] | None = None + + @property + def url(self) -> str: + return f"http://{self._host}:{self._port}{MCP_PATH}" + + @property + def running(self) -> bool: + return self._task is not None and not self._task.done() + + async def start(self, *, timeout_s: float = 10.0) -> str: + if self.running: + return self.url + app = self._server.streamable_http_app( + streamable_http_path=MCP_PATH, host=self._host, stateless_http=True, + ) + config = uvicorn.Config(app, host=self._host, port=self._port, log_level="warning") + self._uvicorn = uvicorn.Server(config) + self._task = asyncio.create_task(self._uvicorn.serve()) + deadline = asyncio.get_running_loop().time() + timeout_s + while not self._uvicorn.started: + if self._task.done(): + self._task.result() # re-raise startup failure + raise RuntimeError("MCP host exited before starting") + if asyncio.get_running_loop().time() > deadline: + raise TimeoutError("MCP host did not start in time") + await asyncio.sleep(0.02) + return self.url + + async def stop(self) -> None: + if self._uvicorn is not None: + self._uvicorn.should_exit = True + if self._task is not None: + try: + await asyncio.wait_for(self._task, timeout=10.0) + except (asyncio.TimeoutError, asyncio.CancelledError): + self._task.cancel() + self._task = None + self._uvicorn = None diff --git a/src/rle/mcp/ledger.py b/src/rle/mcp/ledger.py new file mode 100644 index 0000000..ab28bea --- /dev/null +++ b/src/rle/mcp/ledger.py @@ -0,0 +1,86 @@ +"""Per-tick write ledger shared between the MCP tools and the harness. + +Tool-using harnesses act *during* their turn (they need tool results to +decide the next call), so nothing is left for the loop to execute. The ledger +records every write attempted in the current tick and turns it into the +``StepResult`` the environment scores. No ``mcp`` import here — the harness +base and the scorer depend on this module even when the MCP extra is absent. +""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from typing import Any + +from rle.agents.actions import Action, ActionOutcome, ActionPlan, ExecutionResult +from rle.harness.protocol import StepResult + + +class NoActiveTickError(RuntimeError): + """A write arrived while the environment was not inside a harness step.""" + + +@dataclass +class TickLedger: + harness_name: str = "mcp" + active: bool = False + tick: int = -1 + game_tick: int = 0 + actions: list[Action] = field(default_factory=list) + outcomes: list[ActionOutcome] = field(default_factory=list) + summary: str = "" + extras: dict[str, Any] = field(default_factory=dict) + turn_done: asyncio.Event = field(default_factory=asyncio.Event) + + def begin(self, tick: int, game_tick: int) -> None: + self.active = True + self.tick = tick + self.game_tick = game_tick + self.actions = [] + self.outcomes = [] + self.summary = "" + self.extras = {} + self.turn_done = asyncio.Event() + + def require_active(self) -> None: + if not self.active: + raise NoActiveTickError( + "No tick is in progress — the environment is between turns. " + "Wait for the next prompt before acting.", + ) + + def record(self, action: Action, outcome: ActionOutcome | None) -> None: + self.require_active() + self.actions.append(action) + if outcome is not None: + self.outcomes.append(outcome) + + def end_turn(self, summary: str = "") -> None: + if summary: + self.summary = summary + self.turn_done.set() + + def finish(self) -> StepResult: + """Close the tick and package what happened as a StepResult.""" + self.active = False + executed = sum(1 for o in self.outcomes if o.success) + failed = len(self.outcomes) - executed + plan = ActionPlan( + role=self.harness_name, + tick=self.game_tick, + actions=list(self.actions), + summary=self.summary or f"{len(self.actions)} tool call(s)", + ) + execution = ExecutionResult( + executed=executed, + failed=failed, + total=len(self.outcomes), + outcomes=tuple(self.outcomes), + ) + return StepResult( + plan=plan, + execution=execution, + proposals=(plan,), + extras={"turn_ended": self.turn_done.is_set(), **self.extras}, + ) diff --git a/src/rle/mcp/server.py b/src/rle/mcp/server.py new file mode 100644 index 0000000..b730540 --- /dev/null +++ b/src/rle/mcp/server.py @@ -0,0 +1,110 @@ +"""RimAPI as an MCP tool server (requires the ``mcp`` extra). + +One tool per ``WRITE_CATALOG`` entry (executed immediately through +``ActionExecutor`` and recorded in the tick ledger), a generic read tool over +``READ_CATALOG``, the harness-neutral brief, and ``end_turn``. Any MCP-capable +coding agent can attach and play; the harness packages that do so live in +their own repositories. +""" + +from __future__ import annotations + +import json +from typing import Any, cast + +from mcp.server.mcpserver import MCPServer + +from rle.mcp.session import McpSession +from rle.rimapi.api_catalog import READ_CATALOG, WRITE_CATALOG + +SERVER_NAME = "rle" +INSTRUCTIONS = ( + "You are managing a RimWorld colony through the RLE benchmark environment. " + "Each turn: call get_brief to see the scenario, colony state, MAP_SUMMARY and " + "available actions; issue writes with the action tools (use MAP_SUMMARY " + "coordinates verbatim, never invent positions; pawn ids are integers); then " + "call end_turn with a one-line summary. Writes outside a turn are rejected." +) + + +def _dumps(value: Any) -> str: + return json.dumps(value, indent=2, default=str) + + +def build_server(session: McpSession) -> MCPServer: + server = MCPServer(SERVER_NAME, instructions=INSTRUCTIONS) + + @server.tool( + description="Scenario goals, colony state, MAP_SUMMARY and action catalog for this turn.", + ) + def get_brief() -> str: + return session.brief_text() + + @server.tool( + description="Current colony state as JSON (same data as get_brief, machine-readable).", + ) + def get_state() -> str: + return _dumps(session.state_json()) + + @server.tool(description="List every write action with its parameter shape.") + def list_actions() -> str: + return _dumps(session.actions()) + + @server.tool(description="List RIMAPI read endpoints available to rimapi_read.") + def list_reads() -> str: + return _dumps({ + name: cast(dict[str, Any], entry).get("description", "") + for name, entry in sorted(READ_CATALOG.items()) + }) + + @server.tool( + description=( + "Read a RIMAPI endpoint by catalog name (see list_reads). " + "params become query-string parameters, e.g. {\"map_id\": 0}." + ), + ) + async def rimapi_read(endpoint: str, params: dict[str, Any] | None = None) -> str: + try: + return _dumps(await session.read(endpoint, params)) + except Exception as exc: # surfaced to the agent, never crashes the server + return _dumps({"error": str(exc)}) + + @server.tool( + description=( + "Finish this turn. Call once you have issued all writes for the tick; " + "the environment then advances the game and scores the tick." + ), + ) + def end_turn(summary: str = "") -> str: + session.ledger.end_turn(summary) + n = len(session.ledger.actions) + return f"Turn ended after {n} action(s)." + + for name, raw in sorted(WRITE_CATALOG.items()): + entry = cast(dict[str, Any], raw) + server.add_tool( + _make_action_tool(session, name), + name=name, + description=( + f"{entry.get('description', name)}. " + f"parameters shape: {json.dumps(entry.get('params', {}), default=str)}. " + "Executes immediately and returns {ok, error}." + ), + ) + return server + + +def _make_action_tool(session: McpSession, action_type: str) -> Any: + async def tool( + parameters: dict[str, Any] | None = None, + target_colonist_id: str | None = None, + reason: str = "", + ) -> str: + try: + result = await session.act(action_type, parameters, target_colonist_id, reason) + except Exception as exc: + result = {"ok": False, "action_type": action_type, "error": str(exc)} + return _dumps(result) + + tool.__name__ = action_type + return tool diff --git a/src/rle/mcp/session.py b/src/rle/mcp/session.py new file mode 100644 index 0000000..c4a76e9 --- /dev/null +++ b/src/rle/mcp/session.py @@ -0,0 +1,130 @@ +"""Shared state between the environment and the MCP tool handlers.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any, cast + +from rle.agents.actions import Action, ActionOutcome, ActionPlan, resolve_endpoint +from rle.harness.brief import ScenarioBrief, action_catalog +from rle.mcp.ledger import TickLedger +from rle.orchestration.action_executor import NEEDS_PAWN, ActionExecutor +from rle.rimapi.api_catalog import READ_CATALOG +from rle.rimapi.client import RimAPIClient +from rle.rimapi.schemas import GameState +from rle.tracking.event_log import EventType + +EmitFn = Callable[..., None] + + +def _noop_emit(*_args: Any, **_kwargs: Any) -> None: + return None + + +@dataclass +class McpSession: + """What the tools need: a RIMAPI client, an executor, the current tick's + brief/state, and the ledger to record into.""" + + client: RimAPIClient + executor: ActionExecutor + ledger: TickLedger + brief: ScenarioBrief | None = None + state: GameState | None = None + emit: EmitFn = _noop_emit + extras: dict[str, Any] = field(default_factory=dict) + + def begin_tick( + self, tick: int, state: GameState, brief: ScenarioBrief | None, + ) -> None: + self.state = state + self.brief = brief + self.ledger.begin(tick, state.colony.tick) + + async def act( + self, + action_type: str, + parameters: dict[str, Any] | None = None, + target_colonist_id: str | None = None, + reason: str = "", + ) -> dict[str, Any]: + """Execute one write immediately, record it, and return the outcome.""" + self.ledger.require_active() + action = Action( + action_type=action_type, + target_colonist_id=target_colonist_id, + parameters=dict(parameters or {}), + reason=reason, + ) + endpoint = resolve_endpoint(action_type) + if endpoint == "no_action": + self.ledger.record(action, None) + return {"ok": True, "action_type": action_type, "note": "no-op recorded"} + outcome: ActionOutcome + if endpoint in NEEDS_PAWN and not target_colonist_id: + # The executor would silently skip this; tell the agent instead. + outcome = ActionOutcome( + action_type=action_type, endpoint=endpoint, target_colonist_id=None, + success=False, error="target_colonist_id is required for this action", + parameters=action.parameters, + ) + self.ledger.record(action, outcome) + return {"ok": False, "action_type": action_type, "endpoint": endpoint, + "error": outcome.error} + result = await self.executor.execute( + ActionPlan(role=self.ledger.harness_name, tick=self.ledger.game_tick, actions=[action]), + ) + if result.outcomes: + outcome = result.outcomes[0] + else: + # The executor skipped it (unknown endpoint). + outcome = ActionOutcome( + action_type=action_type, endpoint=endpoint, + target_colonist_id=target_colonist_id, success=False, + error="skipped by executor (unknown action)", + parameters=action.parameters, + ) + self.ledger.record(action, outcome) + self.emit( + EventType.ACTION_EXEC, self.ledger.tick, + action_type=outcome.action_type, target=outcome.target_colonist_id, + success=outcome.success, error=outcome.error, parameters=outcome.parameters, + via="mcp", + ) + return { + "ok": outcome.success, + "action_type": action_type, + "endpoint": endpoint, + "error": outcome.error, + } + + async def read(self, endpoint: str, params: dict[str, Any] | None = None) -> Any: + """GET a READ_CATALOG endpoint by name.""" + raw = READ_CATALOG.get(endpoint) + if raw is None: + known = ", ".join(sorted(READ_CATALOG)) + raise KeyError(f"Unknown read endpoint {endpoint!r}. Known: {known}") + entry = cast(dict[str, Any], raw) + path = str(entry["path"]) + if params: + query = "&".join(f"{k}={v}" for k, v in params.items()) + path = f"{path}?{query}" + return await self.client.call(str(entry.get("method", "GET")), path) + + def brief_text(self) -> str: + if not self.ledger.active or self.brief is None: + return ( + "No tick is in progress. The environment will prompt you when the " + "next turn starts." + ) + return self.brief.to_text() + + def state_json(self) -> dict[str, Any]: + if self.brief is None: + return {} + return dict(self.brief.state) + + @staticmethod + def actions() -> list[dict[str, Any]]: + return action_catalog() diff --git a/src/rle/orchestration/action_executor.py b/src/rle/orchestration/action_executor.py index add1602..b1478eb 100644 --- a/src/rle/orchestration/action_executor.py +++ b/src/rle/orchestration/action_executor.py @@ -6,12 +6,18 @@ import logging from typing import Any, cast -from pydantic import BaseModel, ConfigDict - -from rle.agents.actions import Action, ActionPlan, resolve_endpoint +from rle.agents.actions import ( + Action, + ActionOutcome, + ActionPlan, + ExecutionResult, + resolve_endpoint, +) from rle.rimapi.api_catalog import WRITE_CATALOG from rle.rimapi.client import RimAPIClient, RimAPIResponseError +__all__ = ["NEEDS_PAWN", "ActionExecutor", "ActionOutcome", "ExecutionResult"] + logger = logging.getLogger(__name__) # Endpoints that require a valid colonist/pawn ID. @@ -25,32 +31,8 @@ "bed_rest", "tend", }) - - -class ActionOutcome(BaseModel): - """Per-action execution result. Captures failure detail so the next - tick's deliberation context can surface it to the agent that proposed it. - """ - - model_config = ConfigDict(frozen=True) - - action_type: str - endpoint: str - target_colonist_id: str | None = None - success: bool - error: str | None = None - parameters: dict[str, Any] = {} - - -class ExecutionResult(BaseModel): - """Summary of action execution for one tick.""" - - model_config = ConfigDict(frozen=True) - - executed: int - failed: int - total: int - outcomes: tuple[ActionOutcome, ...] = () +# Public alias for callers that validate before dispatch (MCP tools). +NEEDS_PAWN = _NEEDS_PAWN def _extract_rimapi_error(detail: str) -> str: diff --git a/src/rle/orchestration/game_loop.py b/src/rle/orchestration/game_loop.py index 8b36458..ca15d54 100644 --- a/src/rle/orchestration/game_loop.py +++ b/src/rle/orchestration/game_loop.py @@ -1,4 +1,11 @@ -"""RLE game loop — turn-based orchestrator.""" +"""RLE game loop — the environment side of a run. + +The loop owns the game: pause/unpause, state refresh, action execution, +scoring, scenario evaluation, dashboard export. Deciding *what* to do each +tick is delegated to a :class:`~rle.harness.protocol.BaseHarness`, so the same +loop benchmarks the Felix multi-agent stack, an unmanaged baseline, or an +external coding agent attached over MCP without knowing which one it has. +""" from __future__ import annotations @@ -7,25 +14,29 @@ import json import logging import time as _time +from collections.abc import Sequence from pathlib import Path +from typing import Any -from felix_agent_sdk.communication import CentralPost, MessageType, SpokeManager -from felix_agent_sdk.providers import ProviderError -from felix_agent_sdk.visualization import HelixVisualizer from pydantic import BaseModel, ConfigDict -from rle.agents.actions import ActionPlan, ActionPlanParseError, resolve_endpoint -from rle.agents.base_role import RimWorldRoleAgent +from rle.agents.actions import ActionPlan, resolve_endpoint from rle.config import RLEConfig +from rle.harness.compat import build_legacy_harness +from rle.harness.protocol import ( + BaseHarness, + HarnessContext, + HarnessStepError, + StepResult, +) from rle.orchestration.action_executor import ActionExecutor, ExecutionResult -from rle.orchestration.action_resolver import ActionResolver from rle.orchestration.camera_director import CameraDirector from rle.orchestration.state_manager import GameStateManager from rle.rimapi.client import RimAPIClient from rle.rimapi.schemas import GameState from rle.rimapi.sse_client import RimAPISSEClient from rle.scenarios.evaluator import EvaluationResult, ScenarioEvaluator -from rle.scenarios.schema import TriggeredIncident +from rle.scenarios.schema import ScenarioConfig, TriggeredIncident from rle.scoring.composite import CompositeScorer, ScoreSnapshot from rle.scoring.metrics import MetricContext from rle.scoring.recorder import TimeSeriesRecorder @@ -34,19 +45,6 @@ logger = logging.getLogger(__name__) -# Truncation limits for human-readable text persisted to the event log / -# deliberation log. Keep these tight so the JSONL stays grep-able and small; -# the per-scenario *_deliberations.jsonl carries the full raw reasoning. -_ACTION_REASON_CHARS = 200 -_PLAN_SUMMARY_CHARS = 300 -_PARSE_FAILURE_RAW_CHARS = 500 -# Full LLM completion text on successful deliberation (PROVIDER_CALL events). -# 16 KB headroom: frontier models (Fable 5) emit multi-KB structured plans -# and the verbatim transcripts are first-class analysis artifacts; longer -# completions are tail-truncated rather than head-truncated so the parsed -# action JSON remains visible. -_RAW_OUTPUT_CHARS = 16384 - class TickResult(BaseModel): """Summary of a single game tick.""" @@ -59,23 +57,26 @@ class TickResult(BaseModel): plan: ActionPlan execution: ExecutionResult score: ScoreSnapshot | None = None + harness: str = "" + step_latency_s: float = 0.0 + extras: dict[str, Any] = {} class RLEGameLoop: - """Turn-based game loop: pause → read → deliberate → resolve → execute → score → unpause.""" + """Turn-based loop: pause → read → harness.step → execute → score → unpause.""" def __init__( self, config: RLEConfig, client: RimAPIClient, - agents: list[RimWorldRoleAgent], + agents: Sequence[Any] | None = None, expected_duration_days: int = 60, scorer: CompositeScorer | None = None, recorder: TimeSeriesRecorder | None = None, evaluator: ScenarioEvaluator | None = None, initial_population: int = 3, initial_wealth: float = 0.0, - visualizer: HelixVisualizer | None = None, + visualizer: Any | None = None, parallel: bool = True, sse_client: RimAPISSEClient | None = None, dashboard_export_dir: Path | None = None, @@ -88,23 +89,23 @@ def __init__( auto_dismiss_dialogs: bool = True, camera_director: CameraDirector | None = None, speed_keepalive_s: float = 10.0, + *, + harness: BaseHarness | None = None, + harness_context: HarnessContext | None = None, + scenario: ScenarioConfig | None = None, ) -> None: + """``harness`` is the modern entry point (pass the ``harness_context`` + it was created with so both sides share one client/event log). + ``agents`` / ``no_agent`` / ``parallel`` / ``visualizer`` are legacy + arguments that build a Felix or baseline harness for you (see + ``rle.harness.compat``).""" + if harness is not None and agents: + raise ValueError("Pass either harness= or the legacy agents= argument, not both") self._config = config self._client = client - self._agents = agents - # Separate MapAnalyst (runs first) from role agents (run in parallel) - self._map_analyst: RimWorldRoleAgent | None = None - self._role_agents: list[RimWorldRoleAgent] = [] - for agent in agents: - if agent.ROLE_NAME == "map_analyst": - self._map_analyst = agent - else: - self._role_agents.append(agent) - self._no_agent = no_agent self._no_pause = no_pause self._state_manager = GameStateManager(client, expected_duration_days, sse_client) self._executor = ActionExecutor(client) - self._resolver = ActionResolver() self._scorer = scorer self._recorder = recorder self._evaluator = evaluator @@ -115,16 +116,7 @@ def __init__( initial_population=initial_population, initial_wealth=initial_wealth, ) - self._parse_successes = 0 - self._parse_failures = 0 - self._role_timeout_s = config.role_timeout_s - self._log_dir: Path | None = None - self._deliberation_log: list[dict[str, object]] = [] - self._parallel = parallel - self._last_phase: str = "" self._dashboard_export_dir = dashboard_export_dir - - self._visualizer = visualizer self._event_log = event_log self._cost_tracker = cost_tracker self._triggered_incidents = triggered_incidents or [] @@ -133,12 +125,41 @@ def __init__( self._camera_director = camera_director self._speed_keepalive_s = speed_keepalive_s - # Hub-spoke communication — agents read messages from their spokes - self._hub = CentralPost(max_agents=len(agents)) - self._spoke_manager = SpokeManager(self._hub) - for agent in agents: - spoke = self._spoke_manager.create_spoke(agent.agent_id, agent=agent) - agent.attach_spoke(spoke) + self._harness: BaseHarness = harness or build_legacy_harness( + agents, + no_agent=no_agent, + parallel=parallel, + visualizer=visualizer, + role_timeout_s=config.role_timeout_s, + ) + if harness_context is None: + harness_context = HarnessContext( + config=config, + client=client, + expected_duration_days=expected_duration_days, + initial_population=initial_population, + scenario=scenario, + event_log=event_log, + cost_tracker=cost_tracker, + tick_timeout_s=config.tick_timeout_s, + ) + else: + # The loop is authoritative for run-shaped facts the caller may + # not have known when it built the context. + harness_context.expected_duration_days = expected_duration_days + harness_context.initial_population = initial_population + if scenario is not None: + harness_context.scenario = scenario + if harness_context.event_log is None: + harness_context.event_log = event_log + if harness_context.cost_tracker is None: + harness_context.cost_tracker = cost_tracker + self._harness_ctx = harness_context + self._setup_done = False + + # ------------------------------------------------------------------ + # Plumbing + # ------------------------------------------------------------------ def _emit( self, event_type: EventType, tick: int, @@ -148,10 +169,23 @@ def _emit( if self._event_log is not None: self._event_log.emit(event_type, tick, agent=agent, **data) + @property + def harness(self) -> BaseHarness: + return self._harness + + @property + def harness_context(self) -> HarnessContext: + return self._harness_ctx + @property def cost_tracker(self) -> CostTracker | None: return self._cost_tracker + async def _ensure_setup(self) -> None: + if not self._setup_done: + await self._harness.setup(self._harness_ctx) + self._setup_done = True + async def _dismiss_blocking_dialogs(self, tick_num: int) -> None: """Close force-pause popups that stall unattended runs (issue #33). @@ -172,207 +206,69 @@ async def _dismiss_blocking_dialogs(self, tick_num: int) -> None: EventType.TICK_START, tick_num, dismissed_windows=list(closed), ) - def _update_visualizer_agent( - self, agent: RimWorldRoleAgent, plan: ActionPlan, macro_time: float, - ) -> None: - """Push one agent's post-deliberation state to the visualizer.""" - if not self._visualizer: - return - self._visualizer.update( - agent.agent_id, - progress=macro_time, - confidence=plan.confidence, - phase=agent.position.phase, - status=f"{len(plan.actions)} actions", - ) - - def _render_visualizer( - self, tick: int, day: int, exec_result: ExecutionResult, - snapshot: ScoreSnapshot | None, - ) -> None: - """Render the helix visualization for this tick.""" - if not self._visualizer: - return - extra: dict[str, str] = { - "actions": f"{exec_result.executed}/{exec_result.total}", - } - if snapshot: - extra["score"] = f"{snapshot.composite:.3f}" - self._visualizer.render(tick=tick, day=day, extra_info=extra) - - async def _deliberate_agent_with_timeout( - self, agent: RimWorldRoleAgent, state: object, - current_time: float, tick_num: int, - ) -> tuple[RimWorldRoleAgent, ActionPlan | None]: - """Run one agent's deliberation with a hard timeout. - - Deliberation is natively async (felix 0.3.0 ``acomplete()``), so a - timeout cancels the in-flight provider request instead of leaving an - orphaned worker thread. On timeout: emits a deliberation_timeout - ERROR event, records it in _deliberation_log, and returns - (agent, None) so the tick can continue. - """ - try: - return await asyncio.wait_for( - self._deliberate_agent(agent, state, current_time, tick_num), - timeout=self._role_timeout_s, - ) - except asyncio.TimeoutError: - logger.warning( - "Agent %s deliberation timed out after %.1fs (tick %d)", - agent.ROLE_NAME, self._role_timeout_s, tick_num, - ) - self._parse_failures += 1 - self._deliberation_log.append({ - "tick": tick_num, "agent": agent.ROLE_NAME, - "status": "deliberation_timeout", - "reason": f"timed out after {self._role_timeout_s}s", - }) - self._emit( - EventType.ERROR, tick_num, agent=agent.ROLE_NAME, - error_type="deliberation_timeout", - reason=f"timed out after {self._role_timeout_s}s", - timeout_s=self._role_timeout_s, - ) - return agent, None - - async def _deliberate_parallel( - self, state: object, current_time: float, tick_num: int, - ) -> list[tuple[RimWorldRoleAgent, ActionPlan | None]]: - """Run role agents concurrently on the event loop with per-task timeout.""" - return list(await asyncio.gather(*[ - self._deliberate_agent_with_timeout(a, state, current_time, tick_num) - for a in self._role_agents - ])) - - async def _deliberate_sequential( - self, state: object, current_time: float, tick_num: int, - ) -> list[tuple[RimWorldRoleAgent, ActionPlan | None]]: - """Run role agents one at a time. Agents read context from their spokes.""" - results: list[tuple[RimWorldRoleAgent, ActionPlan | None]] = [] - for agent in self._role_agents: - agent_result, plan = await self._deliberate_agent_with_timeout( - agent, state, current_time, tick_num, - ) - results.append((agent_result, plan)) - return results - - async def _deliberate_agent( - self, agent: RimWorldRoleAgent, state: object, - current_time: float, tick_num: int, - ) -> tuple[RimWorldRoleAgent, ActionPlan | None]: - """Run one agent's deliberation. + async def _step_harness( + self, state: GameState, tick_num: int, macro_time: float, + ) -> tuple[StepResult, float]: + """Run the harness for one tick under the loop-level timeout. - Agents read inter-agent context from their CentralPost spoke internally. + ``HarnessStepError`` and a timeout degrade to an empty step plus an + ERROR event so a flaky harness produces a scored (bad) tick instead of + aborting the run. Any other exception is a bug and propagates. """ + events = self._state_manager.pending_events + empty = StepResult( + plan=ActionPlan( + role=self._harness.name, tick=state.colony.tick, actions=[], + summary="harness produced no step", + ), + ) t0 = _time.monotonic() + timeout = self._harness_ctx.tick_timeout_s try: - plan = await agent.adeliberate(state, current_time) # type: ignore[arg-type] - except ActionPlanParseError as e: - latency_ms = round((_time.monotonic() - t0) * 1000, 1) + coro = self._harness.step(state, tick_num, macro_time, events) + step = await (asyncio.wait_for(coro, timeout=timeout) if timeout else coro) + except asyncio.TimeoutError: logger.warning( - "Agent %s parse failure (tick %d): %s", - agent.ROLE_NAME, tick_num, e.reason, + "Harness %s step timed out after %.1fs (tick %d)", + self._harness.name, timeout or 0.0, tick_num, ) - self._parse_failures += 1 - raw_truncated = ( - e.raw_content[:_PARSE_FAILURE_RAW_CHARS] if e.raw_content else None - ) - self._deliberation_log.append({ - "tick": tick_num, "agent": agent.ROLE_NAME, - "status": "parse_failure", "reason": e.reason, - "raw": raw_truncated, - }) self._emit( - EventType.ERROR, tick_num, agent=agent.ROLE_NAME, - error_type="parse_failure", reason=e.reason, latency_ms=latency_ms, - raw=raw_truncated, + EventType.ERROR, tick_num, agent=self._harness.name, + error_type="harness_timeout", timeout_s=timeout, ) - return agent, None - except ProviderError as e: - latency_ms = round((_time.monotonic() - t0) * 1000, 1) + step = empty + except HarnessStepError as exc: logger.warning( - "Agent %s provider error (tick %d): %s", - agent.ROLE_NAME, tick_num, e, + "Harness %s step failed (tick %d): %s", self._harness.name, tick_num, exc, ) - self._parse_failures += 1 - self._deliberation_log.append({ - "tick": tick_num, "agent": agent.ROLE_NAME, - "status": "provider_error", "reason": str(e), - }) self._emit( - EventType.ERROR, tick_num, agent=agent.ROLE_NAME, - error_type="provider_error", reason=str(e), latency_ms=latency_ms, + EventType.ERROR, tick_num, agent=self._harness.name, + error_type="harness_error", reason=str(exc), ) - return agent, None - - latency_ms = round((_time.monotonic() - t0) * 1000, 1) - self._parse_successes += 1 - actions_payload = [ - {"type": a.action_type, "target": a.target_colonist_id, - "priority": a.priority, "reason": a.reason[:_ACTION_REASON_CHARS]} - for a in plan.actions - ] - summary_truncated = plan.summary[:_PLAN_SUMMARY_CHARS] - self._deliberation_log.append({ - "tick": tick_num, "agent": plan.role, - "status": "success", "confidence": plan.confidence, - "num_actions": len(plan.actions), - "actions": actions_payload, - "summary": summary_truncated, - }) - self._emit( - EventType.DELIBERATION, tick_num, agent=plan.role, - latency_ms=latency_ms, confidence=plan.confidence, - num_actions=len(plan.actions), - actions=actions_payload, - summary=summary_truncated, - ) - - # Record token usage for cost tracking and event log - usage = agent._last_usage - if usage and isinstance(usage, dict): - pt = usage.get("prompt_tokens", 0) - ct = usage.get("completion_tokens", 0) - rt = usage.get("reasoning_tokens", 0) - if not isinstance(rt, int): - rt = 0 - if isinstance(pt, int) and isinstance(ct, int): - if self._cost_tracker: - self._cost_tracker.record_raw(pt, ct, rt) - raw_output = agent._last_raw_output - raw_output_truncated = ( - raw_output[:_RAW_OUTPUT_CHARS] if raw_output else None - ) - was_truncated = ( - raw_output is not None - and len(raw_output) > _RAW_OUTPUT_CHARS - ) - self._emit( - EventType.PROVIDER_CALL, tick_num, agent=plan.role, - prompt_tokens=pt, completion_tokens=ct, - reasoning_tokens=rt, - raw_output=raw_output_truncated, - raw_output_truncated=was_truncated, - ) - - return agent, plan + step = empty + return step, _time.monotonic() - t0 def _export_tick_json( - self, plans: list[ActionPlan], resolved: ActionPlan, - exec_result: ExecutionResult, snapshot: ScoreSnapshot | None, + self, step: StepResult, exec_result: ExecutionResult, snapshot: ScoreSnapshot | None, tick: int, day: int, macro_time: float, screenshot_data_uri: str | None = None, ) -> None: - """Write tick data as JSON for the rimapi-dashboard to consume.""" + """Write tick data as JSON for the rimapi-dashboard to consume. + + Harness-neutral schema: ``agents`` lists whatever sub-plans the harness + reported (seven for Felix, one for a single agent, none for baseline); + harness-specific telemetry rides in ``extras``. + """ if not self._dashboard_export_dir: return self._dashboard_export_dir.mkdir(parents=True, exist_ok=True) + resolved = step.plan data = { "tick": tick, "day": day, "macro_time": macro_time, - "phase": self._last_phase, + "harness": self._harness.name, + "phase": str(step.extras.get("phase", "")), "agents": [ { "role": p.role, @@ -389,7 +285,7 @@ def _export_tick_json( for a in p.actions ], } - for p in plans + for p in step.proposals ], "resolved": { "role": resolved.role, @@ -413,9 +309,10 @@ def _export_tick_json( "metrics": snapshot.metrics, } if snapshot else None, "screenshot_data_uri": screenshot_data_uri, + "extras": step.extras, } (self._dashboard_export_dir / "latest_tick.json").write_text( - json.dumps(data, indent=2), + json.dumps(data, indent=2, default=str), ) def _update_metric_context( @@ -459,22 +356,6 @@ def _record_draft_response( threat.threat_id, max(0, tick_num - seen), ) - def _broadcast_phase_if_changed(self, current_time: float) -> None: - """Broadcast PHASE_ANNOUNCE when macro_time crosses a phase boundary.""" - if current_time < 0.4: - phase = "exploration" - elif current_time < 0.7: - phase = "analysis" - else: - phase = "synthesis" - if phase != self._last_phase: - self._spoke_manager.broadcast_message( - MessageType.PHASE_ANNOUNCE, - {"phase": phase, "depth_ratio": current_time}, - sender_id="hub", - ) - self._last_phase = phase - async def _fire_scheduled_incidents(self, tick_num: int) -> None: """Fire any triggered_incidents whose tick_offset matches.""" for incident in self._triggered_incidents: @@ -500,12 +381,17 @@ async def _fire_scheduled_incidents(self, tick_num: int) -> None: incident.name, exc_info=True, ) + # ------------------------------------------------------------------ + # Tick + # ------------------------------------------------------------------ + async def run_tick(self) -> TickResult: """Execute one turn. - In pause mode (default): pause → read → deliberate → execute → unpause. - In no-pause mode: read → fire deliberation + sleep concurrently → execute. + In pause mode (default): pause → read → step → execute → unpause. + In no-pause mode: read → step → execute while the game keeps running. """ + await self._ensure_setup() tick_num = len(self._tick_results) # 1. Pause (skip in no-pause mode — game keeps running) @@ -538,200 +424,56 @@ async def run_tick(self) -> TickResult: tick_num, state, self._state_manager.pending_events, _time.time(), ) - # 3. Route previous tick's messages to agent spokes + broadcast phase changes - messages_before = self._hub.total_messages_processed - self._spoke_manager.process_all_messages() - self._broadcast_phase_if_changed(current_time) - - # 4-6. Agent deliberation + conflict resolution (skipped in no-agent mode) - plans: list[ActionPlan] = [] - if self._no_agent: - # Baseline mode: no deliberation, no actions. Colony runs unmanaged. - resolved = ActionPlan( - role="baseline", tick=state.colony.tick, actions=[], summary="No agents", - ) - else: - # Inject SSE events into all agents (including MapAnalyst) - pending_events = self._state_manager.pending_events - for agent in self._agents: - agent.set_pending_events(pending_events) - for evt in pending_events: - self._emit( - EventType.SSE_EVENT, tick_num, - sse_type=evt.event_type, sse_data=str(evt.data)[:200], - ) + # 3. Harness decides (and, for tool-using harnesses, may already act) + step, step_latency = await self._step_harness(state, tick_num, current_time) - # 4a. MapAnalyst deliberates FIRST (sequential, timeout-wrapped) - if self._map_analyst: - ma_agent, ma_plan = await self._deliberate_agent_with_timeout( - self._map_analyst, state, current_time, tick_num, + # 4. Execute — unless the harness already applied its writes + if step.execution is None: + exec_result = await self._executor.execute(step.plan) + for outcome in exec_result.outcomes: + self._emit( + EventType.ACTION_EXEC, tick_num, + action_type=outcome.action_type, + target=outcome.target_colonist_id, + success=outcome.success, + error=outcome.error, + parameters=outcome.parameters, ) - if ma_plan is not None: - plans.append(ma_plan) - self._update_visualizer_agent(ma_agent, ma_plan, current_time) - spoke = self._spoke_manager.get_spoke(ma_agent.agent_id) - if spoke and spoke.is_connected: - spoke.send_message( - MessageType.TASK_COMPLETE, - { - "role": ma_plan.role, - "summary": ma_plan.summary, - "confidence": ma_plan.confidence, - "num_actions": len(ma_plan.actions), - "action_types": [ - a.action_type for a in ma_plan.actions - ], - }, - ) - # Route MapAnalyst output to role agent spokes immediately - self._spoke_manager.process_all_messages() - - # 4b. Role agents deliberate (parallel or sequential) - # Snapshot which agents have pending spoke messages (for messages_acted_on) - agents_with_messages: set[str] = set() - for ra in self._role_agents: - spoke = self._spoke_manager.get_spoke(ra.agent_id) - if spoke and spoke.has_pending_messages(): - agents_with_messages.add(ra.agent_id) - - if self._parallel: - results = await self._deliberate_parallel(state, current_time, tick_num) - else: - results = await self._deliberate_sequential(state, current_time, tick_num) - - # Collect plans, update visualizer, send via CentralPost - for agent, plan in results: - if plan is None: - continue - plans.append(plan) - if agent.agent_id in agents_with_messages: - self._metric_context.messages_acted_on += 1 - self._update_visualizer_agent(agent, plan, current_time) - spoke = self._spoke_manager.get_spoke(agent.agent_id) - if spoke and spoke.is_connected: - spoke.send_message( - MessageType.TASK_COMPLETE, - { - "role": plan.role, - "summary": plan.summary, - "confidence": plan.confidence, - "num_actions": len(plan.actions), - "action_types": [ - a.action_type for a in plan.actions - ], - }, - ) - - # Capture generation IDs from every provider call this tick — - # parse retries and failed deliberations bill tokens too — for - # billed-cost reconciliation against OpenRouter (the token-count - # estimator diverged up to 4x on the v0.3.0 spread). - if self._cost_tracker: - for any_agent in self._agents: - for gen_id in any_agent.drain_generation_ids(): - self._cost_tracker.record_generation_id(gen_id) - - # Resolve conflicts - resolved, resolver_stats = self._resolver.resolve(plans, state) - self._metric_context.conflicts_total += resolver_stats.conflicts_total - self._metric_context.conflicts_resolved += resolver_stats.conflicts_resolved - self._emit( - EventType.CONFLICT, tick_num, - input_plans=len(plans), - output_actions=len(resolved.actions), - conflicts_detected=resolver_stats.conflicts_total, - conflicts_resolved=resolver_stats.conflicts_resolved, - ) - - # Track message effectiveness - messages_after = self._hub.total_messages_processed - new_messages = messages_after - messages_before - self._metric_context.messages_sent += new_messages - - # 7. Execute merged plan - exec_result = await self._executor.execute(resolved) - for outcome in exec_result.outcomes: - self._emit( - EventType.ACTION_EXEC, tick_num, - action_type=outcome.action_type, - target=outcome.target_colonist_id, - success=outcome.success, - error=outcome.error, - parameters=outcome.parameters, - ) + else: + exec_result = step.execution - # 7a. Track draft responses for the threat_response metric (issue #25) + # 4a. Track draft responses for the threat_response metric (issue #25) self._record_draft_response(exec_result, tick_num) - # 7b. Surface per-action errors back to agents so they can avoid - # re-proposing the same invalid action next tick (e.g. researching - # an already-finished project, setting priority for a disabled work type). - failed_outcomes = [o for o in exec_result.outcomes if not o.success and o.error] - if failed_outcomes: - error_summary = "; ".join( - f"{o.action_type}({o.target_colonist_id or '-'}): {o.error}" - for o in failed_outcomes[:10] - ) - self._spoke_manager.broadcast_message( - MessageType.STATUS_UPDATE, - { - "tick": state.colony.tick, - "summary": f"Last tick action errors — DO NOT REPEAT: {error_summary}", - "action_errors": [ - { - "action_type": o.action_type, - "target_colonist_id": o.target_colonist_id, - "error": o.error, - } - for o in failed_outcomes - ], - }, - sender_id="hub", - ) - - # 8. Score this tick + # 5. Score this tick snapshot: ScoreSnapshot | None = None if self._scorer: snapshot = self._scorer.score(state, self._metric_context) if self._recorder: self._recorder.record(snapshot) - if snapshot: - self._emit( - EventType.SCORE, tick_num, - composite=snapshot.composite, metrics=snapshot.metrics, - ) - - # 9. Broadcast score to all agents via CentralPost - if snapshot: - self._spoke_manager.broadcast_message( - MessageType.STATUS_UPDATE, - { - "tick": state.colony.tick, - "day": state.colony.day, - "composite_score": snapshot.composite, - "metrics": snapshot.metrics, - }, - sender_id="hub", + self._emit( + EventType.SCORE, tick_num, + composite=snapshot.composite, metrics=snapshot.metrics, ) - # 10. Render visualization - self._render_visualizer(state.colony.tick, state.colony.day, exec_result, snapshot) + # 6. Feedback to the harness (action errors, score, visualisers) + await self._harness.on_tick_end(tick_num, state, step, exec_result, snapshot) - # 11. Capture screenshot (opt-in, before export so it's in the JSON) + # 7. Capture screenshot (opt-in, before export so it's in the JSON) screenshot_uri: str | None = None if self._screenshots_enabled: ss = await self._client.take_screenshot() if ss is not None: screenshot_uri = ss.data_uri - # 12. Export tick data for dashboard + # 8. Export tick data for dashboard self._export_tick_json( - plans, resolved, exec_result, snapshot, + step, exec_result, snapshot, state.colony.tick, state.colony.day, current_time, screenshot_data_uri=screenshot_uri, ) - # 13. Unpause (skip in no-pause mode — game was never paused) + # 9. Unpause (skip in no-pause mode — game was never paused) if not self._no_pause: await self._client.unpause_game() @@ -739,13 +481,16 @@ async def run_tick(self) -> TickResult: tick=state.colony.tick, day=state.colony.day, macro_time=current_time, - plan=resolved, + plan=step.plan, execution=exec_result, score=snapshot, + harness=self._harness.name, + step_latency_s=round(step_latency, 3), + extras=step.extras, ) self._tick_results.append(result) - # 9. Update metric context and evaluate scenario + # 10. Update metric context and evaluate scenario self._update_metric_context(result, state, tick_num) if self._evaluator: eval_result = self._evaluator.evaluate( @@ -777,6 +522,7 @@ async def _speed_keepalive(self) -> None: async def run(self, max_ticks: int | None = None) -> list[TickResult]: """Run the game loop for N ticks or until stopped.""" + await self._ensure_setup() self._running = True if not await self._client.window_endpoints_available(): logger.warning( @@ -793,34 +539,30 @@ async def run(self, max_ticks: int | None = None) -> list[TickResult]: keepalive = asyncio.create_task(self._speed_keepalive()) tick_count = 0 try: - while self._running: - result = await self.run_tick() - tick_count += 1 - score_str = "" - if result.score: - score_str = f" | score={result.score.composite:.3f}" - logger.info( - "Tick %d (day %d): %d actions, %d executed%s", - tick_count, - result.day, - result.execution.total, - result.execution.executed, - score_str, - ) - if max_ticks and tick_count >= max_ticks: - break - await asyncio.sleep(self._config.tick_interval) + with self._harness.run_context(): + while self._running: + result = await self.run_tick() + tick_count += 1 + score_str = "" + if result.score: + score_str = f" | score={result.score.composite:.3f}" + logger.info( + "Tick %d (day %d): %d actions, %d executed%s", + tick_count, + result.day, + result.execution.total, + result.execution.executed, + score_str, + ) + if max_ticks and tick_count >= max_ticks: + break + await asyncio.sleep(self._config.tick_interval) finally: if keepalive is not None: keepalive.cancel() with contextlib.suppress(asyncio.CancelledError): await keepalive - # Final drain: a deliberation that timed out on the last tick may - # have completed in its worker thread after the per-tick drain. - if self._cost_tracker: - for agent in self._agents: - for gen_id in agent.drain_generation_ids(): - self._cost_tracker.record_generation_id(gen_id) + await self._harness.teardown() return self._tick_results def stop(self) -> None: @@ -841,10 +583,18 @@ def metric_context(self) -> MetricContext: @property def deliberation_log(self) -> list[dict[str, object]]: - """Per-tick agent deliberation records (status, actions, reasons, summary). + """Per-tick harness deliberation records (status, actions, reasons, summary). Returns a shallow copy. Each entry has keys: tick, agent, status, plus status-specific fields (actions, summary, confidence for success; raw, reason for parse_failure; reason for provider_error). """ - return list(self._deliberation_log) + return list(self._harness.deliberation_log) + + @property + def parse_successes(self) -> int: + return self._harness.parse_successes + + @property + def parse_failures(self) -> int: + return self._harness.parse_failures diff --git a/src/rle/orchestration/save_loader.py b/src/rle/orchestration/save_loader.py new file mode 100644 index 0000000..2c2eb48 --- /dev/null +++ b/src/rle/orchestration/save_loader.py @@ -0,0 +1,65 @@ +"""Load a scenario save and wait until the game is actually ready. + +``load_game`` returns HTTP 200 before Unity's main thread has applied the +load. Writes that race the settle window get 500'd and, worse, can start a +null-ref cascade that poisons the rest of the session. Every entry point +(single scenario, benchmark matrix, baseline reloads) must use this helper +rather than a fixed sleep so agent and baseline runs start from the same +settled state. +""" + +from __future__ import annotations + +import asyncio +import logging + +from rle.docker import wait_for_rimapi +from rle.rimapi.client import RimAPIClient + +logger = logging.getLogger(__name__) + +# The colony population must be > 0 and unchanged for this many consecutive +# polls (2 s apart) before we consider the load settled (~10 s). +STABLE_POLLS_REQUIRED = 5 +POLL_INTERVAL_S = 2.0 +MAX_POLLS = 30 + + +async def load_save_and_settle( + client: RimAPIClient, + rimapi_url: str, + save_name: str, + *, + unforbid_items: bool = True, + rimapi_timeout_s: float = 30.0, +) -> int: + """Load ``save_name`` and block until the colony is stable. + + Returns the number of starting items unforbidden (0 when disabled). + Raises whatever ``load_game`` / ``wait_for_rimapi`` raise so callers can + decide whether to skip the run. + """ + await client.load_game(save_name) + await wait_for_rimapi(rimapi_url, timeout=rimapi_timeout_s) + stable_count = 0 + last_population = -1 + for _ in range(MAX_POLLS): + await asyncio.sleep(POLL_INTERVAL_S) + try: + colony = await client.get_colony() + except Exception: + stable_count = 0 + continue + if colony.population > 0 and colony.population == last_population: + stable_count += 1 + if stable_count >= STABLE_POLLS_REQUIRED: + break + else: + stable_count = 0 + last_population = colony.population + else: + logger.warning("Save %s never reported a stable population; continuing", save_name) + if not unforbid_items: + return 0 + count = await client.unforbid_all_items() + return int(count or 0) diff --git a/src/rle/providers/__init__.py b/src/rle/providers/__init__.py deleted file mode 100644 index 362afb2..0000000 --- a/src/rle/providers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""RLE-local LLM providers (beyond those shipped with felix-agent-sdk).""" - -from rle.providers.claude_code import ClaudeCodeProvider - -__all__ = ["ClaudeCodeProvider"] diff --git a/src/rle/py.typed b/src/rle/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/rle/scenarios/definitions/01_crashlanded_survival.yaml b/src/rle/scenarios/definitions/01_crashlanded_survival.yaml index e4e0079..438eb36 100644 --- a/src/rle/scenarios/definitions/01_crashlanded_survival.yaml +++ b/src/rle/scenarios/definitions/01_crashlanded_survival.yaml @@ -17,14 +17,13 @@ failure_conditions: operator: == value: 1 scoring_weights: - survival: 0.24 - food_security: 0.12 + survival: 0.28 + food_security: 0.16 mood: 0.12 threat_response: 0.08 wealth: 0.04 research: 0.04 - self_sufficiency: 0.12 + self_sufficiency: 0.16 efficiency: 0.04 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: 29530cd8e5f373b50f9ee51617bc0b036de97f95c9fd3264a2bbd5939a133d4e diff --git a/src/rle/scenarios/definitions/02_first_winter.yaml b/src/rle/scenarios/definitions/02_first_winter.yaml index def07f6..edbd71f 100644 --- a/src/rle/scenarios/definitions/02_first_winter.yaml +++ b/src/rle/scenarios/definitions/02_first_winter.yaml @@ -20,14 +20,13 @@ failure_conditions: operator: <= value: 0 scoring_weights: - survival: 0.2 - food_security: 0.16 - self_sufficiency: 0.12 + survival: 0.24 + food_security: 0.2 + self_sufficiency: 0.16 mood: 0.08 wealth: 0.08 research: 0.04 threat_response: 0.08 efficiency: 0.04 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: 22d52f88b3398f4d3c61de65a93fc809e3c946c41b1b19b8f0f6a0b99e93361d diff --git a/src/rle/scenarios/definitions/03_toxic_fallout.yaml b/src/rle/scenarios/definitions/03_toxic_fallout.yaml index 06cdc81..f9abed4 100644 --- a/src/rle/scenarios/definitions/03_toxic_fallout.yaml +++ b/src/rle/scenarios/definitions/03_toxic_fallout.yaml @@ -23,14 +23,13 @@ triggered_incidents: - tick_offset: 1 name: ToxicFallout scoring_weights: - survival: 0.28 - food_security: 0.12 - mood: 0.16 + survival: 0.32 + food_security: 0.16 + mood: 0.2 self_sufficiency: 0.08 wealth: 0.04 research: 0.04 threat_response: 0.04 efficiency: 0.04 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: c8a73b3beb3a6df4988ed4c100886584c376cd61814831ffe78fec9b30228ba5 diff --git a/src/rle/scenarios/definitions/04_raid_defense.yaml b/src/rle/scenarios/definitions/04_raid_defense.yaml index fd66f5e..766f29f 100644 --- a/src/rle/scenarios/definitions/04_raid_defense.yaml +++ b/src/rle/scenarios/definitions/04_raid_defense.yaml @@ -25,14 +25,13 @@ triggered_incidents: incident_parms: points: 500 scoring_weights: - survival: 0.2 - threat_response: 0.24 + survival: 0.26 + threat_response: 0.3 mood: 0.08 food_security: 0.04 wealth: 0.04 research: 0.04 self_sufficiency: 0.08 efficiency: 0.08 - coordination: 0.15 - communication_efficiency: 0.05 + plan_coherence: 0.08 save_sha256: c11ff2d4476d7f4da7c3bdcbb07f96610ae7588c642af3f5696278c0530666e2 diff --git a/src/rle/scenarios/definitions/05_plague_response.yaml b/src/rle/scenarios/definitions/05_plague_response.yaml index f9588e8..4f9f19d 100644 --- a/src/rle/scenarios/definitions/05_plague_response.yaml +++ b/src/rle/scenarios/definitions/05_plague_response.yaml @@ -23,14 +23,13 @@ triggered_incidents: - tick_offset: 1 name: Plague scoring_weights: - survival: 0.28 - mood: 0.12 + survival: 0.34 + mood: 0.18 food_security: 0.08 threat_response: 0.08 wealth: 0.04 research: 0.04 self_sufficiency: 0.08 efficiency: 0.08 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: a31a1eb31913fee79cb39b91009e4058029eac133a30ad3e0fc0e53a31a79e6d diff --git a/src/rle/scenarios/definitions/06_ship_launch.yaml b/src/rle/scenarios/definitions/06_ship_launch.yaml index 1735fe3..6e5b716 100644 --- a/src/rle/scenarios/definitions/06_ship_launch.yaml +++ b/src/rle/scenarios/definitions/06_ship_launch.yaml @@ -14,14 +14,13 @@ failure_conditions: operator: == value: 1 scoring_weights: - research: 0.2 + research: 0.28 survival: 0.16 - wealth: 0.12 + wealth: 0.16 self_sufficiency: 0.12 food_security: 0.04 mood: 0.08 threat_response: 0.04 efficiency: 0.04 - coordination: 0.12 - communication_efficiency: 0.08 + plan_coherence: 0.08 save_sha256: 24db37fec15423b8eb5da56a9be471a4814b0e4040a436a4864d3dba234397ff diff --git a/src/rle/scoring/coherence.py b/src/rle/scoring/coherence.py new file mode 100644 index 0000000..8cf74ed --- /dev/null +++ b/src/rle/scoring/coherence.py @@ -0,0 +1,151 @@ +"""Contradiction detection over a tick's executed writes (``plan_coherence``). + +A harness's job is to hand the game a coherent set of writes each tick. +Whether that coherence comes from a conflict resolver (multi-agent), a single +model's judgement, or a coding agent calling tools one at a time is the +harness's business — this module only looks at what actually reached RIMAPI, +so every harness is scored on the same footing (issue #51). +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import Any + +from rle.agents.actions import ActionOutcome, resolve_endpoint + +Rect = tuple[int, int, int, int] + +_ZONE_ENDPOINTS = frozenset({"growing_zone", "stockpile_zone"}) + + +def _int(value: object) -> int | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int): + return value + if isinstance(value, (str, float)): + try: + return int(value) + except (TypeError, ValueError): + return None + return None + + +def _rect(params: dict[str, Any]) -> Rect | None: + """Mirror the executor's rectangle defaults (x2/z2 default to +5).""" + x1 = _int(params.get("x1", params.get("x"))) + z1 = _int(params.get("z1", params.get("z"))) + if x1 is None or z1 is None: + return None + x2 = _int(params.get("x2", x1 + 5)) + z2 = _int(params.get("z2", z1 + 5)) + if x2 is None or z2 is None: + return None + return (min(x1, x2), min(z1, z2), max(x1, x2), max(z1, z2)) + + +def _rects_overlap(a: Rect, b: Rect) -> bool: + ax1, az1, ax2, az2 = a + bx1, bz1, bx2, bz2 = b + return not (ax2 < bx1 or ax1 > bx2 or az2 < bz1 or az1 > bz2) + + +def _all_conflicting(groups: Iterable[Sequence[int]], flagged: set[int]) -> None: + for group in groups: + if len(group) > 1: + flagged.update(group) + + +def count_contradictions(outcomes: Sequence[ActionOutcome]) -> tuple[int, int]: + """Return ``(contradictory, executed)`` for one tick's outcomes. + + Only successful writes count — a failed write never reached the game, so + it cannot contradict anything (it is already penalised by ``efficiency``). + A write is contradictory when, within the same tick, another executed + write targets the same thing with an incompatible intent: + + * ``draft`` on the same pawn with different ``is_drafted`` values + * ``move`` and ``job_assign`` on the same pawn + * ``work_priority`` on the same pawn setting the same work type to + different priorities + * ``growing_zone`` / ``stockpile_zone`` rectangles that overlap + * ``blueprint`` twice on the same cell + * more than one distinct ``research_target`` project + """ + executed = [(i, o) for i, o in enumerate(outcomes) if o.success] + if not executed: + return 0, 0 + + flagged: set[int] = set() + + draft_by_pawn: dict[str, dict[bool, list[int]]] = {} + move_or_job_by_pawn: dict[str, dict[str, list[int]]] = {} + work_by_pawn_type: dict[tuple[str, str], dict[int, list[int]]] = {} + zones: list[tuple[int, Rect]] = [] + blueprint_by_cell: dict[tuple[int, int], list[int]] = {} + research_by_project: dict[str, list[int]] = {} + + for idx, o in executed: + endpoint = resolve_endpoint(o.action_type) + pawn = o.target_colonist_id or "" + params = o.parameters + if endpoint == "draft": + state = bool(params.get("is_drafted", True)) + draft_by_pawn.setdefault(pawn, {}).setdefault(state, []).append(idx) + elif endpoint in ("move", "job_assign"): + move_or_job_by_pawn.setdefault(pawn, {}).setdefault(endpoint, []).append(idx) + elif endpoint == "work_priority": + for work, pri in _work_priorities(params).items(): + work_by_pawn_type.setdefault((pawn, work), {}).setdefault(pri, []).append(idx) + elif endpoint in _ZONE_ENDPOINTS: + rect = _rect(params) + if rect is not None: + zones.append((idx, rect)) + elif endpoint == "blueprint": + try: + cell = (int(params["x"]), int(params["z"])) + except (KeyError, TypeError, ValueError): + continue + blueprint_by_cell.setdefault(cell, []).append(idx) + elif endpoint == "research_target": + project = str(params.get("project", params.get("name", ""))) + research_by_project.setdefault(project, []).append(idx) + + for by_state in draft_by_pawn.values(): + if len(by_state) > 1: + for idxs in by_state.values(): + flagged.update(idxs) + for by_kind in move_or_job_by_pawn.values(): + if len(by_kind) > 1: + for idxs in by_kind.values(): + flagged.update(idxs) + for by_pri in work_by_pawn_type.values(): + if len(by_pri) > 1: + for idxs in by_pri.values(): + flagged.update(idxs) + for i, (idx_a, rect_a) in enumerate(zones): + for idx_b, rect_b in zones[i + 1:]: + if _rects_overlap(rect_a, rect_b): + flagged.add(idx_a) + flagged.add(idx_b) + _all_conflicting(blueprint_by_cell.values(), flagged) + if len(research_by_project) > 1: + for idxs in research_by_project.values(): + flagged.update(idxs) + + return len(flagged), len(executed) + + +def _work_priorities(params: dict[str, Any]) -> dict[str, int]: + """Mirror of the executor's accepted shapes, tolerant of garbage.""" + nested = params.get("work_priorities") + if isinstance(nested, dict): + return {str(w): int(p) for w, p in nested.items() if isinstance(p, int)} + if "work_type" in params: + pri = params.get("priority", 1) + return {str(params["work_type"]): int(pri)} if isinstance(pri, int) else {} + return { + str(w): p for w, p in params.items() + if isinstance(p, int) and not isinstance(p, bool) + } diff --git a/src/rle/scoring/composite.py b/src/rle/scoring/composite.py index 046c057..e1db8c8 100644 --- a/src/rle/scoring/composite.py +++ b/src/rle/scoring/composite.py @@ -7,17 +7,20 @@ from rle.rimapi.schemas import GameState from rle.scoring.metrics import ALL_METRICS, MetricContext +# Scoring 1.2 (#51): one harness-agnostic process metric (plan_coherence) +# replaces coordination + communication_efficiency; the freed weight returns +# to colony outcomes. Bump SCORING_VERSION in tracking/metadata.py whenever +# this table changes. DEFAULT_WEIGHTS: dict[str, float] = { - "survival": 0.20, - "threat_response": 0.12, + "survival": 0.24, + "threat_response": 0.14, "mood": 0.12, - "food_security": 0.08, + "food_security": 0.10, "wealth": 0.08, "research": 0.08, - "self_sufficiency": 0.08, - "efficiency": 0.04, - "coordination": 0.12, - "communication_efficiency": 0.08, + "self_sufficiency": 0.10, + "efficiency": 0.06, + "plan_coherence": 0.08, } diff --git a/src/rle/scoring/metrics.py b/src/rle/scoring/metrics.py index 4dad794..fbeb150 100644 --- a/src/rle/scoring/metrics.py +++ b/src/rle/scoring/metrics.py @@ -6,10 +6,16 @@ from typing import TYPE_CHECKING from rle.rimapi.schemas import GameState, ThreatData +from rle.scoring.coherence import count_contradictions if TYPE_CHECKING: from rle.orchestration.game_loop import TickResult +# Value returned by process metrics when there is nothing to judge (no ticks +# yet, or a tick that issued zero writes). Neutral rather than 1.0 so an +# unmanaged baseline does not bank free points on process metrics (#51). +NEUTRAL = 0.5 + @dataclass class MetricContext: @@ -24,11 +30,6 @@ class MetricContext: # Response delay in loop ticks per threat, recorded when a draft executes first_draft_tick: dict[str, int] = field(default_factory=dict) initial_wealth: float = 0.0 - # Process metrics (populated by game loop after conflict resolution) - conflicts_total: int = 0 - conflicts_resolved: int = 0 - messages_sent: int = 0 - messages_acted_on: int = 0 def survival(state: GameState, ctx: MetricContext) -> float: @@ -93,31 +94,33 @@ def self_sufficiency(state: GameState, ctx: MetricContext) -> float: def efficiency(state: GameState, ctx: MetricContext) -> float: - """Average action execution rate across all ticks.""" + """Average action execution rate across ticks. Ticks with no writes are neutral.""" if not ctx.tick_results: - return 1.0 + return NEUTRAL rates = [] for tr in ctx.tick_results: total = tr.execution.total if total > 0: rates.append(tr.execution.executed / total) else: - rates.append(1.0) + rates.append(NEUTRAL) return sum(rates) / len(rates) -def coordination(state: GameState, ctx: MetricContext) -> float: - """Ratio of conflicts resolved peacefully. 1.0 = no conflicts or all resolved.""" - if ctx.conflicts_total == 0: - return 1.0 - return min(1.0, ctx.conflicts_resolved / ctx.conflicts_total) - - -def communication_efficiency(state: GameState, ctx: MetricContext) -> float: - """Ratio of inter-agent messages that led to action changes. 1.0 = all useful.""" - if ctx.messages_sent == 0: - return 1.0 - return min(1.0, ctx.messages_acted_on / ctx.messages_sent) +def plan_coherence(state: GameState, ctx: MetricContext) -> float: + """Fraction of executed writes that did not contradict another write in + the same tick, averaged across ticks. Harness-agnostic: computed from what + reached RIMAPI, not from any harness's internal messaging (#51).""" + if not ctx.tick_results: + return NEUTRAL + scores = [] + for tr in ctx.tick_results: + contradictory, executed = count_contradictions(tr.execution.outcomes) + if executed == 0: + scores.append(NEUTRAL) + else: + scores.append(1.0 - contradictory / executed) + return sum(scores) / len(scores) ALL_METRICS = { @@ -129,6 +132,5 @@ def communication_efficiency(state: GameState, ctx: MetricContext) -> float: "research": research, "self_sufficiency": self_sufficiency, "efficiency": efficiency, - "coordination": coordination, - "communication_efficiency": communication_efficiency, + "plan_coherence": plan_coherence, } diff --git a/src/rle/testing/__init__.py b/src/rle/testing/__init__.py new file mode 100644 index 0000000..6ae77d3 --- /dev/null +++ b/src/rle/testing/__init__.py @@ -0,0 +1,23 @@ +"""Test utilities exported for harness plugin authors. + +External harness packages depend on ``rimworld-learning-environment`` and use +these helpers so they never copy RLE internals: + +- :class:`MockRimAPI` / :func:`make_mock_transport` — fake RIMAPI transport +- :func:`run_harness_smoke` — drive a plugin through ``RLEGameLoop`` for a + few ticks against the mock and return the tick results +- ``rle.testing.scripted_agent.ScriptedMcpHarness`` — a fake coding agent + that plays a fixed tool script through the RLE MCP server (needs the + ``mcp`` extra; imported separately so this package stays extra-free) +""" + +from rle.testing.mock_rimapi import MOCK_ROUTES, MockRimAPI, make_mock_transport +from rle.testing.smoke import SmokeReport, run_harness_smoke + +__all__ = [ + "MOCK_ROUTES", + "MockRimAPI", + "SmokeReport", + "make_mock_transport", + "run_harness_smoke", +] diff --git a/src/rle/testing/mock_rimapi.py b/src/rle/testing/mock_rimapi.py new file mode 100644 index 0000000..585eee2 --- /dev/null +++ b/src/rle/testing/mock_rimapi.py @@ -0,0 +1,138 @@ +"""In-memory stand-in for RIMAPI so harnesses can be exercised without RimWorld. + +Used by ``--smoke-test`` in the CLIs and by external harness packages' test +suites (``rle.testing.run_harness_smoke``). State never changes between +ticks — this proves plumbing, not colony management. +""" + +from __future__ import annotations + +import json +from typing import Any + +import httpx + +from rle.rimapi.client import RimAPIClient + +MOCK_ROUTES: dict[str, dict[str, Any] | list[Any]] = { + "/api/v1/colonists": [ + { + "colonist_id": "col_01", "name": "Tynan", "health": 0.95, + "mood": 0.72, "skills": {"shooting": 8, "construction": 5, + "cooking": 3, "mining": 6, "intellectual": 4}, + "traits": ["industrious"], "current_job": "mining", + "is_drafted": False, "needs": {"food": 0.6, "rest": 0.8}, + "injuries": [], "position": [42, 18], + }, + { + "colonist_id": "col_02", "name": "Cassandra", "health": 0.88, + "mood": 0.65, "skills": {"shooting": 3, "construction": 7, + "cooking": 6, "growing": 8, "intellectual": 6}, + "traits": ["kind"], "current_job": "growing", + "is_drafted": False, "needs": {"food": 0.5, "rest": 0.7}, + "injuries": [], "position": [30, 22], + }, + { + "colonist_id": "col_03", "name": "Randy", "health": 0.92, + "mood": 0.58, "skills": {"shooting": 10, "melee": 7, + "construction": 3, "cooking": 2}, + "traits": ["tough", "brawler"], "current_job": None, + "is_drafted": False, "needs": {"food": 0.4, "rest": 0.6}, + "injuries": [], "position": [50, 10], + }, + ], + "/api/v1/resources/summary?map_id=0": { + "total_items": 800, "total_market_value": 8000.0, + "critical_resources": { + "food_summary": {"food_total": 85}, + "medicine_total": 5, "weapon_count": 2, + }, + }, + "/api/v1/map/buildings?map_id=0": [ + {"id": "s_01", "def_name": "Wall", "position": {"x": 10, "y": 0, "z": 10}, + "hit_points": 300.0, "max_hit_points": 300.0}, + ], + "/api/v1/research/summary": { + "current_project": "electricity", "progress": 0.45, + "completed": ["stonecutting"], "available": ["electricity", "battery", "smithing"], + }, + "/api/v1/incidents?map_id=0": {"incidents": []}, + "/api/v1/game/state": { + "name": "New Hope", "wealth": 8000.0, "day": 5, "tick": 300000, + "population": 3, "mood_average": 0.65, "food_days": 7.0, + }, + "/api/v1/map/weather?map_id=0": { + "weather": "clear", "temperature": 22.0, + }, + "/api/v1/map/zones?map_id=0": [], + "/api/v1/map/rooms?map_id=0": [], + "/api/v1/map/ore?map_id=0": [], + "/api/v1/map/farm/summary?map_id=0": { + "total_growing_zones": 0, "planted_cells": 0, + "harvestable_cells": 0, "crops": {}, + }, + "/api/v1/map/terrain?map_id=0": { + "width": 10, "height": 10, + "palette": ["Soil", "WaterMovingShallow", "SoilRich", "Granite_Rough"], + "grid": [100, 0], + "floor_palette": [], "floor_grid": [100, 0], + }, + "/api/v1/resources/stored?map_id=0": { + "Resources": [ + {"def_name": "WoodLog", "stack_count": 200}, + {"def_name": "Steel", "stack_count": 100}, + {"def_name": "ComponentIndustrial", "stack_count": 10}, + ], + }, + "/api/v1/map/power/info?map_id=0": { + "current_power": 0.0, + "total_consumption": 0.0, + "currently_stored_power": 0.0, + "total_power_storage": 0.0, + }, + "/api/v1/factions": [], + "/api/v1/ui/alerts?map_id=0": [], +} + +_POST_OK = b'{"success": true, "errors": [], "warnings": []}' + + +class MockRimAPI: + """Records every POST so tests can assert what a harness wrote.""" + + def __init__(self, routes: dict[str, dict[str, Any] | list[Any]] | None = None) -> None: + self.routes = dict(MOCK_ROUTES if routes is None else routes) + self.posts: list[tuple[str, Any]] = [] + + def handler(self, request: httpx.Request) -> httpx.Response: + raw = request.url.raw_path.decode() + if request.method == "POST": + body: Any = None + if request.content: + try: + body = json.loads(request.content) + except json.JSONDecodeError: + body = request.content.decode(errors="replace") + self.posts.append((raw.split("?")[0], body)) + return httpx.Response( + 200, content=_POST_OK, headers={"content-type": "application/json"}, + ) + for key in (raw, raw.split("?")[0]): + if key in self.routes: + return httpx.Response( + 200, content=json.dumps(self.routes[key]).encode(), + headers={"content-type": "application/json"}, + ) + return httpx.Response(404, content=b"Not found") + + def transport(self) -> httpx.MockTransport: + return httpx.MockTransport(self.handler) + + def attach(self, client: RimAPIClient, base_url: str = "http://mock") -> RimAPIClient: + """Point an (entered) RimAPIClient at this mock.""" + client._client = httpx.AsyncClient(transport=self.transport(), base_url=base_url) + return client + + +def make_mock_transport() -> httpx.MockTransport: + return MockRimAPI().transport() diff --git a/src/rle/testing/scripted_agent.py b/src/rle/testing/scripted_agent.py new file mode 100644 index 0000000..5394611 --- /dev/null +++ b/src/rle/testing/scripted_agent.py @@ -0,0 +1,67 @@ +"""A scripted stand-in for an external coding agent (smoke tests). + +Connects to the RLE MCP server like a real agent would, reads the brief, +issues a fixed script of tool calls, and ends the turn. External harness +packages use :class:`ScriptedMcpHarness` for their ``plugin.smoke`` so the +full MCP round trip is exercised in CI without the real binary. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any, ClassVar + +from mcp.client.client import Client + +from rle.harness.cli_base import HeadlessCliHarness, HeadlessCliOptions, TurnResult + +DEFAULT_SCRIPT: tuple[tuple[str, dict[str, Any]], ...] = ( + ("get_brief", {}), + ("research_target", {"parameters": {"project": "Electricity"}, "reason": "smoke"}), + ("end_turn", {"summary": "scripted smoke turn"}), +) + + +class ScriptedMcpHarness(HeadlessCliHarness): + """Plays a fixed tool-call script through the MCP server each turn.""" + + name: ClassVar[str] = "scripted-mcp" + + def __init__( + self, + options: HeadlessCliOptions | None = None, + script: Sequence[tuple[str, dict[str, Any]]] = DEFAULT_SCRIPT, + *, + name: str | None = None, + ) -> None: + super().__init__(options or HeadlessCliOptions()) + self.script = list(script) + self.turns: list[str] = [] + self.calls: list[tuple[str, dict[str, Any]]] = [] + if name: + self.name = name # type: ignore[misc] # per-instance override for plugins + self._url: str | None = None + + async def start_agent(self, mcp_url: str) -> None: + self._url = mcp_url + + async def send_turn(self, prompt: str) -> TurnResult: + assert self._url is not None + self.turns.append(prompt) + texts: list[str] = [] + async with Client(self._url) as client: + for tool, args in self.script: + result = await client.call_tool(tool, args) + self.calls.append((tool, args)) + content = getattr(result, "content", None) or [] + if content: + texts.append(str(getattr(content[0], "text", ""))) + return TurnResult( + text="\n".join(texts), prompt_tokens=len(prompt) // 4, completion_tokens=32, + ) + + async def stop_agent(self) -> None: + self._url = None + + def agent_versions(self) -> dict[str, str]: + return {"scripted_agent": "1"} diff --git a/src/rle/testing/smoke.py b/src/rle/testing/smoke.py new file mode 100644 index 0000000..06e0838 --- /dev/null +++ b/src/rle/testing/smoke.py @@ -0,0 +1,68 @@ +"""Run any harness plugin through the real loop against the mock RIMAPI.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from pydantic import BaseModel + +from rle.config import RLEConfig +from rle.harness.protocol import HarnessContext, HarnessPlugin +from rle.harness.registry import get_plugin, validate_options +from rle.orchestration.game_loop import RLEGameLoop, TickResult +from rle.rimapi.client import RimAPIClient +from rle.scoring.composite import CompositeScorer +from rle.scoring.recorder import TimeSeriesRecorder +from rle.testing.mock_rimapi import MockRimAPI + + +@dataclass +class SmokeReport: + harness: str + ticks: list[TickResult] + posts: list[tuple[str, Any]] + final_composite: float | None + describe: dict[str, str] = field(default_factory=dict) + + @property + def ok(self) -> bool: + return len(self.ticks) > 0 + + +async def run_harness_smoke( + plugin: HarnessPlugin | str, + *, + options: dict[str, Any] | BaseModel | None = None, + ticks: int = 3, + config: RLEConfig | None = None, + mock: MockRimAPI | None = None, +) -> SmokeReport: + """Build ``plugin.smoke(...)`` and run it for ``ticks`` ticks. + + This is the contract test every harness package should run in CI: if it + passes, the plugin loads, validates options, produces ``StepResult``s the + loop can execute/score, and tears down cleanly. + """ + if isinstance(plugin, str): + plugin = get_plugin(plugin) + cfg = config or RLEConfig(tick_interval=0.0) + mock = mock or MockRimAPI() + async with RimAPIClient("http://mock") as client: + mock.attach(client) + ctx = HarnessContext(config=cfg, client=client, smoke=True) + harness = plugin.smoke(ctx, validate_options(plugin, options)) + recorder = TimeSeriesRecorder() + loop = RLEGameLoop( + cfg, client, harness=harness, harness_context=ctx, + scorer=CompositeScorer(), recorder=recorder, + ) + results = await loop.run(max_ticks=ticks) + final = recorder.snapshots[-1].composite if recorder.snapshots else None + return SmokeReport( + harness=harness.name, + ticks=results, + posts=list(mock.posts), + final_composite=final, + describe=harness.describe(), + ) diff --git a/src/rle/tracking/hf_logger.py b/src/rle/tracking/hf_logger.py index 1c2feac..ad54177 100644 --- a/src/rle/tracking/hf_logger.py +++ b/src/rle/tracking/hf_logger.py @@ -35,18 +35,25 @@ # RLE — RimWorld Learning Environment Benchmarks -Can 7 role-specialized LLM agents keep a RimWorld colony alive? RLE is a -multi-agent coordination benchmark: MapAnalyst + 6 domain agents -(resources, defense, research, social, construction, medical) manage a -live colony through a REST API, scored on a 10-metric weighted composite -against a no-agent baseline (RimWorld's built-in pawn AI, static -4-seed reference). +Can an LLM agent harness keep a RimWorld colony alive? RLE is a +harness x model benchmark: a swappable harness (the Felix 7-agent stack, +a coding agent attached over MCP, ...) manages a live colony through a +REST API and is scored on a weighted composite against the unmanaged +baseline (RimWorld's built-in pawn AI). Rows are keyed by harness and +model; scoring is harness-agnostic (see SCORING_VERSION in each run). - Site + featured runs: https://rle.appsprout.dev -- Harness: https://github.com/AppSprout-dev/RLE +- Environment + harness registry: https://github.com/AppSprout-dev/RLE """ +def _row_label(row: dict[str, Any]) -> str: + """``harness/model`` when the row knows its harness, else just the model.""" + harness = row.get("harness") + model = str(row.get("model", "?")) + return f"{harness}/{model}" if harness else model + + def build_dataset_card(board: dict[str, Any], date: str) -> str: """Render the dataset card README from a spread's leaderboard.json. @@ -66,14 +73,14 @@ def build_dataset_card(board: dict[str, Any], date: str) -> str: f"Baseline: no-agent, {baseline.get('n_runs', '?')} seeds, mean " f"time-to-end {baseline.get('mean_time_to_end_days', '?')} days.", "", - "| # | model | mean | final | vs baseline | ticks > base | action ok | cost |", - "|---|-------|------|-------|-------------|--------------|-----------|------|", + "| # | harness/model | mean | final | vs baseline | ticks > base | action ok | cost |", + "|---|---------------|------|-------|-------------|--------------|-----------|------|", ] for i, r in enumerate(rows, 1): real = r.get("real_cost_usd") cost = f"${real:.2f}" if real is not None else f"~${r.get('est_cost_usd', 0):.2f}" lines.append( - f"| {i} | {r['model']} | {r['mean_composite']:.3f} " + f"| {i} | {_row_label(r)} | {r['mean_composite']:.3f} " f"| {r['final_composite']:.3f} | {r['vs_baseline_mean_delta']:+.3f} " f"| {r['ticks_above_baseline']} | {r['raw_action_success']:.0%} | {cost} |" ) @@ -81,9 +88,9 @@ def build_dataset_card(board: dict[str, Any], date: str) -> str: above = [r for r in rows if r.get("vs_baseline_mean_delta", 0) > 0] lines += [ "", - f"**{len(above)} of {len(rows)} models beat the no-agent baseline.**" + f"**{len(above)} of {len(rows)} harness/model rows beat the unmanaged baseline.**" + ( - " (" + ", ".join(r["model"] for r in above) + ")" + " (" + ", ".join(_row_label(r) for r in above) + ")" if above else "" ), "", diff --git a/src/rle/tracking/history.py b/src/rle/tracking/history.py index c164948..787988f 100644 --- a/src/rle/tracking/history.py +++ b/src/rle/tracking/history.py @@ -43,16 +43,21 @@ def load_history() -> list[dict[str, object]]: def update_baseline(summary: dict[str, Any]) -> tuple[bool, float | None]: - """Update baseline if this run's avg score is a new best for the model. + """Update baseline if this run's avg score is a new best for the + harness x model pair. Quarantined (harness-failure) scenarios are ignored. Returns (is_new_best, previous_score_or_None). """ BASELINES_DIR.mkdir(parents=True, exist_ok=True) - model = summary.get("model", "unknown") - slug = re.sub(r"[^\w\-]", "_", model) + model = str(summary.get("model", "unknown")) + harness = str(summary.get("harness") or "felix") + slug = re.sub(r"[^\w\-]", "_", f"{harness}__{model}" if harness != "felix" else model) baseline_path = BASELINES_DIR / f"{slug}.json" - scenarios = summary.get("scenarios", []) + scenarios = [ + s for s in summary.get("scenarios", []) + if isinstance(s, dict) and not s.get("harness_failed") + ] if not scenarios: return False, None avg_score = sum(s.get("score", 0) for s in scenarios) / len(scenarios) @@ -66,6 +71,7 @@ def update_baseline(summary: dict[str, Any]) -> tuple[bool, float | None]: baseline = { "model": model, + "harness": harness, "avg_score": round(avg_score, 4), "timestamp": summary.get("timestamp", ""), "git_commit": summary.get("git_commit", ""), diff --git a/src/rle/tracking/leaderboard.py b/src/rle/tracking/leaderboard.py index dc7f8c7..98519f1 100644 --- a/src/rle/tracking/leaderboard.py +++ b/src/rle/tracking/leaderboard.py @@ -1,7 +1,9 @@ """Leaderboard generation from RLE benchmark history. -Builds model x scenario results matrix with significance markers, -cost-normalized rankings, and Pareto frontier computation. +Rows are keyed by **harness x model**: the same model under two harnesses is +two rows, because the harness is a benchmark variable. Scenario scores are +averaged across runs (quarantined harness-failure runs excluded), with +bootstrap CIs when N >= 2, plus cost and mean step latency as Pareto axes. """ from __future__ import annotations @@ -12,33 +14,54 @@ from rle.scoring.bootstrap import bootstrap_ci +# History entries written before the harness layer existed were all Felix runs. +LEGACY_HARNESS = "felix" + class LeaderboardEntry(BaseModel): - """One model's benchmark results.""" + """One harness x model's benchmark results.""" model_config = ConfigDict(frozen=True) model: str + harness: str = LEGACY_HARNESS composite_score: float composite_ci: tuple[float, float] | None = None total_cost_usd: float = 0.0 cost_per_scenario: float = 0.0 total_tokens: int = 0 total_wall_time_s: float = 0.0 + mean_step_latency_s: float = 0.0 n_runs: int = 1 + n_quarantined: int = 0 scenarios: dict[str, float] = {} significance_vs_baseline: dict[str, str] = {} timestamp: str = "" git_commit: str = "" + @property + def label(self) -> str: + return f"{self.harness}/{self.model}" + + +def _run_harness(entry: dict[str, Any]) -> str: + harness = entry.get("harness") + return str(harness) if harness else LEGACY_HARNESS + -def _collect_scenario_scores( - runs: list[dict[str, Any]], -) -> dict[str, list[float]]: - """Group scenario scores across multiple runs for one model.""" +def _clean_scenarios(run: dict[str, Any]) -> list[dict[str, Any]]: + """Scenario results minus those flagged as harness/plumbing failures.""" + return [ + sc for sc in run.get("scenarios", []) + if isinstance(sc, dict) and not sc.get("harness_failed") + ] + + +def _collect_scenario_scores(runs: list[dict[str, Any]]) -> dict[str, list[float]]: + """Group scenario scores across multiple runs for one harness x model.""" scores: dict[str, list[float]] = {} for run in runs: - for sc in run.get("scenarios", []): + for sc in _clean_scenarios(run): name = sc.get("name", "") score = sc.get("score") if name and isinstance(score, (int, float)): @@ -51,8 +74,7 @@ def _per_run_composites(runs: list[dict[str, Any]]) -> list[float]: composites: list[float] = [] for run in runs: scenario_scores = [ - sc["score"] - for sc in run.get("scenarios", []) + sc["score"] for sc in _clean_scenarios(run) if isinstance(sc.get("score"), (int, float)) ] if scenario_scores: @@ -60,18 +82,32 @@ def _per_run_composites(runs: list[dict[str, Any]]) -> list[float]: return composites +def _mean_latency(runs: list[dict[str, Any]]) -> float: + latencies = [ + float(sc["mean_step_latency_s"]) + for run in runs for sc in _clean_scenarios(run) + if isinstance(sc.get("mean_step_latency_s"), (int, float)) + ] + return round(sum(latencies) / len(latencies), 3) if latencies else 0.0 + + +def _cost_block(run: dict[str, Any]) -> dict[str, Any]: + block = run.get("cost_snapshot") or run.get("cost") or {} + return block if isinstance(block, dict) else {} + + class Leaderboard: """Manages the RLE benchmark leaderboard.""" def from_history(self, history: list[dict[str, Any]]) -> list[LeaderboardEntry]: """Build sorted leaderboard from benchmark_history.jsonl entries.""" - by_model: dict[str, list[dict[str, Any]]] = {} + by_key: dict[tuple[str, str], list[dict[str, Any]]] = {} for entry in history: - model = entry.get("model", "unknown") - by_model.setdefault(model, []).append(entry) + key = (_run_harness(entry), str(entry.get("model", "unknown"))) + by_key.setdefault(key, []).append(entry) entries: list[LeaderboardEntry] = [] - for model, runs in by_model.items(): + for (harness, model), runs in by_key.items(): latest = runs[-1] scenario_scores = _collect_scenario_scores(runs) scenario_means = {k: sum(v) / len(v) for k, v in scenario_scores.items()} @@ -79,26 +115,34 @@ def from_history(self, history: list[dict[str, Any]]) -> list[LeaderboardEntry]: composites = _per_run_composites(runs) composite = sum(composites) / len(composites) if composites else 0.0 n_runs = len(runs) + n_quarantined = sum( + 1 for run in runs for sc in run.get("scenarios", []) + if isinstance(sc, dict) and sc.get("harness_failed") + ) ci: tuple[float, float] | None = None if len(composites) >= 2: bci = bootstrap_ci(composites) ci = (bci.ci_lower, bci.ci_upper) - cost = float(latest.get("cost", {}).get("estimated_cost_usd", 0.0)) - tokens = int(latest.get("cost", {}).get("total_tokens", 0)) - wall = float(latest.get("cost", {}).get("wall_time_s", 0.0)) + cost_block = _cost_block(latest) + cost = float(cost_block.get("estimated_cost_usd", 0.0)) + tokens = int(cost_block.get("total_tokens", 0)) + wall = float(cost_block.get("wall_time_s", 0.0)) n_scenarios = max(len(scenario_means), 1) entries.append(LeaderboardEntry( model=model, + harness=harness, composite_score=round(composite, 4), composite_ci=ci, total_cost_usd=cost, cost_per_scenario=round(cost / n_scenarios, 4), total_tokens=tokens, total_wall_time_s=wall, + mean_step_latency_s=_mean_latency(runs), n_runs=n_runs, + n_quarantined=n_quarantined, scenarios=scenario_means, timestamp=str(latest.get("timestamp", "")), git_commit=str(latest.get("git_commit", "")), @@ -108,27 +152,33 @@ def from_history(self, history: list[dict[str, Any]]) -> list[LeaderboardEntry]: return entries def to_markdown(self, entries: list[LeaderboardEntry]) -> str: - """Render model x scenario matrix as Markdown table.""" + """Render harness x model x scenario matrix as Markdown table.""" if not entries: return "" - all_scenarios = sorted( - {s for e in entries for s in e.scenarios} - ) + all_scenarios = sorted({s for e in entries for s in e.scenarios}) short_names = [s.split()[0] if " " in s else s[:12] for s in all_scenarios] - header = "| Model | " + " | ".join(short_names) + " | Avg | Cost |" - sep = "|" + "|".join("---" for _ in range(len(short_names) + 3)) + "|" + header = ( + "| Harness | Model | " + " | ".join(short_names) + + " | Avg | Cost | s/step | N |" + ) + sep = "|" + "|".join("---" for _ in range(len(short_names) + 6)) + "|" rows = [header, sep] for entry in entries: - cells = [entry.model] + cells = [entry.harness, entry.model] for scenario in all_scenarios: score = entry.scenarios.get(scenario) sig = entry.significance_vs_baseline.get(scenario, "") cells.append(f"{score:.2f}{sig}" if score is not None else "—") cells.append(f"{entry.composite_score:.2f}") cells.append(f"${entry.total_cost_usd:.2f}") + cells.append(f"{entry.mean_step_latency_s:.1f}") + n = str(entry.n_runs) + if entry.n_quarantined: + n += f" ({entry.n_quarantined} quarantined)" + cells.append(n) rows.append("| " + " | ".join(cells) + " |") return "\n".join(rows) @@ -139,17 +189,22 @@ def to_csv(self, entries: list[LeaderboardEntry], path: str) -> None: return all_scenarios = sorted({s for e in entries for s in e.scenarios}) - header = ["model"] + all_scenarios + ["avg", "cost_usd", "n_runs"] + header = ( + ["harness", "model"] + all_scenarios + + ["avg", "cost_usd", "mean_step_latency_s", "n_runs", "n_quarantined"] + ) lines = [",".join(header)] for entry in entries: - row = [entry.model] + row = [entry.harness, entry.model] for s in all_scenarios: score = entry.scenarios.get(s) row.append(f"{score:.4f}" if score is not None else "") row.append(f"{entry.composite_score:.4f}") row.append(f"{entry.total_cost_usd:.4f}") + row.append(f"{entry.mean_step_latency_s:.3f}") row.append(str(entry.n_runs)) + row.append(str(entry.n_quarantined)) lines.append(",".join(row)) with open(path, "w", encoding="utf-8") as f: diff --git a/src/rle/tracking/metadata.py b/src/rle/tracking/metadata.py index 7e85d6d..337f186 100644 --- a/src/rle/tracking/metadata.py +++ b/src/rle/tracking/metadata.py @@ -15,13 +15,17 @@ # metric implementations, or composite math change in a way that makes scores # from older runs not directly comparable. The leaderboard re-scores artifacts # at the current version on render; mismatches are surfaced, not silently -# elided. 1.1 (issue #25): threat_response now tracks actual draft responses +# elided. +# 1.1 (issue #25): threat_response now tracks actual draft responses # (first_draft_tick wired, was permanently 0.0 once any threat registered) # and null incident placeholders (enemy_count=0, threat_level=0.0) no longer -# count as threats — 1.0 scores with non-empty threats_seen are not -# comparable. coordination + communication_efficiency remain the broken -# implementations to be repaired in Phase C. -SCORING_VERSION = "1.1" +# count as threats. +# 1.2 (issue #51, "Phase C"): coordination + communication_efficiency removed +# (both were ~1.0 by construction and Felix-specific); plan_coherence added +# (contradictory executed writes per tick, harness-agnostic); efficiency and +# plan_coherence return a neutral 0.5 for ticks with no writes so an unmanaged +# baseline no longer banks free process points; weights redistributed. +SCORING_VERSION = "1.2" # Conventional install path for the RIMAPI Workshop mod we deploy our fork DLL # over. Best-effort — if Steam lives elsewhere set the RIMAPI_DLL_PATH env var. @@ -30,13 +34,21 @@ ) -def collect_metadata(random_seed: int | None = None) -> dict[str, object]: +def collect_metadata( + random_seed: int | None = None, + harness_describe: dict[str, str] | None = None, +) -> dict[str, object]: """Gather reproducibility metadata for a benchmark run. The random_seed argument is the seed the caller passed to ``random.seed`` (or None if no seed was set). It controls only RLE-side stochasticity (json_repair fallbacks, resolver tiebreaks); RimWorld's own RNG is unaffected — that lives inside the game and is not reproducible from here. + + ``harness_describe`` is whatever the harness reports about itself + (``BaseHarness.describe()``): SDK versions, agent roster, external tool + versions. Recorded as ``harness_versions`` so a leaderboard row can be + traced to the exact harness build, whichever framework it used. """ dll_path = _rimapi_dll_path() return { @@ -46,7 +58,7 @@ def collect_metadata(random_seed: int | None = None) -> dict[str, object]: "git_branch": _git("branch", "--show-current"), "git_dirty": _git("status", "--porcelain") != "", "rle_version": _version("rimworld-learning-environment"), - "felix_sdk_version": _version("felix-agent-sdk"), + "harness_versions": dict(harness_describe or {}), "platform": sys.platform, "python_version": platform.python_version(), "docker_mode": False, diff --git a/tests/conftest.py b/tests/conftest.py index ed76706..596f324 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,12 +3,11 @@ from __future__ import annotations import json +from importlib.util import find_spec +from typing import Any from unittest.mock import MagicMock import pytest -from felix_agent_sdk.core import HelixConfig, HelixGeometry -from felix_agent_sdk.providers.base import BaseProvider -from felix_agent_sdk.providers.types import CompletionResult from rle.config import RLEConfig from rle.rimapi.schemas import ( @@ -23,6 +22,28 @@ WeatherData, ) +# The Felix harness is an optional extra. Tests that need it either live in +# a module listed below (skipped from collection entirely) or carry the +# `requires_felix` marker; core tests must pass without it. +FELIX_AVAILABLE = find_spec("felix_agent_sdk") is not None +requires_felix = pytest.mark.skipif( + not FELIX_AVAILABLE, reason="felix-agent-sdk not installed (uv sync --extra felix)", +) + +# Modules that import the Felix SDK (or rle.harness.felix internals) at +# module level. Without the extra they cannot even be collected, so they are +# excluded wholesale; the zero-Felix CI job runs everything else. +_FELIX_ONLY_TEST_MODULES = [ + "unit/test_base_role.py", + "unit/test_claude_code_provider.py", + "unit/test_felix_provider_factory.py", + "unit/test_role_agents.py", + "unit/test_visualizer_integration.py", + "integration/test_game_loop.py", + "integration/test_scenario_run.py", +] +collect_ignore = [] if FELIX_AVAILABLE else list(_FELIX_ONLY_TEST_MODULES) + # ------------------------------------------------------------------ # Config # ------------------------------------------------------------------ @@ -35,7 +56,6 @@ def mock_config() -> RLEConfig: provider="anthropic", model="claude-sonnet-4-5", tick_interval=0.5, - helix_preset="default", max_agents=7, log_level="DEBUG", ) @@ -283,15 +303,18 @@ def sample_action_plan_json() -> str: @pytest.fixture -def helix() -> HelixGeometry: - return HelixConfig.default().to_geometry() +def helix() -> Any: + felix_core = pytest.importorskip("felix_agent_sdk.core") + return felix_core.HelixConfig.default().to_geometry() @pytest.fixture def mock_provider() -> MagicMock: - """Provider mock that returns a valid JSON action plan.""" - provider = MagicMock(spec=BaseProvider) - provider.complete.return_value = CompletionResult( + """Felix provider mock that returns a valid JSON action plan.""" + providers_base = pytest.importorskip("felix_agent_sdk.providers.base") + providers_types = pytest.importorskip("felix_agent_sdk.providers.types") + provider = MagicMock(spec=providers_base.BaseProvider) + provider.complete.return_value = providers_types.CompletionResult( content=SAMPLE_ACTION_PLAN_JSON, model="mock-model", usage={"prompt_tokens": 100, "completion_tokens": 50, "total_tokens": 150}, diff --git a/tests/integration/test_cli_base_harness.py b/tests/integration/test_cli_base_harness.py new file mode 100644 index 0000000..7a8a7bf --- /dev/null +++ b/tests/integration/test_cli_base_harness.py @@ -0,0 +1,110 @@ +"""HeadlessCliHarness driven by the scripted MCP agent through the real loop.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any, ClassVar + +import pytest + +from rle.config import RLEConfig +from rle.harness import HarnessContext, HarnessStepError +from rle.orchestration.game_loop import RLEGameLoop +from rle.rimapi.client import RimAPIClient +from rle.scoring.composite import CompositeScorer +from rle.testing import MockRimAPI +from rle.tracking.event_log import EventLog, EventType + +cli_base = pytest.importorskip("rle.harness.cli_base") +scripted = pytest.importorskip("rle.testing.scripted_agent") + + +@asynccontextmanager +async def _env() -> AsyncIterator[tuple[RimAPIClient, MockRimAPI]]: + mock = MockRimAPI() + async with RimAPIClient("http://mock") as client: + mock.attach(client) + yield client, mock + + +class TestScriptedMcpHarness: + async def test_full_round_trip_through_loop(self, tmp_path: Path) -> None: + log = EventLog(tmp_path / "events.jsonl") + harness = scripted.ScriptedMcpHarness() + async with _env() as (client, mock): + config = RLEConfig(tick_interval=0.0) + ctx = HarnessContext(config=config, client=client, event_log=log) + loop = RLEGameLoop( + config, client, harness=harness, harness_context=ctx, + scorer=CompositeScorer(), event_log=log, + ) + results = await loop.run(max_ticks=2) + + assert len(results) == 2 + # research_target went to RIMAPI via the MCP tool, once per tick + research_posts = [p for p, _ in mock.posts if "research" in p] + assert len(research_posts) == 2 + # Ledger reported the execution; the loop did not re-execute + assert results[0].execution.executed == 1 + assert results[0].plan.role == "scripted-mcp" + assert results[0].plan.summary == "scripted smoke turn" + assert results[0].extras["status"] == "success" + assert results[0].extras["turn_ended"] is True + assert harness.parse_successes == 2 + assert len(harness.turns) == 2 and "get_brief" in harness.turns[0] + kinds = {e.event_type for e in log.events} + assert {EventType.DELIBERATION, EventType.ACTION_EXEC, EventType.PROVIDER_CALL} <= kinds + # MCP host torn down with the loop + assert not harness._host.running # type: ignore[union-attr] + + async def test_describe_includes_agent_versions(self) -> None: + harness = scripted.ScriptedMcpHarness( + cli_base.HeadlessCliOptions(model="some/model"), name="my-tool", + ) + async with _env() as (client, _mock): + ctx = HarnessContext(config=RLEConfig(tick_interval=0.0), client=client) + await harness.setup(ctx) + try: + info = harness.describe() + finally: + await harness.teardown() + assert info == {"harness": "my-tool", "model": "some/model", "scripted_agent": "1"} + + +class _NeverEndsTurn(scripted.ScriptedMcpHarness): # type: ignore[misc] + name: ClassVar[str] = "never-ends" + + async def send_turn(self, prompt: str) -> Any: + await asyncio.sleep(10) + return cli_base.TurnResult() + + +class _Crashes(scripted.ScriptedMcpHarness): # type: ignore[misc] + name: ClassVar[str] = "crashes" + + async def send_turn(self, prompt: str) -> Any: + raise HarnessStepError("binary exited 1") + + +class TestTurnFailures: + async def test_turn_timeout_scores_empty_tick(self) -> None: + harness = _NeverEndsTurn( + cli_base.HeadlessCliOptions(turn_timeout_s=0.2, idle_grace_s=0.05), + ) + async with _env() as (client, _mock): + loop = RLEGameLoop(RLEConfig(tick_interval=0.0), client, harness=harness) + result = await loop.run(max_ticks=1) + assert result[0].extras["status"] == "turn_timeout" + assert result[0].execution.total == 0 + assert harness.parse_failures == 1 + + async def test_agent_error_scores_empty_tick(self) -> None: + harness = _Crashes(cli_base.HeadlessCliOptions(idle_grace_s=0.05)) + async with _env() as (client, _mock): + loop = RLEGameLoop(RLEConfig(tick_interval=0.0), client, harness=harness) + result = await loop.run(max_ticks=1) + assert result[0].extras["status"] == "agent_error" + assert harness.parse_failures == 1 diff --git a/tests/integration/test_game_loop.py b/tests/integration/test_game_loop.py index 3a76054..3bfc541 100644 --- a/tests/integration/test_game_loop.py +++ b/tests/integration/test_game_loop.py @@ -12,14 +12,14 @@ from felix_agent_sdk.providers.base import BaseProvider from felix_agent_sdk.providers.types import CompletionResult -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer from rle.config import RLEConfig +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.map_analyst import MapAnalyst +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer from rle.orchestration.game_loop import RLEGameLoop, TickResult from rle.rimapi.client import RimAPIClient from rle.scenarios.evaluator import ScenarioEvaluator @@ -593,8 +593,8 @@ async def test_parallel_all_agents_produce_plans(self) -> None: assert provider.acomplete.call_count == 7 # 1 MapAnalyst + 6 role agents assert result.plan.role == "orchestrator" - assert loop._parse_successes == 7 - assert loop._parse_failures == 0 + assert loop.parse_successes == 7 + assert loop.parse_failures == 0 async def test_spoke_messages_routed_after_tick(self) -> None: """After tick 1, agents should have TASK_COMPLETE messages from other agents.""" @@ -615,7 +615,7 @@ async def test_spoke_messages_routed_after_tick(self) -> None: await loop.run_tick() # Verify hub processed messages (6 TASK_COMPLETE from tick 1 + 6 from tick 2) - assert loop._hub.total_messages_processed >= 6 + assert loop.harness.hub.total_messages_processed >= 6 async def test_sequential_mode_still_works(self) -> None: provider = _make_mock_provider() @@ -631,7 +631,7 @@ async def test_sequential_mode_still_works(self) -> None: assert provider.acomplete.call_count == 7 # 1 MapAnalyst + 6 role agents assert result.plan.role == "orchestrator" - assert loop._parse_successes == 7 + assert loop.parse_successes == 7 async def test_sequential_3_ticks(self) -> None: provider = _make_mock_provider() @@ -684,8 +684,8 @@ async def test_task_complete_messages_sent(self) -> None: await loop.run_tick() # 7 agents sent TASK_COMPLETE messages - assert loop._hub.message_queue_size >= 0 # may have been processed - assert loop._hub.total_messages_processed >= 0 + assert loop.harness.hub.message_queue_size >= 0 # may have been processed + assert loop.harness.hub.total_messages_processed >= 0 # Each agent's spoke should have sent at least 1 message for agent in agents: assert agent._spoke.messages_sent >= 1 @@ -704,7 +704,7 @@ async def test_phase_announce_broadcast(self) -> None: await loop.run_tick() # First tick should have broadcast an initial phase - assert loop._last_phase != "" + assert loop.harness.last_phase != "" async def test_score_broadcast(self) -> None: """STATUS_UPDATE with scores should broadcast after scoring.""" diff --git a/tests/integration/test_harness_loop.py b/tests/integration/test_harness_loop.py new file mode 100644 index 0000000..c69fd98 --- /dev/null +++ b/tests/integration/test_harness_loop.py @@ -0,0 +1,207 @@ +"""RLEGameLoop driven by non-Felix harnesses (the swappable-harness contract).""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import ClassVar + +from rle.agents.actions import Action, ActionPlan +from rle.config import RLEConfig +from rle.harness import BaseHarness, HarnessStepError, StepResult +from rle.harness.baseline import BaselineHarness +from rle.orchestration.action_executor import ActionOutcome, ExecutionResult +from rle.orchestration.game_loop import RLEGameLoop +from rle.rimapi.client import RimAPIClient +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from rle.scoring.composite import CompositeScorer +from rle.scoring.metrics import NEUTRAL +from rle.scoring.recorder import TimeSeriesRecorder +from rle.testing import MockRimAPI +from rle.tracking.event_log import EventLog, EventType + + +@asynccontextmanager +async def _client() -> AsyncIterator[RimAPIClient]: + async with RimAPIClient("http://mock") as client: + MockRimAPI().attach(client) + yield client + + +class _ScriptedHarness(BaseHarness): + """Proposes one work_priority write per tick; the loop executes it.""" + + name: ClassVar[str] = "scripted" + + def __init__(self) -> None: + super().__init__() + self.seen_events: list[list[RimAPIEvent]] = [] + self.tick_ends: list[float | None] = [] + + async def step( + self, state: GameState, tick: int, macro_time: float, events: list[RimAPIEvent], + ) -> StepResult: + self.seen_events.append(events) + plan = ActionPlan( + role="scripted", tick=state.colony.tick, + actions=[Action( + action_type="work_priority", target_colonist_id="col_01", + parameters={"Growing": 1}, reason="script", + )], + ) + self.parse_successes += 1 + return StepResult(plan=plan, proposals=(plan,), extras={"phase": "scripted"}) + + async def on_tick_end(self, tick, state, step, execution, score) -> None: # type: ignore[no-untyped-def] + self.tick_ends.append(score.composite if score else None) + + +class _PreExecutedHarness(BaseHarness): + """Applies its own writes (MCP-style) and reports the execution back.""" + + name: ClassVar[str] = "pre-executed" + + async def step( + self, state: GameState, tick: int, macro_time: float, events: list[RimAPIEvent], + ) -> StepResult: + outcomes = ( + ActionOutcome( + action_type="draft", endpoint="draft", target_colonist_id="col_01", + success=True, parameters={"is_drafted": True}, + ), + ActionOutcome( + action_type="draft", endpoint="draft", target_colonist_id="col_01", + success=True, parameters={"is_drafted": False}, + ), + ) + plan = ActionPlan(role="pre-executed", tick=state.colony.tick, actions=[]) + return StepResult( + plan=plan, + execution=ExecutionResult(executed=2, failed=0, total=2, outcomes=outcomes), + ) + + +class _FlakyHarness(BaseHarness): + name: ClassVar[str] = "flaky" + + async def step(self, state, tick, macro_time, events): # type: ignore[no-untyped-def] + raise HarnessStepError("upstream agent crashed") + + +class _SlowHarness(BaseHarness): + name: ClassVar[str] = "slow" + + async def step(self, state, tick, macro_time, events): # type: ignore[no-untyped-def] + await asyncio.sleep(5) + raise AssertionError("should have timed out") + + +class TestScriptedHarness: + async def test_loop_executes_harness_plan(self) -> None: + harness = _ScriptedHarness() + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=harness, + scorer=CompositeScorer(), recorder=TimeSeriesRecorder(), + ) + results = await loop.run(max_ticks=2) + + assert len(results) == 2 + assert all(r.harness == "scripted" for r in results) + assert results[0].execution.executed == 1 + assert results[0].extras == {"phase": "scripted"} + assert harness.tick_ends and harness.tick_ends[0] is not None + assert loop.parse_successes == 2 + assert len(harness.seen_events) == 2 + + async def test_dashboard_export_is_harness_neutral(self, tmp_path: Path) -> None: + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=_ScriptedHarness(), + dashboard_export_dir=tmp_path, + ) + await loop.run_tick() + data = json.loads((tmp_path / "latest_tick.json").read_text()) + assert data["harness"] == "scripted" + assert data["phase"] == "scripted" + assert len(data["agents"]) == 1 + assert data["extras"] == {"phase": "scripted"} + + +class TestPreExecutedHarness: + async def test_loop_skips_executor_and_scores_coherence(self) -> None: + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=_PreExecutedHarness(), + scorer=CompositeScorer(), + ) + await loop.run_tick() + second = await loop.run_tick() + + # Execution came from the harness untouched + assert second.execution.executed == 2 + # Tick 2 scores tick 1's contradictory draft/undraft: coherence 0.0 + assert second.score is not None + assert second.score.metrics["plan_coherence"] == 0.0 + assert second.score.metrics["efficiency"] == 1.0 + + +class TestBaselineThroughLoop: + async def test_baseline_neutral_process_metrics(self) -> None: + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=BaselineHarness(), + scorer=CompositeScorer(), + ) + await loop.run_tick() + second = await loop.run_tick() + assert second.score is not None + assert second.score.metrics["efficiency"] == NEUTRAL + assert second.score.metrics["plan_coherence"] == NEUTRAL + + async def test_legacy_no_agent_flag_still_works(self) -> None: + async with _client() as client: + loop = RLEGameLoop(RLEConfig(tick_interval=0.0), client, no_agent=True) + result = await loop.run_tick() + assert loop.harness.name == "baseline" + assert result.plan.actions == [] + + +class TestHarnessFailures: + async def test_step_error_degrades_to_empty_tick(self, tmp_path: Path) -> None: + log = EventLog(tmp_path / "events.jsonl") + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0), client, harness=_FlakyHarness(), event_log=log, + ) + result = await loop.run_tick() + assert result.plan.actions == [] + errors = [e for e in log.events if e.event_type == EventType.ERROR] + assert errors and errors[0].data["error_type"] == "harness_error" + + async def test_step_timeout_degrades_to_empty_tick(self, tmp_path: Path) -> None: + log = EventLog(tmp_path / "events.jsonl") + async with _client() as client: + loop = RLEGameLoop( + RLEConfig(tick_interval=0.0, tick_timeout_s=0.05), client, + harness=_SlowHarness(), event_log=log, + ) + result = await loop.run_tick() + assert result.plan.actions == [] + errors = [e for e in log.events if e.event_type == EventType.ERROR] + assert errors and errors[0].data["error_type"] == "harness_timeout" + + async def test_harness_and_agents_are_exclusive(self) -> None: + async with _client() as client: + try: + RLEGameLoop( + RLEConfig(tick_interval=0.0), client, [object()], harness=BaselineHarness(), + ) + except ValueError as exc: + assert "not both" in str(exc) + else: # pragma: no cover + raise AssertionError("expected ValueError") diff --git a/tests/integration/test_scenario_run.py b/tests/integration/test_scenario_run.py index 7f61b56..31dd981 100644 --- a/tests/integration/test_scenario_run.py +++ b/tests/integration/test_scenario_run.py @@ -12,13 +12,13 @@ from felix_agent_sdk.providers.base import BaseProvider from felix_agent_sdk.providers.types import CompletionResult -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer from rle.config import RLEConfig +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer from rle.orchestration.game_loop import RLEGameLoop from rle.rimapi.client import RimAPIClient from rle.scenarios.evaluator import ScenarioEvaluator diff --git a/tests/unit/test_base_role.py b/tests/unit/test_base_role.py index a1dc8de..938ca30 100644 --- a/tests/unit/test_base_role.py +++ b/tests/unit/test_base_role.py @@ -13,7 +13,7 @@ from felix_agent_sdk.providers.types import CompletionResult, MessageRole from rle.agents.actions import ActionPlan, ActionPlanParseError -from rle.agents.base_role import RimWorldRoleAgent +from rle.harness.felix.agents.base_role import RimWorldRoleAgent from rle.rimapi.schemas import GameState # ------------------------------------------------------------------ diff --git a/tests/unit/test_brief.py b/tests/unit/test_brief.py new file mode 100644 index 0000000..0940fec --- /dev/null +++ b/tests/unit/test_brief.py @@ -0,0 +1,90 @@ +"""Harness-neutral scenario brief.""" + +from __future__ import annotations + +from rle.harness.brief import action_catalog, build_brief, build_map_summary +from rle.rimapi.schemas import ( + AreaRect, + ColonyData, + GameState, + MapData, + ResearchData, + ResourceData, + TerrainSummary, + WeatherData, +) +from rle.rimapi.sse_client import RimAPIEvent +from rle.scenarios.loader import list_scenarios + + +def _state(with_terrain: bool = True) -> GameState: + terrain = TerrainSummary( + colony_center=(50, 50), + recommended_shelter=AreaRect(x1=40, z1=40, x2=46, z2=46), + recommended_farm=AreaRect(x1=60, z1=40, x2=67, z2=47), + recommended_stockpile=AreaRect(x1=48, z1=52, x2=52, z2=56), + water_areas=[AreaRect(x1=0, z1=0, x2=5, z2=90)], + ) if with_terrain else None + return GameState( + colony=ColonyData( + name="T", wealth=5000.0, day=3, tick=180000, + population=3, mood_average=0.6, food_days=4.0, + ), + colonists=[], + resources=ResourceData( + food=40.0, medicine=2, steel=50, wood=120, components=3, silver=0, power_net=0.0, + ), + map=MapData( + size=(250, 250), biome="temperate_forest", season="spring", + temperature=15.0, structures=[], terrain=terrain, + ), + research=ResearchData(current_project=None, progress=0.0, completed=[], available=["a"]), + threats=[], + weather=WeatherData(condition="clear", temperature=15.0, outdoor_severity=0.0), + timestamp=0.0, + ) + + +class TestMapSummary: + def test_none_without_terrain(self) -> None: + assert build_map_summary(_state(with_terrain=False)) is None + + def test_contains_verified_sites_and_water(self) -> None: + text = build_map_summary(_state()) + assert text is not None + assert "SHELTER SITE" in text and "(40,40)-(46,46)" in text + assert "FARM SITE" in text and "x1=60,z1=40,x2=67,z2=47" in text + assert "STOCKPILE SITE" in text + assert "WATER (do NOT build here)" in text + assert "Zones: NONE" in text and "Rooms: NONE" in text + + +class TestActionCatalog: + def test_includes_no_action_and_every_write(self) -> None: + names = {a["action_type"] for a in action_catalog()} + assert "no_action" in names + assert {"work_priority", "draft", "blueprint", "growing_zone"} <= names + + +class TestBrief: + def test_brief_carries_goals_state_events_and_actions(self) -> None: + scenario = list_scenarios()[0] + events = [RimAPIEvent(event_type="raid", data={"points": 500}, timestamp=1.0)] + brief = build_brief( + _state(), tick=2, macro_time=0.1, scenario=scenario, events=events, + ) + assert brief.tick == 2 and brief.day == 3 + assert brief.goals["name"] == scenario.name + assert brief.goals["victory"] + assert brief.state["colony"]["population"] == 3 + assert brief.recent_events[0]["event_type"] == "raid" + assert brief.map_summary and "SHELTER SITE" in brief.map_summary + text = brief.to_text() + assert "## Scenario" in text and "## MAP_SUMMARY" in text + assert "## Actions available" in text and "- draft:" in text + + def test_brief_without_scenario(self) -> None: + brief = build_brief(_state(with_terrain=False), tick=0, macro_time=0.0) + assert brief.goals == {} + assert brief.map_summary is None + assert "## Scenario" not in brief.to_text() diff --git a/tests/unit/test_claude_code_provider.py b/tests/unit/test_claude_code_provider.py index 6911e6c..2348ab5 100644 --- a/tests/unit/test_claude_code_provider.py +++ b/tests/unit/test_claude_code_provider.py @@ -12,7 +12,7 @@ from felix_agent_sdk.providers.errors import ProviderError from felix_agent_sdk.providers.types import ChatMessage, MessageRole -from rle.providers.claude_code import ClaudeCodeProvider +from rle.harness.felix.providers.claude_code import ClaudeCodeProvider MESSAGES = [ ChatMessage(role=MessageRole.SYSTEM, content="You are a test agent."), @@ -52,8 +52,8 @@ def _run_complete( ) -> tuple[Any, MagicMock]: """Call provider.complete with mocked CLI resolution + subprocess.""" with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", - ), patch("rle.providers.claude_code.subprocess.run") as mock_run: + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", + ), patch("rle.harness.felix.providers.claude_code.subprocess.run") as mock_run: if side_effect is not None: mock_run.side_effect = side_effect else: @@ -131,7 +131,7 @@ def test_timeout_raises(self) -> None: def test_missing_cli_raises(self) -> None: provider = ClaudeCodeProvider() - with patch("rle.providers.claude_code.shutil.which", return_value=None): + with patch("rle.harness.felix.providers.claude_code.shutil.which", return_value=None): with pytest.raises(ProviderError, match="not found on PATH"): provider.complete(MESSAGES) @@ -142,8 +142,8 @@ def test_assistant_messages_ignored(self) -> None: ChatMessage(role=MessageRole.ASSISTANT, content=""), ] with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", - ), patch("rle.providers.claude_code.subprocess.run") as mock_run: + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", + ), patch("rle.harness.felix.providers.claude_code.subprocess.run") as mock_run: mock_run.return_value = _mock_proc(_cli_envelope()) provider.complete(messages) assert mock_run.call_args[1]["input"] == "Do the thing." @@ -167,9 +167,9 @@ async def test_parses_result_and_usage(self) -> None: provider = ClaudeCodeProvider() proc = _mock_async_proc(_cli_envelope(result="hello", input_tokens=50, output_tokens=7)) with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", ), patch( - "rle.providers.claude_code.asyncio.create_subprocess_exec", + "rle.harness.felix.providers.claude_code.asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc), ): result = await provider.acomplete(MESSAGES) @@ -180,9 +180,9 @@ async def test_nonzero_exit_raises(self) -> None: provider = ClaudeCodeProvider() proc = _mock_async_proc("", returncode=1, stderr="boom") with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", ), patch( - "rle.providers.claude_code.asyncio.create_subprocess_exec", + "rle.harness.felix.providers.claude_code.asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc), ): with pytest.raises(ProviderError, match="exited with code 1"): @@ -193,9 +193,9 @@ async def test_timeout_kills_subprocess(self) -> None: proc = _mock_async_proc("") proc.communicate = AsyncMock(side_effect=asyncio.TimeoutError) with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", ), patch( - "rle.providers.claude_code.asyncio.create_subprocess_exec", + "rle.harness.felix.providers.claude_code.asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc), ): with pytest.raises(ProviderError, match="timed out"): @@ -212,9 +212,9 @@ async def _hang(*args: Any, **kwargs: Any) -> tuple[bytes, bytes]: proc.communicate = AsyncMock(side_effect=_hang) with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", ), patch( - "rle.providers.claude_code.asyncio.create_subprocess_exec", + "rle.harness.felix.providers.claude_code.asyncio.create_subprocess_exec", new=AsyncMock(return_value=proc), ): task = asyncio.ensure_future(provider.acomplete(MESSAGES)) @@ -229,8 +229,8 @@ class TestStreamAndTokens: def test_stream_yields_content_then_final(self) -> None: provider = ClaudeCodeProvider() with patch( - "rle.providers.claude_code.shutil.which", return_value="claude", - ), patch("rle.providers.claude_code.subprocess.run") as mock_run: + "rle.harness.felix.providers.claude_code.shutil.which", return_value="claude", + ), patch("rle.harness.felix.providers.claude_code.subprocess.run") as mock_run: mock_run.return_value = _mock_proc(_cli_envelope(result="streamed")) chunks = list(provider.stream(MESSAGES)) assert chunks[0].text == "streamed" diff --git a/tests/unit/test_composite_scorer.py b/tests/unit/test_composite_scorer.py index e941ad3..f0c507b 100644 --- a/tests/unit/test_composite_scorer.py +++ b/tests/unit/test_composite_scorer.py @@ -4,6 +4,9 @@ import pytest +from rle.agents.actions import ActionPlan +from rle.orchestration.action_executor import ExecutionResult +from rle.orchestration.game_loop import TickResult from rle.rimapi.schemas import ( ColonyData, GameState, @@ -13,7 +16,7 @@ WeatherData, ) from rle.scoring.composite import DEFAULT_WEIGHTS, CompositeScorer, ScoreSnapshot -from rle.scoring.metrics import MetricContext +from rle.scoring.metrics import ALL_METRICS, NEUTRAL, MetricContext def _state() -> GameState: @@ -48,8 +51,35 @@ class TestDefaultWeights: def test_sum_to_one(self) -> None: assert sum(DEFAULT_WEIGHTS.values()) == pytest.approx(1.0) - def test_ten_metrics(self) -> None: - assert len(DEFAULT_WEIGHTS) == 10 + def test_nine_metrics(self) -> None: + assert len(DEFAULT_WEIGHTS) == 9 + + def test_weights_cover_exactly_the_registered_metrics(self) -> None: + assert set(DEFAULT_WEIGHTS) == set(ALL_METRICS) + + def test_legacy_process_metrics_gone(self) -> None: + """Scoring 1.2 (#51): always-1.0 Felix-specific metrics removed.""" + assert "coordination" not in DEFAULT_WEIGHTS + assert "communication_efficiency" not in DEFAULT_WEIGHTS + assert "plan_coherence" in DEFAULT_WEIGHTS + + def test_outcome_metrics_dominate(self) -> None: + process = DEFAULT_WEIGHTS["efficiency"] + DEFAULT_WEIGHTS["plan_coherence"] + assert process <= 0.15 + + +class TestBaselineNeutrality: + def test_unmanaged_baseline_gets_no_free_process_points(self) -> None: + """A run with zero writes scores NEUTRAL, not 1.0, on process metrics.""" + ctx = _ctx() + ctx.tick_results.append(TickResult( + tick=1, day=1, macro_time=0.0, + plan=ActionPlan(role="baseline", tick=1, actions=[]), + execution=ExecutionResult(executed=0, failed=0, total=0), + )) + snap = CompositeScorer().score(_state(), ctx) + assert snap.metrics["efficiency"] == pytest.approx(NEUTRAL) + assert snap.metrics["plan_coherence"] == pytest.approx(NEUTRAL) class TestCompositeScorer: @@ -59,7 +89,7 @@ def test_score_returns_snapshot(self) -> None: assert isinstance(snap, ScoreSnapshot) assert snap.tick == 600000 assert snap.day == 10 - assert len(snap.metrics) == 10 + assert len(snap.metrics) == 9 assert 0.0 <= snap.composite <= 1.0 def test_custom_weights(self) -> None: @@ -89,7 +119,7 @@ def test_averages_snapshots(self) -> None: metrics={"survival": 1.0, "mood": 0.8, "food_security": 0.6, "wealth": 0.5, "research": 0.0, "threat_response": 1.0, "self_sufficiency": 0.5, "efficiency": 1.0, - "coordination": 0.9, "communication_efficiency": 0.8}, + "plan_coherence": 0.9}, composite=0.7, ), ScoreSnapshot( @@ -97,7 +127,7 @@ def test_averages_snapshots(self) -> None: metrics={"survival": 0.5, "mood": 0.6, "food_security": 0.4, "wealth": 0.3, "research": 0.5, "threat_response": 0.5, "self_sufficiency": 0.5, "efficiency": 0.5, - "coordination": 0.7, "communication_efficiency": 0.6}, + "plan_coherence": 0.7}, composite=0.5, ), ] diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index fd54e9e..a437770 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -33,14 +33,16 @@ def test_noop_when_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: assert "ANTHROPIC_API_KEY" not in os.environ -class TestProviderRegistry: - def test_claude_code_provider_registered(self) -> None: - from rle.providers.claude_code import ClaudeCodeProvider - - config = RLEConfig(provider="claude-code", model="claude-fable-5") - provider = config.get_provider() - assert isinstance(provider, ClaudeCodeProvider) - assert provider.model == "claude-fable-5" +class TestConfigIsFrameworkFree: + def test_harness_defaults(self) -> None: + config = RLEConfig() + assert config.harness == "felix" + assert config.harness_options == {} + assert config.tick_timeout_s is None + + def test_no_felix_symbols_on_config(self) -> None: + assert not hasattr(RLEConfig(), "get_provider") + assert not hasattr(RLEConfig(), "get_helix_config") class TestBridgeOpenRouterKey: diff --git a/tests/unit/test_felix_provider_factory.py b/tests/unit/test_felix_provider_factory.py new file mode 100644 index 0000000..862e64a --- /dev/null +++ b/tests/unit/test_felix_provider_factory.py @@ -0,0 +1,28 @@ +"""Felix provider/helix construction (moved off RLEConfig).""" + +from __future__ import annotations + +import pytest + +from rle.harness.felix.provider_factory import build_helix, build_provider +from rle.harness.felix.providers.claude_code import ClaudeCodeProvider + + +class TestProviderRegistry: + def test_claude_code_provider_registered(self) -> None: + provider = build_provider("claude-code", "claude-fable-5") + assert isinstance(provider, ClaudeCodeProvider) + assert provider.model == "claude-fable-5" + + def test_unknown_provider_lists_choices(self) -> None: + with pytest.raises(ValueError, match="anthropic"): + build_provider("nope", "x") + + +class TestHelixPresets: + def test_default(self) -> None: + assert build_helix("default") is not None + + def test_unknown_preset(self) -> None: + with pytest.raises(ValueError, match="research_heavy"): + build_helix("spiral") diff --git a/tests/unit/test_harness_registry.py b/tests/unit/test_harness_registry.py new file mode 100644 index 0000000..c3673c1 --- /dev/null +++ b/tests/unit/test_harness_registry.py @@ -0,0 +1,196 @@ +"""Tests for the harness plugin registry and the harness protocol surface.""" + +from __future__ import annotations + +from typing import ClassVar + +import pytest +from pydantic import BaseModel, ConfigDict + +import rle.harness.registry as registry +from rle.agents.actions import ActionPlan +from rle.config import RLEConfig +from rle.harness import ( + Availability, + BaseHarness, + EmptyOptions, + HarnessContext, + HarnessNotFoundError, + HarnessOptionsError, + HarnessUnavailableError, + StepResult, + create_harness, + get_plugin, + harness_names, + list_harnesses, + parse_option_pairs, + validate_options, +) +from rle.harness.baseline import BaselineHarness +from rle.harness.compat import build_legacy_harness +from rle.rimapi.client import RimAPIClient +from rle.rimapi.schemas import GameState +from rle.rimapi.sse_client import RimAPIEvent +from tests.conftest import FELIX_AVAILABLE, requires_felix + + +def _ctx(**overrides: object) -> HarnessContext: + return HarnessContext( + config=RLEConfig(tick_interval=0.0), + client=RimAPIClient("http://test"), + **overrides, # type: ignore[arg-type] + ) + + +class TestBuiltinRegistration: + def test_builtins_discoverable(self) -> None: + names = harness_names() + assert "baseline" in names + assert "felix" in names + + def test_list_reports_package_and_availability(self) -> None: + infos = {i.name: i for i in list_harnesses()} + assert infos["baseline"].availability.ok + assert infos["baseline"].package == "rimworld-learning-environment" + assert infos["felix"].availability.ok is FELIX_AVAILABLE + assert "Felix" in infos["felix"].description + if not FELIX_AVAILABLE: + assert "extra felix" in infos["felix"].availability.reason + + def test_unknown_name_lists_installed(self) -> None: + with pytest.raises(HarnessNotFoundError, match="baseline"): + get_plugin("does-not-exist") + + def test_create_baseline(self) -> None: + harness = create_harness("baseline", _ctx()) + assert isinstance(harness, BaselineHarness) + assert harness.name == "baseline" + + @requires_felix + def test_create_felix_smoke_builds_seven_agents(self) -> None: + harness = create_harness("felix", _ctx(), {"no_think": True}, smoke=True) + assert harness.name == "felix" + assert len(harness.agents) == 7 # type: ignore[attr-defined] + + @requires_felix + def test_felix_options_validated(self) -> None: + with pytest.raises(HarnessOptionsError, match="bogus"): + create_harness("felix", _ctx(), {"bogus": 1}, smoke=True) + + def test_baseline_rejects_options(self) -> None: + with pytest.raises(HarnessOptionsError): + create_harness("baseline", _ctx(), {"anything": True}) + + +class TestOptionParsing: + def test_json_coercion(self) -> None: + parsed = parse_option_pairs(["parallel=false", "role_timeout_s=12.5", "name=abc"]) + assert parsed == {"parallel": False, "role_timeout_s": 12.5, "name": "abc"} + + def test_bad_pair(self) -> None: + with pytest.raises(HarnessOptionsError): + parse_option_pairs(["novalue"]) + + def test_validate_passthrough_model(self) -> None: + plugin = get_plugin("baseline") + opts = EmptyOptions() + assert validate_options(plugin, opts) is opts + + +class _Opts(BaseModel): + model_config = ConfigDict(extra="forbid") + shout: bool = False + + +class _EchoHarness(BaseHarness): + name: ClassVar[str] = "echo" + + def __init__(self, shout: bool) -> None: + super().__init__() + self.shout = shout + self.steps = 0 + self.ended = 0 + self.torn_down = False + + async def step( + self, state: GameState, tick: int, macro_time: float, events: list[RimAPIEvent], + ) -> StepResult: + self.steps += 1 + return StepResult( + plan=ActionPlan(role="echo", tick=state.colony.tick, actions=[]), + extras={"shout": self.shout}, + ) + + async def on_tick_end(self, tick, state, step, execution, score) -> None: # type: ignore[no-untyped-def] + self.ended += 1 + await super().on_tick_end(tick, state, step, execution, score) + + async def teardown(self) -> None: + self.torn_down = True + + +class _UnavailablePlugin: + name = "ghost" + description = "never runs" + + def available(self) -> Availability: + return Availability.missing("ghost binary not on PATH") + + def option_schema(self) -> type[BaseModel]: + return EmptyOptions + + def create(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + raise AssertionError("must not be called") + + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + return BaselineHarness() + + def describe(self) -> dict[str, str]: + return {} + + +class _SmokeNeedsDep(_UnavailablePlugin): + def smoke(self, ctx: HarnessContext, options: BaseModel) -> BaseHarness: + raise ImportError("No module named 'ghosttool'") + + +class TestUnavailable: + def test_unavailable_plugin_raises_with_reason(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(registry, "get_plugin", lambda name: _UnavailablePlugin()) + with pytest.raises(HarnessUnavailableError, match="ghost binary"): + registry.create_harness("ghost", _ctx()) + + def test_smoke_does_not_require_the_tool(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Smoke variants exist precisely so CI can run without the binary.""" + monkeypatch.setattr(registry, "get_plugin", lambda name: _UnavailablePlugin()) + assert isinstance(registry.create_harness("ghost", _ctx(), smoke=True), BaselineHarness) + + def test_smoke_missing_dependency_is_unavailable(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(registry, "get_plugin", lambda name: _SmokeNeedsDep()) + with pytest.raises(HarnessUnavailableError, match="ghost binary"): + registry.create_harness("ghost", _ctx(), smoke=True) + + +class TestCompatShim: + def test_no_agent_builds_baseline(self) -> None: + assert isinstance(build_legacy_harness([], no_agent=True), BaselineHarness) + assert isinstance(build_legacy_harness(None), BaselineHarness) + + @requires_felix + def test_agents_build_felix(self) -> None: + felix = create_harness("felix", _ctx(), smoke=True) + rebuilt = build_legacy_harness(felix.agents, parallel=False) # type: ignore[attr-defined] + assert rebuilt.name == "felix" + + +class TestBaseHarnessDefaults: + def test_ctx_before_setup_raises(self) -> None: + with pytest.raises(RuntimeError): + _ = _EchoHarness(shout=False).ctx + + async def test_setup_stores_ctx(self) -> None: + h = _EchoHarness(shout=True) + ctx = _ctx() + await h.setup(ctx) + assert h.ctx is ctx + assert h.describe() == {"harness": "echo"} diff --git a/tests/unit/test_hf_logger.py b/tests/unit/test_hf_logger.py index 226927f..c720764 100644 --- a/tests/unit/test_hf_logger.py +++ b/tests/unit/test_hf_logger.py @@ -61,7 +61,7 @@ def test_real_cost_unmarked_estimate_marked(self) -> None: def test_baseline_beaters_called_out(self) -> None: card = build_dataset_card(_BOARD, "2026-06-11") - assert "1 of 3 models beat the no-agent baseline." in card + assert "1 of 3 harness/model rows beat the unmanaged baseline." in card assert "(z-ai/glm-5.1)" in card def test_date_and_baseline_framing(self) -> None: @@ -76,4 +76,19 @@ def test_no_baseline_beaters_omits_paren(self) -> None: "rows": [r for r in _BOARD["rows"] if r["model"] != "z-ai/glm-5.1"], } card = build_dataset_card(board, "2026-06-11") - assert "0 of 2 models beat the no-agent baseline." in card + assert "0 of 2 harness/model rows beat the unmanaged baseline." in card + + +class TestHarnessRows: + def test_rows_with_harness_are_labelled_harness_slash_model(self) -> None: + board = { + "baseline": _BOARD["baseline"], + "rows": [ + {**_BOARD["rows"][1], "harness": "felix"}, + {**_BOARD["rows"][0], "harness": "some-tool"}, + ], + } + card = build_dataset_card(board, "2026-09-04") + assert "| felix/z-ai/glm-5.1 |" in card + assert "| some-tool/x-ai/grok-4.3 |" in card + assert "(felix/z-ai/glm-5.1)" in card diff --git a/tests/unit/test_leaderboard.py b/tests/unit/test_leaderboard.py index ee30a02..0dbdd81 100644 --- a/tests/unit/test_leaderboard.py +++ b/tests/unit/test_leaderboard.py @@ -72,7 +72,7 @@ def test_produces_valid_markdown(self) -> None: md = lb.to_markdown(entries) lines = md.strip().split("\n") assert len(lines) >= 3 # header + separator + at least 1 row - assert "| Model |" in lines[0] + assert "| Harness | Model |" in lines[0] assert "---" in lines[1] def test_includes_cost_column(self) -> None: @@ -130,3 +130,38 @@ def test_single_entry(self) -> None: def test_empty(self) -> None: lb = Leaderboard() assert lb.pareto_frontier([]) == [] + + +class TestHarnessKeys: + def test_same_model_two_harnesses_are_two_rows(self) -> None: + history = [ + {**_history_entry("gpt-4o", [_scenario("S1", 0.8)]), "harness": "felix"}, + {**_history_entry("gpt-4o", [_scenario("S1", 0.6)]), "harness": "some-tool"}, + ] + entries = Leaderboard().from_history(history) + keys = {(e.harness, e.model) for e in entries} + assert keys == {("felix", "gpt-4o"), ("some-tool", "gpt-4o")} + assert entries[0].label == "felix/gpt-4o" + + def test_legacy_entries_default_to_felix(self) -> None: + entries = Leaderboard().from_history(HISTORY) + assert all(e.harness == "felix" for e in entries) + + def test_quarantined_scenarios_excluded_and_counted(self) -> None: + run = _history_entry("m", [ + {"name": "S1", "score": 0.9, "mean_step_latency_s": 2.0}, + {"name": "S2", "score": 0.1, "harness_failed": True, "mean_step_latency_s": 50.0}, + ]) + entry = Leaderboard().from_history([run])[0] + assert entry.scenarios == {"S1": 0.9} + assert entry.composite_score == 0.9 + assert entry.n_quarantined == 1 + assert entry.mean_step_latency_s == 2.0 + md = Leaderboard().to_markdown([entry]) + assert "1 (1 quarantined)" in md and "s/step" in md + + def test_reads_cost_snapshot_block(self) -> None: + run = {**_history_entry("m", [_scenario("S1", 0.5)])} + run["cost_snapshot"] = run.pop("cost") + run["cost_snapshot"]["estimated_cost_usd"] = 3.5 + assert Leaderboard().from_history([run])[0].total_cost_usd == 3.5 diff --git a/tests/unit/test_mcp_server.py b/tests/unit/test_mcp_server.py new file mode 100644 index 0000000..8fa9689 --- /dev/null +++ b/tests/unit/test_mcp_server.py @@ -0,0 +1,157 @@ +"""RimAPI MCP server: tools execute immediately and land in the tick ledger.""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import pytest + +from rle.agents.actions import Action, ActionOutcome +from rle.harness.brief import build_brief +from rle.mcp import McpSession, NoActiveTickError, TickLedger +from rle.orchestration.action_executor import ActionExecutor +from rle.orchestration.state_manager import GameStateManager +from rle.rimapi.client import RimAPIClient +from rle.scenarios.loader import list_scenarios +from rle.testing import MockRimAPI + +mcp_server = pytest.importorskip("rle.mcp.server") +mcp_host = pytest.importorskip("rle.mcp.host") +mcp_client = pytest.importorskip("mcp.client.client") + + +@asynccontextmanager +async def _session() -> AsyncIterator[tuple[McpSession, MockRimAPI]]: + mock = MockRimAPI() + async with RimAPIClient("http://mock") as client: + mock.attach(client) + ledger = TickLedger(harness_name="test-tool") + session = McpSession(client=client, executor=ActionExecutor(client), ledger=ledger) + state = await GameStateManager(client, 30).refresh() + brief = build_brief(state, tick=0, macro_time=0.0, scenario=list_scenarios()[0]) + session.begin_tick(0, state, brief) + yield session, mock + + +def _text(result: object) -> str: + content = getattr(result, "content", None) + assert content, f"no content in {result!r}" + return str(content[0].text) + + +class TestLedger: + def test_writes_outside_tick_rejected(self) -> None: + ledger = TickLedger() + with pytest.raises(NoActiveTickError): + ledger.require_active() + + def test_finish_packages_step_result(self) -> None: + ledger = TickLedger(harness_name="x") + ledger.begin(3, 1800) + ledger.record( + Action(action_type="draft", target_colonist_id="1", parameters={"is_drafted": True}), + ActionOutcome(action_type="draft", endpoint="draft", target_colonist_id="1", + success=True, parameters={"is_drafted": True}), + ) + ledger.record( + Action(action_type="blueprint", parameters={}), + ActionOutcome(action_type="blueprint", endpoint="blueprint", success=False, + error="no coords"), + ) + ledger.end_turn("done") + step = ledger.finish() + assert not ledger.active + assert step.plan.role == "x" and step.plan.tick == 1800 + assert step.plan.summary == "done" + assert step.execution is not None + assert (step.execution.executed, step.execution.failed, step.execution.total) == (1, 1, 2) + assert step.extras["turn_ended"] is True + + +class TestToolsInMemory: + async def test_brief_state_actions_and_reads(self) -> None: + async with _session() as (session, _mock): + server = mcp_server.build_server(session) + names = {t.name for t in await server.list_tools()} + assert {"get_brief", "get_state", "list_actions", "rimapi_read", "end_turn"} <= names + assert {"work_priority", "draft", "blueprint", "growing_zone"} <= names + + brief = _text(await server.call_tool("get_brief", {})) + assert "## Scenario" in brief and "## Actions available" in brief + state = json.loads(_text(await server.call_tool("get_state", {}))) + assert state["colony"]["population"] == 3 + reads = json.loads(_text(await server.call_tool("list_reads", {}))) + assert "colonists" in reads + colonists = json.loads(_text(await server.call_tool( + "rimapi_read", {"endpoint": "colonists"}, + ))) + assert isinstance(colonists, list) and colonists[0]["name"] == "Tynan" + bad = json.loads(_text(await server.call_tool( + "rimapi_read", {"endpoint": "nope"}, + ))) + assert "Unknown read endpoint" in bad["error"] + + async def test_write_tool_executes_and_records(self) -> None: + async with _session() as (session, mock): + server = mcp_server.build_server(session) + out = json.loads(_text(await server.call_tool( + "work_priority", + {"parameters": {"Growing": 1}, "target_colonist_id": "col_01", "reason": "food"}, + ))) + assert out["ok"] is True + assert any(p.endswith("/colonist/work-priority") for p, _ in mock.posts) + assert len(session.ledger.actions) == 1 + assert session.ledger.outcomes[0].success + + ended = _text(await server.call_tool("end_turn", {"summary": "ok"})) + assert "Turn ended after 1 action(s)" in ended + step = session.ledger.finish() + assert step.execution is not None and step.execution.executed == 1 + assert step.plan.summary == "ok" + + async def test_write_after_turn_closed_is_rejected(self) -> None: + async with _session() as (session, _mock): + server = mcp_server.build_server(session) + session.ledger.finish() + out = json.loads(_text(await server.call_tool( + "draft", {"parameters": {"is_drafted": True}, "target_colonist_id": "col_01"}, + ))) + assert out["ok"] is False and "No tick is in progress" in out["error"] + assert "No tick is in progress" in _text(await server.call_tool("get_brief", {})) + + async def test_no_action_is_recorded_without_execution(self) -> None: + async with _session() as (session, mock): + before = len(mock.posts) + result = await session.act("no_action", reason="nothing to do") + assert result["ok"] is True + assert len(mock.posts) == before + assert len(session.ledger.actions) == 1 and session.ledger.outcomes == [] + + async def test_missing_pawn_id_reports_failure(self) -> None: + async with _session() as (session, _mock): + result = await session.act("draft", {"is_drafted": True}) + assert result["ok"] is False + assert "target_colonist_id is required" in (result["error"] or "") + assert session.ledger.outcomes[0].success is False + + +class TestHttpHost: + async def test_client_over_streamable_http(self) -> None: + async with _session() as (session, _mock): + host = mcp_host.McpHost(mcp_server.build_server(session)) + url = await host.start() + try: + assert url.endswith("/mcp") + async with mcp_client.Client(url) as client: + tools = await client.list_tools() + assert any(t.name == "end_turn" for t in tools.tools) + result = await client.call_tool( + "research_target", {"parameters": {"project": "Electricity"}}, + ) + assert json.loads(_text(result))["ok"] is True + assert len(session.ledger.actions) == 1 + finally: + await host.stop() + assert not host.running diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index c6a1fba..1a19672 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -17,7 +17,7 @@ def test_scoring_version_pins_a_string() -> None: that requires this test (and the dataset card) to be updated.""" assert isinstance(SCORING_VERSION, str) assert SCORING_VERSION - assert SCORING_VERSION == "1.1" + assert SCORING_VERSION == "1.2" def test_file_sha256_returns_none_for_missing_path() -> None: @@ -46,7 +46,7 @@ def test_collect_metadata_includes_scoring_version_and_seed() -> None: "git_branch", "git_dirty", "rle_version", - "felix_sdk_version", + "harness_versions", "platform", "python_version", "rimapi_dll_path", @@ -56,6 +56,12 @@ def test_collect_metadata_includes_scoring_version_and_seed() -> None: assert key in md, f"missing metadata field: {key}" +def test_collect_metadata_records_harness_describe() -> None: + md = collect_metadata(harness_describe={"harness": "x", "tool": "1.2"}) + assert md["harness_versions"] == {"harness": "x", "tool": "1.2"} + assert collect_metadata()["harness_versions"] == {} + + def test_collect_metadata_default_seed_is_none() -> None: md = collect_metadata() assert md["random_seed"] is None diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py index b893a87..10a9f8f 100644 --- a/tests/unit/test_metrics.py +++ b/tests/unit/test_metrics.py @@ -6,7 +6,7 @@ # Using the ActionPlan import for TickResult construction from rle.agents.actions import ActionPlan -from rle.orchestration.action_executor import ExecutionResult +from rle.orchestration.action_executor import ActionOutcome, ExecutionResult from rle.orchestration.game_loop import TickResult from rle.rimapi.schemas import ( ColonistData, @@ -19,12 +19,12 @@ WeatherData, ) from rle.scoring.metrics import ( + NEUTRAL, MetricContext, - communication_efficiency, - coordination, efficiency, food_security, mood, + plan_coherence, research, self_sufficiency, survival, @@ -95,14 +95,29 @@ def _ctx( ) -def _tick_result(executed: int, total: int) -> TickResult: +def _tick_result( + executed: int, total: int, outcomes: tuple[ActionOutcome, ...] = (), +) -> TickResult: return TickResult( tick=1, day=1, macro_time=0.1, plan=ActionPlan(role="test", tick=1, actions=[]), - execution=ExecutionResult(executed=executed, failed=total - executed, total=total), + execution=ExecutionResult( + executed=executed, failed=total - executed, total=total, outcomes=outcomes, + ), ) +def _ok(action_type: str, pawn: str | None = None, **params: object) -> ActionOutcome: + return ActionOutcome( + action_type=action_type, endpoint=action_type, target_colonist_id=pawn, + success=True, parameters=dict(params), + ) + + +def _coherence_tick(*outcomes: ActionOutcome) -> TickResult: + return _tick_result(len(outcomes), len(outcomes), outcomes) + + class TestSurvival: def test_all_alive(self) -> None: assert survival(_state(population=3), _ctx(initial_pop=3)) == pytest.approx(1.0) @@ -222,57 +237,109 @@ def test_mixed(self) -> None: # (0.5 + 1.0) / 2 = 0.75 assert efficiency(_state(), ctx) == pytest.approx(0.75) - def test_no_ticks(self) -> None: - assert efficiency(_state(), _ctx()) == pytest.approx(1.0) + def test_no_ticks_is_neutral(self) -> None: + assert efficiency(_state(), _ctx()) == pytest.approx(NEUTRAL) - def test_empty_plan(self) -> None: + def test_empty_plan_is_neutral(self) -> None: + """An unmanaged baseline issues no writes — it must not bank 1.0 (#51).""" ctx = _ctx(tick_results=[_tick_result(0, 0)]) - assert efficiency(_state(), ctx) == pytest.approx(1.0) + assert efficiency(_state(), ctx) == pytest.approx(NEUTRAL) + +class TestPlanCoherence: + def test_no_ticks_is_neutral(self) -> None: + assert plan_coherence(_state(), _ctx()) == pytest.approx(NEUTRAL) -class TestCoordination: - def test_no_conflicts(self) -> None: - ctx = _ctx() - assert coordination(_state(), ctx) == pytest.approx(1.0) - - def test_all_resolved(self) -> None: - ctx = _ctx() - ctx.conflicts_total = 10 - ctx.conflicts_resolved = 10 - assert coordination(_state(), ctx) == pytest.approx(1.0) - - def test_half_resolved(self) -> None: - ctx = _ctx() - ctx.conflicts_total = 8 - ctx.conflicts_resolved = 4 - assert coordination(_state(), ctx) == pytest.approx(0.5) - - def test_none_resolved(self) -> None: - ctx = _ctx() - ctx.conflicts_total = 5 - ctx.conflicts_resolved = 0 - assert coordination(_state(), ctx) == pytest.approx(0.0) - - -class TestCommunicationEfficiency: - def test_no_messages(self) -> None: - ctx = _ctx() - assert communication_efficiency(_state(), ctx) == pytest.approx(1.0) - - def test_all_acted_on(self) -> None: - ctx = _ctx() - ctx.messages_sent = 12 - ctx.messages_acted_on = 12 - assert communication_efficiency(_state(), ctx) == pytest.approx(1.0) - - def test_half_acted_on(self) -> None: - ctx = _ctx() - ctx.messages_sent = 10 - ctx.messages_acted_on = 5 - assert communication_efficiency(_state(), ctx) == pytest.approx(0.5) - - def test_none_acted_on(self) -> None: - ctx = _ctx() - ctx.messages_sent = 7 - ctx.messages_acted_on = 0 - assert communication_efficiency(_state(), ctx) == pytest.approx(0.0) + def test_no_writes_is_neutral(self) -> None: + ctx = _ctx(tick_results=[_tick_result(0, 0)]) + assert plan_coherence(_state(), ctx) == pytest.approx(NEUTRAL) + + def test_independent_writes_are_coherent(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("draft", "1", is_drafted=True), + _ok("work_priority", "2", Growing=1), + _ok("research_target", project="Electricity"), + _ok("blueprint", x=10, z=10, def_name="Wall"), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(1.0) + + def test_draft_undraft_same_pawn(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("draft", "1", is_drafted=True), + _ok("draft", "1", is_drafted=False), + _ok("work_priority", "2", Mining=1), + )]) + # 2 of 3 executed writes contradict each other + assert plan_coherence(_state(), ctx) == pytest.approx(1 / 3) + + def test_move_and_job_same_pawn(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("move", "1", x=5, z=5), + _ok("job_assign", "1", job_def="Mine"), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(0.0) + + def test_legacy_alias_resolves(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("draft_colonist", "1", is_drafted=True), + _ok("undraft_colonist", "1", is_drafted=False), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(0.0) + + def test_conflicting_work_priority(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("work_priority", "1", Growing=1), + _ok("work_priority", "1", Growing=4), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(0.0) + + def test_same_work_priority_twice_is_redundant_not_contradictory(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("work_priority", "1", Growing=1), + _ok("work_priority", "1", Growing=1), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(1.0) + + def test_overlapping_zones(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("growing_zone", x1=0, z1=0, x2=8, z2=8), + _ok("stockpile_zone", x1=4, z1=4, x2=9, z2=9), + _ok("growing_zone", x1=20, z1=20, x2=28, z2=28), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(1 / 3) + + def test_duplicate_blueprint_cell(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("blueprint", x=1, z=1, def_name="Wall"), + _ok("blueprint", x=1, z=1, def_name="Door"), + _ok("blueprint", x=2, z=1, def_name="Wall"), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(1 / 3) + + def test_competing_research_targets(self) -> None: + ctx = _ctx(tick_results=[_coherence_tick( + _ok("research_target", project="Electricity"), + _ok("research_target", project="Batteries"), + )]) + assert plan_coherence(_state(), ctx) == pytest.approx(0.0) + + def test_failed_writes_do_not_count(self) -> None: + failed = ActionOutcome( + action_type="draft", endpoint="draft", target_colonist_id="1", + success=False, error="boom", parameters={"is_drafted": False}, + ) + ctx = _ctx(tick_results=[_tick_result(1, 2, ( + _ok("draft", "1", is_drafted=True), failed, + ))]) + assert plan_coherence(_state(), ctx) == pytest.approx(1.0) + + def test_averages_across_ticks(self) -> None: + ctx = _ctx(tick_results=[ + _coherence_tick(_ok("draft", "1", is_drafted=True)), + _coherence_tick( + _ok("draft", "1", is_drafted=True), + _ok("draft", "1", is_drafted=False), + ), + _tick_result(0, 0), + ]) + assert plan_coherence(_state(), ctx) == pytest.approx((1.0 + 0.0 + NEUTRAL) / 3) diff --git a/tests/unit/test_role_agents.py b/tests/unit/test_role_agents.py index 710015f..e9dd16d 100644 --- a/tests/unit/test_role_agents.py +++ b/tests/unit/test_role_agents.py @@ -7,16 +7,16 @@ from felix_agent_sdk import AgentFactory from felix_agent_sdk.core import HelixConfig, HelixGeometry -from rle.agents import register_rle_agents from rle.agents.actions import ActionPlan -from rle.agents.base_role import _SHARED_SYSTEM_PREFIX -from rle.agents.construction_planner import ConstructionPlanner -from rle.agents.defense_commander import DefenseCommander -from rle.agents.map_analyst import MapAnalyst -from rle.agents.medical_officer import MedicalOfficer -from rle.agents.research_director import ResearchDirector -from rle.agents.resource_manager import ResourceManager -from rle.agents.social_overseer import SocialOverseer +from rle.harness.felix.agents import register_rle_agents +from rle.harness.felix.agents.base_role import _SHARED_SYSTEM_PREFIX +from rle.harness.felix.agents.construction_planner import ConstructionPlanner +from rle.harness.felix.agents.defense_commander import DefenseCommander +from rle.harness.felix.agents.map_analyst import MapAnalyst +from rle.harness.felix.agents.medical_officer import MedicalOfficer +from rle.harness.felix.agents.research_director import ResearchDirector +from rle.harness.felix.agents.resource_manager import ResourceManager +from rle.harness.felix.agents.social_overseer import SocialOverseer from rle.rimapi.schemas import GameState # ================================================================== diff --git a/tests/unit/test_scenario_loader.py b/tests/unit/test_scenario_loader.py index 9136378..19ef1af 100644 --- a/tests/unit/test_scenario_loader.py +++ b/tests/unit/test_scenario_loader.py @@ -34,7 +34,7 @@ def test_load_ship_launch(self) -> None: def test_scoring_weights_override(self) -> None: path = DEFINITIONS_DIR / "04_raid_defense.yaml" scenario = load_scenario(path) - assert scenario.scoring_weights["threat_response"] == 0.24 + assert scenario.scoring_weights["threat_response"] == 0.3 def test_invalid_path_raises(self) -> None: with pytest.raises(FileNotFoundError): diff --git a/tests/unit/test_testing_smoke.py b/tests/unit/test_testing_smoke.py new file mode 100644 index 0000000..f4cff29 --- /dev/null +++ b/tests/unit/test_testing_smoke.py @@ -0,0 +1,34 @@ +"""rle.testing.run_harness_smoke — the contract test external plugins run in CI.""" + +from __future__ import annotations + +import pytest + +from rle.testing import MockRimAPI, run_harness_smoke +from tests.conftest import requires_felix + + +@pytest.mark.parametrize( + "name", ["baseline", pytest.param("felix", marks=requires_felix)], +) +async def test_builtin_plugins_pass_smoke(name: str) -> None: + report = await run_harness_smoke(name, ticks=2) + assert report.ok + assert report.harness == name + assert len(report.ticks) == 2 + assert report.final_composite is not None + assert report.describe["harness"] == name + + +async def test_smoke_records_posts() -> None: + mock = MockRimAPI() + report = await run_harness_smoke("baseline", ticks=1, mock=mock) + # pause + unpause at minimum went through the mock + assert any(path.startswith("/api/v1/game/speed") for path, _ in report.posts) + + +async def test_smoke_validates_options() -> None: + from rle.harness import HarnessOptionsError + + with pytest.raises(HarnessOptionsError): + await run_harness_smoke("baseline", options={"nope": 1}) diff --git a/tests/unit/test_visualizer_integration.py b/tests/unit/test_visualizer_integration.py index 565b8a2..ba05853 100644 --- a/tests/unit/test_visualizer_integration.py +++ b/tests/unit/test_visualizer_integration.py @@ -5,7 +5,7 @@ from felix_agent_sdk.core import HelixConfig from felix_agent_sdk.visualization import HelixVisualizer -from rle.agents import AGENT_DISPLAY +from rle.harness.felix.agents import AGENT_DISPLAY class TestAgentDisplay: diff --git a/uv.lock b/uv.lock index fc4c818..d9053fe 100644 --- a/uv.lock +++ b/uv.lock @@ -51,6 +51,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "certifi" version = "2026.2.25" @@ -60,6 +69,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, ] +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.7" @@ -194,6 +262,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, ] +[[package]] +name = "cryptography" +version = "50.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/ad/5d6702db60b1e40b41ef513b6967ff5848f307d50f8449baf1634f5908f1/cryptography-50.0.1.tar.gz", hash = "sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20", size = 880381, upload-time = "2026-08-25T19:45:45.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/19/797e2aaac9df6a66f1550f49979dc1b1e39ecd2077501c30efa81e8d5d67/cryptography-50.0.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986", size = 4010153, upload-time = "2026-08-25T19:44:03.155Z" }, + { url = "https://files.pythonhosted.org/packages/90/34/9ce9a62ed9dc82ca9fd6a34445b6904af56e5f38b3eae2ed32e49c36053d/cryptography-50.0.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f", size = 4723133, upload-time = "2026-08-25T19:44:05.461Z" }, + { url = "https://files.pythonhosted.org/packages/57/26/e6d4fc8512a51a5f9ee7bfdbfb853bce1197087df40c9ad993ad370b846f/cryptography-50.0.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef", size = 4712478, upload-time = "2026-08-25T19:44:07.375Z" }, + { url = "https://files.pythonhosted.org/packages/e6/de/d3cdc2815697aae84126cbd6a030ca7b6b452e28a88b501b836bd3aa7a86/cryptography-50.0.1-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8", size = 4730726, upload-time = "2026-08-25T19:44:09.294Z" }, + { url = "https://files.pythonhosted.org/packages/55/32/38c0d344b98c06d34b5df8946565a9c0d6dbf32c8e0730a7f05f0a3c6cab/cryptography-50.0.1-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45", size = 5353524, upload-time = "2026-08-25T19:44:11.96Z" }, + { url = "https://files.pythonhosted.org/packages/e1/1b/82f0f0d8858d4432be1af790477edf62aef90324041aa07c57e57bef1af7/cryptography-50.0.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad", size = 4746720, upload-time = "2026-08-25T19:44:14.051Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/042ca458b8c64348c768284b5d23e69b92ed53d057ab779fee628564676d/cryptography-50.0.1-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49", size = 4361866, upload-time = "2026-08-25T19:44:16.167Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/e96c1ef71edef71057c7e3c3d982ce8fda554e0c52d0cc19c18845cde3eb/cryptography-50.0.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f", size = 4730028, upload-time = "2026-08-25T19:44:18.085Z" }, + { url = "https://files.pythonhosted.org/packages/e3/38/45abd72ef63f2e7d0754a6cacf97bd8b69512ace7f6130d24c39ece65da2/cryptography-50.0.1-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527", size = 5308405, upload-time = "2026-08-25T19:44:20.197Z" }, + { url = "https://files.pythonhosted.org/packages/85/66/6ccca4722987ddedaa7fc9c3f4708af7431f5535666c174350830888c6b7/cryptography-50.0.1-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a", size = 4746230, upload-time = "2026-08-25T19:44:22.376Z" }, + { url = "https://files.pythonhosted.org/packages/13/0e/b1f92e013228111413f2e6743948b80bc24dfd3c1b87ba98ceea16f5df89/cryptography-50.0.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959", size = 4862596, upload-time = "2026-08-25T19:44:24.472Z" }, + { url = "https://files.pythonhosted.org/packages/7e/22/c3654cccc856e9d682817b04ac3ee79731cb09ca6f95996a95c904de2883/cryptography-50.0.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b", size = 5014082, upload-time = "2026-08-25T19:44:26.709Z" }, + { url = "https://files.pythonhosted.org/packages/42/8b/cb12b1b60c91b074ca6bf0fdd59aa8f10d8bc5f73af8faece86ef0421b37/cryptography-50.0.1-cp311-abi3-win_amd64.whl", hash = "sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648", size = 3842826, upload-time = "2026-08-25T19:44:28.784Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f0/424cb557d99aa86ac55da5e2add02e2882e44047b6264f93ade1b975a993/cryptography-50.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f", size = 3973525, upload-time = "2026-08-25T19:44:30.7Z" }, + { url = "https://files.pythonhosted.org/packages/4d/72/3a2711d967977ab5fc80b782837c7e8d1ac7445e764c20c381a265c57ef3/cryptography-50.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a", size = 4708817, upload-time = "2026-08-25T19:44:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f2/bb1f56e10815b789df0b409a69fa4992ff3d3fef9c72747f4a6b26fed38e/cryptography-50.0.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367", size = 4697300, upload-time = "2026-08-25T19:44:35.144Z" }, + { url = "https://files.pythonhosted.org/packages/08/bd/ed5396be499ffcf8807a585bfe38b71a1fbdd1c342b4f9b6d0ef5162a946/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5", size = 4716039, upload-time = "2026-08-25T19:44:37.192Z" }, + { url = "https://files.pythonhosted.org/packages/f6/6e/1cf405c5c8e8df7545378048e954792f00b7f2367af8863ce8b8f3e10607/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9", size = 5332388, upload-time = "2026-08-25T19:44:39.16Z" }, + { url = "https://files.pythonhosted.org/packages/47/92/b4317e8c32c4f47b062f5398bd79106b220a124546f42be83bf32b761e2a/cryptography-50.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0", size = 4730293, upload-time = "2026-08-25T19:44:41.298Z" }, + { url = "https://files.pythonhosted.org/packages/39/0d/a1e7633e2c744d0f2983320a27e924ef2264c79c56e1a58d5fb0a1cfd413/cryptography-50.0.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc", size = 4346031, upload-time = "2026-08-25T19:44:43.245Z" }, + { url = "https://files.pythonhosted.org/packages/88/dd/b215616f9bab3fc18510c78a4e5c9f362d77838503c363dc747c7d4f5c6f/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17", size = 4715344, upload-time = "2026-08-25T19:44:45.291Z" }, + { url = "https://files.pythonhosted.org/packages/b1/1b/ec3ebd31741d0e963612c4fe43caa39341b9b1e031e469820e42e4c83918/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6", size = 5287201, upload-time = "2026-08-25T19:44:47.297Z" }, + { url = "https://files.pythonhosted.org/packages/1a/01/0127d11a762b31a9ee0221894f540318761783f3fdc4bc5d057698caebd5/cryptography-50.0.1-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3", size = 4730023, upload-time = "2026-08-25T19:44:49.435Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b9/e7425ebfb599241a0c1d7000f1b466c3062da66c19d9525031315dff7213/cryptography-50.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6", size = 4847362, upload-time = "2026-08-25T19:44:51.94Z" }, + { url = "https://files.pythonhosted.org/packages/2d/fd/60d0ddf4defa12e482c9d5e0f554384d6e8ab25341fd15f060028fd92e6a/cryptography-50.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149", size = 4999247, upload-time = "2026-08-25T19:44:53.876Z" }, + { url = "https://files.pythonhosted.org/packages/4d/56/bc4f2b209e766c93372cfcd59b781a0b2b59700f62a969580415b699c2b2/cryptography-50.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf", size = 3825806, upload-time = "2026-08-25T19:44:56.209Z" }, + { url = "https://files.pythonhosted.org/packages/84/a9/ee16a903f13755e914d1eecc482fe64d1f10761c3960e5d8fa6837377aff/cryptography-50.0.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0", size = 4035307, upload-time = "2026-08-25T19:44:58.305Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a5/9ec7e81e8526c0d7a387d73386b2daed3f39e10d81a85930bd1b6bfba65c/cryptography-50.0.1-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23", size = 4751900, upload-time = "2026-08-25T19:45:00.401Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3c/0e77bd5ffcf078e9dd27d3074aad6c030d9b10d0bf69329d573c927a188c/cryptography-50.0.1-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733", size = 4738357, upload-time = "2026-08-25T19:45:02.786Z" }, + { url = "https://files.pythonhosted.org/packages/27/3a/3c5f80daa4dcd47323c7af8a2fcb90de27a33564d4fcac69846c0972691a/cryptography-50.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88", size = 4758474, upload-time = "2026-08-25T19:45:04.889Z" }, + { url = "https://files.pythonhosted.org/packages/6e/2b/214cf0cf93db9628c3c20c896b229f327f6fb1b20e4b3743d8ad3f00af8b/cryptography-50.0.1-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054", size = 5375862, upload-time = "2026-08-25T19:45:07.163Z" }, + { url = "https://files.pythonhosted.org/packages/d6/51/3f9701867a46b6c1740c9b52fc4d3bed6cbdcfedcc9b6e64305c07f39cff/cryptography-50.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5", size = 4772942, upload-time = "2026-08-25T19:45:09.396Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5c/13ea642e08e2544d0f5396122055f4820cfacb3203562197b5967125ea97/cryptography-50.0.1-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361", size = 4383347, upload-time = "2026-08-25T19:45:11.659Z" }, + { url = "https://files.pythonhosted.org/packages/84/d5/7d1fe1cb93f91c428093ff234e128c89ba8ea61a6f26aab406081f9b996e/cryptography-50.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71", size = 4758050, upload-time = "2026-08-25T19:45:13.745Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/557fc5ead96a829e0bc812a3b9dc4a52a2f27e4f7f5950da7ff27653a805/cryptography-50.0.1-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80", size = 5332955, upload-time = "2026-08-25T19:45:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/eb/5d7124083e8d8cda8f5b348f544b71ad6f707ad63193758ef4d8e569da02/cryptography-50.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239", size = 4772694, upload-time = "2026-08-25T19:45:18.315Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/f1f955e0921dd2b6d22eae7e8d24a4c4b638d10735ffbf6a71f99eb0fcb8/cryptography-50.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558", size = 4888413, upload-time = "2026-08-25T19:45:20.4Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ab/89e2b798d2c3925f82e2bb72d5979f3d2f6da2dd22ef4a8cd8b70d920039/cryptography-50.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e", size = 5044355, upload-time = "2026-08-25T19:45:22.353Z" }, + { url = "https://files.pythonhosted.org/packages/99/89/87ef49ffe383ef4e147d27b7bf2088fb0b54ea409dd87b5a89442e5828a5/cryptography-50.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2", size = 3875429, upload-time = "2026-08-25T19:45:24.418Z" }, +] + [[package]] name = "cycler" version = "0.12.1" @@ -364,6 +482,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -379,6 +510,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "huggingface-hub" version = "1.9.2" @@ -401,11 +557,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] [[package]] @@ -450,6 +606,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/8e/7def204fea9f9be8b3c21a6f2dd6c020cf56c7d5ff753e0e23ed7f9ea57e/jiter-0.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2c26cf47e2cad140fa23b6d58d435a7c0161f5c514284802f25e87fddfe11024", size = 187152, upload-time = "2026-02-02T12:37:22.124Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "kiwisolver" version = "1.5.0" @@ -567,6 +750,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/49/d651878698a0b67f23aa28e17f45a6d6dd3d3f933fa29087fa4ce5947b5a/matplotlib-3.10.8-cp314-cp314t-win_arm64.whl", hash = "sha256:113bb52413ea508ce954a02c10ffd0d565f9c3bc7f2eddc27dfe1731e71c7b5f", size = 8192560, upload-time = "2025-12-10T22:56:38.008Z" }, ] +[[package]] +name = "mcp" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx2" }, + { name = "jsonschema" }, + { name = "mcp-types" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/6e/21fb8e5d579dbe21d96ea4d5034200d46d8bdf2261053b5bd041f3c2f612/mcp-2.1.1.tar.gz", hash = "sha256:50b7ba1ebbe117008ea7bdd288234043e69c20b403d6851d19661e6d431a75ef", size = 3984589, upload-time = "2026-08-25T16:14:02.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/af/8644cc5fa26a59afd2df2e98eeb19e72926887fa4b7441aba4ff661140db/mcp-2.1.1-py3-none-any.whl", hash = "sha256:1c6c31c5d6471c58db76af3af8af67f46d11d01f0a59077d0a308cbdb3d3e915", size = 357912, upload-time = "2026-08-25T16:13:59.024Z" }, +] + +[[package]] +name = "mcp-types" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/dd/1c4417dc0b722c23a1669032d5f044e41170fe5d4773b488a50fcce98c32/mcp_types-2.1.1.tar.gz", hash = "sha256:77dcbe48fba73cca71a673f2646a5f037a017b7a0a07ac89cec1113028890eda", size = 66674, upload-time = "2026-08-25T16:14:03.861Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/d0/242e63c510f4a17381f55b1549a3f94f5687a0595984febd2b6f87a687a0/mcp_types-2.1.1-py3-none-any.whl", hash = "sha256:26f9f7f03f2a5730717a5b98e2ab7eb640ac352d05a00cdc725c311864778295", size = 69656, upload-time = "2026-08-25T16:14:00.667Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -662,6 +883,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/bc/a8f7c3aa03452fedbb9af8be83e959adba96a6b4a35e416faffcc959c568/openai-2.31.0-py3-none-any.whl", hash = "sha256:44e1344d87e56a493d649b17e2fac519d1368cbb0745f59f1957c4c26de50a0a", size = 1153479, upload-time = "2026-04-08T21:01:39.217Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -746,6 +979,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -823,6 +1065,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pyparsing" version = "3.3.2" @@ -895,6 +1151,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] +[[package]] +name = "python-multipart" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, +] + +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -921,6 +1199,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + [[package]] name = "requests" version = "2.33.1" @@ -951,10 +1242,9 @@ wheels = [ [[package]] name = "rimworld-learning-environment" -version = "0.3.0" +version = "0.4.1" source = { editable = "." } dependencies = [ - { name = "felix-agent-sdk" }, { name = "httpx" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -976,9 +1266,15 @@ dev = [ { name = "ruff" }, { name = "types-pyyaml" }, ] +felix = [ + { name = "felix-agent-sdk" }, +] local = [ { name = "felix-agent-sdk", extra = ["local"] }, ] +mcp = [ + { name = "mcp" }, +] openai = [ { name = "felix-agent-sdk", extra = ["openai"] }, ] @@ -992,7 +1288,7 @@ viz = [ [package.metadata] requires-dist = [ - { name = "felix-agent-sdk", specifier = ">=0.3.0" }, + { name = "felix-agent-sdk", marker = "extra == 'felix'", specifier = ">=0.3.0" }, { name = "felix-agent-sdk", extras = ["all"], marker = "extra == 'all'" }, { name = "felix-agent-sdk", extras = ["anthropic"], marker = "extra == 'anthropic'" }, { name = "felix-agent-sdk", extras = ["local"], marker = "extra == 'local'" }, @@ -1000,6 +1296,7 @@ requires-dist = [ { name = "httpx", specifier = ">=0.24" }, { name = "huggingface-hub", marker = "extra == 'tracking'", specifier = ">=0.20" }, { name = "matplotlib", marker = "extra == 'viz'", specifier = ">=3.5" }, + { name = "mcp", marker = "extra == 'mcp'", specifier = ">=2.1" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pydantic-settings", specifier = ">=2.0" }, @@ -1011,7 +1308,73 @@ requires-dist = [ { name = "types-pyyaml", marker = "extra == 'dev'", specifier = ">=6.0" }, { name = "wandb", marker = "extra == 'tracking'", specifier = ">=0.15" }, ] -provides-extras = ["anthropic", "openai", "local", "all", "dev", "viz", "tracking"] +provides-extras = ["felix", "anthropic", "openai", "local", "all", "mcp", "dev", "viz", "tracking"] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, +] [[package]] name = "ruff" @@ -1087,6 +1450,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sse-starlette" +version = "3.4.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/8a41e88e825ea26c44333897c7ffe35fe60153a2cfc097a5bd1d209ad281/sse_starlette-3.4.10.tar.gz", hash = "sha256:c6c87280d8feb4e55a8d79633782766b9cac6a26da5c79a145d00aa404117a86", size = 33720, upload-time = "2026-09-03T09:36:24.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/3c/96018a51c7301a64f7b0579d9ce8f9b69dd39ca8ed5aa100ba3feadee503/sse_starlette-3.4.10-py3-none-any.whl", hash = "sha256:710f5f5b0527409903a22a91699db02f76f4c2eb9204e882e4ee7cada76bdf75", size = 17120, upload-time = "2026-09-03T09:36:22.56Z" }, +] + +[[package]] +name = "starlette" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/b4/205b0d5241d934e8add0c38aa924c4f9fb7330834ff11e5444db964ec3f9/starlette-1.6.0.tar.gz", hash = "sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b", size = 2716969, upload-time = "2026-08-08T18:27:57.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl", hash = "sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c", size = 75969, upload-time = "2026-08-08T18:27:56.196Z" }, +] + [[package]] name = "tqdm" version = "4.67.3" @@ -1099,6 +1487,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "typer" version = "0.24.1" @@ -1153,6 +1550,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, ] +[[package]] +name = "uvicorn" +version = "0.52.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/0f/3f86e61397dd33bf2ccf28188c40db6a740658aeebbbf6e7dbc101a1f487/uvicorn-0.52.4.tar.gz", hash = "sha256:73acfee47a0b133c5de13d219492d62d8a31e935f4fe6e41a232451a15379f86", size = 100627, upload-time = "2026-08-19T06:27:41.821Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/79/4a20b54ab0491485ccd8c077db2d39187c7f12b3e15485d38a7be37c81b4/uvicorn-0.52.4-py3-none-any.whl", hash = "sha256:f86e41a149d7d05a9969337e3946a9c171c06a5d42680896daaba624aeac8da1", size = 79871, upload-time = "2026-08-19T06:27:40.36Z" }, +] + [[package]] name = "wandb" version = "0.25.1"