feat(sdk): implement the outbound Evaluator v2 worker runtime

sdk/python/CHANGELOG.md

@@ -16,6 +16,10 @@ moved the version here automatically; nothing has landed against `0.0.1b2` yet.
 Add entries as changes merge — this section becomes the GitHub Release body when
 it ships.
 
+- Retire the old inbound evaluator boundary and add evaluator authoring plus the
+  outbound-only v2 worker runtime under the lazy `failproofai_sdk.evaluator`
+  namespace.
+
 ## 0.0.1b1 — 2026-08-24
 
 The first release under this name. Everything below describes the package as it

sdk/python/README.md

@@ -10,6 +10,14 @@ the platform.
 - **Dependencies:** none. Standard library only, so installing it constrains
   nothing else in your environment.
 
+## Evaluator v2 status
+
+The legacy inbound `agenteye-evaluator` package has been retired; do not build new
+evaluator services against its server-push HTTP contract. Evaluator v2 authoring
+and its customer-hosted, outbound-only worker runtime live under the lazy
+`failproofai_sdk.evaluator` namespace. Importing the top-level tracing SDK does not
+import or start the evaluator runtime.
+
 ## Installation
 
 ```bash

sdk/python/examples/evaluator_worker.py

@@ -0,0 +1,121 @@
+"""Customer evaluator with deterministic and optional async judge checks."""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import json
+import os
+from urllib.parse import urlsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+from failproofai_sdk.evaluator import (
+    ConditionResult,
+    EvalResult,
+    Evaluator,
+    Metric,
+    Score,
+)
+
+app = Evaluator(name="customer-production", version="2026.08.1")
+
+
+class _RejectRedirects(HTTPRedirectHandler):
+    def redirect_request(self, request, file_pointer, code, message, headers, new_url):
+        return None
+
+
+@app.eval(
+    "tool_efficiency",
+    version="1.0.0",
+    labels=["tools", "deterministic"],
+    when=lambda session: ConditionResult(
+        session.count("tool_use") > 0, "no_tool_calls"
+    ),
+)
+def tool_efficiency(session):
+    calls = session.events_of_type("tool_use")
+    distinct = {
+        event.payload.get("tool_name")
+        for event in calls
+        if event.payload.get("tool_name")
+    }
+    value = len(distinct) / len(calls)
+    return EvalResult(
+        score=Score(value, passed=value >= 0.7),
+        metrics={
+            "tool_call_count": Metric(len(calls), unit="events"),
+            "distinct_tool_count": Metric(len(distinct), unit="tools"),
+        },
+        reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls",
+    )
+
+
+def _judge_configured(session):
+    configured = bool(os.environ.get("EXAMPLE_JUDGE_URL"))
+    return ConditionResult(configured, "judge_not_configured")
+
+
+def _last_content(session, event_type):
+    events = session.events_of_type(event_type)
+    if not events:
+        return None
+    payload = events[-1].payload
+    fields = {
+        "human_input": ("response",),
+        "model_response": ("content",),
+        "agent_end": ("summary",),
+    }.get(event_type, ("content", "summary", "response"))
+    return next((payload.get(field) for field in fields if payload.get(field)), None)
+
+
+def _call_judge(question, answer):
+    url = os.environ["EXAMPLE_JUDGE_URL"]
+    parsed = urlsplit(url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise ValueError("EXAMPLE_JUDGE_URL must be an absolute http(s) URL")
+    hostname = parsed.hostname
+    loopback = hostname == "localhost"
+    if hostname is not None and not loopback:
+        try:
+            loopback = ipaddress.ip_address(hostname).is_loopback
+        except ValueError:
+            loopback = False
+    if parsed.scheme != "https" and not loopback:
+        raise ValueError("EXAMPLE_JUDGE_URL must use https unless it targets loopback")
+    token = os.environ.get("EXAMPLE_JUDGE_TOKEN")
+    body = json.dumps({"question": question, "answer": answer}).encode("utf-8")
+    headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    request = Request(url, data=body, headers=headers, method="POST")
+    with build_opener(_RejectRedirects()).open(request, timeout=25) as response:  # nosec B310
+        result = json.loads(response.read(64 * 1024))
+    return float(result["score"]), str(
+        result.get("reasoning") or "Judge returned no reasoning"
+    )
+
+
+@app.eval(
+    "answer_relevance",
+    version="judge-api-v1",
+    labels=["llm_judge", "relevance"],
+    when=_judge_configured,
+    timeout_seconds=30,
+)
+async def answer_relevance(session):
+    question = _last_content(session, "human_input")
+    answer = _last_content(session, "model_response")
+    if question is None or answer is None:
+        raise ValueError("answer relevance requires human input and model output")
+    value, reasoning = await asyncio.to_thread(_call_judge, question, answer)
+    value = min(max(value, 0.0), 1.0)
+    return EvalResult(
+        score=Score(value, passed=value >= 0.7),
+        reasoning=reasoning,
+        labels=("llm_judge", "relevance"),
+    )
+
+
+if __name__ == "__main__":
+    app.run_from_env()

sdk/python/failproofai_sdk/evaluator/__init__.py

@@ -0,0 +1,101 @@
+"""Authoring and worker primitives for FailproofAI Evaluator v2.
+
+This namespace is intentionally lazy relative to :mod:`failproofai_sdk`: users
+who only emit telemetry do not import evaluator networking or runtime code.
+"""
+
+from failproofai_sdk.evaluator.authoring import (
+    Assertion,
+    ConditionResult,
+    EvalDefinition,
+    EvalResult,
+    Evaluator,
+    Metric,
+    Score,
+)
+from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient
+from failproofai_sdk.evaluator.protocol import (
+    Assignment,
+    AssignmentDefinition,
+    CatalogDefinition,
+    ClaimRequest,
+    ClaimResponse,
+    ErrorResponse,
+    EvalSelection,
+    ExecutionMode,
+    EvaluatorKind,
+    HeartbeatRequest,
+    HeartbeatResponse,
+    HeartbeatRun,
+    PlannedRun,
+    PlanRequest,
+    PlanResponse,
+    DefinitionsResponse,
+    ProtocolError,
+    RegisterRequest,
+    RegisterResponse,
+    RemoteError,
+    ResultItem,
+    ResultKind,
+    ResultRequest,
+    ResultResponse,
+    SessionTranscript,
+    SkippedEval,
+    TerminalRunStatus,
+    TranscriptEvent,
+    UnsupportedProtocolVersion,
+)
+from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime
+from failproofai_sdk.evaluator.source import (
+    UnsafeEvaluatorSource,
+    compile_condition,
+    compile_evaluator,
+    source_checksum,
+)
+
+__all__ = [
+    "Assertion",
+    "Assignment",
+    "AssignmentDefinition",
+    "CatalogDefinition",
+    "ClaimRequest",
+    "ClaimResponse",
+    "ConditionResult",
+    "ErrorResponse",
+    "EvalDefinition",
+    "EvalResult",
+    "EvalSelection",
+    "ExecutionMode",
+    "Evaluator",
+    "EvaluatorAPIError",
+    "EvaluatorClient",
+    "EvaluatorKind",
+    "HeartbeatRequest",
+    "HeartbeatResponse",
+    "HeartbeatRun",
+    "Metric",
+    "PlanRequest",
+    "PlanResponse",
+    "DefinitionsResponse",
+    "PlannedRun",
+    "ProtocolError",
+    "RegisterRequest",
+    "RegisterResponse",
+    "RemoteError",
+    "ResultItem",
+    "ResultKind",
+    "ResultRequest",
+    "ResultResponse",
+    "Score",
+    "SessionTranscript",
+    "SkippedEval",
+    "TerminalRunStatus",
+    "TranscriptEvent",
+    "UnsupportedProtocolVersion",
+    "WorkerConfig",
+    "WorkerRuntime",
+    "UnsafeEvaluatorSource",
+    "compile_condition",
+    "compile_evaluator",
+    "source_checksum",
+]

sdk/python/failproofai_sdk/evaluator/__main__.py

@@ -0,0 +1,49 @@
+"""Run an evaluator declared as ``module:attribute``."""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import os
+from collections.abc import Sequence
+
+from failproofai_sdk.evaluator.authoring import Evaluator
+
+
+def load_evaluator(spec: str) -> Evaluator:
+    module_name, separator, attribute = spec.partition(":")
+    if not module_name:
+        raise ValueError("evaluator module must not be empty")
+    if not separator:
+        attribute = "app"
+    if not attribute:
+        raise ValueError("evaluator attribute must not be empty")
+    module = importlib.import_module(module_name)
+    try:
+        evaluator = getattr(module, attribute)
+    except AttributeError as error:
+        raise ValueError(f"{spec!r} does not define {attribute!r}") from error
+    if not isinstance(evaluator, Evaluator):
+        raise TypeError(
+            f"{spec!r} resolved to {type(evaluator).__name__}, not Evaluator"
+        )
+    return evaluator
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m failproofai_sdk.evaluator")
+    parser.add_argument(
+        "module",
+        nargs="?",
+        default=os.environ.get("FAILPROOFAI_EVALUATOR_MODULE"),
+        help="Python module and optional attribute (for example my_evals:app)",
+    )
+    args = parser.parse_args(argv)
+    if not args.module:
+        parser.error("module is required (or set FAILPROOFAI_EVALUATOR_MODULE)")
+    load_evaluator(args.module).run_from_env()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())