diff --git a/.agents/plans/03-assurance-hardening/decisions.tsv b/.agents/plans/03-assurance-hardening/decisions.tsv index 220cc55..80df1b5 100644 --- a/.agents/plans/03-assurance-hardening/decisions.tsv +++ b/.agents/plans/03-assurance-hardening/decisions.tsv @@ -23,3 +23,11 @@ ts phase decision why evidence result 2026-08-28T01:50:22Z phase-3 kept zero-inode files ineligible the same path and opened handle cannot be proven identical without a usable filesystem identity; accepting metadata similarity would weaken the phase guarantee architecture judge; three reviewers; Node filesystem contract fail closed, with macOS and Windows filesystem suites added to CI 2026-08-28T01:56:55Z phase-3 built deterministic filesystem race checkpoints static symlink tests did not execute the replacement and mutation branches the assurance claim depends on tests/test-results.test.ts leaf replacement before and after open, parent replacement, growth, and truncation all fail closed 2026-08-28T01:56:55Z phase-3 completed filesystem report assurance the final diff passed deslop, comment review, three-model interrogate, full product verification, and cassette replay bun run check; bun run replay 572 pass, 1 intentional skip, 13 of 13 replays, 892 source bytes headroom +2026-08-28T02:12:45Z phase-3 merged stable report assurance the exact head reproduced both parent regressions and passed independent shipping verification plus Windows and macOS CI PR 50; merge 51ea890 merged to main +2026-08-28T02:12:45Z phase-4 selected narrow runtime, evaluator, and ledger boundaries the independent judge chose explicit provider and host values, hard evaluator failures, and campaign-stopping persistence failures without synthetic attempts three architecture candidates; independent phase2_ship_verify judge candidate A base with B discriminated result and C integrity semantics +2026-08-28T02:12:45Z phase-4 captured failure-origin regressions before implementation the runner and qualifier must not hide grader or store failures as environment gaps four focused test files four failures and one missing-module error as expected +2026-08-28T02:50:17Z phase-4 made failure origin structural across every eval runner optional environment and error flags let one broad catch rewrite grader, provider, host, and store failures as host gaps evals/failure-origin.ts; evals/run.ts; evals/benchmark-run.ts; evals/reviewer-run.ts provider errors come from assistant error events, host operations are tagged, unknown processing defaults evaluator, store errors stop campaigns +2026-08-28T02:50:17Z phase-4 made evaluator integrity failures hard and non-retryable an evaluator crash cannot become a smaller sample or spend reserve attempts evals/analysis.ts; evals/harness.ts; focused failure and scheduler tests NOT VERIFIED for required evaluator failures; paid queues drain in-flight work and start nothing new +2026-08-28T02:50:17Z phase-4 kept persistence outside durable attempt evidence a store that cannot publish the attempt cannot truthfully publish a second attempt claiming the store failed evals/report.ts; persistEvaluation; preservePrimaryFailure persistence is a campaign stop only; primary errors retain cleanup diagnostics; fabricated persistence attempts are schema-invalid +2026-08-28T02:50:17Z phase-4 completed failure-origin verification the corrected design passed deslop, comment review, three-model interrogation, full product verification, and replay bun run check; bun run replay 588 pass, 1 intentional skip, 13 of 13 replays, 892 source bytes headroom +2026-08-28T02:50:59Z phase-4 closed final concurrent and provenance review findings last-pass reviewers found a lost in-flight persistence error, silent host reads, one extra paired arm, and provider fidelity still labelled host evals/harness.ts; evals/benchmark-run.ts; evals/cassette.ts; focused concurrency and cassette tests 589 pass, 1 intentional skip, 13 of 13 replays, no remaining Sol or 5.4 findings diff --git a/docs/release-qualification.md b/docs/release-qualification.md index 504184c..f56466a 100644 --- a/docs/release-qualification.md +++ b/docs/release-qualification.md @@ -33,10 +33,9 @@ forces a decision about what its result is allowed to mean. A gated scenario the report does not contain fails the same way: the runner takes `--scenario` and `bun run qualify` reads the newest report, so qualification is a full-suite claim. -An excluded attempt is not a smaller sample but a missing one: the runner drops one -that aborted mid-flight, or asked where the scenario does not allow it, so a gated -pair below the floor — or holding any abort — means re-running it, not reading the -remainder as its rate. +A non-product attempt never shrinks the required sample. Provider or host failure, +or an unallowed ask, leaves an evidence gap. Evaluator failure is `NOT VERIFIED`; +persistence failure stops without a finalized report. Re-run only external gaps. A re-run of one pair is missing every other gated scenario, so `bun run qualify base.json rerun.json` takes the pairs the later report measured and diff --git a/evals/analysis.ts b/evals/analysis.ts index 36a7a83..4ee1045 100644 --- a/evals/analysis.ts +++ b/evals/analysis.ts @@ -79,6 +79,7 @@ export type DecisionReason = { | "false-completion" | "unsubmitted-review" | "below-pass-rate" + | "campaign-integrity-failure" | "campaign-stopped" | "missing-attempt" | "unscored-attempt" @@ -479,6 +480,17 @@ export function deriveReleaseDecision(input: { ) .map((cell) => cell.cellId), ); + if ( + report.completion.status === "stopped" && + report.completion.cause === "persistence" + ) { + decisionReason( + reasons, + "hard", + "campaign-integrity-failure", + `Campaign stopped after a ${report.completion.cause} failure.`, + ); + } if ( promotionArtifact && !samePackedArtifact(expected.artifact, promotionArtifact) @@ -508,6 +520,19 @@ export function deriveReleaseDecision(input: { if (!requiredKeys.has(`${attempt.caseId}\u0000${attempt.caseVersion}`)) { continue; } + if ( + attempt.outcome.kind === "failure" && + attempt.outcome.origin === "evaluator" + ) { + decisionReason( + reasons, + "hard", + "campaign-integrity-failure", + `Attempt ${attempt.attemptId} failed in ${attempt.outcome.origin} code.`, + attempt.caseId, + attempt.caseVersion, + ); + } if (attempt.outcome.kind !== "product") continue; const evidence = attempt.outcome.evidence; if ("falseCompletion" in evidence && evidence.falseCompletion) { diff --git a/evals/benchmark-run.ts b/evals/benchmark-run.ts index 348f6de..d3fe5e9 100644 --- a/evals/benchmark-run.ts +++ b/evals/benchmark-run.ts @@ -17,6 +17,18 @@ import { revealPairedAnalysis, scanPairedTranscript, } from "./experiment.js"; +import { + type AttemptFailure, + type DurableFailureOrigin, + EvaluationPersistenceError, + EvaluationPhaseError, + evaluationPhase, + evaluatorFailure, + failureOutcome, + isEvaluatorFailure, + persistEvaluation, + preservePrimaryFailure, +} from "./failure-origin.js"; import { type CommandEnd, EvalHost, @@ -312,10 +324,14 @@ async function main(): Promise { "results", `paired-${new Date().toISOString().replace(/[:.]/g, "-")}.v2`, ); - await mkdir(join(root, "evals", "results"), { recursive: true }); + await persistEvaluation("report-directory", () => + mkdir(join(root, "evals", "results"), { recursive: true }), + ); const store = createReportStore({ directory, catalog }); - await store.initialize(experiment.plan); - await store.writeCatalog(catalog); + await persistEvaluation("initialize", () => + store.initialize(experiment.plan), + ); + await persistEvaluation("catalog", () => store.writeCatalog(catalog)); const packDir = await mkdtemp(join(tmpdir(), "flow-paired-pack-")); const attempts: AttemptRecordV2[] = []; const scans: ReturnType[] = []; @@ -328,7 +344,7 @@ async function main(): Promise { repositoryRoot: root, tarballPath: tarball, }); - await store.writeArtifact(tarball); + await persistEvaluation("artifact", () => store.writeArtifact(tarball)); const evaluator = evaluatorIdentity({ sourceCommit: artifact.sourceCommit, caseCatalog: selected.map((c) => ({ @@ -401,10 +417,10 @@ async function main(): Promise { ): Promise<{ readonly nonProduct: boolean; readonly budget: boolean; - readonly origin: "host" | "evaluator" | null; + readonly origin: DurableFailureOrigin | null; }> => { let nonProduct = false; - let blockOrigin: "host" | "evaluator" | null = null; + let blockOrigin: DurableFailureOrigin | null = null; for (const cell of block.cells) { if ( attempts.length >= experiment.plan.budget.maxAttempts || @@ -418,123 +434,163 @@ async function main(): Promise { const cellStarted = Date.now(); let host: EvalHost | null = null; let recorded = false; - let failureOrigin: "host" | "evaluator" = "host"; + let runFailure: AttemptFailure | null = null; let failureUsage: AttemptRecordV2["usage"] = { durationMs: 0, outputTokens: 0, costUsd: null, }; - try { - host = await EvalHost.start({ - toolchain, - packageCache: cache, - opencodeVersion, - files: benchmark.files, - withFlow: flow, - }); - const session = await host.createSession("paired task"); - let error: string | null = null; - let commandEnd: CommandEnd = "quiet"; - try { - commandEnd = flow - ? await host.runCommand( - session, - "flow-auto", - benchmark.prompt, - options.model, - ) - : await host.runPrompt(session, benchmark.prompt, options.model); - } catch (caught) { - error = caught instanceof Error ? caught.message : String(caught); - } - const outcome = await host.outcome( - [session], - Date.now() - cellStarted, - ); - failureUsage = { - durationMs: outcome.durationMs, - outputTokens: outcome.tokens.output, - costUsd: outcome.costUsd, - }; - if (error || outcome.hostError) { - throw new Error(error ?? outcome.hostError ?? "host-error"); - } - failureOrigin = "evaluator"; - const grade = await benchmark.grade(host.project); - const transcript = redactTranscript({ - projectPath: host.project, - value: { calls: outcome.allCalls, finalText: outcome.finalText }, - }); - const stored = await store.writeTranscript({ - attemptId: `attempt-${cell.cellId}`, - text: transcript.text, - }); - scans.push(scanPairedTranscript(transcript.text)); - const attempt = productAttempt({ - cell, - benchmark, - outcome, - artifact: flow ? artifact : { kind: "ordinary-opencode" }, - evaluator, - hostConfig: hostConfigSha256({ - opencodeVersion, - model: options.model, - flow, - }), - transcript: stored, - requested, - flow, - hiddenCorrectness: grade.passed, - gradeIssues: grade.issues, - endedBy: commandEnd, - }); - await store.writeAttempt(attempt); - attempts.push(attempt); - recorded = true; - } catch (caught) { - if (recorded) throw caught; - const message = - caught instanceof Error ? caught.message : String(caught); - const failure: AttemptRecordV2 = { - schemaVersion: 2, - attemptId: `attempt-${cell.cellId}`, - cellId: cell.cellId, - blockId: cell.blockId, - caseId: cell.caseId, - caseVersion: cell.caseVersion, - armToken: cell.armToken, - repetition: cell.repetition, - artifact: flow ? artifact : { kind: "ordinary-opencode" }, - evaluator, - hostConfigSha256: hostConfigSha256({ - opencodeVersion, - model: options.model, - flow, - }), - actors: [], - instructions: [], - transcript: null, - outcome: { - kind: "failure", - origin: failureOrigin, - code: message.slice(0, 512), - retryable: true, - }, - usage: - failureUsage.durationMs > 0 - ? failureUsage - : { - durationMs: Date.now() - cellStarted, - outputTokens: 0, - costUsd: null, - }, - }; - await store.writeAttempt(failure); - attempts.push(failure); - nonProduct = true; - blockOrigin = failureOrigin; - } finally { - await host?.stop(); + await preservePrimaryFailure( + async () => { + try { + host = await EvalHost.start({ + toolchain, + packageCache: cache, + opencodeVersion, + files: benchmark.files, + withFlow: flow, + }); + const activeHost = host; + const session = await evaluationPhase( + "host", + "session-create-failed", + true, + () => activeHost.createSession("paired task"), + ); + let commandEnd: CommandEnd = "quiet"; + try { + commandEnd = flow + ? await evaluationPhase("host", "command-aborted", true, () => + activeHost.runCommand( + session, + "flow-auto", + benchmark.prompt, + options.model, + ), + ) + : await evaluationPhase("host", "command-aborted", true, () => + activeHost.runPrompt( + session, + benchmark.prompt, + options.model, + ), + ); + } catch (caught) { + runFailure = evaluatorFailure(caught, "command-aborted"); + } + const outcome = await evaluationPhase( + "evaluator", + "outcome-collection-threw", + false, + () => activeHost.outcome([session], Date.now() - cellStarted), + ); + failureUsage = { + durationMs: outcome.durationMs, + outputTokens: outcome.tokens.output, + costUsd: outcome.costUsd, + }; + runFailure ??= outcome.providerError; + if (runFailure) + throw new EvaluationPhaseError(runFailure, runFailure); + const grade = await evaluationPhase( + "evaluator", + "benchmark-grade-threw", + false, + () => benchmark.grade(activeHost.project), + ); + const transcript = redactTranscript({ + projectPath: activeHost.project, + value: { + calls: outcome.allCalls, + finalText: outcome.finalText, + }, + }); + const stored = await persistEvaluation("transcript", () => + store.writeTranscript({ + attemptId: `attempt-${cell.cellId}`, + text: transcript.text, + }), + ); + scans.push(scanPairedTranscript(transcript.text)); + const attempt = productAttempt({ + cell, + benchmark, + outcome, + artifact: flow ? artifact : { kind: "ordinary-opencode" }, + evaluator, + hostConfig: hostConfigSha256({ + opencodeVersion, + model: options.model, + flow, + }), + transcript: stored, + requested, + flow, + hiddenCorrectness: grade.passed, + gradeIssues: grade.issues, + endedBy: commandEnd, + }); + await persistEvaluation("attempt", () => + store.writeAttempt(attempt), + ); + attempts.push(attempt); + recorded = true; + } catch (caught) { + if (caught instanceof EvaluationPersistenceError) throw caught; + if (recorded) throw caught; + const classified = evaluatorFailure(caught); + const failed = + classified.origin === "evaluator" + ? classified + : (runFailure ?? classified); + const failure: AttemptRecordV2 = { + schemaVersion: 2, + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: cell.armToken, + repetition: cell.repetition, + artifact: flow ? artifact : { kind: "ordinary-opencode" }, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + model: options.model, + flow, + }), + actors: [], + instructions: [], + transcript: null, + outcome: failureOutcome(failed), + usage: + failureUsage.durationMs > 0 + ? failureUsage + : { + durationMs: Date.now() - cellStarted, + outputTokens: 0, + costUsd: null, + }, + }; + await persistEvaluation("attempt", () => + store.writeAttempt(failure), + ); + attempts.push(failure); + nonProduct = true; + blockOrigin = failed.origin; + } + }, + async () => { + const cleanupHost = host; + if (cleanupHost) { + await evaluationPhase("host", "host-cleanup-failed", true, () => + cleanupHost.stop(), + ); + } + }, + ); + if (isEvaluatorFailure(blockOrigin)) { + return { nonProduct: true, budget: false, origin: blockOrigin }; } if (budgetExceeded()) { return { nonProduct: true, budget: true, origin: blockOrigin }; @@ -543,10 +599,14 @@ async function main(): Promise { return { nonProduct, budget: false, origin: blockOrigin }; }; let budgetStopped = false; - let incompleteCause: "host" | "evaluator" = "host"; + let incompleteCause: DurableFailureOrigin = "host"; for (const block of primary) { let result = await runBlock(block); if (result.origin) incompleteCause = result.origin; + if (isEvaluatorFailure(result.origin)) { + unresolved = true; + break; + } if (result.budget) { budgetStopped = true; unresolved = true; @@ -566,13 +626,17 @@ async function main(): Promise { ); result = await runBlock(reserve); if (result.origin) incompleteCause = result.origin; + if (isEvaluatorFailure(result.origin)) { + unresolved = true; + break; + } if (result.budget) { budgetStopped = true; unresolved = true; break; } } - if (budgetStopped) break; + if (budgetStopped || isEvaluatorFailure(result.origin)) break; } const finishedAt = new Date().toISOString(); const outputTokens = attempts.reduce( @@ -606,40 +670,46 @@ async function main(): Promise { !unresolved && !finishedBudgetExceeded && completePairs === primary.length; - const report = await store.finalize({ - reportId: `paired-${Date.now()}`, - completion: { - status: complete ? "complete" : "stopped", - cause: complete - ? "fixed-target" - : finishedBudgetExceeded - ? "budget" - : incompleteCause, - startedAt, - finishedAt, - activatedReserveCellIds, - observed: { - attempts: attempts.length, - outputTokens, - costUsd, - wallClockMs, + const report = await persistEvaluation("finalize", () => + store.finalize({ + reportId: `paired-${Date.now()}`, + completion: { + status: complete ? "complete" : "stopped", + cause: complete + ? "fixed-target" + : finishedBudgetExceeded + ? "budget" + : incompleteCause, + startedAt, + finishedAt, + activatedReserveCellIds, + observed: { + attempts: attempts.length, + outputTokens, + costUsd, + wallClockMs, + }, }, - }, - allocationCommitmentSha256: experiment.allocationCommitmentSha256, - }); + allocationCommitmentSha256: experiment.allocationCommitmentSha256, + }), + ); const masked = freezeMaskedAnalysis({ report, scans, frozenAt: new Date().toISOString(), }); - await store.writeMaskedAnalysis(masked); + await persistEvaluation("masked-analysis", () => + store.writeMaskedAnalysis(masked), + ); const revealed = revealPairedAnalysis({ report, masked, secret: experiment.secret, revealedAt: new Date().toISOString(), }); - await store.writeAllocation(revealed.allocation); + await persistEvaluation("allocation", () => + store.writeAllocation(revealed.allocation), + ); console.log(`Paired V2 report: ${join(directory, "report.json")}`); console.log(`Masked analysis: ${join(directory, "masked-analysis.json")}`); console.log(`Allocation: ${join(directory, "allocation.json")}`); diff --git a/evals/cassette.ts b/evals/cassette.ts index ee2b4dd..0947aac 100644 --- a/evals/cassette.ts +++ b/evals/cassette.ts @@ -94,7 +94,9 @@ export type FidelityNote = | "no-flow-calls" | "run-aborted" | "run-unscored" - | "host-error"; + | "host-error" + | "provider-error" + | "evaluator-error"; export type Cassette = Readonly<{ cassetteVersion: number; diff --git a/evals/failure-origin.ts b/evals/failure-origin.ts new file mode 100644 index 0000000..aa78083 --- /dev/null +++ b/evals/failure-origin.ts @@ -0,0 +1,196 @@ +import type { AttemptOutcome } from "./report.js"; + +export type DurableFailureOrigin = Extract< + AttemptOutcome, + { kind: "failure" } +>["origin"]; +export type FailureOrigin = DurableFailureOrigin | "persistence"; + +export type AttemptFailure = + Readonly<{ + origin: Origin; + code: string; + detail: string; + retryable: boolean; + }>; + +function detail(error: unknown): string { + const message = + error instanceof Error + ? error.message + : typeof error === "string" + ? error + : JSON.stringify(error) || String(error); + return message.slice(0, 4096); +} + +export function attemptFailure( + origin: Origin, + code: string, + error: unknown, + retryable: boolean, +): AttemptFailure { + return { origin, code, detail: detail(error), retryable }; +} + +export function failureOutcome( + failure: AttemptFailure, +): Extract { + return { + kind: "failure", + origin: failure.origin, + code: failure.code, + retryable: failure.retryable, + }; +} + +export function providerFailure(error: unknown): AttemptFailure<"provider"> { + return attemptFailure("provider", "provider-rejected-turn", error, true); +} + +export function evaluateScenario( + check: (outcome: T) => readonly string[], + outcome: T, +): + | Readonly<{ kind: "evaluated"; issues: readonly string[] }> + | Readonly<{ + kind: "failure"; + failure: AttemptFailure<"evaluator">; + }> { + try { + return { kind: "evaluated", issues: check(outcome) }; + } catch (error) { + return { + kind: "failure", + failure: attemptFailure( + "evaluator", + "scenario-check-threw", + error, + false, + ), + }; + } +} + +export class EvaluationPhaseError extends Error { + readonly failure: AttemptFailure; + + constructor(failure: AttemptFailure, cause: unknown) { + super(failure.detail, { cause }); + this.failure = failure; + } +} + +export async function evaluationPhase( + origin: DurableFailureOrigin, + code: string, + retryable: boolean, + operation: () => Promise, +): Promise { + try { + return await operation(); + } catch (error) { + if ( + error instanceof EvaluationPhaseError || + error instanceof EvaluationPersistenceError + ) + throw error; + throw new EvaluationPhaseError( + attemptFailure(origin, code, error, retryable), + error, + ); + } +} + +export function evaluatorFailure( + error: unknown, + code = "evaluator-transform-threw", +): AttemptFailure { + return error instanceof EvaluationPhaseError + ? error.failure + : attemptFailure("evaluator", code, error, false); +} + +export function strongestFailureOrigin( + origins: readonly (FailureOrigin | null | undefined)[], +): FailureOrigin | null { + for (const origin of [ + "persistence", + "evaluator", + "host", + "provider", + ] as const) { + if (origins.includes(origin)) return origin; + } + return null; +} + +export function isEvaluatorFailure( + origin: FailureOrigin | null | undefined, +): origin is "evaluator" { + return origin === "evaluator"; +} + +export class EvaluationPersistenceError extends Error { + readonly failure: AttemptFailure<"persistence">; + + constructor(phase: string, cause: unknown) { + const failure = attemptFailure( + "persistence", + `${phase}-write-failed`, + cause, + false, + ); + super(failure.detail, { cause }); + this.failure = failure; + } +} + +export async function persistEvaluation( + phase: string, + operation: () => Promise, +): Promise { + try { + return await operation(); + } catch (error) { + throw new EvaluationPersistenceError(phase, error); + } +} + +export async function preservePrimaryFailure( + operation: () => Promise, + cleanup: () => Promise, +): Promise { + let result: T | undefined; + let primary: unknown; + let cleanupFailure: unknown; + let failed = false; + try { + result = await operation(); + } catch (error) { + failed = true; + primary = error; + } + try { + await cleanup(); + } catch (error) { + if (!failed) throw error; + cleanupFailure = error; + } + if (failed) { + if (primary instanceof Error && cleanupFailure !== undefined) { + Object.defineProperty(primary, "cause", { + configurable: true, + value: new AggregateError( + [ + ...(primary.cause === undefined ? [] : [primary.cause]), + cleanupFailure, + ], + "Cleanup also failed.", + ), + }); + } + throw primary; + } + return result as T; +} diff --git a/evals/harness.ts b/evals/harness.ts index 532ce8b..b727e7e 100644 --- a/evals/harness.ts +++ b/evals/harness.ts @@ -25,6 +25,14 @@ import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import packageJson from "../package.json" with { type: "json" }; import { type BunToolchain, runPinnedBunSync } from "./bun-toolchain.js"; +import { + type AttemptFailure, + attemptFailure, + EvaluationPhaseError, + evaluationPhase, + preservePrimaryFailure, + providerFailure, +} from "./failure-origin.js"; import { extractObservedActor, guidanceLoad, @@ -234,8 +242,7 @@ export type Outcome = { readonly costUsd: number | null; readonly assistantMessages: number; readonly durationMs: number; - /** True when the host reported an error on any assistant message. */ - readonly hostError: string | null; + readonly providerError: AttemptFailure<"provider"> | null; }; /** @@ -756,24 +763,42 @@ export async function runQueues( queues: readonly (readonly Job[])[], concurrency: number, run: (job: Job) => Promise, + shouldStop?: (result: Result) => boolean, ): Promise { const results: Result[] = []; let next = 0; + let stopped = false; + let failed = false; + let failure: unknown; await Promise.all( Array.from( { length: Math.max(1, Math.min(concurrency, queues.length)) }, async () => { for (;;) { + if (stopped) return; // Read and advance in one synchronous step, so no two workers can claim // the same queue. const queue = queues[next]; next += 1; if (!queue) return; - for (const job of queue) results.push(await run(job)); + for (const job of queue) { + if (stopped) return; + try { + const result = await run(job); + results.push(result); + if (shouldStop?.(result)) stopped = true; + } catch (error) { + if (!failed) failure = error; + stopped = true; + failed = true; + return; + } + } } }, ), ); + if (failed) throw failure; return results; } @@ -829,24 +854,20 @@ export function sessionBoundaries( } /** - * Passes per attempt for each scenario and model pair, in run order. Attempts that - * were not scored -- an allowed ask, a lost host -- are counted apart rather than - * dropped, so a scenario whose every attempt went unscored still gets a row saying - * so instead of leaving the table without a trace. + * Passes per attempt for each scenario and model pair, in run order. Provider and + * host failures or an unallowed ask are excluded gaps; evaluator failures are + * aborted measurements. Every scenario still gets a row. * - * An abort is counted apart for the same reason and was not: a wedged attempt ends - * with no issues and `passed: false`, which is indistinguishable in a rate from a run - * that reached the wrong outcome. One such attempt produced the only failing - * threshold in a measured report, on a guarantee that never ran. */ export function passRates( results: readonly { readonly scenario: string; readonly model: string; readonly passed: boolean; - readonly environment?: boolean; readonly unscored?: boolean; - readonly error?: string; + readonly failure?: { + readonly origin: "provider" | "host" | "evaluator"; + }; }[], ): [string, PassRate][] { const rates = new Map(); @@ -858,8 +879,13 @@ export function passRates( unscored: 0, aborted: 0, }; - if (result.environment || result.unscored) rate.unscored += 1; - else if (result.error !== undefined) rate.aborted += 1; + if ( + result.unscored || + result.failure?.origin === "provider" || + result.failure?.origin === "host" + ) + rate.unscored += 1; + else if (result.failure?.origin === "evaluator") rate.aborted += 1; else { rate.attempts += 1; if (result.passed) rate.passed += 1; @@ -1094,7 +1120,12 @@ export class EvalHost { const project = join(scratch, "project"); await mkdir(childHome, { recursive: true }); await mkdir(join(project, ".opencode"), { recursive: true }); - const credentialPaths = await carryProviderCredentials(childData); + const credentialPaths = await evaluationPhase( + "host", + "credential-copy-failed", + true, + () => carryProviderCredentials(childData), + ); // Flow derives source identity from git, so the fixture must be a repo. for (const [relative, contents] of Object.entries(options.files)) { @@ -1140,67 +1171,71 @@ export class EvalHost { const host = new EvalHost(project, scratch); host.credentialPaths = credentialPaths; - const port = await availablePort(); - host.baseUrl = `http://127.0.0.1:${port}`; - host.server = spawn( - options.toolchain.executable, - [ - "x", - `opencode-ai@${options.opencodeVersion}`, - "serve", - "--port", - String(port), - "--hostname", - "127.0.0.1", - ], - { - cwd: project, - env: { - ...options.toolchain.environment, - ...(options.reviewerModel - ? { OPENCODE_FLOW_REVIEWER_MODEL: options.reviewerModel } - : {}), - HOME: childHome, - XDG_CACHE_HOME: childCache, - XDG_CONFIG_HOME: join(childHome, ".config"), - XDG_DATA_HOME: childData, - XDG_STATE_HOME: join(childHome, ".local", "state"), + return evaluationPhase("host", "host-start-failed", true, async () => { + const port = await availablePort(); + host.baseUrl = `http://127.0.0.1:${port}`; + host.server = spawn( + options.toolchain.executable, + [ + "x", + `opencode-ai@${options.opencodeVersion}`, + "serve", + "--port", + String(port), + "--hostname", + "127.0.0.1", + ], + { + cwd: project, + env: { + ...options.toolchain.environment, + ...(options.reviewerModel + ? { OPENCODE_FLOW_REVIEWER_MODEL: options.reviewerModel } + : {}), + HOME: childHome, + XDG_CACHE_HOME: childCache, + XDG_CONFIG_HOME: join(childHome, ".config"), + XDG_DATA_HOME: childData, + XDG_STATE_HOME: join(childHome, ".local", "state"), + }, + stdio: ["ignore", "pipe", "pipe"], }, - stdio: ["ignore", "pipe", "pipe"], - }, - ); - const record = (chunk: unknown) => { - host.serverLog += String(chunk); - }; - host.server.stdout?.on("data", record); - host.server.stderr?.on("data", record); + ); + const record = (chunk: unknown) => { + host.serverLog += String(chunk); + }; + host.server.stdout?.on("data", record); + host.server.stderr?.on("data", record); - try { - const deadline = Date.now() + STARTUP_TIMEOUT_MS; - for (;;) { - try { - const health = (await fetchJson( - `${host.baseUrl}/global/health`, - 3_000, - )) as { - healthy?: boolean; - }; - if (health.healthy) break; - } catch { - // still starting - } - if (Date.now() > deadline) { - throw new Error( - `OpenCode did not become healthy.\n${host.serverLog}`, - ); + try { + const deadline = Date.now() + STARTUP_TIMEOUT_MS; + for (;;) { + try { + const health = (await fetchJson( + `${host.baseUrl}/global/health`, + 3_000, + )) as { + healthy?: boolean; + }; + if (health.healthy) break; + } catch { + // still starting + } + if (Date.now() > deadline) { + throw new Error( + `OpenCode did not become healthy.\n${host.serverLog}`, + ); + } + await Bun.sleep(500); } - await Bun.sleep(500); + return host; + } catch (error) { + return preservePrimaryFailure( + () => Promise.reject(error), + () => host.stop(), + ); } - return host; - } catch (error) { - await host.stop(); - throw error; - } + }); } get log(): string { @@ -1542,8 +1577,11 @@ export class EvalHost { let entries: MessageEntry[] | null; try { entries = (await this.messages(sessionId)) as MessageEntry[]; - } catch { - entries = null; + } catch (error) { + throw new EvaluationPhaseError( + attemptFailure("host", "session-messages-read-failed", error, true), + error, + ); } const session = sessionRecords[sessionIndex]; if (session) sessionMessages.push({ ...session, messages: entries }); @@ -1567,7 +1605,7 @@ export class EvalHost { let costUsd = 0; let costReported = false; let assistantMessages = 0; - let hostError: string | null = null; + let providerError: AttemptFailure<"provider"> | null = null; let finalText = ""; const guidanceLoads: ObservedGuidanceLoad[] = []; let guidanceSequence = 0; @@ -1593,14 +1631,14 @@ export class EvalHost { const created = entry.info.time?.created; if ( entry.info.error && - !hostError && + !providerError && !isSelfAbortError( entry.info.error, this.lastSelfAbortAt > 0 && (created === undefined || created <= this.lastSelfAbortAt), ) ) - hostError = JSON.stringify(entry.info.error); + providerError = providerFailure(entry.info.error); } for (const part of entry.parts) { if ( @@ -1689,7 +1727,7 @@ export class EvalHost { costUsd: reportedCost(costReported ? costUsd : null, tokens.output), assistantMessages, durationMs, - hostError, + providerError, }; } @@ -1701,8 +1739,16 @@ export class EvalHost { string, unknown >; - } catch { - return null; + } catch (error) { + if ( + error instanceof SyntaxError || + (error instanceof Error && "code" in error && error.code === "ENOENT") + ) + return null; + throw new EvaluationPhaseError( + attemptFailure("host", "workspace-read-failed", error, true), + error, + ); } } @@ -1711,8 +1757,13 @@ export class EvalHost { let names: string[]; try { names = await readdir(history); - } catch { - return []; + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") + return []; + throw new EvaluationPhaseError( + attemptFailure("host", "archive-directory-read-failed", error, true), + error, + ); } const documents: Record[] = []; for (const name of names diff --git a/evals/replay.ts b/evals/replay.ts index b374ed7..fa072e3 100644 --- a/evals/replay.ts +++ b/evals/replay.ts @@ -435,7 +435,7 @@ export async function replayCassette( costUsd: null, assistantMessages: cassette.assistantMessages, durationMs: Date.now() - started, - hostError: null, + providerError: null, }; return { cassette, outcome, divergences }; } finally { diff --git a/evals/report.ts b/evals/report.ts index a735ad6..0dbf91a 100644 --- a/evals/report.ts +++ b/evals/report.ts @@ -287,6 +287,7 @@ const CampaignCompletionSchema = z "provider", "host", "evaluator", + "persistence", "operator", ]), startedAt: TimestampSchema, diff --git a/evals/reviewer-run.ts b/evals/reviewer-run.ts index 121a17c..f199146 100644 --- a/evals/reviewer-run.ts +++ b/evals/reviewer-run.ts @@ -7,6 +7,20 @@ import packageJson from "../package.json" with { type: "json" }; import { currentBunToolchain } from "./bun-toolchain.js"; import { canonicalSha256 } from "./canonical-json.js"; import { parseCaseCatalog } from "./catalog.js"; +import { + type AttemptFailure, + attemptFailure, + type DurableFailureOrigin, + EvaluationPersistenceError, + EvaluationPhaseError, + evaluationPhase, + evaluatorFailure, + failureOutcome, + isEvaluatorFailure, + persistEvaluation, + preservePrimaryFailure, + strongestFailureOrigin, +} from "./failure-origin.js"; import { type CommandEnd, EvalHost, @@ -215,15 +229,17 @@ async function main(): Promise { const opencodeVersion = packageJson.devDependencies["@opencode-ai/plugin"]; const packDir = await mkdtemp(join(tmpdir(), "flow-reviewer-pack-")); const reportDir = join(repositoryRoot, "evals", "results"); - await mkdir(reportDir, { recursive: true }); + await persistEvaluation("report-directory", () => + mkdir(reportDir, { recursive: true }), + ); const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const campaignDirectory = join(reportDir, `reviewer-${stamp}.v2`); const catalogInput = catalogFor(cases); const catalog = parseCatalog(catalogInput); const plan = planFor(cases, model); const store = createReportStore({ directory: campaignDirectory, catalog }); - await store.initialize(plan); - await store.writeCatalog(catalog); + await persistEvaluation("initialize", () => store.initialize(plan)); + await persistEvaluation("catalog", () => store.writeCatalog(catalog)); const startedAt = new Date().toISOString(); try { const tarball = await packPlugin(repositoryRoot, packDir, toolchain); @@ -231,7 +247,7 @@ async function main(): Promise { repositoryRoot, tarballPath: tarball, }); - await store.writeArtifact(tarball); + await persistEvaluation("artifact", () => store.writeArtifact(tarball)); const evaluator = evaluatorIdentity({ sourceCommit: artifact.sourceCommit, caseCatalog: cases.map((entry) => ({ @@ -248,119 +264,188 @@ async function main(): Promise { if (!cell) throw new Error("Reviewer campaign cell is missing."); const started = Date.now(); let host: EvalHost | null = null; - let attempt: AttemptRecordV2; - try { - host = await EvalHost.start({ - toolchain, - packageCache, - opencodeVersion, - files: entry.files, - reviewerModel: options.model, - }); - const catalogModels = await host.catalogModels(); - if (!catalogModels.includes(options.model)) { - throw new Error( - `Reviewer model ${options.model} is absent from the host catalog.`, + let attempt: AttemptRecordV2 | null = null; + let runFailure: AttemptFailure | null = null; + await preservePrimaryFailure( + async () => { + try { + host = await EvalHost.start({ + toolchain, + packageCache, + opencodeVersion, + files: entry.files, + reviewerModel: options.model, + }); + const activeHost = host; + const catalogModels = await evaluationPhase( + "host", + "model-catalog-failed", + true, + () => activeHost.catalogModels(), + ); + if (!catalogModels.includes(options.model)) { + runFailure = attemptFailure( + "provider", + "model-unavailable", + `Reviewer model ${options.model} is absent from the host catalog.`, + false, + ); + throw new EvaluationPhaseError(runFailure, runFailure); + } + const seed = await evaluationPhase( + "evaluator", + "reviewer-seed-failed", + false, + () => + seedReviewerAssignment({ + workspace: activeHost.project, + fixture: entry, + }), + ); + const sessionId = await evaluationPhase( + "host", + "session-create-failed", + true, + () => activeHost.createSession("reviewer evaluation"), + ); + const commandEnd = await evaluationPhase( + "host", + "command-aborted", + true, + () => + activeHost.runCommand( + sessionId, + "flow-review", + seed.assignmentId, + options.model, + ), + ); + const outcome = await evaluationPhase( + "evaluator", + "outcome-collection-threw", + false, + () => activeHost.outcome([sessionId], Date.now() - started), + ); + if (outcome.providerError) { + runFailure = outcome.providerError; + throw new EvaluationPhaseError(runFailure, runFailure); + } + const submission = await evaluationPhase( + "evaluator", + "reviewer-grade-threw", + false, + () => + readDurableReviewerSubmission({ + workspace: activeHost.project, + seed, + }), + ); + const transcript = redactTranscript({ + projectPath: activeHost.project, + value: { calls: outcome.allCalls, finalText: outcome.finalText }, + }); + const storedTranscript = await persistEvaluation("transcript", () => + store.writeTranscript({ + attemptId: `attempt-${cell.cellId}`, + text: transcript.text, + }), + ); + const instructions: InstructionDelivery[] = [ + instructionDelivery({ + source: "command", + name: "flow-review", + sequence: 0, + text: seed.assignmentId, + }), + ]; + const actor = reportReviewerActor(model, outcome); + attempt = { + schemaVersion: 2, + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: null, + repetition: 0, + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + reviewerModel: options.model, + }), + actors: actor ? [actor] : [], + instructions, + transcript: { + sha256: storedTranscript.sha256, + artifact: storedTranscript.artifact, + }, + outcome: reviewerOutcome(entry, submission, commandEnd), + usage: { + durationMs: outcome.durationMs, + outputTokens: outcome.tokens.output, + costUsd: outcome.costUsd, + }, + }; + } catch (error) { + if (error instanceof EvaluationPersistenceError) throw error; + const classified = evaluatorFailure(error); + const failed = + classified.origin === "evaluator" + ? classified + : (runFailure ?? classified); + attempt = { + schemaVersion: 2, + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: null, + repetition: 0, + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + reviewerModel: options.model, + }), + actors: [], + instructions: [], + transcript: null, + outcome: failureOutcome(failed), + usage: { + durationMs: Date.now() - started, + outputTokens: 0, + costUsd: null, + }, + }; + } + const finalizedAttempt = attempt; + if (!finalizedAttempt) + throw new Error("Reviewer attempt was not constructed."); + await persistEvaluation("attempt", () => + store.writeAttempt(finalizedAttempt), ); - } - const seed = await seedReviewerAssignment({ - workspace: host.project, - fixture: entry, - }); - const sessionId = await host.createSession("reviewer evaluation"); - const commandEnd = await host.runCommand( - sessionId, - "flow-review", - seed.assignmentId, - options.model, - ); - const outcome = await host.outcome([sessionId], Date.now() - started); - const submission = await readDurableReviewerSubmission({ - workspace: host.project, - seed, - }); - const transcript = redactTranscript({ - projectPath: host.project, - value: { calls: outcome.allCalls, finalText: outcome.finalText }, - }); - const storedTranscript = await store.writeTranscript({ - attemptId: `attempt-${cell.cellId}`, - text: transcript.text, - }); - const instructions: InstructionDelivery[] = [ - instructionDelivery({ - source: "command", - name: "flow-review", - sequence: 0, - text: seed.assignmentId, - }), - ]; - const actor = reportReviewerActor(model, outcome); - attempt = { - schemaVersion: 2, - attemptId: `attempt-${cell.cellId}`, - cellId: cell.cellId, - blockId: cell.blockId, - caseId: cell.caseId, - caseVersion: cell.caseVersion, - armToken: null, - repetition: 0, - artifact, - evaluator, - hostConfigSha256: hostConfigSha256({ - opencodeVersion, - reviewerModel: options.model, - }), - actors: actor ? [actor] : [], - instructions, - transcript: { - sha256: storedTranscript.sha256, - artifact: storedTranscript.artifact, - }, - outcome: reviewerOutcome(entry, submission, commandEnd), - usage: { - durationMs: outcome.durationMs, - outputTokens: outcome.tokens.output, - costUsd: outcome.costUsd, - }, - }; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - attempt = { - schemaVersion: 2, - attemptId: `attempt-${cell.cellId}`, - cellId: cell.cellId, - blockId: cell.blockId, - caseId: cell.caseId, - caseVersion: cell.caseVersion, - armToken: null, - repetition: 0, - artifact, - evaluator, - hostConfigSha256: hostConfigSha256({ - opencodeVersion, - reviewerModel: options.model, - }), - actors: [], - instructions: [], - transcript: null, - outcome: { - kind: "failure", - origin: "host", - code: message.slice(0, 512), - retryable: true, - }, - usage: { - durationMs: Date.now() - started, - outputTokens: 0, - costUsd: null, - }, - }; - } finally { - await host?.stop(); + attempts.push(finalizedAttempt); + }, + async () => { + const cleanupHost = host; + if (cleanupHost) { + await evaluationPhase("host", "host-cleanup-failed", true, () => + cleanupHost.stop(), + ); + } + }, + ); + const recordedAttempt = attempts.at(-1); + if (!recordedAttempt || recordedAttempt.cellId !== cell.cellId) { + throw new Error("Reviewer attempt was not persisted."); } - await store.writeAttempt(attempt); - attempts.push(attempt); + if ( + recordedAttempt.outcome.kind === "failure" && + isEvaluatorFailure(recordedAttempt.outcome.origin) + ) + break; } const finishedAt = new Date().toISOString(); const complete = attempts.every( @@ -372,29 +457,36 @@ async function main(): Promise { (total, attempt) => total + (attempt.usage.costUsd ?? 0), 0, ); - await store.finalize({ - reportId: `flow-reviewer-${stamp}`, - completion: { - status: complete ? "complete" : "stopped", - cause: complete ? "fixed-target" : "host", - startedAt, - finishedAt, - activatedReserveCellIds: [], - observed: { - attempts: attempts.length, - outputTokens: attempts.reduce( - (total, attempt) => total + attempt.usage.outputTokens, - 0, - ), - costUsd, - wallClockMs: Math.max( - Date.parse(finishedAt) - Date.parse(startedAt), - ...attempts.map((attempt) => attempt.usage.durationMs), - ), + const stoppedOrigin = strongestFailureOrigin( + attempts.map((attempt) => + attempt.outcome.kind === "failure" ? attempt.outcome.origin : null, + ), + ); + await persistEvaluation("finalize", () => + store.finalize({ + reportId: `flow-reviewer-${stamp}`, + completion: { + status: complete ? "complete" : "stopped", + cause: complete || !stoppedOrigin ? "fixed-target" : stoppedOrigin, + startedAt, + finishedAt, + activatedReserveCellIds: [], + observed: { + attempts: attempts.length, + outputTokens: attempts.reduce( + (total, attempt) => total + attempt.usage.outputTokens, + 0, + ), + costUsd, + wallClockMs: Math.max( + Date.parse(finishedAt) - Date.parse(startedAt), + ...attempts.map((attempt) => attempt.usage.durationMs), + ), + }, }, - }, - allocationCommitmentSha256: null, - }); + allocationCommitmentSha256: null, + }), + ); console.log( `Reviewer V2 report: ${join(campaignDirectory, "report.json")}`, ); diff --git a/evals/run.ts b/evals/run.ts index 5a604fa..afe1188 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -26,6 +26,19 @@ import { type FidelityNote, } from "./cassette.js"; import { parseCaseCatalog, type ValidatedCaseCatalog } from "./catalog.js"; +import { + type AttemptFailure, + type DurableFailureOrigin, + EvaluationPersistenceError, + evaluateScenario, + evaluationPhase, + evaluatorFailure, + failureOutcome, + isEvaluatorFailure, + persistEvaluation, + preservePrimaryFailure, + strongestFailureOrigin, +} from "./failure-origin.js"; import { askedQuestions, askedScoring, @@ -87,18 +100,10 @@ const SURFACES: FlowPromptSurfaceName[] = [ "flow-worker", ]; -type RunResult = { +type RunResultCommon = { scenario: string; model: string; attempt: number; - passed: boolean; - /** - * True when the run never reached the model: a host that would not boot, a - * failed dependency install, a lost network. Such a run is no evidence about - * the prompts either way, so it is excluded from the pass rate rather than - * counted as a regression. - */ - environment?: boolean; /** True when the model asked the user and stopped, scored or not. */ escalated?: boolean; /** @@ -108,7 +113,6 @@ type RunResult = { * against the prompts. */ unscored?: boolean; - issues: readonly string[]; tokens: Outcome["tokens"]; costUsd: number | null; assistantMessages: number; @@ -168,9 +172,22 @@ type RunResult = { readonly instructions: readonly InstructionDelivery[]; readonly transcript: { readonly sha256: string; readonly text: string }; }; - error?: string; }; +type RunResult = RunResultCommon & + ( + | { + passed: boolean; + issues: readonly string[]; + failure?: never; + } + | { + passed: false; + issues: readonly []; + failure: AttemptFailure; + } + ); + /** * The most attempts allowed in flight at once, however many models are named. * @@ -342,13 +359,8 @@ function reportActor( } function attemptOutcome(result: RunResult): AttemptRecordV2["outcome"] { - if (result.environment || result.error !== undefined) { - return { - kind: "failure", - origin: "host", - code: result.environment ? "environment" : "attempt-error", - retryable: true, - }; + if (result.failure) { + return failureOutcome(result.failure); } if (result.unscored) { return { @@ -555,8 +567,7 @@ function promptFootprint(): { * outcome and one that reached the only end left to it. */ function verdict(result: RunResult): string { - if (result.environment) return "ENV"; - if (result.error) return "ABORT"; + if (result.failure) return result.failure.origin.toUpperCase(); if (result.unscored) return "ASKED"; return `${result.passed ? "PASS" : "FAIL"}${result.escalated ? "+ASK" : ""}`; } @@ -580,8 +591,8 @@ function formatTable(results: readonly RunResult[]): string { String(result.tokens.output), String(result.assistantMessages), String(Math.round(result.durationMs / 1000)), - result.error - ? `harness: ${result.error}` + result.failure + ? `${result.failure.code}: ${result.failure.detail}` : result.issues.length === 0 ? "-" : result.issues.join("; "), @@ -712,7 +723,9 @@ async function main(): Promise { const packDir = await mkdtemp(join(tmpdir(), "flow-eval-pack-")); const reportDir = join(repositoryRoot, "evals", "results"); - await mkdir(reportDir, { recursive: true }); + await persistEvaluation("report-directory", () => + mkdir(reportDir, { recursive: true }), + ); const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const v2Directory = join(reportDir, `${stamp}.v2`); const v2Catalog = caseCatalogFor(selected); @@ -726,8 +739,8 @@ async function main(): Promise { directory: v2Directory, catalog: v2Catalog, }); - await reportStore.initialize(v2Plan); - await reportStore.writeCatalog(v2Catalog); + await persistEvaluation("initialize", () => reportStore.initialize(v2Plan)); + await persistEvaluation("catalog", () => reportStore.writeCatalog(v2Catalog)); const campaignStartedAt = new Date().toISOString(); const campaignCells = v2Plan.cells; const v2Attempts: AttemptRecordV2[] = []; @@ -742,7 +755,9 @@ async function main(): Promise { repositoryRoot, tarballPath: tarball, }); - await reportStore.writeArtifact(tarball); + await persistEvaluation("artifact", () => + reportStore.writeArtifact(tarball), + ); const evaluator = evaluatorIdentity({ sourceCommit: artifact.sourceCommit, caseCatalog: selected.map((scenario) => ({ @@ -845,265 +860,315 @@ async function main(): Promise { let cassette: Cassette | null = null; const started = Date.now(); let host: EvalHost | null = null; - try { - host = await EvalHost.start({ - toolchain, - packageCache, - opencodeVersion, - files: scenario.files, - }); - const sessionIds = [ - await host.createSession(`flow-eval ${scenario.id}`), - ]; - // A step that times out still produced tokens, messages, and tool - // calls, and those are the only evidence of how far the model got. - // Throwing here would discard them and report a run of zeroes, so - // the failure is remembered and the outcome collected regardless. - let stepError: string | null = null; - const escalatedSteps: number[] = []; - for (const [index, step] of scenario.steps.entries()) { + return preservePrimaryFailure( + async () => { try { - if (step.freshSession) { - sessionIds.push( - await host.createSession(`flow-eval ${scenario.id} resumed`), - ); + host = await EvalHost.start({ + toolchain, + packageCache, + opencodeVersion, + files: scenario.files, + }); + const activeHost = host; + const sessionIds = [ + await evaluationPhase("host", "session-create-failed", true, () => + activeHost.createSession(`flow-eval ${scenario.id}`), + ), + ]; + // A step that times out still produced tokens, messages, and tool + // calls, and those are the only evidence of how far the model got. + // Throwing here would discard them and report a run of zeroes, so + // the failure is remembered and the outcome collected regardless. + let stepFailure: AttemptFailure | null = null; + const escalatedSteps: number[] = []; + for (const [index, step] of scenario.steps.entries()) { + try { + if (step.freshSession) { + sessionIds.push( + await evaluationPhase( + "host", + "session-create-failed", + true, + () => + activeHost.createSession( + `flow-eval ${scenario.id} resumed`, + ), + ), + ); + } + const end = await evaluationPhase( + "host", + "command-aborted", + true, + () => + activeHost.runCommand( + sessionIds[sessionIds.length - 1] ?? "", + step.command, + step.arguments, + model, + ), + ); + if (end === "escalated") { + escalatedSteps.push(index); + // A question at the end of a non-final step is what the next step + // answers: three scenarios open with `flow-plan`, where asking for + // approval is the behaviour `plan-only-stops` gates at 100%, and + // the step that follows says "you have my approval". Ending the run + // there discarded a correct attempt — and since a gated pair needs + // three scored attempts, one such question failed qualification for + // a run that did nothing wrong. Only the last step's question ends + // the run; `runCommand` has already aborted the pending turn, so + // the session is idle and the next prompt is the answer. + if (index === scenario.steps.length - 1) break; + } + } catch (error) { + stepFailure = evaluatorFailure(error, "command-aborted"); + break; + } } - const end = await host.runCommand( - sessionIds[sessionIds.length - 1] ?? "", - step.command, - step.arguments, + const outcome = await evaluationPhase( + "evaluator", + "outcome-collection-threw", + false, + () => activeHost.outcome(sessionIds, Date.now() - started), + ); + const observedFailure = outcome.providerError; + // Asking the user is the designed end of some scenarios, but only at the + // wall. `askedScoring` holds the rule and its reasoning. + const { escalated, unscored } = askedScoring( + escalatedSteps, + scenario.steps.length, + scenario.mayEscalate === true, + ); + // An aborted or unscored step leaves the workflow mid-flight, so `check` + // would report expected-but-meaningless gaps. The stop is the finding; + // the collected evidence explains it. + const evaluation = + stepFailure || observedFailure || unscored + ? null + : evaluateScenario(scenario.check, outcome); + const failure = + stepFailure ?? + observedFailure ?? + (evaluation?.kind === "failure" ? evaluation.failure : null); + const issues = + evaluation?.kind === "evaluated" ? evaluation.issues : []; + const documents = [ + ...(outcome.session ? [outcome.session] : []), + ...outcome.archives, + ] as MetricSession[]; + const actors = (outcome.actors ?? []).map((actor) => ({ + ...actor, + requestedModelId: + actor.role === "manager" ? model : requestedReviewerModel, + requestedModel: legacyRequestedModel( + actor.role === "manager" ? model : requestedReviewerModel, + ), + })); + const instructions = (outcome.guidanceLoads ?? []).map((load) => + instructionDelivery({ + source: "guidance", + name: load.id ?? "unknown-guidance", + sequence: load.sequence, + text: load.rawOutput, + }), + ); + const transcript = redactTranscript({ + projectPath: host.project, + value: { + actors, + guidanceLoads: outcome.guidanceLoads ?? [], + calls: outcome.allCalls, + finalText: outcome.finalText, + }, + }); + const common: RunResultCommon = { + scenario: scenario.id, model, + attempt, + ...(escalated ? { escalated: true } : {}), + ...(unscored ? { unscored: true } : {}), + tokens: outcome.tokens, + costUsd: outcome.costUsd, + assistantMessages: outcome.assistantMessages, + flowCalls: outcome.flowCalls.map((call) => call.tool), + sessionBoundaries: sessionBoundaries(outcome.flowCalls), + documents, + honesty: completionHonesty( + documents.find((document) => document.closure) ?? null, + ), + reviewer: reviewerActivity(documents), + operational: operationalMetrics(documents, { + flowCalls: outcome.flowCalls.map((call) => call.tool), + assistantMessages: outcome.assistantMessages, + durationMs: outcome.durationMs, + }), + refusedBroadScope: refusedBroadScope(outcome.flowCalls), + guidanceSkips: countGuidanceSkips(outcome.flowCalls), + finalText: outcome.finalText, + questions: askedQuestions(outcome), + durationMs: outcome.durationMs, + hostError: outcome.providerError?.detail ?? null, + provenance: { + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + plugin: `opencode-plugin-flow@${packageJson.version}`, + model, + reviewerModel: requestedReviewerModel, + reviewerSteps: requestedReviewerSteps, + platform: hostPlatform, + }), + actors, + instructions, + transcript, + }, + }; + const result: RunResult = failure + ? { ...common, passed: false, issues: [], failure } + : { + ...common, + passed: !unscored && issues.length === 0, + issues, + }; + const fidelity: FidelityNote[] = []; + if (stepFailure) fidelity.push("run-aborted"); + if (unscored) fidelity.push("run-unscored"); + if (failure?.origin === "evaluator") + fidelity.push("evaluator-error"); + // Only a provider error this runner did not cause. `outcome` withholds + // the `MessageAbortedError` left by an abort the harness issued itself. + // Recording those as host errors made 19 + // of 63 cassettes advisory, and every refusal scenario — the runs + // most worth gating — was among them. + if (outcome.providerError) fidelity.push("provider-error"); + cassette = buildCassette({ + flowVersion: packageJson.version, + scenario: scenario.id, + model, + attempt, + hostPlatform, + files: scenario.files, + projectPath: host.project, + calls: outcome.allCalls, + finalText: outcome.finalText, + assistantMessages: outcome.assistantMessages, + verdict: verdict(result), + issues, + falseCompletion: result.honesty.falseCompletion, + documents, + extraFidelity: fidelity, + }); + const scoreLabel = + issues.length === 0 ? "PASS" : `FAIL (${issues.length})`; + console.log( + `- ${label} ... ${ + failure + ? `${failure.origin.toUpperCase()} (${failure.detail.split("\n")[0]})` + : unscored + ? "ASKED (the model asked the user; nothing answers, so the wait ended)" + : escalatedSteps.includes(scenario.steps.length - 1) + ? `${scoreLabel} (asked the user, which this scenario allows)` + : escalated + ? `${scoreLabel} (asked the user; the next step answered)` + : scoreLabel + }`, ); - if (end === "escalated") { - escalatedSteps.push(index); - // A question at the end of a non-final step is what the next step - // answers: three scenarios open with `flow-plan`, where asking for - // approval is the behaviour `plan-only-stops` gates at 100%, and - // the step that follows says "you have my approval". Ending the run - // there discarded a correct attempt — and since a gated pair needs - // three scored attempts, one such question failed qualification for - // a run that did nothing wrong. Only the last step's question ends - // the run; `runCommand` has already aborted the pending turn, so - // the session is idle and the next prompt is the answer. - if (index === scenario.steps.length - 1) break; - } + const cell = campaignCells[job.slot]; + if (!cell) + throw new Error(`Missing v2 campaign cell for slot ${job.slot}.`); + await persistEvaluation("attempt", () => + persistV2Attempt(result, cell, scenario), + ); + return { slot: job.slot, result, cassette }; } catch (error) { - stepError = error instanceof Error ? error.message : String(error); - break; - } - } - const outcome = await host.outcome(sessionIds, Date.now() - started); - // A host-level error (bad model id, missing credentials) is not a - // prompt result, so it must not be reported as a scenario failure. - if (outcome.hostError && outcome.flowCalls.length === 0) { - throw new Error(`host rejected the turn: ${outcome.hostError}`); - } - // Asking the user is the designed end of some scenarios, but only at the - // wall. `askedScoring` holds the rule and its reasoning. - const { escalated, unscored } = askedScoring( - escalatedSteps, - scenario.steps.length, - scenario.mayEscalate === true, - ); - // An aborted or unscored step leaves the workflow mid-flight, so `check` - // would report expected-but-meaningless gaps. The stop is the finding; - // the collected evidence explains it. - const issues = stepError || unscored ? [] : scenario.check(outcome); - const documents = [ - ...(outcome.session ? [outcome.session] : []), - ...outcome.archives, - ] as MetricSession[]; - const actors = (outcome.actors ?? []).map((actor) => ({ - ...actor, - requestedModelId: - actor.role === "manager" ? model : requestedReviewerModel, - requestedModel: legacyRequestedModel( - actor.role === "manager" ? model : requestedReviewerModel, - ), - })); - const instructions = (outcome.guidanceLoads ?? []).map((load) => - instructionDelivery({ - source: "guidance", - name: load.id ?? "unknown-guidance", - sequence: load.sequence, - text: load.rawOutput, - }), - ); - const transcript = redactTranscript({ - projectPath: host.project, - value: { - actors, - guidanceLoads: outcome.guidanceLoads ?? [], - calls: outcome.allCalls, - finalText: outcome.finalText, - }, - }); - const result: RunResult = { - scenario: scenario.id, - model, - attempt, - passed: stepError === null && !unscored && issues.length === 0, - ...(escalated ? { escalated: true } : {}), - ...(unscored ? { unscored: true } : {}), - issues, - ...(stepError ? { error: stepError } : {}), - tokens: outcome.tokens, - costUsd: outcome.costUsd, - assistantMessages: outcome.assistantMessages, - flowCalls: outcome.flowCalls.map((call) => call.tool), - sessionBoundaries: sessionBoundaries(outcome.flowCalls), - documents, - honesty: completionHonesty( - documents.find((document) => document.closure) ?? null, - ), - reviewer: reviewerActivity(documents), - operational: operationalMetrics(documents, { - flowCalls: outcome.flowCalls.map((call) => call.tool), - assistantMessages: outcome.assistantMessages, - durationMs: outcome.durationMs, - }), - refusedBroadScope: refusedBroadScope(outcome.flowCalls), - guidanceSkips: countGuidanceSkips(outcome.flowCalls), - finalText: outcome.finalText, - questions: askedQuestions(outcome), - durationMs: outcome.durationMs, - hostError: outcome.hostError, - provenance: { - artifact, - evaluator, - hostConfigSha256: hostConfigSha256({ - opencodeVersion, - plugin: `opencode-plugin-flow@${packageJson.version}`, - model, - reviewerModel: requestedReviewerModel, - reviewerSteps: requestedReviewerSteps, - platform: hostPlatform, - }), - actors, - instructions, - transcript, - }, - }; - const fidelity: FidelityNote[] = []; - if (stepError) fidelity.push("run-aborted"); - if (unscored) fidelity.push("run-unscored"); - // Only a host error this runner did not cause, which `outcome` now - // decides: it withholds the `MessageAbortedError` left by an abort - // the harness issued itself. Recording those as host errors made 19 - // of 63 cassettes advisory, and every refusal scenario — the runs - // most worth gating — was among them. - if (outcome.hostError) fidelity.push("host-error"); - cassette = buildCassette({ - flowVersion: packageJson.version, - scenario: scenario.id, - model, - attempt, - hostPlatform, - files: scenario.files, - projectPath: host.project, - calls: outcome.allCalls, - finalText: outcome.finalText, - assistantMessages: outcome.assistantMessages, - verdict: verdict(result), - issues, - falseCompletion: result.honesty.falseCompletion, - documents, - extraFidelity: fidelity, - }); - const scoreLabel = - issues.length === 0 ? "PASS" : `FAIL (${issues.length})`; - console.log( - `- ${label} ... ${ - stepError - ? `ABORT (${stepError.split("\n")[0]})` - : unscored - ? "ASKED (the model asked the user; nothing answers, so the wait ended)" - : escalatedSteps.includes(scenario.steps.length - 1) - ? `${scoreLabel} (asked the user, which this scenario allows)` - : escalated - ? `${scoreLabel} (asked the user; the next step answered)` - : scoreLabel - }`, - ); - const cell = campaignCells[job.slot]; - if (!cell) - throw new Error(`Missing v2 campaign cell for slot ${job.slot}.`); - await persistV2Attempt(result, cell, scenario); - return { slot: job.slot, result, cassette }; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const transcript = redactTranscript({ - projectPath: host?.project ?? "", - value: { environmentError: message }, - }); - console.log(`- ${label} ... ENVIRONMENT (${message.split("\n")[0]})`); - // Reaching here means the scenario never got a model turn, with one - // exception: a host that answered but rejected every turn is thrown - // above and is equally not a prompt result. - const result: RunResult = { - scenario: scenario.id, - model, - attempt, - passed: false, - environment: true, - issues: [], - tokens: { - input: 0, - output: 0, - reasoning: 0, - cacheRead: 0, - cacheWrite: 0, - }, - costUsd: null, - assistantMessages: 0, - flowCalls: [], - sessionBoundaries: [], - documents: [], - honesty: completionHonesty(null), - reviewer: reviewerActivity([]), - operational: operationalMetrics([], { - flowCalls: [], - assistantMessages: 0, - durationMs: Date.now() - started, - }), - refusedBroadScope: 0, - guidanceSkips: 0, - finalText: "", - questions: [], - durationMs: Date.now() - started, - hostError: null, - provenance: { - artifact, - evaluator, - hostConfigSha256: hostConfigSha256({ - opencodeVersion, - plugin: `opencode-plugin-flow@${packageJson.version}`, + if (error instanceof EvaluationPersistenceError) throw error; + const message = + error instanceof Error ? error.message : String(error); + const failure = evaluatorFailure(error); + const transcript = redactTranscript({ + projectPath: host?.project ?? "", + value: { [`${failure.origin}Error`]: message }, + }); + console.log( + `- ${label} ... ${failure.origin.toUpperCase()} (${message.split("\n")[0]})`, + ); + const result: RunResult = { + scenario: scenario.id, model, - reviewerModel: requestedReviewerModel, - reviewerSteps: requestedReviewerSteps, - platform: hostPlatform, - }), - actors: [], - instructions: [], - transcript, - }, - error: message, - }; - const cell = campaignCells[job.slot]; - if (!cell) - throw new Error(`Missing v2 campaign cell for slot ${job.slot}.`); - await persistV2Attempt(result, cell, scenario); - return { - slot: job.slot, - cassette, - result, - }; - } finally { - await host?.stop(); - } + attempt, + passed: false, + issues: [], + failure, + tokens: { + input: 0, + output: 0, + reasoning: 0, + cacheRead: 0, + cacheWrite: 0, + }, + costUsd: null, + assistantMessages: 0, + flowCalls: [], + sessionBoundaries: [], + documents: [], + honesty: completionHonesty(null), + reviewer: reviewerActivity([]), + operational: operationalMetrics([], { + flowCalls: [], + assistantMessages: 0, + durationMs: Date.now() - started, + }), + refusedBroadScope: 0, + guidanceSkips: 0, + finalText: "", + questions: [], + durationMs: Date.now() - started, + hostError: null, + provenance: { + artifact, + evaluator, + hostConfigSha256: hostConfigSha256({ + opencodeVersion, + plugin: `opencode-plugin-flow@${packageJson.version}`, + model, + reviewerModel: requestedReviewerModel, + reviewerSteps: requestedReviewerSteps, + platform: hostPlatform, + }), + actors: [], + instructions: [], + transcript, + }, + }; + const cell = campaignCells[job.slot]; + if (!cell) + throw new Error(`Missing v2 campaign cell for slot ${job.slot}.`); + await persistEvaluation("attempt", () => + persistV2Attempt(result, cell, scenario), + ); + return { + slot: job.slot, + cassette, + result, + }; + } + }, + async () => { + const cleanupHost = host; + if (cleanupHost) { + await evaluationPhase("host", "host-cleanup-failed", true, () => + cleanupHost.stop(), + ); + } + }, + ); }; - const recorded = await runQueues(queues, concurrency, runAttempt); + const recorded = await runQueues(queues, concurrency, runAttempt, (entry) => + isEvaluatorFailure(entry.result.failure?.origin), + ); for (const entry of recorded.sort( (left, right) => left.slot - right.slot, )) { @@ -1124,14 +1189,15 @@ async function main(): Promise { 0, ); const v2FinishedAt = new Date().toISOString(); + const stoppedOrigin = strongestFailureOrigin( + results.map((result) => result.failure?.origin), + ); const v2Completion: CampaignCompletion = { status: v2Complete ? "complete" : "stopped", cause: v2Complete ? "fixed-target" - : results.some( - (result) => result.environment || result.error !== undefined, - ) - ? "host" + : stoppedOrigin + ? stoppedOrigin : results.some((result) => result.unscored) ? "operator" : "evaluator", @@ -1151,26 +1217,27 @@ async function main(): Promise { ), }, }; - await reportStore.finalize({ - reportId: `flow-v2-${stamp}`, - completion: v2Completion, - allocationCommitmentSha256: null, - }); + await persistEvaluation("finalize", () => + reportStore.finalize({ + reportId: `flow-v2-${stamp}`, + completion: v2Completion, + allocationCommitmentSha256: null, + }), + ); const v2ReportPath = join(v2Directory, "report.json"); console.log(`V2 report: ${v2ReportPath}`); console.log(`\n${formatTable(results)}\n`); - // An abort is excluded for the same reason an allowed ask is: the run never - // reached the outcome the scenario asks about, so counting it as a failure reports - // a measurement that did not happen. One wedged attempt was the only reason a - // measured report came back NOT QUALIFIED. const scored = results.filter( - (result) => - !result.environment && !result.unscored && result.error === undefined, + (result) => !result.failure && !result.unscored, ); - const blocked = results.filter((result) => result.environment).length; + const blocked = results.filter( + (result) => + result.failure?.origin === "provider" || + result.failure?.origin === "host", + ).length; const aborted = results.filter( - (result) => !result.environment && result.error !== undefined, + (result) => result.failure?.origin === "evaluator", ).length; const asked = results.filter((result) => result.escalated).length; const askedUnscored = results.filter((result) => result.unscored).length; @@ -1199,11 +1266,11 @@ async function main(): Promise { console.log( `${passed}/${scored.length} passed | ${totalIn} input (+${totalCached} cached) + ${totalOut} output tokens | ${spend}${ blocked > 0 - ? `\n${blocked} run(s) never reached the model and are excluded; re-run them before trusting this pass rate.` + ? `\n${blocked} provider or host failure(s) are excluded; address the external cause, then re-run them before trusting this pass rate.` : "" }${ aborted > 0 - ? `\n${aborted} run(s) aborted mid-flight and are excluded; the stop is the finding, not the outcome, so re-run them before trusting this pass rate.` + ? `\n${aborted} evaluator failure(s) are excluded; fix the evaluator before starting a fresh campaign.` : "" }${ asked > 0 @@ -1246,7 +1313,11 @@ async function main(): Promise { ); const operational = aggregateOperationalMetrics( results - .filter((result) => !result.environment) + .filter( + (result) => + result.failure?.origin !== "provider" && + result.failure?.origin !== "host", + ) .map((result) => result.operational), ); console.log( @@ -1290,41 +1361,43 @@ async function main(): Promise { } const reportPath = join(reportDir, `${stamp}.json`); - await writeFile( - reportPath, - `${JSON.stringify( - { - flowVersion: packageJson.version, - opencodeVersion, - recordedAt: new Date().toISOString(), - promptFootprint: footprint, - summary: { - passed, - scored: scored.length, - environmentBlocked: blocked, - aborted, - escalated: asked, - escalationExcluded: askedUnscored, - total: results.length, - totalIn, - totalCached, - totalOut, - costUsd: priced.length === 0 ? null : cost, - costReportedRuns: priced.length, - passRates: Object.fromEntries(rates), - closedCompleted, - falseCompletions: falseCompletions.length, - reviewer, - broadScopeRefusals, - guidanceSkipped, - operational, + await persistEvaluation("legacy-report", () => + writeFile( + reportPath, + `${JSON.stringify( + { + flowVersion: packageJson.version, + opencodeVersion, + recordedAt: new Date().toISOString(), + promptFootprint: footprint, + summary: { + passed, + scored: scored.length, + environmentBlocked: blocked, + aborted, + escalated: asked, + escalationExcluded: askedUnscored, + total: results.length, + totalIn, + totalCached, + totalOut, + costUsd: priced.length === 0 ? null : cost, + costReportedRuns: priced.length, + passRates: Object.fromEntries(rates), + closedCompleted, + falseCompletions: falseCompletions.length, + reviewer, + broadScopeRefusals, + guidanceSkipped, + operational, + }, + results, }, - results, - }, - null, - 2, - )}\n`, - "utf8", + null, + 2, + )}\n`, + "utf8", + ), ); console.log(`Report: ${reportPath}`); @@ -1333,19 +1406,23 @@ async function main(): Promise { // which decisions are worth pinning. if (cassettes.length > 0) { const cassetteDir = join(reportDir, `${stamp}.cassettes`); - await mkdir(cassetteDir, { recursive: true }); + await persistEvaluation("cassette-directory", () => + mkdir(cassetteDir, { recursive: true }), + ); for (const cassette of cassettes) { - await writeFile( - join( - cassetteDir, - cassetteFileName(cassette.scenario, cassette.model, cassette.attempt), + await persistEvaluation("cassette", () => + writeFile( + join( + cassetteDir, + cassetteFileName( + cassette.scenario, + cassette.model, + cassette.attempt, + ), + ), + `${JSON.stringify(cassette, null, "\t")}\n`, + "utf8", ), - // Tabs, because a pinned cassette lives under `evals/` and the repo - // formatter checks it there. Two-space candidates meant every copy into - // `evals/cassettes/` failed lint until it was reformatted, which is a - // step between reading a run and keeping it. - `${JSON.stringify(cassette, null, "\t")}\n`, - "utf8", ); } const gated = cassettes.filter( diff --git a/tests/atomic-analysis.test.ts b/tests/atomic-analysis.test.ts index eae4ff3..4dee5a7 100644 --- a/tests/atomic-analysis.test.ts +++ b/tests/atomic-analysis.test.ts @@ -67,7 +67,10 @@ type RateRow = { readonly unsubmittedReviews?: number; } | { readonly kind: "unscored" } - | { readonly kind: "failure" }; + | { + readonly kind: "failure"; + readonly origin?: "provider" | "host" | "evaluator"; + }; }; function rateOutcome(row: RateRow) { @@ -80,7 +83,7 @@ function rateOutcome(row: RateRow) { if (row.outcome.kind === "failure") { return { kind: "failure" as const, - origin: "evaluator" as const, + origin: row.outcome.origin ?? ("evaluator" as const), code: "fixture-failure", retryable: false, }; @@ -364,6 +367,54 @@ describe("v2 atomic release decisions", () => { ); }); + test("makes evaluator integrity failures hard", () => { + const fixture = parsedRate([ + { + provider: "provider-a", + outcome: { kind: "failure", origin: "evaluator" }, + }, + { provider: "provider-b", outcome: { kind: "product", passed: true } }, + ]); + const decision = deriveReleaseDecision(fixture); + expect(decision.verdict).toBe("NOT VERIFIED"); + expect(decision.reasons.map((reason) => reason.code)).toContain( + "campaign-integrity-failure", + ); + }); + + test("makes a persistence stop hard even when no failure attempt was writable", () => { + const raw = buildRateReport(); + const attempts = raw.attempts.slice(0, 1); + const catalog = mustCatalog(rateCatalog()); + const report = mustReport( + { + ...raw, + attempts, + completion: { + ...raw.completion, + status: "stopped", + cause: "persistence", + observed: { + ...raw.completion.observed, + attempts: attempts.length, + outputTokens: 10, + costUsd: 0.1, + }, + }, + }, + catalog, + ); + const decision = deriveReleaseDecision({ + catalog, + report, + expected: releaseExpected(report), + }); + expect(decision.verdict).toBe("NOT VERIFIED"); + expect(decision.reasons.map((reason) => reason.code)).toContain( + "campaign-integrity-failure", + ); + }); + test("accepts distinct frozen host configurations for different model cells", () => { const raw = buildRateReport(); const second = raw.attempts[1]; diff --git a/tests/eval-failure-origin.test.ts b/tests/eval-failure-origin.test.ts new file mode 100644 index 0000000..1c29ae1 --- /dev/null +++ b/tests/eval-failure-origin.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, test } from "bun:test"; +import { + attemptFailure, + EvaluationPersistenceError, + evaluateScenario, + evaluationPhase, + evaluatorFailure, + failureOutcome, + isEvaluatorFailure, + persistEvaluation, + preservePrimaryFailure, + providerFailure, + strongestFailureOrigin, +} from "../evals/failure-origin.js"; + +describe("eval failure origins", () => { + test("turns every grader throw into a non-retryable evaluator failure", () => { + for (const thrown of [new Error("grader exploded"), "grader exploded"]) { + const evaluated = evaluateScenario( + () => { + throw thrown; + }, + { durable: true }, + ); + expect(evaluated).toEqual({ + kind: "failure", + failure: { + origin: "evaluator", + code: "scenario-check-threw", + detail: "grader exploded", + retryable: false, + }, + }); + } + }); + + test("returns grader issues without manufacturing a failure", () => { + expect(evaluateScenario(() => ["missing closure"], {})).toEqual({ + kind: "evaluated", + issues: ["missing closure"], + }); + }); + + test("preserves every failure origin in the durable outcome", () => { + for (const origin of ["provider", "host", "evaluator"] as const) { + expect( + failureOutcome(attemptFailure(origin, "fixture", origin, false)), + ).toEqual({ + kind: "failure", + origin, + code: "fixture", + retryable: false, + }); + } + }); + + test("attributes assistant-message errors to the provider boundary", () => { + expect( + providerFailure({ name: "APIError", providerID: "xai" }), + ).toMatchObject({ + origin: "provider", + code: "provider-rejected-turn", + }); + }); + + test("keeps tagged host failures external and defaults unknown code to evaluator", async () => { + let tagged: unknown; + try { + await evaluationPhase("host", "session-create-failed", true, () => + Promise.reject(new Error("connection lost")), + ); + } catch (error) { + tagged = error; + } + expect(evaluatorFailure(tagged)).toMatchObject({ + origin: "host", + code: "session-create-failed", + retryable: true, + }); + expect(evaluatorFailure(new Error("parser bug"))).toMatchObject({ + origin: "evaluator", + code: "evaluator-transform-threw", + retryable: false, + }); + }); + + test("stops on one persistence failure without retrying the write", async () => { + let writes = 0; + await expect( + persistEvaluation("attempt", async () => { + writes += 1; + throw new Error("disk full"); + }), + ).rejects.toMatchObject({ + failure: { + origin: "persistence", + code: "attempt-write-failed", + detail: "disk full", + retryable: false, + }, + }); + expect(writes).toBe(1); + }); + + test("preserves a primary persistence failure when cleanup also fails", async () => { + let caught: unknown; + try { + await preservePrimaryFailure( + () => Promise.reject(new EvaluationPersistenceError("attempt", "disk")), + () => Promise.reject(new Error("cleanup")), + ); + } catch (error) { + caught = error; + } + expect(caught).toMatchObject({ + failure: { origin: "persistence", code: "attempt-write-failed" }, + }); + expect(caught).toBeInstanceOf(EvaluationPersistenceError); + expect(caught instanceof Error ? caught.cause : null).toBeInstanceOf( + AggregateError, + ); + }); + + test("uses the strongest campaign stop independent of attempt order", () => { + expect(strongestFailureOrigin(["provider", "evaluator", "host"])).toBe( + "evaluator", + ); + expect(strongestFailureOrigin(["host", "persistence"])).toBe("persistence"); + expect(strongestFailureOrigin([])).toBeNull(); + }); + + test("marks only evaluator failures as paid-work stop signals", () => { + expect(isEvaluatorFailure("evaluator")).toBe(true); + expect(isEvaluatorFailure("provider")).toBe(false); + expect(isEvaluatorFailure("host")).toBe(false); + expect(isEvaluatorFailure("persistence")).toBe(false); + }); +}); diff --git a/tests/eval-replay.test.ts b/tests/eval-replay.test.ts index 66ec8b4..e0eb437 100644 --- a/tests/eval-replay.test.ts +++ b/tests/eval-replay.test.ts @@ -13,6 +13,7 @@ import { describe, expect, test } from "bun:test"; import { readFile } from "node:fs/promises"; import { + buildCassette, CASSETTE_VERSION, type Cassette, cassetteFileName, @@ -211,6 +212,28 @@ function honestyOf(outcome: Awaited>) { } describe("decision-layer replay", () => { + test("retains provider failures as provider fidelity", () => { + const cassette = buildCassette({ + flowVersion: "test", + scenario: "provider-failure", + model: "provider/model", + attempt: 1, + hostPlatform: "linux", + files: {}, + projectPath: "/workspace", + calls: [], + finalText: "", + assistantMessages: 0, + verdict: "PROVIDER", + issues: [], + falseCompletion: false, + documents: [], + extraFidelity: ["provider-error"], + }); + expect(cassette.fidelity).toContain("provider-error"); + expect(cassette.fidelity).not.toContain("host-error"); + }); + test("reproduces a passing happy-path run with no model and no host", async () => { const result = await replayCassette(happyPathCassette()); expect(result.divergences).toEqual([]); diff --git a/tests/eval-report.test.ts b/tests/eval-report.test.ts index 7b6bad9..0551d64 100644 --- a/tests/eval-report.test.ts +++ b/tests/eval-report.test.ts @@ -314,6 +314,47 @@ function itemAt(values: readonly T[], index: number): T { } describe("eval report boundary", () => { + test("keeps persistence as a campaign stop, never a fabricated attempt", () => { + const fixture = report(); + const fabricatedAttempt: unknown = { + ...fixture, + attempts: fixture.attempts.map((attempt, index) => + index === 0 + ? { + ...attempt, + actors: [], + instructions: [], + transcript: null, + outcome: { + kind: "failure", + origin: "persistence", + code: "attempt-write-failed", + retryable: false, + }, + } + : attempt, + ), + }; + expect(parseReport(fabricatedAttempt, caseCatalog()).ok).toBe(false); + + const persistenceStop: unknown = { + ...fixture, + attempts: [], + completion: { + ...fixture.completion, + status: "stopped", + cause: "persistence", + observed: { + ...fixture.completion.observed, + attempts: 0, + outputTokens: 0, + costUsd: 0, + }, + }, + }; + expect(parseReport(persistenceStop, caseCatalog()).ok).toBe(true); + }); + test("accepts a complete ledger and derives a canonical plan hash", () => { const value = report(); const parsed = result(value); diff --git a/tests/eval-reporting.test.ts b/tests/eval-reporting.test.ts index d389ca4..5d37b05 100644 --- a/tests/eval-reporting.test.ts +++ b/tests/eval-reporting.test.ts @@ -113,6 +113,85 @@ describe("eval run classification", () => { expect(peak).toBe(2); }); + test("stops unclaimed jobs after a fatal failure and drains in-flight work", async () => { + const started: string[] = []; + const finished: string[] = []; + let releaseSecond: (() => void) | undefined; + const secondStarted = new Promise((resolve) => { + releaseSecond = resolve; + }); + await expect( + runQueues( + [ + ["a1", "a2"], + ["b1", "b2"], + ], + 2, + async (job) => { + started.push(job); + if (job === "a1") { + await secondStarted; + throw new Error("fatal persistence"); + } + releaseSecond?.(); + await Bun.sleep(5); + finished.push(job); + return job; + }, + ), + ).rejects.toThrow("fatal persistence"); + expect(started.sort()).toEqual(["a1", "b1"]); + expect(finished).toEqual(["b1"]); + }); + + test("stops after a returned integrity failure while preserving its result", async () => { + const started: string[] = []; + const done = await runQueues( + [ + ["a1", "a2"], + ["b1", "b2"], + ], + 1, + async (job) => { + started.push(job); + return { job, fatal: job === "a1" }; + }, + (result) => result.fatal, + ); + expect(started).toEqual(["a1"]); + expect(done).toEqual([{ job: "a1", fatal: true }]); + }); + + test("preserves an in-flight persistence error after an integrity stop", async () => { + let startSecond: (() => void) | undefined; + let releaseSecond: (() => void) | undefined; + const secondStarted = new Promise((resolve) => { + startSecond = resolve; + }); + const integrityStopped = new Promise((resolve) => { + releaseSecond = resolve; + }); + await expect( + runQueues( + [["evaluator"], ["persistence"]], + 2, + async (job) => { + if (job === "evaluator") { + await secondStarted; + return { fatal: true }; + } + startSecond?.(); + await integrityStopped; + throw new Error("attempt write failed"); + }, + (result) => { + if (result.fatal) releaseSecond?.(); + return result.fatal; + }, + ), + ).rejects.toThrow("attempt write failed"); + }); + test("never runs two jobs from one queue at once", async () => { // The whole point of keying a queue by model: overlap inside one queue would // race one provider's rate limit against itself. @@ -493,8 +572,18 @@ describe("eval pass rates", () => { const attempt = ( scenario: string, passed: boolean, - extra: { unscored?: boolean; environment?: boolean; error?: string } = {}, + extra: { + unscored?: boolean; + failure?: { + origin: "provider" | "host" | "evaluator"; + code: string; + detail: string; + retryable: boolean; + }; + } = {}, ) => ({ scenario, model: "m", passed, ...extra }); + const failure = (origin: "provider" | "host" | "evaluator") => + ({ origin, code: "fixture", detail: "fixture", retryable: false }) as const; test("counts passes against scored attempts only", () => { expect( @@ -514,7 +603,7 @@ describe("eval pass rates", () => { // rather than as unmeasured. const rates = passRates([ attempt("gate", false, { unscored: true }), - attempt("gate", false, { environment: true }), + attempt("gate", false, { failure: failure("host") }), ]); expect(rates).toEqual([ ["gate @ m", { passed: 0, attempts: 0, unscored: 2, aborted: 0 }], @@ -533,7 +622,7 @@ describe("eval pass rates", () => { passRates([ attempt("gate", true), attempt("gate", true), - attempt("gate", false, { error: "wedged: bash:running" }), + attempt("gate", false, { failure: failure("evaluator") }), ]), ).toEqual([ ["gate @ m", { passed: 2, attempts: 2, unscored: 0, aborted: 1 }], @@ -541,14 +630,22 @@ describe("eval pass rates", () => { expect( formatRate({ passed: 2, attempts: 2, unscored: 0, aborted: 1 }), ).toBe("2/2 1 aborted"); - // A lost host is still environment-blocked rather than an abort, though it - // carries the same `error` field. + expect( + passRates([attempt("gate", false, { failure: failure("provider") })]), + ).toEqual([ + ["gate @ m", { passed: 0, attempts: 0, unscored: 1, aborted: 0 }], + ]); + }); + + test("never hides evaluator failures as environment exclusions", () => { expect( passRates([ - attempt("gate", false, { environment: true, error: "no credentials" }), + { ...attempt("gate", false), failure: failure("provider") }, + { ...attempt("gate", false), failure: failure("host") }, + { ...attempt("gate", false), failure: failure("evaluator") }, ]), ).toEqual([ - ["gate @ m", { passed: 0, attempts: 0, unscored: 1, aborted: 0 }], + ["gate @ m", { passed: 0, attempts: 0, unscored: 2, aborted: 1 }], ]); }); diff --git a/tests/eval-scenario-checks.test.ts b/tests/eval-scenario-checks.test.ts index f926312..8d643d4 100644 --- a/tests/eval-scenario-checks.test.ts +++ b/tests/eval-scenario-checks.test.ts @@ -21,7 +21,7 @@ function outcome(overrides: Partial): Outcome { costUsd: null, assistantMessages: 0, durationMs: 0, - hostError: null, + providerError: null, ...overrides, }; }