From 3298e60f175fc6d720755242fd3c664f01464a95 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Fri, 28 Aug 2026 07:30:40 +0200 Subject: [PATCH] Seal regradable qualification bundles --- .../03-assurance-hardening/decisions.tsv | 8 + .github/workflows/evals.yml | 21 +- docs/development.md | 9 +- docs/maintainer-contract.md | 5 +- docs/release-qualification.md | 20 +- evals/conformance-evidence.ts | 85 ++ evals/grader-input.ts | 140 ++++ evals/harness.ts | 5 +- evals/provenance.ts | 5 +- evals/qualification-bundle.ts | 734 ++++++++++++++++++ evals/qualification/README.md | 19 +- evals/release-policy.ts | 18 +- evals/report-store.ts | 13 +- evals/run.ts | 128 ++- evals/scenarios.ts | 47 +- scripts/qualify-release.ts | 458 ++++++++++- scripts/release-metadata.ts | 2 +- tests/documentation-contract.test.ts | 7 +- tests/grader-input.test.ts | 163 ++++ tests/provenance.test.ts | 4 + tests/qualification-bundle.test.ts | 341 ++++++++ tests/qualification-cli.test.ts | 431 ++++++++++ tests/release-qualification.test.ts | 69 ++ 23 files changed, 2547 insertions(+), 185 deletions(-) create mode 100644 evals/conformance-evidence.ts create mode 100644 evals/grader-input.ts create mode 100644 evals/qualification-bundle.ts create mode 100644 tests/grader-input.test.ts create mode 100644 tests/qualification-bundle.test.ts create mode 100644 tests/qualification-cli.test.ts diff --git a/.agents/plans/03-assurance-hardening/decisions.tsv b/.agents/plans/03-assurance-hardening/decisions.tsv index e41d087..9c41940 100644 --- a/.agents/plans/03-assurance-hardening/decisions.tsv +++ b/.agents/plans/03-assurance-hardening/decisions.tsv @@ -49,3 +49,11 @@ ts phase decision why evidence result 2026-08-28T04:23:41Z phase-6 closed the final transcript and lineage review findings flat synthetic transcripts and unlinked reviewer observations remained broader than the real OpenCode evidence boundary root export messages plus tool parts; parent-child-model lineage match; final three-model review 586 pass, 1 intentional skip, 13 of 13 replays, pinned live smoke pass; all final reviewers green 2026-08-28T04:32:34Z phase-6 merged evidence-derived canary qualification the exact PR head passed two CI runs and isolated shipping verification PR 53; merge 269bd855 merged to main 2026-08-28T04:32:34Z phase-6-lineage captured and closed cross-session transcript splicing late review showed an authentic runtime status could be pooled with lifecycle calls from another manager session ObservedCall session lineage; focused red splice test all canary proof calls now come from the one manager session named by the reviewer task lineage +2026-08-28T04:58:19Z phase-6-lineage merged coherent canary session lineage the exact three-file follow-up passed CI and isolated shipping verification PR 54; merge 65f1211 merged to main +2026-08-28T04:58:19Z phase-7 selected a sealed complete evidence bundle independent judge required attempts, redacted transcripts, expected provenance, exact artifact and canary, decision, and full grader source to make regrading real three architecture candidates; independent judge candidate C base with A manifest-last layout and B evidence retention; V3 object store and package-lock closure rejected +2026-08-28T04:58:19Z phase-7 captured bundle and redaction regressions first bundle sealing, replay, interruption, missing roles, secrets, raw IDs, corruption, concurrency, symlinks, and normalized grader inputs needed executable stop gates new focused tests missing modules and raw session ID retention failed red as expected +2026-08-28T04:58:19Z phase-7 implemented regradable qualification packaging digest-only decisions could not be independently reproduced normalized grader inputs; source closure; stable campaign ingestion; content-addressed manifest-last bundle qualifier regrades every retained attempt and seals all evidence only with an exact derived canary and VERIFIED decision +2026-08-28T05:21:31Z phase-7 fixed all actionable first-wave review findings transcripts could be reused or substituted, duplicate attempts hidden, full outcome/provenance fields trusted, canary bytes raced, artifact secrets skipped, and parent paths replaced canonical report-store paths; shared conformance derivation; canary ref recheck; bounded tar scan; component identity snapshots all claimed product outcome, actor, instruction, artifact, and evidence fields now reproduce from retained bytes +2026-08-28T05:21:31Z phase-7 proved the complete qualifier CLI path placeholder bundle tests did not exercise the canonical 76-cell campaign and exact canary wiring real packed plugin; eight passing cassette replays expanded to 76 cells; evidence-derived canary; real CLI positive end-to-end bundle seals with 12 fixed roles, 76 attempt/transcript pairs, and complete source closure +2026-08-28T05:30:13Z phase-7 closed the final outcome and independence gaps second review found empty reviewer observations, unretained usage, repeatable host evidence, extra sidecars, and weaker tar text scanning filtered actor observations; schema-bound attempt identity and usage; unique manager pseudonyms; completion rederivation; strict root closure all budget-relevant usage, product outcomes, provenance, and per-cell independence now reproduce from retained evidence +2026-08-28T05:30:13Z phase-7 completed immutable bundle verification the corrected implementation needed whole-product, replay, live-host, and multi-model review evidence bun run check; bun run replay; pinned OpenCode smoke; three final reviewers 599 pass, 1 intentional skip, 13 of 13 replays, live smoke pass, all reviewers green diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 1fed824..06def6b 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -132,30 +132,17 @@ jobs: eval-v2.log retention-days: 90 - - name: Apply release-qualification thresholds - id: qualify + - name: Record sealed-bundle prerequisite if: always() && steps.matrix.outputs.models != '' && steps.run.outputs.report != '' - continue-on-error: true run: | - set +e - mkdir -p evals/decisions - bun run qualify -- --report "${{ steps.run.outputs.report }}" --catalog "${{ steps.run.outputs.catalog }}" --artifact "${{ steps.run.outputs.artifact }}" --decisions-dir evals/decisions 2>&1 | tee qualification.log - status=${PIPESTATUS[0]} - decision_path="$(sed -n -E 's/^(VERIFIED|NOT VERIFIED|INCONCLUSIVE): (.*)$/\2/p' qualification.log | tail -n 1)" - if [ -z "$decision_path" ] || [ ! -f "$decision_path" ]; then - echo "::error::Qualifier did not print an existing decision path." - exit 1 - fi - printf 'decision=%s\n' "$decision_path" >> "$GITHUB_OUTPUT" - exit "$status" + echo "INCONCLUSIVE: sealed qualification requires the exact-artifact canary." | tee qualification.log - - name: Publish the decision artifacts + - name: Publish qualification inputs if: always() && steps.matrix.outputs.models != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: eval-v2-decision + name: eval-v2-qualification-input path: | qualification.log - ${{ steps.qualify.outputs.decision }} if-no-files-found: warn retention-days: 90 diff --git a/docs/development.md b/docs/development.md index b1e5020..46d7070 100644 --- a/docs/development.md +++ b/docs/development.md @@ -144,10 +144,11 @@ deterministic hook and lifecycle gates. ## Release -A release needs a V2 qualification report from at least two providers, a canary of -the exact packed artifact, and a canary-bound decision. Pass `--report`, `--catalog`, -`--artifact`, and `--canary` to `bun run qualify`. Commit `evals/canary` and -`evals/decisions` before tagging. [Release qualification](release-qualification.md) +A release needs a complete V2 campaign from at least two providers and a canary of +the exact packed artifact. Pass `--campaign-dir` and `--canary` to +`bun run qualify`; it seals the report, attempts, transcripts, artifact, canary, +expected provenance, decision, and grader source into one immutable bundle. Commit +the bundle and canary before tagging. [Release qualification](release-qualification.md) publishes the thresholds and commands. Release tags use `v`. Blocking release checks include the diff --git a/docs/maintainer-contract.md b/docs/maintainer-contract.md index f269f20..1180119 100644 --- a/docs/maintainer-contract.md +++ b/docs/maintainer-contract.md @@ -338,9 +338,8 @@ Model evals are the one exception, admitted by [ADR 0010](adr/0010-declared-canonical-gate.md) against the prompt prose the declared gate replaced. They run weekly and on demand in one workflow, never in a gate a contributor waits on, and skip themselves without a configured matrix or -credentials. `bun run qualify` applies the published thresholds to a report; a -scenario with no published threshold, or a gated scenario the report never ran, -fails qualification. +credentials. `bun run qualify` seals a complete campaign and exact canary. A +scenario without a threshold, or a required scenario the report omitted, fails. Bounded-wave coverage should test the real agent permissions, manager guidance, and host-visible configuration without adding a scheduler or tests-of-tests. diff --git a/docs/release-qualification.md b/docs/release-qualification.md index bd1e5a8..fa3c623 100644 --- a/docs/release-qualification.md +++ b/docs/release-qualification.md @@ -61,10 +61,10 @@ direction. The cadence follows from that: - **Freeze on the public surface** while the guarantees are being measured: tools, commands, guides, agents, and the Session v5 shape. Additive optional fields are allowed; removals and renames are not. -- **No release** without a committed exact V2 decision and fresh canary. - `bun run qualify` writes the decision under `evals/decisions`; release metadata - refuses a tag without its matching `evals/canary` evidence. A `CHANGELOG` entry - states the schema impact explicitly. +- **No release** without a sealed V2 qualification bundle and fresh canary. + The bundle retains every attempt, transcript, grader source, and exact artifact + needed to reproduce its decision. A `CHANGELOG` entry states the schema impact + explicitly. - **Patch releases** for defects and host-compatibility fixes, which is what the weekly OpenCode compatibility smoke exists to catch early. - **Deprecate before removing.** A surface that is going away is announced in one @@ -78,9 +78,7 @@ bun run eval -- --release --model --model bun run eval:canary -- prepare --report /report.json --out # Run the prepared fixture, then record its session and transcript. bun run eval:canary -- record -bun run qualify -- --report /report.json \ - --catalog /catalog.json \ - --artifact /artifact.tgz \ +bun run qualify -- --campaign-dir \ --canary evals/canary/.json ``` @@ -93,7 +91,7 @@ described with their prices in `bun run benchmark -- --model --repeat 3 --seed ` compares Flow with ordinary OpenCode on hidden-graded tasks. It is not a qualification input. -The scheduled workflow (`.github/workflows/evals.yml`) does the same weekly and -publishes the report as an artifact. It skips itself when no model matrix or -provider credentials are configured, because an unconfigured fork is a -configuration state and not a failure. +The scheduled workflow runs the paid campaign weekly and publishes its complete +campaign directory. Sealing waits for the exact-artifact canary, so the workflow +reports qualification as inconclusive rather than manufacturing a partial bundle. +It skips itself when no model matrix or provider credentials are configured. diff --git a/evals/conformance-evidence.ts b/evals/conformance-evidence.ts new file mode 100644 index 0000000..d39412b --- /dev/null +++ b/evals/conformance-evidence.ts @@ -0,0 +1,85 @@ +import type { + RetainedScenarioEvidence, + ScenarioGradeInput, +} from "./grader-input.js"; +import { askedQuestions } from "./harness.js"; +import { + completionHonesty, + type MetricSession, + reviewerActivity, +} from "./metrics.js"; +import { instructionDelivery } from "./provenance.js"; +import type { + ActorIdentity, + AttemptRecordV2, + InstructionDelivery, +} from "./report.js"; + +export function retainedReportActors( + evidence: RetainedScenarioEvidence, +): ActorIdentity[] { + return evidence.actors.map((actor) => ({ + role: actor.role, + requestedModel: actor.requestedModel, + actualModel: + actor.actualModel.kind === "observed" + ? { + kind: "unobserved" as const, + reason: `Host observed providerID=${actor.actualModel.value.providerID} modelID=${actor.actualModel.value.modelID}; full family, gateway, and revision identity is unavailable.`, + } + : actor.actualModel, + sessionIds: [...actor.sessionIds], + })); +} + +export function retainedInstructions( + evidence: RetainedScenarioEvidence, +): InstructionDelivery[] { + return evidence.guidanceLoads.map((load) => + instructionDelivery({ + source: "guidance", + name: load.id ?? "unknown-guidance", + sequence: load.sequence, + text: load.rawOutput, + }), + ); +} + +export function deriveConformanceOutcome(input: { + readonly evidence: RetainedScenarioEvidence; + readonly check: (gradeInput: ScenarioGradeInput) => readonly string[]; + readonly scenarioId: string; + readonly model: string; + readonly attempt: number; +}): Extract { + const issues = [...input.check(input.evidence.gradeInput)]; + const documents = [ + ...(input.evidence.gradeInput.session + ? [input.evidence.gradeInput.session] + : []), + ...input.evidence.gradeInput.archives, + ] as MetricSession[]; + return { + kind: "product", + passed: issues.length === 0, + endedBy: + askedQuestions(input.evidence.gradeInput).length > 0 + ? "user-escalation" + : "quiet", + issues: issues.length > 0 ? issues : [], + evidence: { + kind: "conformance", + falseCompletion: completionHonesty( + documents.find((document) => document.closure) ?? null, + ).falseCompletion, + unsubmittedReviews: reviewerActivity(documents).unsubmitted, + facts: { + scenario: input.scenarioId, + model: input.model, + attempt: input.attempt, + flowCalls: input.evidence.gradeInput.flowCalls.length, + guidanceLoads: input.evidence.guidanceLoads.length, + }, + }, + }; +} diff --git a/evals/grader-input.ts b/evals/grader-input.ts new file mode 100644 index 0000000..3667163 --- /dev/null +++ b/evals/grader-input.ts @@ -0,0 +1,140 @@ +import { z } from "zod"; +import { canonicalSha256 } from "./canonical-json.js"; +import { mapStrings } from "./cassette.js"; +import type { ObservedToolCall, Outcome } from "./harness.js"; + +const TextSchema = z + .string() + .min(1) + .max(4 * 1024 * 1024) + .regex(/\S/); +const JsonRecordSchema = z.record(z.string(), z.unknown()); +const ToolCallSchema: z.ZodType = z + .object({ + tool: TextSchema, + status: z.enum(["pending", "running", "completed", "error"]), + sessionIndex: z.number().int().safe().nonnegative(), + agent: TextSchema, + input: JsonRecordSchema, + output: z.unknown(), + rawOutput: z.string().max(4 * 1024 * 1024), + metadata: JsonRecordSchema, + }) + .strict(); + +export const ScenarioGradeInputSchema = z + .object({ + schemaVersion: z.literal(1), + flowCalls: z.array(ToolCallSchema).max(4096), + allCalls: z.array(ToolCallSchema).max(4096), + session: JsonRecordSchema.nullable(), + archives: z.array(JsonRecordSchema).max(512), + finalText: z.string().max(4 * 1024 * 1024), + }) + .strict(); + +const ObservedModelSchema = z.discriminatedUnion("kind", [ + z + .object({ + kind: z.literal("observed"), + value: z.object({ providerID: TextSchema, modelID: TextSchema }).strict(), + }) + .strict(), + z.object({ kind: z.literal("unobserved"), reason: TextSchema }).strict(), +]); +const RequestedModelSchema = z + .object({ + routeProvider: TextSchema, + gateway: TextSchema.nullable(), + family: TextSchema, + model: TextSchema, + revision: TextSchema.nullable(), + }) + .strict(); +const RetainedActorSchema = z + .object({ + role: z.enum(["manager", "reviewer"]), + sessionIds: z.array(TextSchema).min(1), + actualModel: ObservedModelSchema, + requestedModelId: TextSchema, + requestedModel: RequestedModelSchema, + }) + .strict(); +const GuidanceLoadSchema = z + .object({ + sequence: z.number().int().safe().nonnegative(), + sessionIndex: z.number().int().safe().nonnegative(), + agent: TextSchema, + id: TextSchema.nullable(), + rawOutput: z.string().max(4 * 1024 * 1024), + utf8Bytes: z.number().int().safe().nonnegative(), + }) + .strict(); + +export const RetainedScenarioEvidenceSchema = z + .object({ + schemaVersion: z.literal(1), + attempt: z + .object({ + attemptId: TextSchema, + cellId: TextSchema, + caseId: TextSchema, + repetition: z.number().int().safe().nonnegative(), + model: RequestedModelSchema, + }) + .strict(), + actors: z.array(RetainedActorSchema).max(64), + guidanceLoads: z.array(GuidanceLoadSchema).max(4096), + gradeInput: ScenarioGradeInputSchema, + usage: z + .object({ + durationMs: z.number().int().safe().nonnegative(), + outputTokens: z.number().int().safe().nonnegative(), + costUsd: z.number().finite().nonnegative().nullable(), + }) + .strict(), + }) + .strict(); + +export type ScenarioGradeInput = Pick< + Outcome, + "flowCalls" | "allCalls" | "session" | "archives" | "finalText" +>; +export type RetainedScenarioGradeInput = z.infer< + typeof ScenarioGradeInputSchema +>; +export type RetainedScenarioEvidence = z.infer< + typeof RetainedScenarioEvidenceSchema +>; + +export function pseudonymousEvalId(id: string): string { + return `id_${canonicalSha256("flow-eval-redacted-id-v1", id).slice("sha256:".length, "sha256:".length + 16)}`; +} + +export function pseudonymizeEvalIds(value: T): T { + return mapStrings(value, (text) => + text.replace( + /\b(?:ses_[A-Za-z0-9]+|(?:session|review):[A-Za-z0-9-]+)\b/g, + pseudonymousEvalId, + ), + ) as T; +} + +export function actorsWithSessions< + Actor extends { readonly sessionIds: readonly string[] }, +>(actors: readonly Actor[]): Actor[] { + return actors.filter((actor) => actor.sessionIds.length > 0); +} + +export function scenarioGradeInput( + outcome: Outcome, +): RetainedScenarioGradeInput { + return ScenarioGradeInputSchema.parse({ + schemaVersion: 1, + flowCalls: outcome.flowCalls, + allCalls: outcome.allCalls, + session: outcome.session, + archives: outcome.archives, + finalText: outcome.finalText, + }); +} diff --git a/evals/harness.ts b/evals/harness.ts index b727e7e..af7622e 100644 --- a/evals/harness.ts +++ b/evals/harness.ts @@ -33,6 +33,7 @@ import { preservePrimaryFailure, providerFailure, } from "./failure-origin.js"; +import type { ScenarioGradeInput } from "./grader-input.js"; import { extractObservedActor, guidanceLoad, @@ -292,7 +293,7 @@ export type Scenario = { */ readonly mayEscalate?: boolean; /** Returns a list of failures. Empty means the scenario passed. */ - readonly check: (outcome: Outcome) => readonly string[]; + readonly check: (outcome: ScenarioGradeInput) => readonly string[]; }; /** @@ -810,7 +811,7 @@ export async function runQueues( * both for a human judging whether asking was right and for a check that reads * whether the blocker was named at all. */ -export function askedQuestions(outcome: Outcome): string[] { +export function askedQuestions(outcome: Pick): string[] { return outcome.allCalls .filter((call) => call.tool === "question") .map((call) => JSON.stringify(call.input)); diff --git a/evals/provenance.ts b/evals/provenance.ts index 090a0d4..b5129c1 100644 --- a/evals/provenance.ts +++ b/evals/provenance.ts @@ -6,6 +6,7 @@ import { promisify } from "node:util"; import { createFileSourceIdentityProvider } from "../src/infrastructure/fs/source-identity.js"; import { canonicalJson, canonicalSha256 } from "./canonical-json.js"; import { normalizeRecorded, REDACTED } from "./cassette.js"; +import { pseudonymizeEvalIds } from "./grader-input.js"; import type { ArtifactIdentity, EvaluatorIdentity, @@ -345,7 +346,9 @@ export function redactTranscript(input: { readonly projectPath: string; }): RedactedTranscript { const text = canonicalJson( - redactSensitiveFields(normalizeRecorded(input.value, input.projectPath)), + pseudonymizeEvalIds( + redactSensitiveFields(normalizeRecorded(input.value, input.projectPath)), + ), ); return { text, sha256: sha256(new TextEncoder().encode(text)) }; } diff --git a/evals/qualification-bundle.ts b/evals/qualification-bundle.ts new file mode 100644 index 0000000..2c594a4 --- /dev/null +++ b/evals/qualification-bundle.ts @@ -0,0 +1,734 @@ +import { spawnSync } from "node:child_process"; +import { constants } from "node:fs"; +import { + link, + lstat, + mkdir, + mkdtemp, + open, + readdir, + realpath, + rm, + unlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join, relative, resolve, sep, win32 } from "node:path"; +import { z } from "zod"; +import { canonicalJson, canonicalSha256 } from "./canonical-json.js"; +import { scrubSecrets } from "./cassette.js"; + +const MAX_OBJECT_BYTES = 16 * 1024 * 1024; +const MAX_BUNDLE_BYTES = 128 * 1024 * 1024; +const DigestSchema = z.string().regex(/^sha256:[a-f0-9]{64}$/); +const TextSchema = z.string().min(1).max(4096).regex(/\S/); +const RoleSchema = z.enum([ + "report", + "catalog", + "policy", + "plan", + "completion", + "expected-provenance", + "decision", + "artifact", + "attempt", + "transcript", + "canary-record", + "canary-installation", + "canary-session", + "canary-transcript", + "authority-source", +]); +const MediaTypeSchema = z.enum([ + "application/json", + "application/gzip", + "text/typescript", +]); +const BundleFileSchema = z + .object({ + role: RoleSchema, + id: TextSchema.optional(), + mediaType: MediaTypeSchema, + object: z.string().regex(/^objects\/sha256-[a-f0-9]{64}$/), + sha256: DigestSchema, + bytes: z.number().int().safe().nonnegative().max(MAX_OBJECT_BYTES), + }) + .strict(); + +export const QualificationBundleManifestSchema = z + .object({ + schemaVersion: z.literal(1), + kind: z.literal("release-qualification"), + bundleId: z.string().regex(/^qb1-[a-f0-9]{64}$/), + bundleSha256: DigestSchema, + reportId: TextSchema, + packageVersion: TextSchema, + verdict: z.enum(["VERIFIED", "NOT VERIFIED", "INCONCLUSIVE"]), + files: z.array(BundleFileSchema).min(1).max(1024), + }) + .strict(); + +export type QualificationBundleManifest = z.infer< + typeof QualificationBundleManifestSchema +>; +export type QualificationBundleFile = Readonly<{ + role: z.infer; + id?: string | undefined; + mediaType: z.infer; + bytes: Uint8Array; +}>; +export type QualificationBundleInput = Readonly<{ + reportId: string; + packageVersion: string; + verdict: "VERIFIED" | "NOT VERIFIED" | "INCONCLUSIVE"; + files: readonly QualificationBundleFile[]; +}>; + +const fixedRoles = [ + "report", + "catalog", + "policy", + "plan", + "completion", + "expected-provenance", + "decision", + "artifact", + "canary-record", + "canary-installation", + "canary-session", + "canary-transcript", +] as const; + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${new Bun.CryptoHasher("sha256").update(bytes).digest("hex")}`; +} + +function manifestSha256( + manifest: Omit, +): string { + return canonicalSha256("flow-qualification-bundle-v1", manifest); +} + +function fileKey(file: Pick): string { + return `${file.role}\0${file.id ?? ""}`; +} + +function validateRoles(files: readonly QualificationBundleFile[]): void { + const keys = files.map(fileKey); + if (new Set(keys).size !== keys.length) + throw new Error("Qualification bundle contains a duplicate role and id."); + for (const file of files) { + if (!file.id) continue; + if ( + file.role === "authority-source" + ? isAbsolute(file.id) || + win32.isAbsolute(file.id) || + file.id.split(/[\\/]/).includes("..") + : /[\\/]/.test(file.id) + ) { + throw new Error("Qualification bundle role id is not a safe identifier."); + } + } + for (const role of fixedRoles) { + if (files.filter((file) => file.role === role && !file.id).length !== 1) { + throw new Error( + `Qualification bundle role ${role} must occur exactly once.`, + ); + } + } + const attempts = new Set( + files + .filter((file) => file.role === "attempt" && file.id) + .map((file) => file.id), + ); + const transcripts = new Set( + files + .filter((file) => file.role === "transcript" && file.id) + .map((file) => file.id), + ); + if ( + attempts.size === 0 || + canonicalJson([...attempts].sort()) !== + canonicalJson([...transcripts].sort()) + ) { + throw new Error( + "Qualification bundle attempt and transcript roles must be complete.", + ); + } + if (!files.some((file) => file.role === "authority-source" && file.id)) { + throw new Error("Qualification bundle authority source roles are missing."); + } +} + +function assertSafeText(text: string): void { + if (scrubSecrets(text) !== text) + throw new Error("Qualification bundle contains secret-shaped evidence."); + if (/\b(?:ses_[A-Za-z0-9]+|(?:session|review):[A-Za-z0-9-]+)\b/.test(text)) + throw new Error("Qualification bundle contains a raw session identifier."); + if ( + /(?:\/Users\/[^/\s]+|\/home\/[^/\s]+|[A-Za-z]:\\Users\\[^\\\s]+)/.test(text) + ) + throw new Error("Qualification bundle contains an absolute user path."); +} + +function assertSafeSource(text: string): void { + if ( + /\b(?:sk-ant|sk-proj|sk-or|xai|gsk|hf|dop_v1|shpat)[-_][A-Za-z0-9_-]{16,}/.test( + text, + ) || + /\b(?:github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9]{16}|AKIA[0-9A-Z]{16})\b/.test( + text, + ) || + /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/.test(text) || + /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/i.test(text) + ) { + throw new Error("Qualification bundle contains secret-shaped source."); + } +} + +async function assertSafeArtifact(bytes: Uint8Array): Promise { + const directory = await mkdtemp(join(tmpdir(), "flow-bundle-artifact-")); + const path = join(directory, "artifact.tgz"); + try { + await writeFile(path, bytes, { mode: 0o600 }); + const listed = spawnSync("tar", ["-tzf", path], { + encoding: "utf8", + maxBuffer: MAX_BUNDLE_BYTES, + }); + if (listed.status !== 0) + throw new Error("Qualification artifact is not a readable tarball."); + const members = listed.stdout.split(/\r?\n/).filter(Boolean); + const verbose = spawnSync("tar", ["-tvzf", path], { + encoding: "utf8", + maxBuffer: MAX_BUNDLE_BYTES, + }); + const memberTypes = verbose.stdout.split(/\r?\n/).filter(Boolean); + if ( + verbose.status !== 0 || + memberTypes.length !== members.length || + memberTypes.some((line) => line[0] !== "-" && line[0] !== "d") + ) { + throw new Error( + "Qualification artifact contains an unsupported member type.", + ); + } + if (new Set(members).size !== members.length) + throw new Error("Qualification artifact contains duplicate members."); + let total = 0; + for (const member of members) { + if ( + member.startsWith("-") || + member.startsWith("/") || + member.split("/").includes("..") || + member.includes("\\") + ) + throw new Error( + "Qualification artifact contains an unsafe member path.", + ); + if (member.endsWith("/")) continue; + const extracted = spawnSync("tar", ["-xOzf", path, "--", member], { + encoding: "buffer", + maxBuffer: MAX_OBJECT_BYTES + 1, + }); + if (extracted.status !== 0 || !Buffer.isBuffer(extracted.stdout)) + throw new Error("Qualification artifact member is unreadable."); + if (extracted.stdout.byteLength > MAX_OBJECT_BYTES) + throw new Error( + "Qualification artifact member exceeds its byte limit.", + ); + total += extracted.stdout.byteLength; + if (total > MAX_BUNDLE_BYTES) + throw new Error( + "Qualification artifact exceeds its expanded byte limit.", + ); + const source = extracted.stdout.toString("utf8"); + assertSafeSource(source); + try { + assertSafeText( + new TextDecoder("utf-8", { fatal: true }).decode(extracted.stdout), + ); + } catch (error) { + if (!(error instanceof TypeError)) throw error; + } + } + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +function normalizedBytes(file: QualificationBundleFile): Buffer { + const bytes = Buffer.from(file.bytes); + if (bytes.byteLength > MAX_OBJECT_BYTES) + throw new Error("Qualification bundle object exceeds its byte limit."); + if (file.mediaType === "application/json") { + let parsed: unknown; + try { + parsed = JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(bytes), + ); + } catch { + throw new Error("Qualification bundle JSON is malformed."); + } + const canonical = canonicalJson(parsed); + assertSafeText(canonical); + return Buffer.from(canonical); + } + if (file.mediaType === "text/typescript") { + const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + assertSafeSource(text); + return Buffer.from(text); + } + return bytes; +} + +function buildManifest(input: QualificationBundleInput): { + readonly manifest: QualificationBundleManifest; + readonly objects: ReadonlyMap; +} { + validateRoles(input.files); + const objects = new Map(); + const files = input.files + .map((file) => { + const bytes = normalizedBytes(file); + const digest = sha256(bytes); + const object = `objects/sha256-${digest.slice("sha256:".length)}`; + const prior = objects.get(object); + if (prior && !prior.equals(bytes)) + throw new Error("Qualification bundle object digest collision."); + objects.set(object, bytes); + return { + role: file.role, + ...(file.id ? { id: file.id } : {}), + mediaType: file.mediaType, + object, + sha256: digest, + bytes: bytes.byteLength, + }; + }) + .sort((left, right) => fileKey(left).localeCompare(fileKey(right))); + const total = files.reduce((sum, file) => sum + file.bytes, 0); + if (total > MAX_BUNDLE_BYTES) + throw new Error("Qualification bundle exceeds its total byte limit."); + const base = { + schemaVersion: 1 as const, + kind: "release-qualification" as const, + reportId: input.reportId, + packageVersion: input.packageVersion, + verdict: input.verdict, + files, + }; + const bundleSha256 = manifestSha256(base); + const manifest = QualificationBundleManifestSchema.parse({ + ...base, + bundleId: `qb1-${bundleSha256.slice("sha256:".length)}`, + bundleSha256, + }); + return { manifest, objects }; +} + +async function syncDirectory(path: string): Promise { + const handle = await open(path, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function writeImmutable( + path: string, + bytes: Buffer, + temporaryRoot: string, +): Promise<"written" | "replayed"> { + try { + const existing = await readSettledImmutable(path, bytes.byteLength + 1); + if (existing.equals(bytes)) return "replayed"; + throw new Error(`Immutable qualification object conflicts: ${path}`); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const temporary = join(temporaryRoot, `.bundle-${crypto.randomUUID()}`); + const handle = await open(temporary, "wx", 0o600); + try { + await handle.writeFile(bytes); + await handle.sync(); + } finally { + await handle.close(); + } + try { + await link(temporary, path); + return "written"; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const existing = await readSettledImmutable(path, bytes.byteLength + 1); + if (!existing.equals(bytes)) + throw new Error(`Immutable qualification object conflicts: ${path}`); + return "replayed"; + } finally { + await unlink(temporary).catch(() => {}); + } +} + +async function readSettledImmutable( + path: string, + maxBytes: number, +): Promise { + let last: unknown; + for (let attempt = 0; attempt < 3; attempt += 1) { + try { + return await readStable(path, maxBytes); + } catch (error) { + last = error; + if (!String(error).includes("changed while reading")) throw error; + await Bun.sleep(1); + } + } + throw last; +} + +export async function writeQualificationBundle(input: { + readonly input: QualificationBundleInput; + readonly outputRoot: string; + readonly checkpoint?: + | ((stage: "before-seal", path: string) => void | Promise) + | undefined; +}): Promise<{ + readonly kind: "written" | "replayed"; + readonly path: string; + readonly manifest: QualificationBundleManifest; +}> { + const artifact = input.input.files.find((file) => file.role === "artifact"); + if (!artifact) + throw new Error("Qualification bundle artifact role is missing."); + await assertSafeArtifact(artifact.bytes); + const built = buildManifest(input.input); + await mkdir(input.outputRoot, { recursive: true, mode: 0o700 }); + const outputIdentity = await realDirectoryIdentity(input.outputRoot); + const path = join(input.outputRoot, built.manifest.bundleId); + await mkdir(join(path, "objects"), { recursive: true, mode: 0o700 }); + const bundleIdentity = await realDirectoryIdentity(path); + const objectsIdentity = await realDirectoryIdentity(join(path, "objects")); + let kind: "written" | "replayed" = "replayed"; + for (const [object, bytes] of built.objects) { + if ( + (await writeImmutable(join(path, object), bytes, input.outputRoot)) === + "written" + ) { + kind = "written"; + } + } + await syncDirectory(join(path, "objects")); + await input.checkpoint?.("before-seal", path); + const seal = Buffer.from(canonicalJson(built.manifest)); + assertSafeText(seal.toString("utf8")); + if ( + (await writeImmutable( + join(path, "bundle.json"), + seal, + input.outputRoot, + )) === "written" + ) { + kind = "written"; + } + await syncDirectory(path); + await syncDirectory(input.outputRoot); + await readQualificationBundle(path); + if ( + !(await sameDirectoryIdentity(outputIdentity)) || + !(await sameDirectoryIdentity(bundleIdentity)) || + !(await sameDirectoryIdentity(objectsIdentity)) + ) + throw new Error( + "Qualification bundle directory changed during publication.", + ); + return { kind, path, manifest: built.manifest }; +} + +async function readStable(path: string, maxBytes: number): Promise { + const beforePath = await lstat(path, { bigint: true }); + if ( + !beforePath.isFile() || + beforePath.isSymbolicLink() || + beforePath.ino === 0n + ) + throw new Error( + "Qualification bundle object is not a stable regular file.", + ); + const flags = + process.platform === "win32" + ? constants.O_RDONLY + : constants.O_RDONLY | constants.O_NOFOLLOW; + const handle = await open(path, flags); + try { + const before = await handle.stat({ bigint: true }); + if ( + !before.isFile() || + before.ino === 0n || + before.dev !== beforePath.dev || + before.ino !== beforePath.ino || + before.size > BigInt(maxBytes) + ) + throw new Error("Qualification bundle object exceeds its byte limit."); + const bytes = Buffer.allocUnsafe(Number(before.size) + 1); + let length = 0; + while (length < bytes.length) { + const read = await handle.read( + bytes, + length, + bytes.length - length, + length, + ); + if (read.bytesRead === 0) break; + length += read.bytesRead; + } + const after = await handle.stat({ bigint: true }); + const afterPath = await lstat(path, { bigint: true }); + if ( + length > maxBytes || + afterPath.isSymbolicLink() || + before.dev !== afterPath.dev || + before.ino !== afterPath.ino || + before.dev !== after.dev || + before.ino !== after.ino || + before.mode !== after.mode || + before.size !== after.size || + before.ctimeNs !== after.ctimeNs || + before.mtimeNs !== after.mtimeNs || + BigInt(length) !== after.size + ) { + throw new Error("Qualification bundle object changed while reading."); + } + return bytes.subarray(0, length); + } finally { + await handle.close(); + } +} + +type DirectoryIdentity = Readonly<{ path: string; dev: bigint; ino: bigint }>; + +async function realDirectoryIdentity(path: string): Promise { + const info = await lstat(path, { bigint: true }); + if (info.isSymbolicLink() || !info.isDirectory() || info.ino === 0n) + throw new Error("Qualification bundle directory must not be a symlink."); + return { path, dev: info.dev, ino: info.ino }; +} + +async function sameDirectoryIdentity( + identity: DirectoryIdentity, +): Promise { + const current = await realDirectoryIdentity(identity.path); + return current.dev === identity.dev && current.ino === identity.ino; +} + +async function safeInputPath( + rootPath: string, + relativePath: string, + kind: "file" | "directory", +): Promise<{ + readonly path: string; + readonly identities: readonly Readonly<{ + path: string; + dev: bigint; + ino: bigint; + }>[]; +}> { + if (isAbsolute(relativePath) || win32.isAbsolute(relativePath)) + throw new Error("Qualification input path must be relative."); + const requestedRoot = resolve(rootPath); + const inside = relative(requestedRoot, resolve(requestedRoot, relativePath)); + if (!inside || inside.split(/[\\/]/)[0] === ".." || isAbsolute(inside)) + throw new Error("Qualification input path escapes its root."); + const rootInfo = await lstat(requestedRoot, { bigint: true }); + if ( + rootInfo.isSymbolicLink() || + !rootInfo.isDirectory() || + rootInfo.ino === 0n + ) + throw new Error("Qualification input root must be a real directory."); + const root = await realpath(requestedRoot); + const identities = [{ path: root, dev: rootInfo.dev, ino: rootInfo.ino }]; + let current = root; + const parts = inside.split(sep); + for (const [index, part] of parts.entries()) { + current = join(current, part); + const info = await lstat(current, { bigint: true }); + const final = index === parts.length - 1; + if ( + info.isSymbolicLink() || + info.ino === 0n || + (final + ? kind === "file" + ? !info.isFile() + : !info.isDirectory() + : !info.isDirectory()) + ) { + throw new Error( + "Qualification input path is not a stable expected type.", + ); + } + identities.push({ path: current, dev: info.dev, ino: info.ino }); + } + if ( + (await realpath(current)).split(sep).join("/") !== + current.split(sep).join("/") + ) + throw new Error("Qualification input path changed during inspection."); + return { path: current, identities }; +} + +function sameIdentities( + left: readonly { + readonly path: string; + readonly dev: bigint; + readonly ino: bigint; + }[], + right: readonly { + readonly path: string; + readonly dev: bigint; + readonly ino: bigint; + }[], +): boolean { + return ( + left.length === right.length && + left.every((identity, index) => { + const next = right[index]; + return ( + next?.path === identity.path && + next.dev === identity.dev && + next.ino === identity.ino + ); + }) + ); +} + +export async function readStableQualificationInput( + root: string, + relativePath: string, + maxBytes = MAX_OBJECT_BYTES, + checkpoint?: + | ((stage: "inspected" | "read") => void | Promise) + | undefined, +): Promise { + const before = await safeInputPath(root, relativePath, "file"); + await checkpoint?.("inspected"); + const bytes = await readStable(before.path, maxBytes); + await checkpoint?.("read"); + const after = await safeInputPath(root, relativePath, "file"); + if (!sameIdentities(before.identities, after.identities)) + throw new Error("Qualification input path changed while reading."); + return bytes; +} + +export async function listStableQualificationDirectory( + root: string, + relativePath: string, +): Promise { + const before = await safeInputPath(root, relativePath, "directory"); + const entries = await readdir(before.path, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || entry.isSymbolicLink()) + throw new Error( + "Qualification input directory contains a non-file entry.", + ); + } + const after = await safeInputPath(root, relativePath, "directory"); + if (!sameIdentities(before.identities, after.identities)) + throw new Error("Qualification input directory changed while listing."); + return entries.map((entry) => entry.name).sort(); +} + +export async function readQualificationBundle(path: string): Promise<{ + readonly manifest: QualificationBundleManifest; + readonly files: readonly Readonly<{ + ref: z.infer; + bytes: Buffer; + }>[]; +}> { + const bundleIdentity = await realDirectoryIdentity(path); + const objectsIdentity = await realDirectoryIdentity(join(path, "objects")); + const rootEntries = await readdir(path, { withFileTypes: true }); + if (!rootEntries.some((entry) => entry.name === "bundle.json")) + throw new Error("Qualification bundle seal is missing."); + if ( + rootEntries.length !== 2 || + !rootEntries.some( + (entry) => + entry.name === "bundle.json" && + entry.isFile() && + !entry.isSymbolicLink(), + ) || + !rootEntries.some( + (entry) => + entry.name === "objects" && + entry.isDirectory() && + !entry.isSymbolicLink(), + ) + ) { + throw new Error( + "Qualification bundle contains unexpected top-level entries.", + ); + } + let seal: Buffer; + try { + seal = await readStable(join(path, "bundle.json"), MAX_OBJECT_BYTES); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + throw new Error("Qualification bundle seal is missing."); + throw error; + } + const manifest = QualificationBundleManifestSchema.parse( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(seal)), + ); + assertSafeText(canonicalJson(manifest)); + const { + bundleId: _bundleId, + bundleSha256: _bundleSha256, + ...base + } = manifest; + const expected = manifestSha256(base); + if ( + manifest.bundleSha256 !== expected || + manifest.bundleId !== `qb1-${expected.slice("sha256:".length)}` || + manifest.bundleId !== path.split(/[\\/]/).at(-1) + ) { + throw new Error("Qualification bundle manifest digest is invalid."); + } + validateRoles( + manifest.files.map((file) => ({ + role: file.role, + ...(file.id ? { id: file.id } : {}), + mediaType: file.mediaType, + bytes: new Uint8Array(), + })), + ); + const objectNames = (await readdir(join(path, "objects"))).sort(); + const expectedNames = [ + ...new Set( + manifest.files.map((file) => file.object.slice("objects/".length)), + ), + ].sort(); + if (canonicalJson(objectNames) !== canonicalJson(expectedNames)) + throw new Error("Qualification bundle contains missing or extra objects."); + const files = await Promise.all( + manifest.files.map(async (ref) => { + const bytes = await readStable(join(path, ref.object), MAX_OBJECT_BYTES); + if (bytes.byteLength !== ref.bytes || sha256(bytes) !== ref.sha256) + throw new Error( + "Qualification bundle object digest or size is invalid.", + ); + normalizedBytes({ + role: ref.role, + ...(ref.id ? { id: ref.id } : {}), + mediaType: ref.mediaType, + bytes, + }); + return { ref, bytes }; + }), + ); + if ( + !(await sameDirectoryIdentity(bundleIdentity)) || + !(await sameDirectoryIdentity(objectsIdentity)) + ) + throw new Error("Qualification bundle directory changed while reading."); + return { manifest, files }; +} diff --git a/evals/qualification/README.md b/evals/qualification/README.md index 226a6dd..ac2299a 100644 --- a/evals/qualification/README.md +++ b/evals/qualification/README.md @@ -1,10 +1,13 @@ -# Qualification records +# Qualification bundles -One JSON file per major release, written by `bun run qualify -- --record ` -when a report clears every published threshold and its `flowVersion` matches -this repository. `scripts/release-metadata.ts` -refuses an `x.0.0` tag whose record is missing, mismatched, or not `QUALIFIED`. +`bun run qualify -- --campaign-dir --canary ` writes a sealed, +content-addressed directory under `bundles/`. -This is a checklist with a filename, not a forged-proof gate. A human can write -the file by hand. The point is that a major tag cannot be cut without one, so -skipping the qualification run has to show up in the release diff. +The seal binds the canonical report, catalog, policy, plan, completion, every +attempt and redacted transcript, expected provenance, exact artifact, canary and +its evidence, derived decision, and the complete grader source closure. The seal is +written last. An interrupted directory is not qualification evidence; an identical +retry completes or replays it, while conflicting bytes are refused. + +Bundle creation is not release authorization. The bundle retains the inputs that +release verification will independently regrade and rederive. diff --git a/evals/release-policy.ts b/evals/release-policy.ts index 1ad81dd..22caa9b 100644 --- a/evals/release-policy.ts +++ b/evals/release-policy.ts @@ -281,7 +281,7 @@ export function releaseCaseCatalogSha256( ); } -export function releaseGraderBundle(repositoryRoot: string) { +export function releaseGraderSourceBundle(repositoryRoot: string) { const root = resolve(repositoryRoot); const pending = ["evals/run.ts", "scripts/qualify-release.ts"]; const files = new Map(); @@ -300,6 +300,10 @@ export function releaseGraderBundle(repositoryRoot: string) { const scanSource = source.startsWith("#!") ? source.slice(source.indexOf("\n") + 1) : source; + for (const match of scanSource.matchAll(/\bimport\s*\(([^)]*)\)/g)) { + if (!/^\s*["'][^"']+["']\s*$/.test(match[1] ?? "")) + throw new Error(`Release grader has a non-literal import: ${path}`); + } const specifiers = transpiler .scanImports(scanSource) .map((item) => item.path) @@ -321,9 +325,17 @@ export function releaseGraderBundle(repositoryRoot: string) { return { files: [...files] .sort(([left], [right]) => left.localeCompare(right)) - .map(([path, source]) => ({ + .map(([path, source]) => ({ path, source })), + }; +} + +export function releaseGraderBundle(repositoryRoot: string) { + return { + files: releaseGraderSourceBundle(repositoryRoot).files.map( + ({ path, source }) => ({ path, sha256: canonicalSha256("flow-release-grader-file-v1", source), - })), + }), + ), }; } diff --git a/evals/report-store.ts b/evals/report-store.ts index 73e0be8..0613040 100644 --- a/evals/report-store.ts +++ b/evals/report-store.ts @@ -57,11 +57,11 @@ function cellId(value: unknown): string | null { : null; } -function attemptFileName(attemptId: string): string { +export function reportStoreAttemptFileName(attemptId: string): string { return `${Buffer.from(attemptId).toString("base64url")}.json`; } -function cellFileName(cellId: string): string { +export function reportStoreCellFileName(cellId: string): string { return `${Buffer.from(cellId).toString("base64url")}.json`; } @@ -237,7 +237,7 @@ export class ReportStore { fail(`Attempt references an unknown plan cell: ${attempt.cellId}.`); } return writeImmutable( - join(this.attemptsDirectory, cellFileName(attempt.cellId)), + join(this.attemptsDirectory, reportStoreCellFileName(attempt.cellId)), Buffer.from(canonicalJson(attempt)), this.hooks, ); @@ -248,10 +248,13 @@ export class ReportStore { readonly text: string; }): Promise<{ readonly artifact: string; readonly sha256: string }> { await mkdir(this.transcriptsDirectory, { recursive: true, mode: 0o700 }); - const artifact = `transcripts/${attemptFileName(input.attemptId)}`; + const artifact = `transcripts/${reportStoreAttemptFileName(input.attemptId)}`; const bytes = Buffer.from(input.text, "utf8"); await writeImmutable( - join(this.transcriptsDirectory, attemptFileName(input.attemptId)), + join( + this.transcriptsDirectory, + reportStoreAttemptFileName(input.attemptId), + ), bytes, this.hooks, ); diff --git a/evals/run.ts b/evals/run.ts index 6cb3551..eb11dd4 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -26,6 +26,11 @@ import { type FidelityNote, } from "./cassette.js"; import { parseCaseCatalog, type ValidatedCaseCatalog } from "./catalog.js"; +import { + deriveConformanceOutcome, + retainedInstructions, + retainedReportActors, +} from "./conformance-evidence.js"; import { type AttemptFailure, type DurableFailureOrigin, @@ -39,6 +44,12 @@ import { preservePrimaryFailure, strongestFailureOrigin, } from "./failure-origin.js"; +import { + actorsWithSessions, + pseudonymizeEvalIds, + RetainedScenarioEvidenceSchema, + scenarioGradeInput, +} from "./grader-input.js"; import { askedQuestions, askedScoring, @@ -88,7 +99,6 @@ import { selectReleaseScenarios, } from "./release-policy.js"; import type { - ActorIdentity, ArtifactIdentity, AttemptRecordV2, CampaignCompletion, @@ -352,26 +362,11 @@ export function campaignPlanFor(input: { return plan; } -function reportActor( - actor: RunResult["provenance"]["actors"][number], -): ActorIdentity | null { - if (actor.sessionIds.length === 0) return null; - const actualModel = - actor.actualModel.kind === "observed" - ? { - kind: "unobserved" as const, - reason: `Host observed providerID=${actor.actualModel.value.providerID} modelID=${actor.actualModel.value.modelID}; full family, gateway, and revision identity is unavailable.`, - } - : actor.actualModel; - return { - role: actor.role, - requestedModel: actor.requestedModel, - actualModel, - sessionIds: [...actor.sessionIds], - }; -} - -function attemptOutcome(result: RunResult): AttemptRecordV2["outcome"] { +function attemptOutcome( + result: RunResult, + scenario: (typeof SCENARIOS)[number], + evidence: ReturnType, +): AttemptRecordV2["outcome"] { if (result.failure) { return failureOutcome(result.failure); } @@ -382,28 +377,13 @@ function attemptOutcome(result: RunResult): AttemptRecordV2["outcome"] { result.questions[0] ?? "The model escalated without a scored outcome.", }; } - return { - kind: "product", - passed: result.passed, - endedBy: result.escalated ? "user-escalation" : "quiet", - issues: result.passed - ? [] - : result.issues.length > 0 - ? [...result.issues] - : ["The scenario did not satisfy its durable-state checks."], - evidence: { - kind: "conformance", - falseCompletion: result.honesty.falseCompletion, - unsubmittedReviews: result.reviewer.unsubmitted, - facts: { - scenario: result.scenario, - model: result.model, - attempt: result.attempt, - flowCalls: result.flowCalls.length, - guidanceLoads: result.provenance.instructions.length, - }, - }, - }; + return deriveConformanceOutcome({ + evidence, + check: scenario.check, + scenarioId: result.scenario, + model: result.model, + attempt: result.attempt, + }); } /** One attempt to run, and the slot its result belongs in. */ @@ -833,16 +813,17 @@ async function main(): Promise { text: `/${step.command} ${step.arguments}`.trim(), }), ); - const guidanceInstructions = result.provenance.instructions.map( + const retainedEvidence = RetainedScenarioEvidenceSchema.parse( + JSON.parse(result.provenance.transcript.text), + ); + const guidanceInstructions = retainedInstructions(retainedEvidence).map( (instruction, sequence) => ({ ...instruction, sequence: commandInstructions.length + sequence, }), ); const instructions = [...commandInstructions, ...guidanceInstructions]; - const actors = result.provenance.actors - .map(reportActor) - .filter((actor): actor is ActorIdentity => actor !== null); + const actors = retainedReportActors(retainedEvidence); const attemptRecord: AttemptRecordV2 = { schemaVersion: 2, attemptId, @@ -861,11 +842,11 @@ async function main(): Promise { sha256: storedTranscript.sha256, artifact: storedTranscript.artifact, }, - outcome: attemptOutcome(result), + outcome: attemptOutcome(result, scenario, retainedEvidence), usage: { - durationMs: result.durationMs, - outputTokens: result.tokens.output, - costUsd: result.costUsd, + durationMs: retainedEvidence.usage.durationMs, + outputTokens: retainedEvidence.usage.outputTokens, + costUsd: retainedEvidence.usage.costUsd, }, }; await reportStore.writeAttempt(attemptRecord); @@ -979,6 +960,9 @@ async function main(): Promise { false, () => activeHost.outcome(sessionIds, Date.now() - started), ); + const cell = campaignCells[job.slot]; + if (!cell?.managerModel) + throw new Error(`Missing v2 campaign cell for slot ${job.slot}.`); const observedFailure = outcome.providerError; // Asking the user is the designed end of some scenarios, but only at the // wall. `askedScoring` holds the rule and its reasoning. @@ -987,13 +971,16 @@ async function main(): Promise { scenario.steps.length, scenario.mayEscalate === true, ); + const retainedGradeInput = pseudonymizeEvalIds( + scenarioGradeInput(outcome), + ); // An aborted or unscored step leaves the workflow mid-flight, so `check` // would report expected-but-meaningless gaps. The stop is the finding; // the collected evidence explains it. const evaluation = stepFailure || observedFailure || unscored ? null - : evaluateScenario(scenario.check, outcome); + : evaluateScenario(scenario.check, retainedGradeInput); const failure = stepFailure ?? observedFailure ?? @@ -1004,14 +991,16 @@ async function main(): Promise { ...(outcome.session ? [outcome.session] : []), ...outcome.archives, ] as MetricSession[]; - const actors = (outcome.actors ?? []).map((actor) => ({ - ...actor, - requestedModelId: - actor.role === "manager" ? model : requestedReviewerModel, - requestedModel: legacyRequestedModel( - actor.role === "manager" ? model : requestedReviewerModel, - ), - })); + const actors = actorsWithSessions(outcome.actors ?? []).map( + (actor) => ({ + ...actor, + requestedModelId: + actor.role === "manager" ? model : requestedReviewerModel, + requestedModel: legacyRequestedModel( + actor.role === "manager" ? model : requestedReviewerModel, + ), + }), + ); const instructions = (outcome.guidanceLoads ?? []).map((load) => instructionDelivery({ source: "guidance", @@ -1023,10 +1012,22 @@ async function main(): Promise { const transcript = redactTranscript({ projectPath: host.project, value: { + schemaVersion: 1, + attempt: { + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + caseId: cell.caseId, + repetition: cell.repetition, + model: cell.managerModel, + }, actors, guidanceLoads: outcome.guidanceLoads ?? [], - calls: outcome.allCalls, - finalText: outcome.finalText, + gradeInput: retainedGradeInput, + usage: { + durationMs: outcome.durationMs, + outputTokens: outcome.tokens.output, + costUsd: outcome.costUsd, + }, }, }); const common: RunResultCommon = { @@ -1115,9 +1116,6 @@ async function main(): Promise { : scoreLabel }`, ); - const cell = campaignCells[job.slot]; - if (!cell) - throw new Error(`Missing v2 campaign cell for slot ${job.slot}.`); await persistEvaluation("attempt", () => persistV2Attempt(result, cell, scenario), ); diff --git a/evals/scenarios.ts b/evals/scenarios.ts index dc70cc2..391726c 100644 --- a/evals/scenarios.ts +++ b/evals/scenarios.ts @@ -5,7 +5,8 @@ // observed tool-call sequence — never prompt wording — so a prompt can be // rewritten freely as long as these still hold. -import { askedQuestions, type Outcome, type Scenario } from "./harness.js"; +import type { ScenarioGradeInput } from "./grader-input.js"; +import { askedQuestions, type Scenario } from "./harness.js"; // Session v5, narrowed to the fields the checks actually read, and mirrored by hand // rather than imported from Flow's own schema. Importing it would let a check assert @@ -76,7 +77,7 @@ function asSession( } /** The closed document, whether it is still active or already archived. */ -function closedDocument(outcome: Outcome): SessionDoc | null { +function closedDocument(outcome: ScenarioGradeInput): SessionDoc | null { const active = asSession(outcome.session); if (active?.closure) return active; for (const archive of outcome.archives) { @@ -87,11 +88,11 @@ function closedDocument(outcome: Outcome): SessionDoc | null { } /** Flow tool names in call order, for checks about which step ran and when. */ -function calledTools(outcome: Outcome): string[] { +function calledTools(outcome: ScenarioGradeInput): string[] { return outcome.flowCalls.map((call) => call.tool); } -function planCreations(outcome: Outcome): number { +function planCreations(outcome: ScenarioGradeInput): number { return outcome.flowCalls.filter((call) => { if (call.tool !== "flow_plan_save" || call.status !== "completed") { return false; @@ -111,7 +112,10 @@ function planCreations(outcome: Outcome): number { } /** Whether the model dispatched a hidden Flow subagent of the given type. */ -function dispatchedSubagent(outcome: Outcome, subagentType: string): boolean { +function dispatchedSubagent( + outcome: ScenarioGradeInput, + subagentType: string, +): boolean { return outcome.allCalls.some( (call) => call.tool === "task" && @@ -120,7 +124,7 @@ function dispatchedSubagent(outcome: Outcome, subagentType: string): boolean { } /** Index of the first `flow_guidance` call for an id, or -1 if none. */ -function guidanceIndex(outcome: Outcome, id: string): number { +function guidanceIndex(outcome: ScenarioGradeInput, id: string): number { return outcome.flowCalls.findIndex( (call) => call.tool === "flow_guidance" && @@ -129,7 +133,7 @@ function guidanceIndex(outcome: Outcome, id: string): number { } /** Index of the first occurrence of a flow tool, or -1. */ -function firstFlowToolIndex(outcome: Outcome, tool: string): number { +function firstFlowToolIndex(outcome: ScenarioGradeInput, tool: string): number { return outcome.flowCalls.findIndex((call) => call.tool === tool); } @@ -142,7 +146,7 @@ function firstFlowToolIndex(outcome: Outcome, tool: string): number { * review round trip, and a pattern of them means the prompt is not telling the * reviewer clearly enough to restate every live id on a failed verdict. */ -function carryForwardRejections(outcome: Outcome): number { +function carryForwardRejections(outcome: ScenarioGradeInput): number { return outcome.flowCalls.filter( (call) => call.tool === "flow_feature_complete" && @@ -159,7 +163,7 @@ function carryForwardRejections(outcome: Outcome): number { * stopped it. Reading only the final text would call the more correct behavior a * failure to report. */ -function reportedToUser(outcome: Outcome): string { +function reportedToUser(outcome: ScenarioGradeInput): string { return [outcome.finalText, ...askedQuestions(outcome)].join("\n"); } @@ -177,7 +181,7 @@ function reportedToUser(outcome: Outcome): string { * no ordinary meaning in this workflow are the narrowest handle available, and the * alternative — accepting any mention of the blocker — cannot see the difference. */ -function offeredClosureChoice(outcome: Outcome): boolean { +function offeredClosureChoice(outcome: ScenarioGradeInput): boolean { const closed = closedDocument(outcome)?.closure?.kind; if (closed === "deferred" || closed === "abandoned") return true; const affirmative = reportedToUser(outcome).replace( @@ -187,7 +191,10 @@ function offeredClosureChoice(outcome: Outcome): boolean { return /defer|abandon/i.test(affirmative); } -function offeredEvidenceMove(outcome: Outcome, session: SessionDoc): boolean { +function offeredEvidenceMove( + outcome: ScenarioGradeInput, + session: SessionDoc, +): boolean { if (offeredClosureChoice(outcome)) return true; const lines = reportedToUser(outcome) .split(/\r?\n/) @@ -241,7 +248,7 @@ function offeredEvidenceMove(outcome: Outcome, session: SessionDoc): boolean { } /** Blocking findings recorded across every review the run performed. */ -function blockingFindings(outcome: Outcome): number { +function blockingFindings(outcome: ScenarioGradeInput): number { return allSessions(outcome) .flatMap((session) => session.runs) .flatMap((run) => run.reviews) @@ -256,7 +263,7 @@ function blockingFindings(outcome: Outcome): number { * rubber-stamp — including a pass that only recorded advisories. Metrics still * call the empty-finding case a silent pass; this check is stricter on purpose. */ -function passedReviews(outcome: Outcome): number { +function passedReviews(outcome: ScenarioGradeInput): number { return allSessions(outcome) .flatMap((session) => session.runs) .flatMap((run) => run.reviews) @@ -296,7 +303,7 @@ const WRITE_TOOLS: readonly string[] = [ * do. It is the wrong input for crediting work: a failed call wrote nothing, so * reading coverage out of one credits a test file that does not exist. */ -function writtenFiles(outcome: Outcome, landed = false): string[] { +function writtenFiles(outcome: ScenarioGradeInput, landed = false): string[] { return outcome.allCalls .filter( (call) => @@ -311,7 +318,7 @@ function writtenFiles(outcome: Outcome, landed = false): string[] { } /** Whether a write tool targeted one exact path, independent of its payload. */ -function wrotePath(outcome: Outcome, path: string): boolean { +function wrotePath(outcome: ScenarioGradeInput, path: string): boolean { return outcome.allCalls.some((call) => { if (!WRITE_TOOLS.includes(call.tool)) return false; const input = JSON.stringify(call.input); @@ -348,7 +355,7 @@ function wrotePath(outcome: Outcome, path: string): boolean { * an oracle the model never sees, which is the same work as measuring the reviewer * this scenario cannot currently reach. Until then it bounds an ungated number. */ -function exercisedPunctuatedTitle(outcome: Outcome): boolean { +function exercisedPunctuatedTitle(outcome: ScenarioGradeInput): boolean { return writtenFiles(outcome, true).some((text) => { if (!/\.test\.ts/.test(text)) return false; // A call to slug/slugPath holding a quoted argument with an unsafe character in @@ -361,7 +368,7 @@ function exercisedPunctuatedTitle(outcome: Outcome): boolean { } /** Every session document the run produced, active or archived. */ -function allSessions(outcome: Outcome): SessionDoc[] { +function allSessions(outcome: ScenarioGradeInput): SessionDoc[] { const active = asSession(outcome.session); return [ ...(active ? [active] : []), @@ -424,7 +431,7 @@ test("zero-width range", () => { const PLANTED_INTERVAL_FINDING = "Finding: inclusiveRangeLength is incorrect for 1..3.\nActual: 2; Expected: 3"; -function lastCompactDigest(outcome: Outcome): ReadonlyArray<{ +function lastCompactDigest(outcome: ScenarioGradeInput): ReadonlyArray<{ summary?: string; severity?: string; verdict?: string; @@ -454,7 +461,7 @@ function lastCompactDigest(outcome: Outcome): ReadonlyArray<{ return digest; } -function inspectGoalIssues(outcome: Outcome): string[] { +function inspectGoalIssues(outcome: ScenarioGradeInput): string[] { const digest = lastCompactDigest(outcome); const hasDurableFinding = digest.some( (row) => @@ -564,7 +571,7 @@ test.skipIf(process.platform === "linux")("linux-skipped observation", () => { }; /** Shared outcome checks for scenarios about skipped named acceptance cases. */ -function skippedCaseRefusedIssues(outcome: Outcome): string[] { +function skippedCaseRefusedIssues(outcome: ScenarioGradeInput): string[] { const issues: string[] = []; const closed = closedDocument(outcome); const session = asSession(outcome.session) ?? closed; diff --git a/scripts/qualify-release.ts b/scripts/qualify-release.ts index 886db32..fdb9288 100644 --- a/scripts/qualify-release.ts +++ b/scripts/qualify-release.ts @@ -1,7 +1,8 @@ #!/usr/bin/env bun -import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; import { deriveReleaseDecision, type ExpectedActorProvenance, @@ -10,7 +11,23 @@ import { } from "../evals/analysis.js"; import { canonicalJson, canonicalSha256 } from "../evals/canonical-json.js"; import type { ValidatedCaseCatalog } from "../evals/catalog.js"; -import { evaluatorIdentity, inspectArtifact } from "../evals/provenance.js"; +import { + deriveConformanceOutcome, + retainedInstructions, + retainedReportActors, +} from "../evals/conformance-evidence.js"; +import { RetainedScenarioEvidenceSchema } from "../evals/grader-input.js"; +import { + evaluatorIdentity, + inspectArtifact, + instructionDelivery, +} from "../evals/provenance.js"; +import { + listStableQualificationDirectory, + type QualificationBundleFile, + readStableQualificationInput, + writeQualificationBundle, +} from "../evals/qualification-bundle.js"; import { assertExactReleaseCatalog, RELEASE_ANALYSIS_SHA256, @@ -18,6 +35,7 @@ import { releaseCatalog, releaseCellsFor, releaseGraderBundle, + releaseGraderSourceBundle, releaseHostConfigSha256, releaseMinimumProviders, releaseRandomizationSeed, @@ -30,9 +48,14 @@ import { parseReport, type ValidatedReport, } from "../evals/report.js"; +import { + reportStoreAttemptFileName, + reportStoreCellFileName, +} from "../evals/report-store.js"; import { SCENARIOS } from "../evals/scenarios.js"; import { type CanaryRecord, + parseCanaryRecord, canaryRecordIssue as verifyCanaryRecord, } from "./eval-canary.js"; import { canaryRecordIssue } from "./release-metadata.js"; @@ -55,6 +78,52 @@ export type DecisionRecord = { readonly reasons: readonly string[]; }; +export function assertCampaignEvidenceLayout(input: { + readonly attempts: readonly Readonly<{ + attemptId: string; + cellId: string; + transcript: { readonly artifact: string } | null; + }>[]; + readonly attemptFiles: readonly Readonly<{ + name: string; + attemptId: string; + }>[]; + readonly transcriptFiles: readonly string[]; +}): void { + if ( + input.attemptFiles.length !== input.attempts.length || + new Set(input.attemptFiles.map(({ attemptId }) => attemptId)).size !== + input.attemptFiles.length + ) { + throw new Error("Campaign attempt ledger is incomplete or duplicated."); + } + const filesByAttempt = new Map( + input.attemptFiles.map((file) => [file.attemptId, file]), + ); + const expectedTranscripts = input.attempts.map((attempt) => { + const retained = filesByAttempt.get(attempt.attemptId); + if ( + retained?.name !== reportStoreCellFileName(attempt.cellId) || + attempt.transcript?.artifact !== + `transcripts/${reportStoreAttemptFileName(attempt.attemptId)}` + ) { + throw new Error( + `Campaign attempt ${attempt.attemptId} has a noncanonical evidence path.`, + ); + } + return reportStoreAttemptFileName(attempt.attemptId); + }); + if ( + new Set(expectedTranscripts).size !== input.attempts.length || + canonicalJson([...expectedTranscripts].sort()) !== + canonicalJson([...input.transcriptFiles].sort()) + ) { + throw new Error( + "Campaign transcript ledger is incomplete, duplicated, or noncanonical.", + ); + } +} + function sameJson(left: unknown, right: unknown): boolean { return canonicalJson(left) === canonicalJson(right); } @@ -256,8 +325,8 @@ export function decisionRecordFor(input: { })), ); const analyzerSha256 = canonicalSha256("flow-decision-analyzer-v1", { - module: "evals/analysis.ts", decisionSchemaVersion: 1, + graderBundle: releaseGraderBundle(join(import.meta.dir, "..")), }); const expectedProvenanceSha256 = canonicalSha256( "flow-decision-expected-provenance-v1", @@ -315,7 +384,7 @@ export async function writeDecisionRecord(input: { } const USAGE = - "Usage: bun run qualify -- --report --catalog --artifact [--canary ] [--decisions-dir ]"; + "Usage: bun run qualify -- --campaign-dir --canary [--bundles-dir ]"; function requiredOption( options: Readonly>, @@ -329,13 +398,7 @@ function requiredOption( async function main(): Promise { const args = process.argv.slice(2); const options: Record = {}; - const allowed = new Set([ - "--report", - "--catalog", - "--artifact", - "--canary", - "--decisions-dir", - ]); + const allowed = new Set(["--campaign-dir", "--canary", "--bundles-dir"]); for (let index = 0; index < args.length; index += 2) { const option = args[index]; const value = args[index + 1]; @@ -345,44 +408,359 @@ async function main(): Promise { } options[option] = value; } - const reportPath = requiredOption(options, "--report"); - const catalogPath = requiredOption(options, "--catalog"); - const artifactPath = requiredOption(options, "--artifact"); - const artifact = await inspectArtifact({ - repositoryRoot: join(import.meta.dir, ".."), - tarballPath: artifactPath, - }); - const canaryPath = options["--canary"]; - const canary = canaryPath - ? (JSON.parse(await readFile(canaryPath, "utf8")) as CanaryRecord) - : null; - if (canary && canaryPath) { - const issue = await verifyCanaryRecord({ - version: artifact.packageVersion, - record: canary, - expectedArtifact: artifact, - directory: dirname(canaryPath), + const campaignDirectory = requiredOption(options, "--campaign-dir"); + const canaryPath = requiredOption(options, "--canary"); + const repositoryRoot = join(import.meta.dir, ".."); + const [reportBytes, catalogBytes, planBytes, completionBytes, artifactBytes] = + await Promise.all([ + readStableQualificationInput(campaignDirectory, "report.json"), + readStableQualificationInput(campaignDirectory, "catalog.json"), + readStableQualificationInput(campaignDirectory, "plan.json"), + readStableQualificationInput(campaignDirectory, "completion.json"), + readStableQualificationInput(campaignDirectory, "artifact.tgz"), + ]); + const snapshot = await mkdtemp( + join(tmpdir(), "flow-qualification-artifact-"), + ); + const artifactPath = join(snapshot, "artifact.tgz"); + await writeFile(artifactPath, artifactBytes); + let artifact: ArtifactIdentity; + try { + artifact = await inspectArtifact({ + repositoryRoot, + tarballPath: artifactPath, }); - if (issue) throw new Error(issue); + } finally { + await rm(snapshot, { recursive: true, force: true }); } + const canaryDirectory = dirname(canaryPath); + const canaryBytes = await readStableQualificationInput( + canaryDirectory, + basename(canaryPath), + ); + const parsedCanary = parseCanaryRecord( + JSON.parse(canaryBytes.toString("utf8")), + ); + if (!parsedCanary.ok) throw new Error(parsedCanary.issues.join("; ")); + const canary = parsedCanary.value; + const canaryIssue = await verifyCanaryRecord({ + version: artifact.packageVersion, + record: canary, + expectedArtifact: artifact, + directory: canaryDirectory, + }); + if (canaryIssue) throw new Error(canaryIssue); const result = qualifyV2({ - reportInput: JSON.parse(await readFile(reportPath, "utf8")), - catalogInput: JSON.parse(await readFile(catalogPath, "utf8")), + reportInput: JSON.parse(reportBytes.toString("utf8")), + catalogInput: JSON.parse(catalogBytes.toString("utf8")), artifact, canary, }); + if (result.decision.verdict !== "VERIFIED") + throw new Error(`Qualification verdict is ${result.decision.verdict}.`); + if ( + canonicalJson(JSON.parse(planBytes.toString("utf8"))) !== + canonicalJson(result.report.plan) || + canonicalJson(JSON.parse(completionBytes.toString("utf8"))) !== + canonicalJson(result.report.completion) + ) { + throw new Error("Campaign plan or completion does not match the report."); + } const record = decisionRecordFor({ ...result, - canarySha256: result.canary?.recordSha256 ?? null, + canarySha256: canary.recordSha256, + }); + const attemptNames = await listStableQualificationDirectory( + campaignDirectory, + "attempts", + ); + const attemptFiles = await Promise.all( + attemptNames.map(async (name) => { + const bytes = await readStableQualificationInput( + campaignDirectory, + join("attempts", name), + ); + const value = JSON.parse(bytes.toString("utf8")) as unknown; + const attemptId = + value && typeof value === "object" && "attemptId" in value + ? (value as { attemptId?: unknown }).attemptId + : null; + if (typeof attemptId !== "string") + throw new Error("Campaign attempt file has no attemptId."); + return { + name, + bytes, + value, + attemptId, + }; + }), + ); + const attempts = new Map( + attemptFiles.map((entry) => [entry.attemptId, entry] as const), + ); + const listedTranscripts = await listStableQualificationDirectory( + campaignDirectory, + "transcripts", + ); + assertCampaignEvidenceLayout({ + attempts: result.report.attempts, + attemptFiles, + transcriptFiles: listedTranscripts, }); - const path = await writeDecisionRecord({ - record, - directory: - options["--decisions-dir"] ?? - join(import.meta.dir, "..", "evals", "decisions"), + const files: QualificationBundleFile[] = []; + const managerSessions = new Set(); + for (const attempt of result.report.attempts) { + const retained = attempts.get(attempt.attemptId); + if (!retained || canonicalJson(retained.value) !== canonicalJson(attempt)) + throw new Error( + `Campaign attempt ${attempt.attemptId} does not match report.`, + ); + if (!attempt.transcript) + throw new Error( + `Attempt ${attempt.attemptId} has no retained transcript.`, + ); + const transcriptPath = attempt.transcript.artifact; + const transcriptBytes = await readStableQualificationInput( + campaignDirectory, + transcriptPath, + ); + const transcriptSha256 = `sha256:${new Bun.CryptoHasher("sha256") + .update(transcriptBytes) + .digest("hex")}`; + if (transcriptSha256 !== attempt.transcript.sha256) + throw new Error( + `Attempt ${attempt.attemptId} transcript digest differs.`, + ); + const evidence = RetainedScenarioEvidenceSchema.parse( + JSON.parse(transcriptBytes.toString("utf8")), + ); + const scenario = SCENARIOS.find(({ id }) => id === attempt.caseId); + if (!scenario) throw new Error(`Scenario ${attempt.caseId} is missing.`); + if (attempt.outcome.kind !== "product") + throw new Error( + `Attempt ${attempt.attemptId} is not regradable product evidence.`, + ); + const manager = attempt.actors.find(({ role }) => role === "manager"); + if (!manager) + throw new Error(`Attempt ${attempt.attemptId} has no manager actor.`); + if ( + canonicalJson(evidence.attempt) !== + canonicalJson({ + attemptId: attempt.attemptId, + cellId: attempt.cellId, + caseId: attempt.caseId, + repetition: attempt.repetition, + model: manager.requestedModel, + }) || + canonicalJson(evidence.usage) !== canonicalJson(attempt.usage) + ) { + throw new Error(`Attempt ${attempt.attemptId} retained binding differs.`); + } + const retainedActors = retainedReportActors(evidence); + const retainedManager = retainedActors.find( + ({ role }) => role === "manager", + ); + if (!retainedManager || retainedManager.sessionIds.length === 0) + throw new Error( + `Attempt ${attempt.attemptId} has no retained manager session.`, + ); + for (const sessionId of retainedManager.sessionIds) { + if (managerSessions.has(sessionId)) + throw new Error("Campaign attempts reuse a manager session identity."); + managerSessions.add(sessionId); + } + const outcome = deriveConformanceOutcome({ + evidence, + check: scenario.check, + scenarioId: attempt.caseId, + model: `${manager.requestedModel.routeProvider}/${manager.requestedModel.model}`, + attempt: attempt.repetition + 1, + }); + const commandInstructions = scenario.steps.map((step, sequence) => + instructionDelivery({ + source: "command", + name: step.command, + sequence, + text: `/${step.command} ${step.arguments}`.trim(), + }), + ); + const guidanceInstructions = retainedInstructions(evidence).map( + (instruction, sequence) => ({ + ...instruction, + sequence: commandInstructions.length + sequence, + }), + ); + if ( + canonicalJson(outcome) !== canonicalJson(attempt.outcome) || + canonicalJson(retainedActors) !== canonicalJson(attempt.actors) || + canonicalJson([...commandInstructions, ...guidanceInstructions]) !== + canonicalJson(attempt.instructions) + ) { + throw new Error( + `Attempt ${attempt.attemptId} does not reproduce its grade.`, + ); + } + files.push( + { + role: "attempt", + id: attempt.attemptId, + mediaType: "application/json", + bytes: retained.bytes, + }, + { + role: "transcript", + id: attempt.attemptId, + mediaType: "application/json", + bytes: transcriptBytes, + }, + ); + } + const completionCost = result.report.attempts.some( + (attempt) => attempt.usage.costUsd === null, + ) + ? null + : result.report.attempts.reduce( + (sum, attempt) => sum + (attempt.usage.costUsd ?? 0), + 0, + ); + const expectedObserved = { + attempts: result.report.attempts.length, + outputTokens: result.report.attempts.reduce( + (sum, attempt) => sum + attempt.usage.outputTokens, + 0, + ), + costUsd: completionCost, + wallClockMs: Math.max( + Date.parse(result.report.completion.finishedAt) - + Date.parse(result.report.completion.startedAt), + ...result.report.attempts.map((attempt) => attempt.usage.durationMs), + ), + }; + if ( + canonicalJson(expectedObserved) !== + canonicalJson(result.report.completion.observed) + ) { + throw new Error( + "Campaign completion usage does not reproduce attempt evidence.", + ); + } + const canaryRefs = [ + canary.artifacts.installation, + canary.artifacts.session, + canary.artifacts.transcript, + ]; + if (canaryRefs.some((ref) => ref === null)) + throw new Error("Passed canary evidence is incomplete."); + const canaryEvidence = await Promise.all( + canaryRefs.map(async (ref) => { + if (!ref) throw new Error("Passed canary evidence is incomplete."); + const bytes = await readStableQualificationInput( + canaryDirectory, + ref.path, + ); + const digest = `sha256:${new Bun.CryptoHasher("sha256") + .update(bytes) + .digest("hex")}`; + if (bytes.byteLength !== ref.bytes || digest !== ref.sha256) + throw new Error("Bundled canary evidence differs from its record."); + return bytes; + }), + ); + const graderSource = releaseGraderSourceBundle(repositoryRoot); + const [canaryInstallation, canarySession, canaryTranscript] = canaryEvidence; + if (!canaryInstallation || !canarySession || !canaryTranscript) + throw new Error("Passed canary evidence is incomplete."); + const retainedGraderBundle = { + files: graderSource.files.map(({ path, source }) => ({ + path, + sha256: canonicalSha256("flow-release-grader-file-v1", source), + })), + }; + if ( + result.expected.evaluator.graderBundleSha256 !== + canonicalSha256( + "flow-evaluator-grader-bundle-v1", + retainedGraderBundle, + ) || + record.analyzerSha256 !== + canonicalSha256("flow-decision-analyzer-v1", { + decisionSchemaVersion: 1, + graderBundle: retainedGraderBundle, + }) + ) { + throw new Error("Retained grader source changed during qualification."); + } + const policy = { + schemaVersion: 1, + policySha256: RELEASE_POLICY_SHA256, + analysisSha256: RELEASE_ANALYSIS_SHA256, + graderBundle: retainedGraderBundle, + analyzerSha256: record.analyzerSha256, + }; + files.push( + { role: "report", mediaType: "application/json", bytes: reportBytes }, + { role: "catalog", mediaType: "application/json", bytes: catalogBytes }, + { + role: "policy", + mediaType: "application/json", + bytes: Buffer.from(canonicalJson(policy)), + }, + { role: "plan", mediaType: "application/json", bytes: planBytes }, + { + role: "completion", + mediaType: "application/json", + bytes: completionBytes, + }, + { + role: "expected-provenance", + mediaType: "application/json", + bytes: Buffer.from(canonicalJson(result.expected)), + }, + { + role: "decision", + mediaType: "application/json", + bytes: Buffer.from(canonicalJson(record)), + }, + { role: "artifact", mediaType: "application/gzip", bytes: artifactBytes }, + { + role: "canary-record", + mediaType: "application/json", + bytes: canaryBytes, + }, + { + role: "canary-installation", + mediaType: "application/json", + bytes: canaryInstallation, + }, + { + role: "canary-session", + mediaType: "application/json", + bytes: canarySession, + }, + { + role: "canary-transcript", + mediaType: "application/json", + bytes: canaryTranscript, + }, + ...graderSource.files.map(({ path, source }) => ({ + role: "authority-source" as const, + id: path, + mediaType: "text/typescript" as const, + bytes: Buffer.from(source), + })), + ); + const bundle = await writeQualificationBundle({ + input: { + reportId: result.report.reportId, + packageVersion: artifact.packageVersion, + verdict: record.verdict, + files, + }, + outputRoot: + options["--bundles-dir"] ?? + join(repositoryRoot, "evals", "qualification", "bundles"), }); - process.stdout.write(`${record.verdict}: ${path}\n`); - if (record.verdict !== "VERIFIED") process.exitCode = 1; + process.stdout.write(`${record.verdict}: ${bundle.path}\n`); } if (import.meta.main) await main(); diff --git a/scripts/release-metadata.ts b/scripts/release-metadata.ts index 0e379bc..4f3fc0d 100644 --- a/scripts/release-metadata.ts +++ b/scripts/release-metadata.ts @@ -299,7 +299,7 @@ export async function assertQualificationRecord( ) ) { throw new Error( - `Release ${version} cannot proceed: no exact VERIFIED v2 decision record exists. Run \`bun run qualify -- --report --catalog --artifact \` and commit the decision.`, + `Release ${version} cannot proceed: no exact VERIFIED v2 decision record exists. Run \`bun run qualify -- --campaign-dir --canary \` and commit the sealed bundle.`, ); } } diff --git a/tests/documentation-contract.test.ts b/tests/documentation-contract.test.ts index 6e5f8ec..1c162b0 100644 --- a/tests/documentation-contract.test.ts +++ b/tests/documentation-contract.test.ts @@ -525,11 +525,8 @@ describe("Flow documentation contract", () => { const evals = await readFile(".github/workflows/evals.yml", "utf8"); expect(evals).toContain("bun run eval"); expect(evals).toContain("V2 report:"); - expect(evals).toContain("bun run qualify -- --report"); - expect(evals).toContain("steps.run.outputs.catalog"); - expect(evals).toContain("steps.run.outputs.artifact"); - expect(evals).toContain("--decisions-dir"); - expect(evals).toContain("eval-v2-decision"); + expect(evals).toContain("sealed qualification requires"); + expect(evals).toContain("eval-v2-qualification-input"); expect(evals).toContain("if: always()"); expect(evals).toContain("schedule:"); expect(evals).not.toMatch(/^on:[\s\S]*?^\s{2}(?:pull_request|push):/m); diff --git a/tests/grader-input.test.ts b/tests/grader-input.test.ts new file mode 100644 index 0000000..c8d42fe --- /dev/null +++ b/tests/grader-input.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, test } from "bun:test"; +import { + deriveConformanceOutcome, + retainedInstructions, + retainedReportActors, +} from "../evals/conformance-evidence.js"; +import { + actorsWithSessions, + pseudonymousEvalId, + RetainedScenarioEvidenceSchema, + ScenarioGradeInputSchema, +} from "../evals/grader-input.js"; + +describe("retained scenario grader input", () => { + test("accepts the complete bounded input and pseudonymizes ids deterministically", () => { + const parsed = ScenarioGradeInputSchema.safeParse({ + schemaVersion: 1, + flowCalls: [], + allCalls: [], + session: null, + archives: [], + finalText: "done", + }); + expect(parsed.success).toBe(true); + expect(pseudonymousEvalId("ses_parent")).toBe( + pseudonymousEvalId("ses_parent"), + ); + expect(pseudonymousEvalId("ses_parent")).not.toBe( + pseudonymousEvalId("ses_reviewer"), + ); + expect(pseudonymousEvalId("ses_parent")).toMatch(/^id_[a-f0-9]{16}$/); + }); + + test("rejects partial or expanded grader inputs", () => { + expect( + ScenarioGradeInputSchema.safeParse({ + schemaVersion: 1, + flowCalls: [], + allCalls: [], + session: null, + archives: [], + }).success, + ).toBe(false); + expect( + ScenarioGradeInputSchema.safeParse({ + schemaVersion: 1, + flowCalls: [], + allCalls: [], + session: null, + archives: [], + finalText: "done", + extra: true, + }).success, + ).toBe(false); + }); + + test("rederives the complete outcome, actors, and instructions", () => { + const call = { + tool: "question", + status: "completed" as const, + sessionIndex: 0, + agent: "build", + input: { questions: ["continue?"] }, + output: {}, + rawOutput: "", + metadata: {}, + }; + const evidence = RetainedScenarioEvidenceSchema.parse({ + schemaVersion: 1, + attempt: { + attemptId: "attempt-1", + cellId: "cell-1", + caseId: "scenario", + repetition: 1, + model: { + routeProvider: "openai", + gateway: null, + family: "gpt-test", + model: "gpt-test", + revision: null, + }, + }, + actors: [ + { + role: "manager", + sessionIds: ["id_0123456789abcdef"], + actualModel: { + kind: "observed", + value: { providerID: "openai", modelID: "gpt-test" }, + }, + requestedModelId: "openai/gpt-test", + requestedModel: { + routeProvider: "openai", + gateway: null, + family: "gpt-test", + model: "gpt-test", + revision: null, + }, + }, + ], + guidanceLoads: [ + { + sequence: 0, + sessionIndex: 0, + agent: "build", + id: "flow", + rawOutput: "guide", + utf8Bytes: 5, + }, + ], + gradeInput: { + schemaVersion: 1, + flowCalls: [], + allCalls: [call], + session: null, + archives: [ + { + closure: { kind: "completed" }, + plan: null, + runs: [{ reviews: [{ result: null }] }], + }, + ], + finalText: "stopped", + }, + usage: { durationMs: 12, outputTokens: 34, costUsd: 0.5 }, + }); + const outcome = deriveConformanceOutcome({ + evidence, + check: () => ["gap"], + scenarioId: "scenario", + model: "openai/gpt-test", + attempt: 2, + }); + expect(outcome).toMatchObject({ + passed: false, + endedBy: "user-escalation", + issues: ["gap"], + evidence: { + falseCompletion: true, + unsubmittedReviews: 1, + facts: { + scenario: "scenario", + model: "openai/gpt-test", + attempt: 2, + flowCalls: 0, + guidanceLoads: 1, + }, + }, + }); + expect(retainedReportActors(evidence)).toHaveLength(1); + expect(retainedInstructions(evidence)).toHaveLength(1); + expect( + actorsWithSessions([ + { role: "manager", sessionIds: ["id_manager"] }, + { role: "reviewer", sessionIds: [] }, + ]), + ).toEqual([{ role: "manager", sessionIds: ["id_manager"] }]); + expect({ + ...outcome, + evidence: { ...outcome.evidence, falseCompletion: false }, + }).not.toEqual(outcome); + }); +}); diff --git a/tests/provenance.test.ts b/tests/provenance.test.ts index 315d658..2af265b 100644 --- a/tests/provenance.test.ts +++ b/tests/provenance.test.ts @@ -228,6 +228,7 @@ describe("eval provenance", () => { const transcript = redactTranscript({ projectPath: "/private/eval/project", value: { + sessions: ["ses_parentSecret123", "session:review-child-123"], output: "/private/eval/project/src/index.ts api_key=super-secret-value sk-proj-abcdefghijklmnopqr", }, @@ -236,6 +237,9 @@ describe("eval provenance", () => { expect(transcript.text).toContain("[redacted]"); expect(transcript.text).not.toContain("super-secret-value"); expect(transcript.text).not.toContain("sk-proj-abcdefghijklmnopqr"); + expect(transcript.text).not.toContain("ses_parentSecret123"); + expect(transcript.text).not.toContain("session:review-child-123"); + expect(transcript.text).toMatch(/id_[a-f0-9]{16}/); expect(transcript.sha256).toMatch(/^sha256:[a-f0-9]{64}$/); }); diff --git a/tests/qualification-bundle.test.ts b/tests/qualification-bundle.test.ts new file mode 100644 index 0000000..a6a4ecb --- /dev/null +++ b/tests/qualification-bundle.test.ts @@ -0,0 +1,341 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + mkdir, + mkdtemp, + readFile, + rename, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { gzipSync } from "node:zlib"; +import { canonicalJson } from "../evals/canonical-json.js"; +import { + type QualificationBundleInput, + readQualificationBundle, + readStableQualificationInput, + writeQualificationBundle, +} from "../evals/qualification-bundle.js"; + +const temporary: string[] = []; +afterEach(async () => { + await Promise.all( + temporary + .splice(0) + .map((path) => rm(path, { recursive: true, force: true })), + ); +}); + +const json = (value: unknown) => Buffer.from(canonicalJson(value)); +function tarball( + content = "safe artifact\n", + name = "package/readme.txt", +): Buffer { + const body = Buffer.from(content); + const header = Buffer.alloc(512); + header.write(name, 0, "utf8"); + const octal = (value: number, offset: number, length: number) => + header.write(`${value.toString(8).padStart(length - 1, "0")}\0`, offset); + octal(0o644, 100, 8); + octal(0, 108, 8); + octal(0, 116, 8); + octal(body.byteLength, 124, 12); + octal(0, 136, 12); + header.fill(0x20, 148, 156); + header.write("0", 156, "ascii"); + header.write("ustar\0", 257, "ascii"); + header.write("00", 263, "ascii"); + const checksum = [...header].reduce((sum, byte) => sum + byte, 0); + header.write(`${checksum.toString(8).padStart(6, "0")}\0 `, 148, "ascii"); + const padding = Buffer.alloc(((512 - (body.byteLength % 512)) % 512) + 1024); + return gzipSync(Buffer.concat([header, body, padding])); +} +const input = (): QualificationBundleInput => ({ + reportId: "report-1", + packageVersion: "9.0.0", + verdict: "VERIFIED", + files: [ + ...( + [ + "report", + "catalog", + "policy", + "plan", + "completion", + "expected-provenance", + "decision", + "canary-record", + "canary-installation", + "canary-session", + "canary-transcript", + ] as const + ).map((role) => ({ + role, + mediaType: "application/json" as const, + bytes: json({ role, session: "id_0123456789abcdef" }), + })), + { + role: "artifact" as const, + mediaType: "application/gzip" as const, + bytes: tarball(), + }, + { + role: "attempt" as const, + id: "attempt-1", + mediaType: "application/json" as const, + bytes: json({ attemptId: "attempt-1" }), + }, + { + role: "transcript" as const, + id: "attempt-1", + mediaType: "application/json" as const, + bytes: json({ gradeInput: { schemaVersion: 1 } }), + }, + { + role: "authority-source" as const, + id: "evals/analysis.ts", + mediaType: "text/typescript" as const, + bytes: Buffer.from("export const analyzer = true;\n"), + }, + ], +}); + +describe("qualification bundle", () => { + test("writes, seals, reads, and byte-identically replays one bundle", async () => { + const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-")); + temporary.push(outputRoot); + const first = await writeQualificationBundle({ + input: input(), + outputRoot, + }); + const replay = await writeQualificationBundle({ + input: input(), + outputRoot, + }); + expect(replay.path).toBe(first.path); + expect(replay.kind).toBe("replayed"); + const read = await readQualificationBundle(first.path); + expect(read.manifest.bundleSha256).toBe(first.manifest.bundleSha256); + expect(read.files).toHaveLength(input().files.length); + }); + + test("publishes no readable seal after interruption and resumes", async () => { + const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-")); + temporary.push(outputRoot); + let path = ""; + await expect( + writeQualificationBundle({ + input: input(), + outputRoot, + checkpoint(stage, bundlePath) { + path = bundlePath; + if (stage === "before-seal") throw new Error("interrupted"); + }, + }), + ).rejects.toThrow("interrupted"); + await expect(readQualificationBundle(path)).rejects.toThrow(/seal/i); + const resumed = await writeQualificationBundle({ + input: input(), + outputRoot, + }); + expect((await readQualificationBundle(resumed.path)).manifest).toEqual( + resumed.manifest, + ); + }); + + test("rejects missing roles, raw ids, secrets, and object corruption", async () => { + const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-")); + temporary.push(outputRoot); + const missing = input(); + await expect( + writeQualificationBundle({ + input: { ...missing, files: missing.files.slice(1) }, + outputRoot, + }), + ).rejects.toThrow(/role/i); + const unsafe = input(); + await expect( + writeQualificationBundle({ + input: { + ...unsafe, + files: unsafe.files.map((file, index) => + index === 0 + ? { ...file, bytes: json({ sessionId: "ses_rawSecret" }) } + : file, + ), + }, + outputRoot, + }), + ).rejects.toThrow(/secret|session/i); + const sourceSecret = input(); + await expect( + writeQualificationBundle({ + input: { + ...sourceSecret, + files: sourceSecret.files.map((file) => + file.role === "authority-source" + ? { + ...file, + bytes: Buffer.from( + "export const key = 'sk-proj-abcdefghijklmnopqr';\n", + ), + } + : file, + ), + }, + outputRoot, + }), + ).rejects.toThrow(/secret-shaped source/); + const assignedSecret = input(); + await expect( + writeQualificationBundle({ + input: { + ...assignedSecret, + files: assignedSecret.files.map((file) => + file.role === "artifact" + ? { ...file, bytes: tarball("api_key=super-secret-value") } + : file, + ), + }, + outputRoot, + }), + ).rejects.toThrow(/secret-shaped evidence/); + const optionMember = input(); + await expect( + writeQualificationBundle({ + input: { + ...optionMember, + files: optionMember.files.map((file) => + file.role === "artifact" + ? { ...file, bytes: tarball("safe", "-checkpoint-action=exec") } + : file, + ), + }, + outputRoot, + }), + ).rejects.toThrow(/unsafe member path/); + const artifactSecret = input(); + await expect( + writeQualificationBundle({ + input: { + ...artifactSecret, + files: artifactSecret.files.map((file) => + file.role === "artifact" + ? { ...file, bytes: tarball("sk-proj-abcdefghijklmnopqr") } + : file, + ), + }, + outputRoot, + }), + ).rejects.toThrow(/secret-shaped source/); + const written = await writeQualificationBundle({ + input: input(), + outputRoot, + }); + const sidecar = join(written.path, "unsealed.txt"); + await writeFile(sidecar, "secret-shaped sidecar"); + await expect(readQualificationBundle(written.path)).rejects.toThrow( + /unexpected top-level/, + ); + await rm(sidecar); + const object = written.manifest.files.at(0)?.object; + if (!object) throw new Error("Bundle object fixture is missing."); + await writeFile(join(written.path, object), "corrupt"); + await expect(readQualificationBundle(written.path)).rejects.toThrow( + /digest|size/i, + ); + }); + + test("concurrent identical writers converge", async () => { + const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-")); + temporary.push(outputRoot); + const [left, right] = await Promise.all([ + writeQualificationBundle({ input: input(), outputRoot }), + writeQualificationBundle({ input: input(), outputRoot }), + ]); + expect(left.path).toBe(right.path); + expect(await readFile(join(left.path, "bundle.json"), "utf8")).toBe( + await readFile(join(right.path, "bundle.json"), "utf8"), + ); + }); + + test("rejects unsafe role ids and symlinked input files", async () => { + const outputRoot = await mkdtemp(join(tmpdir(), "flow-bundle-")); + temporary.push(outputRoot); + const unsafe = input(); + await expect( + writeQualificationBundle({ + input: { + ...unsafe, + files: unsafe.files.map((file) => + file.role === "authority-source" + ? { ...file, id: "../outside.ts" } + : file, + ), + }, + outputRoot, + }), + ).rejects.toThrow(/safe identifier/); + const inputRoot = join(outputRoot, "input"); + await mkdir(inputRoot); + await writeFile(join(outputRoot, "outside.json"), "{}"); + await symlink( + join(outputRoot, "outside.json"), + join(inputRoot, "report.json"), + ); + await expect( + readStableQualificationInput(inputRoot, "report.json"), + ).rejects.toThrow(/stable expected type/); + const archiveRoot = join(outputRoot, "archive"); + await mkdir(archiveRoot); + await writeFile(join(archiveRoot, "target.txt"), "safe"); + await symlink("target.txt", join(archiveRoot, "link.txt")); + const unsafeTar = join(outputRoot, "unsafe.tgz"); + const packed = Bun.spawnSync([ + "tar", + "-czf", + unsafeTar, + "-C", + archiveRoot, + "link.txt", + ]); + expect(packed.exitCode).toBe(0); + const unsafeArtifact = input(); + const unsafeTarBytes = await readFile(unsafeTar); + await expect( + writeQualificationBundle({ + input: { + ...unsafeArtifact, + files: unsafeArtifact.files.map((file) => + file.role === "artifact" + ? { ...file, bytes: unsafeTarBytes } + : file, + ), + }, + outputRoot, + }), + ).rejects.toThrow(/unsupported member type/); + }); + + test("rejects parent replacement between inspection and read", async () => { + const root = await mkdtemp(join(tmpdir(), "flow-bundle-input-")); + temporary.push(root); + await mkdir(join(root, "campaign")); + await writeFile(join(root, "campaign", "report.json"), "{}"); + await expect( + readStableQualificationInput( + root, + "campaign/report.json", + undefined, + async (stage) => { + if (stage !== "inspected") return; + await rename(join(root, "campaign"), join(root, "original")); + await mkdir(join(root, "campaign")); + await writeFile(join(root, "campaign", "report.json"), "{}"); + }, + ), + ).rejects.toThrow(/changed while reading/); + }); +}); diff --git a/tests/qualification-cli.test.ts b/tests/qualification-cli.test.ts new file mode 100644 index 0000000..811070c --- /dev/null +++ b/tests/qualification-cli.test.ts @@ -0,0 +1,431 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { currentBunToolchain } from "../evals/bun-toolchain.js"; +import { canonicalJson } from "../evals/canonical-json.js"; +import { + deriveConformanceOutcome, + retainedInstructions, + retainedReportActors, +} from "../evals/conformance-evidence.js"; +import { + pseudonymizeEvalIds, + pseudonymousEvalId, + RetainedScenarioEvidenceSchema, + scenarioGradeInput, +} from "../evals/grader-input.js"; +import { packPlugin } from "../evals/harness.js"; +import { + evaluatorIdentity, + inspectArtifact, + instructionDelivery, +} from "../evals/provenance.js"; +import { readQualificationBundle } from "../evals/qualification-bundle.js"; +import { + releaseCatalog, + releaseGraderBundle, + releaseHostConfigSha256, + releaseScenarioCatalog, +} from "../evals/release-policy.js"; +import { replayCassette } from "../evals/replay.js"; +import { createReportStore } from "../evals/report-store.js"; +import { campaignPlanFor, releaseScenarios } from "../evals/run.js"; +import { SCENARIOS } from "../evals/scenarios.js"; +import packageJson from "../package.json" with { type: "json" }; +import { prepareCanary, recordCanary } from "../scripts/eval-canary.js"; +import { assuranceProjection } from "../src/application/delivery.js"; +import { SessionSchema } from "../src/application/schema.js"; +import { operationInputDigest } from "../src/domain/operation.js"; + +const CASSETTES = { + "happy-path": "happy-path--opencode_claude-sonnet-5--2.json", + "plan-only-stops": "plan-only-stops--openai_gpt-5.6-sol--1.json", + "goal-change-refused": "goal-change-refused--openai_gpt-5.6-sol--3.json", + "continuation-accepted": + "continuation-accepted--fixture_hand-written--1.json", + "failing-gate-blocks": + "failing-gate-blocks--opencode_claude-sonnet-5--1.json", + "resumes-after-interruption": + "resumes-after-interruption--opencode_claude-sonnet-5--3.json", + "unprovable-claim-refused": "unprovable-claim-refused--xai_grok-4.5--1.json", + "skipped-case-named-binding": + "skipped-case-named-binding--fixture_hand-written--1.json", +} as const; + +const FIXED_ROLES = [ + "report", + "catalog", + "policy", + "plan", + "completion", + "expected-provenance", + "decision", + "artifact", + "canary-record", + "canary-installation", + "canary-session", + "canary-transcript", +] as const; + +function repairedCanarySession() { + const session = JSON.parse( + readFileSync( + join(import.meta.dir, "../evals/canary/artifacts/8.1.2-session.json"), + "utf8", + ), + ) as { + id: string; + closure: { + kind: "completed"; + summary: string; + operationId: string; + recordedRevision: number; + }; + operations: Array<{ id: string; inputDigest: string }>; + runs: Array<{ + validations: Array<{ + observedAssertions?: Array<{ status: string }>; + }>; + }>; + }; + for (const run of session.runs) { + for (const validation of run.validations) { + for (const assertion of validation.observedAssertions ?? []) { + assertion.status = "passed"; + } + } + } + const closureOperation = session.operations.find( + (operation) => operation.id === session.closure.operationId, + ); + if (!closureOperation) + throw new Error("Canary closure operation is missing."); + closureOperation.inputDigest = operationInputDigest({ + operationId: session.closure.operationId, + expectedRevision: session.closure.recordedRevision - 1, + sessionId: session.id, + kind: session.closure.kind, + summary: session.closure.summary, + }); + return SessionSchema.parse(session); +} + +function retainedReplayOutcome( + outcome: Awaited>["outcome"], +) { + const retained = structuredClone(outcome); + for (const call of [...retained.flowCalls, ...retained.allCalls]) { + Reflect.deleteProperty(call, "recordedStatus"); + } + return retained; +} + +function canaryTranscript(input: { + readonly fixture: string; + readonly packageVersion: string; + readonly pluginEntrySha256: string; + readonly session: ReturnType; +}) { + const call = (tool: string, output: unknown = {}) => ({ + type: "tool", + tool, + state: { status: "completed", input: {}, output }, + }); + return { + info: { directory: input.fixture, version: "1.18.6" }, + messages: [ + { + info: { + role: "assistant", + agent: "build", + providerID: "fixture-provider", + modelID: "fixture-manager", + sessionID: "ses_manager1", + }, + parts: [ + call("flow_status", { + workflowData: { + runtimeIdentity: { + packageVersion: input.packageVersion, + pluginEntrySha256: input.pluginEntrySha256, + }, + }, + }), + call("flow_plan_save"), + call("flow_validation_start"), + call("flow_review_start"), + { + type: "tool", + tool: "task", + state: { + status: "completed", + input: { subagent_type: "flow-reviewer" }, + output: {}, + metadata: { + model: { + providerID: "fixture-provider", + modelID: "fixture-reviewer", + }, + parentSessionId: "ses_manager1", + sessionId: "ses_reviewer1", + }, + }, + }, + call("flow_session_close", { + workflowData: { + delivery: { assurance: assuranceProjection(input.session) }, + }, + }), + ], + }, + { + info: { + role: "assistant", + agent: "flow-reviewer", + providerID: "fixture-provider", + modelID: "fixture-reviewer", + sessionID: "ses_reviewer1", + }, + parts: [], + }, + ], + }; +} + +test("qualifies and seals a complete exact-artifact campaign through the CLI", async () => { + const repositoryRoot = join(import.meta.dir, ".."); + const temporary = await mkdtemp(join(tmpdir(), "flow-qualification-cli-")); + try { + const artifactPath = await packPlugin( + repositoryRoot, + temporary, + currentBunToolchain(packageJson.packageManager), + ); + const artifact = await inspectArtifact({ + repositoryRoot, + tarballPath: artifactPath, + }); + const scenarios = releaseScenarios(); + const models = ["fixture-alpha/model-a", "fixture-beta/model-b"]; + const plan = campaignPlanFor({ + models, + scenarios, + sampling: { kind: "release" }, + opencodeVersion: "1.18.6", + }); + const evaluator = evaluatorIdentity({ + sourceCommit: artifact.sourceCommit, + caseCatalog: releaseScenarioCatalog(scenarios), + policyCatalog: releaseCatalog(), + graderBundle: releaseGraderBundle(repositoryRoot), + }); + const campaignDirectory = join(temporary, "campaign"); + const store = createReportStore({ + directory: campaignDirectory, + catalog: releaseCatalog(), + }); + await store.initialize(plan); + await store.writeCatalog(releaseCatalog()); + await store.writeArtifact(artifactPath); + + const replayedByScenario = new Map< + string, + Awaited> + >(); + for (const scenario of scenarios) { + const name = CASSETTES[scenario.id as keyof typeof CASSETTES]; + if (!name) throw new Error(`No cassette fixture for ${scenario.id}.`); + const cassette = JSON.parse( + await readFile( + join(repositoryRoot, "evals", "cassettes", name), + "utf8", + ), + ); + const replayed = await replayCassette(cassette); + expect(replayed.divergences, scenario.id).toEqual([]); + expect(scenario.check(replayed.outcome), scenario.id).toEqual([]); + replayedByScenario.set(scenario.id, replayed); + } + + for (const cell of plan.cells) { + const scenario = SCENARIOS.find(({ id }) => id === cell.caseId); + const replayed = replayedByScenario.get(cell.caseId); + const model = cell.managerModel; + if (!scenario || !replayed || !model) { + throw new Error(`Incomplete fixture for ${cell.cellId}.`); + } + const evidence = RetainedScenarioEvidenceSchema.parse({ + schemaVersion: 1, + attempt: { + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + caseId: cell.caseId, + repetition: cell.repetition, + model, + }, + actors: ["manager", "reviewer"].map((role, actorIndex) => ({ + role, + sessionIds: [pseudonymousEvalId(`session:${cell.cellId}-${role}`)], + actualModel: { + kind: "observed", + value: { + providerID: model.routeProvider, + modelID: `${model.model}-${actorIndex}`, + }, + }, + requestedModelId: `${model.routeProvider}/${model.model}`, + requestedModel: model, + })), + guidanceLoads: [], + gradeInput: pseudonymizeEvalIds( + scenarioGradeInput(retainedReplayOutcome(replayed.outcome)), + ), + usage: { durationMs: 1, outputTokens: 1, costUsd: 0 }, + }); + const attemptId = `attempt-${cell.cellId}`; + const transcript = await store.writeTranscript({ + attemptId, + text: canonicalJson(evidence), + }); + const commands = scenario.steps.map((step, sequence) => + instructionDelivery({ + source: "command", + name: step.command, + sequence, + text: `/${step.command} ${step.arguments}`.trim(), + }), + ); + await store.writeAttempt({ + schemaVersion: 2, + attemptId, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: cell.armToken, + repetition: cell.repetition, + artifact, + evaluator, + hostConfigSha256: releaseHostConfigSha256({ + packageVersion: artifact.packageVersion, + model, + }), + actors: retainedReportActors(evidence), + instructions: [...commands, ...retainedInstructions(evidence)], + transcript, + outcome: deriveConformanceOutcome({ + evidence, + check: scenario.check, + scenarioId: scenario.id, + model: `${model.routeProvider}/${model.model}`, + attempt: cell.repetition + 1, + }), + usage: { durationMs: 1, outputTokens: 1, costUsd: 0 }, + }); + } + + const completion = { + status: "complete" as const, + cause: "fixed-target" as const, + startedAt: "2026-08-28T00:00:00.000Z", + finishedAt: "2026-08-28T00:00:01.000Z", + activatedReserveCellIds: [], + observed: { + attempts: plan.cells.length, + outputTokens: plan.cells.length, + costUsd: 0, + wallClockMs: 1_000, + }, + }; + const report = await store.finalize({ + reportId: "qualification-cli-positive", + completion, + allocationCommitmentSha256: null, + }); + expect(report.attempts).toHaveLength(76); + + const preparedDirectory = join(temporary, "prepared-canary"); + await mkdir(preparedDirectory, { recursive: true }); + const recordedAt = new Date(); + const prepared = await prepareCanary({ + repositoryRoot, + artifactPath, + expectedArtifact: artifact, + outputDirectory: preparedDirectory, + preparedAt: recordedAt, + }); + const session = repairedCanarySession(); + const canaryRoot = join(temporary, "canary-root"); + const canary = await recordCanary({ + repositoryRoot: canaryRoot, + prepared, + preparedDirectory, + operator: "qualification-test", + session, + transcript: canaryTranscript({ + fixture: join(preparedDirectory, "fixture"), + packageVersion: artifact.packageVersion, + pluginEntrySha256: prepared.pluginEntrySha256, + session, + }), + recordedAt, + }); + expect(canary.record.status).toBe("passed"); + + const bundlesDirectory = join(temporary, "bundles"); + const qualified = Bun.spawn( + [ + "bun", + "run", + "qualify", + "--", + "--campaign-dir", + campaignDirectory, + "--canary", + canary.path, + "--bundles-dir", + bundlesDirectory, + ], + { cwd: repositoryRoot, stdout: "pipe", stderr: "pipe" }, + ); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(qualified.stdout).text(), + new Response(qualified.stderr).text(), + qualified.exited, + ]); + expect(exitCode, stderr).toBe(0); + expect(stdout).toContain("VERIFIED:"); + const bundlePath = stdout.trim().slice("VERIFIED: ".length); + const bundle = await readQualificationBundle(bundlePath); + expect(bundle.manifest.verdict).toBe("VERIFIED"); + expect(bundle.manifest.packageVersion).toBe(packageJson.version); + for (const role of FIXED_ROLES) { + expect( + bundle.files.filter(({ ref }) => ref.role === role && !ref.id), + role, + ).toHaveLength(1); + } + const attempts = bundle.files.filter(({ ref }) => ref.role === "attempt"); + const transcripts = bundle.files.filter( + ({ ref }) => ref.role === "transcript", + ); + expect(attempts).toHaveLength(76); + expect(transcripts).toHaveLength(76); + expect(attempts.map(({ ref }) => ref.id).sort()).toEqual( + transcripts.map(({ ref }) => ref.id).sort(), + ); + const authority = bundle.files + .filter(({ ref }) => ref.role === "authority-source") + .map(({ ref }) => ref.id) + .sort(); + expect(authority).toEqual( + releaseGraderBundle(repositoryRoot) + .files.map(({ path }) => path) + .sort(), + ); + } finally { + await rm(temporary, { recursive: true, force: true }); + } +}, 120_000); diff --git a/tests/release-qualification.test.ts b/tests/release-qualification.test.ts index 3c50d6b..66e41dd 100644 --- a/tests/release-qualification.test.ts +++ b/tests/release-qualification.test.ts @@ -12,9 +12,14 @@ import { releaseScenarioCatalog, } from "../evals/release-policy.js"; import { campaignPlanSha256 } from "../evals/report.js"; +import { + reportStoreAttemptFileName, + reportStoreCellFileName, +} from "../evals/report-store.js"; import { campaignPlanFor, releaseScenarios } from "../evals/run.js"; import { SCENARIOS } from "../evals/scenarios.js"; import { + assertCampaignEvidenceLayout, decisionRecordFor, qualifyV2, writeDecisionRecord, @@ -135,6 +140,51 @@ function releaseReport(stopped = false) { } describe("repository-owned v2 qualification", () => { + test("requires one canonical attempt and transcript path per cell", () => { + const attempts = ["a", "b", "c"].map((id) => ({ + attemptId: `attempt-${id}`, + cellId: `cell-${id}`, + transcript: { + artifact: `transcripts/${reportStoreAttemptFileName(`attempt-${id}`)}`, + }, + })); + const attemptFiles = attempts.map((attempt) => ({ + name: reportStoreCellFileName(attempt.cellId), + attemptId: attempt.attemptId, + })); + const transcriptFiles = attempts.map((attempt) => + reportStoreAttemptFileName(attempt.attemptId), + ); + const firstAttempt = attempts.at(0); + const firstFile = attemptFiles.at(0); + if (!firstAttempt || !firstFile) + throw new Error("Campaign layout fixture is missing."); + expect(() => + assertCampaignEvidenceLayout({ + attempts, + attemptFiles, + transcriptFiles, + }), + ).not.toThrow(); + expect(() => + assertCampaignEvidenceLayout({ + attempts, + attemptFiles: [...attemptFiles, firstFile], + transcriptFiles, + }), + ).toThrow(/duplicated/); + expect(() => + assertCampaignEvidenceLayout({ + attempts: attempts.map((attempt, index) => + index === 2 + ? { ...attempt, transcript: firstAttempt.transcript } + : attempt, + ), + attemptFiles, + transcriptFiles, + }), + ).toThrow(/noncanonical evidence path/); + }); test("derives all verdicts from the canonical 76-cell policy", () => { const verified = qualifyV2({ reportInput: releaseReport(), @@ -272,6 +322,25 @@ describe("repository-owned v2 qualification", () => { } }); + test("rejects non-literal imports from the grader closure", async () => { + const root = await mkdtemp(join(tmpdir(), "flow-grader-bundle-")); + try { + await mkdir(join(root, "evals"), { recursive: true }); + await mkdir(join(root, "scripts"), { recursive: true }); + await writeFile( + join(root, "evals", "run.ts"), + "const path = './grade.js'; await import(path);\n", + ); + await writeFile( + join(root, "scripts", "qualify-release.ts"), + "export {};\n", + ); + expect(() => releaseGraderBundle(root)).toThrow(/non-literal import/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + test("rejects the old 70-cell plan even after its hash is recomputed", () => { const report = releaseReport(); report.plan.cells = report.plan.cells.filter(