Seal regradable qualification bundles

.agents/plans/03-assurance-hardening/decisions.tsv

@@ -49,3 +49,11 @@ ts	phase	decision	why	evidence	result
 2026-08-28T04:23:41Z	phase-6	closed the final transcript and lineage review findings	flat synthetic transcripts and unlinked reviewer observations remained broader than the real OpenCode evidence boundary	root export messages plus tool parts; parent-child-model lineage match; final three-model review	586 pass, 1 intentional skip, 13 of 13 replays, pinned live smoke pass; all final reviewers green
 2026-08-28T04:32:34Z	phase-6	merged evidence-derived canary qualification	the exact PR head passed two CI runs and isolated shipping verification	PR 53; merge 269bd855	merged to main
 2026-08-28T04:32:34Z	phase-6-lineage	captured and closed cross-session transcript splicing	late review showed an authentic runtime status could be pooled with lifecycle calls from another manager session	ObservedCall session lineage; focused red splice test	all canary proof calls now come from the one manager session named by the reviewer task lineage
+2026-08-28T04:58:19Z	phase-6-lineage	merged coherent canary session lineage	the exact three-file follow-up passed CI and isolated shipping verification	PR 54; merge 65f1211	merged to main
+2026-08-28T04:58:19Z	phase-7	selected a sealed complete evidence bundle	independent judge required attempts, redacted transcripts, expected provenance, exact artifact and canary, decision, and full grader source to make regrading real	three architecture candidates; independent judge	candidate C base with A manifest-last layout and B evidence retention; V3 object store and package-lock closure rejected
+2026-08-28T04:58:19Z	phase-7	captured bundle and redaction regressions first	bundle sealing, replay, interruption, missing roles, secrets, raw IDs, corruption, concurrency, symlinks, and normalized grader inputs needed executable stop gates	new focused tests	missing modules and raw session ID retention failed red as expected
+2026-08-28T04:58:19Z	phase-7	implemented regradable qualification packaging	digest-only decisions could not be independently reproduced	normalized grader inputs; source closure; stable campaign ingestion; content-addressed manifest-last bundle	qualifier regrades every retained attempt and seals all evidence only with an exact derived canary and VERIFIED decision
+2026-08-28T05:21:31Z	phase-7	fixed all actionable first-wave review findings	transcripts could be reused or substituted, duplicate attempts hidden, full outcome/provenance fields trusted, canary bytes raced, artifact secrets skipped, and parent paths replaced	canonical report-store paths; shared conformance derivation; canary ref recheck; bounded tar scan; component identity snapshots	all claimed product outcome, actor, instruction, artifact, and evidence fields now reproduce from retained bytes
+2026-08-28T05:21:31Z	phase-7	proved the complete qualifier CLI path	placeholder bundle tests did not exercise the canonical 76-cell campaign and exact canary wiring	real packed plugin; eight passing cassette replays expanded to 76 cells; evidence-derived canary; real CLI	positive end-to-end bundle seals with 12 fixed roles, 76 attempt/transcript pairs, and complete source closure
+2026-08-28T05:30:13Z	phase-7	closed the final outcome and independence gaps	second review found empty reviewer observations, unretained usage, repeatable host evidence, extra sidecars, and weaker tar text scanning	filtered actor observations; schema-bound attempt identity and usage; unique manager pseudonyms; completion rederivation; strict root closure	all budget-relevant usage, product outcomes, provenance, and per-cell independence now reproduce from retained evidence
+2026-08-28T05:30:13Z	phase-7	completed immutable bundle verification	the corrected implementation needed whole-product, replay, live-host, and multi-model review evidence	bun run check; bun run replay; pinned OpenCode smoke; three final reviewers	599 pass, 1 intentional skip, 13 of 13 replays, live smoke pass, all reviewers green

.github/workflows/evals.yml

@@ -132,30 +132,17 @@ jobs:
             eval-v2.log
           retention-days: 90
 
-      - name: Apply release-qualification thresholds
-        id: qualify
+      - name: Record sealed-bundle prerequisite
         if: always() && steps.matrix.outputs.models != '' && steps.run.outputs.report != ''
-        continue-on-error: true
         run: |
-          set +e
-          mkdir -p evals/decisions
-          bun run qualify -- --report "${{ steps.run.outputs.report }}" --catalog "${{ steps.run.outputs.catalog }}" --artifact "${{ steps.run.outputs.artifact }}" --decisions-dir evals/decisions 2>&1 | tee qualification.log
-          status=${PIPESTATUS[0]}
-          decision_path="$(sed -n -E 's/^(VERIFIED|NOT VERIFIED|INCONCLUSIVE): (.*)$/\2/p' qualification.log | tail -n 1)"
-          if [ -z "$decision_path" ] || [ ! -f "$decision_path" ]; then
-            echo "::error::Qualifier did not print an existing decision path."
-            exit 1
-          fi
-          printf 'decision=%s\n' "$decision_path" >> "$GITHUB_OUTPUT"
-          exit "$status"
+          echo "INCONCLUSIVE: sealed qualification requires the exact-artifact canary." | tee qualification.log
 
-      - name: Publish the decision artifacts
+      - name: Publish qualification inputs
         if: always() && steps.matrix.outputs.models != ''
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: eval-v2-decision
+          name: eval-v2-qualification-input
           path: |
             qualification.log
-            ${{ steps.qualify.outputs.decision }}
           if-no-files-found: warn
           retention-days: 90

docs/development.md

@@ -144,10 +144,11 @@ deterministic hook and lifecycle gates.
 
 ## Release
 
-A release needs a V2 qualification report from at least two providers, a canary of
-the exact packed artifact, and a canary-bound decision. Pass `--report`, `--catalog`,
-`--artifact`, and `--canary` to `bun run qualify`. Commit `evals/canary` and
-`evals/decisions` before tagging. [Release qualification](release-qualification.md)
+A release needs a complete V2 campaign from at least two providers and a canary of
+the exact packed artifact. Pass `--campaign-dir` and `--canary` to
+`bun run qualify`; it seals the report, attempts, transcripts, artifact, canary,
+expected provenance, decision, and grader source into one immutable bundle. Commit
+the bundle and canary before tagging. [Release qualification](release-qualification.md)
 publishes the thresholds and commands.
 
 Release tags use `v<package-version>`. Blocking release checks include the

docs/maintainer-contract.md

@@ -338,9 +338,8 @@ Model evals are the one exception, admitted by
 [ADR 0010](adr/0010-declared-canonical-gate.md) against the prompt prose the
 declared gate replaced. They run weekly and on demand in one workflow, never in a
 gate a contributor waits on, and skip themselves without a configured matrix or
-credentials. `bun run qualify` applies the published thresholds to a report; a
-scenario with no published threshold, or a gated scenario the report never ran,
-fails qualification.
+credentials. `bun run qualify` seals a complete campaign and exact canary. A
+scenario without a threshold, or a required scenario the report omitted, fails.
 
 Bounded-wave coverage should test the real agent permissions, manager guidance, and
 host-visible configuration without adding a scheduler or tests-of-tests.

docs/release-qualification.md

@@ -61,10 +61,10 @@ direction. The cadence follows from that:
 - **Freeze on the public surface** while the guarantees are being measured: tools,
   commands, guides, agents, and the Session v5 shape. Additive optional fields are
   allowed; removals and renames are not.
-- **No release** without a committed exact V2 decision and fresh canary.
-  `bun run qualify` writes the decision under `evals/decisions`; release metadata
-  refuses a tag without its matching `evals/canary` evidence. A `CHANGELOG` entry
-  states the schema impact explicitly.
+- **No release** without a sealed V2 qualification bundle and fresh canary.
+  The bundle retains every attempt, transcript, grader source, and exact artifact
+  needed to reproduce its decision. A `CHANGELOG` entry states the schema impact
+  explicitly.
 - **Patch releases** for defects and host-compatibility fixes, which is what the
   weekly OpenCode compatibility smoke exists to catch early.
 - **Deprecate before removing.** A surface that is going away is announced in one
@@ -78,9 +78,7 @@ bun run eval -- --release --model <anthropic-id> --model <openai-id>
 bun run eval:canary -- prepare --report <campaign-dir>/report.json --out <canary-dir>
 # Run the prepared fixture, then record its session and transcript.
 bun run eval:canary -- record <record-options>
-bun run qualify -- --report <campaign-dir>/report.json \
-  --catalog <campaign-dir>/catalog.json \
-  --artifact <campaign-dir>/artifact.tgz \
+bun run qualify -- --campaign-dir <campaign-dir> \
   --canary evals/canary/<version>.json
 ```
 
@@ -93,7 +91,7 @@ described with their prices in
 `bun run benchmark -- --model <id> --repeat 3 --seed <text>` compares Flow with
 ordinary OpenCode on hidden-graded tasks. It is not a qualification input.
 
-The scheduled workflow (`.github/workflows/evals.yml`) does the same weekly and
-publishes the report as an artifact. It skips itself when no model matrix or
-provider credentials are configured, because an unconfigured fork is a
-configuration state and not a failure.
+The scheduled workflow runs the paid campaign weekly and publishes its complete
+campaign directory. Sealing waits for the exact-artifact canary, so the workflow
+reports qualification as inconclusive rather than manufacturing a partial bundle.
+It skips itself when no model matrix or provider credentials are configured.

evals/conformance-evidence.ts

@@ -0,0 +1,85 @@
+import type {
+	RetainedScenarioEvidence,
+	ScenarioGradeInput,
+} from "./grader-input.js";
+import { askedQuestions } from "./harness.js";
+import {
+	completionHonesty,
+	type MetricSession,
+	reviewerActivity,
+} from "./metrics.js";
+import { instructionDelivery } from "./provenance.js";
+import type {
+	ActorIdentity,
+	AttemptRecordV2,
+	InstructionDelivery,
+} from "./report.js";
+
+export function retainedReportActors(
+	evidence: RetainedScenarioEvidence,
+): ActorIdentity[] {
+	return evidence.actors.map((actor) => ({
+		role: actor.role,
+		requestedModel: actor.requestedModel,
+		actualModel:
+			actor.actualModel.kind === "observed"
+				? {
+						kind: "unobserved" as const,
+						reason: `Host observed providerID=${actor.actualModel.value.providerID} modelID=${actor.actualModel.value.modelID}; full family, gateway, and revision identity is unavailable.`,
+					}
+				: actor.actualModel,
+		sessionIds: [...actor.sessionIds],
+	}));
+}
+
+export function retainedInstructions(
+	evidence: RetainedScenarioEvidence,
+): InstructionDelivery[] {
+	return evidence.guidanceLoads.map((load) =>
+		instructionDelivery({
+			source: "guidance",
+			name: load.id ?? "unknown-guidance",
+			sequence: load.sequence,
+			text: load.rawOutput,
+		}),
+	);
+}
+
+export function deriveConformanceOutcome(input: {
+	readonly evidence: RetainedScenarioEvidence;
+	readonly check: (gradeInput: ScenarioGradeInput) => readonly string[];
+	readonly scenarioId: string;
+	readonly model: string;
+	readonly attempt: number;
+}): Extract<AttemptRecordV2["outcome"], { kind: "product" }> {
+	const issues = [...input.check(input.evidence.gradeInput)];
+	const documents = [
+		...(input.evidence.gradeInput.session
+			? [input.evidence.gradeInput.session]
+			: []),
+		...input.evidence.gradeInput.archives,
+	] as MetricSession[];
+	return {
+		kind: "product",
+		passed: issues.length === 0,
+		endedBy:
+			askedQuestions(input.evidence.gradeInput).length > 0
+				? "user-escalation"
+				: "quiet",
+		issues: issues.length > 0 ? issues : [],
+		evidence: {
+			kind: "conformance",
+			falseCompletion: completionHonesty(
+				documents.find((document) => document.closure) ?? null,
+			).falseCompletion,
+			unsubmittedReviews: reviewerActivity(documents).unsubmitted,
+			facts: {
+				scenario: input.scenarioId,
+				model: input.model,
+				attempt: input.attempt,
+				flowCalls: input.evidence.gradeInput.flowCalls.length,
+				guidanceLoads: input.evidence.guidanceLoads.length,
+			},
+		},
+	};
+}

evals/grader-input.ts

@@ -0,0 +1,140 @@
+import { z } from "zod";
+import { canonicalSha256 } from "./canonical-json.js";
+import { mapStrings } from "./cassette.js";
+import type { ObservedToolCall, Outcome } from "./harness.js";
+
+const TextSchema = z
+	.string()
+	.min(1)
+	.max(4 * 1024 * 1024)
+	.regex(/\S/);
+const JsonRecordSchema = z.record(z.string(), z.unknown());
+const ToolCallSchema: z.ZodType<ObservedToolCall> = z
+	.object({
+		tool: TextSchema,
+		status: z.enum(["pending", "running", "completed", "error"]),
+		sessionIndex: z.number().int().safe().nonnegative(),
+		agent: TextSchema,
+		input: JsonRecordSchema,
+		output: z.unknown(),
+		rawOutput: z.string().max(4 * 1024 * 1024),
+		metadata: JsonRecordSchema,
+	})
+	.strict();
+
+export const ScenarioGradeInputSchema = z
+	.object({
+		schemaVersion: z.literal(1),
+		flowCalls: z.array(ToolCallSchema).max(4096),
+		allCalls: z.array(ToolCallSchema).max(4096),
+		session: JsonRecordSchema.nullable(),
+		archives: z.array(JsonRecordSchema).max(512),
+		finalText: z.string().max(4 * 1024 * 1024),
+	})
+	.strict();
+
+const ObservedModelSchema = z.discriminatedUnion("kind", [
+	z
+		.object({
+			kind: z.literal("observed"),
+			value: z.object({ providerID: TextSchema, modelID: TextSchema }).strict(),
+		})
+		.strict(),
+	z.object({ kind: z.literal("unobserved"), reason: TextSchema }).strict(),
+]);
+const RequestedModelSchema = z
+	.object({
+		routeProvider: TextSchema,
+		gateway: TextSchema.nullable(),
+		family: TextSchema,
+		model: TextSchema,
+		revision: TextSchema.nullable(),
+	})
+	.strict();
+const RetainedActorSchema = z
+	.object({
+		role: z.enum(["manager", "reviewer"]),
+		sessionIds: z.array(TextSchema).min(1),
+		actualModel: ObservedModelSchema,
+		requestedModelId: TextSchema,
+		requestedModel: RequestedModelSchema,
+	})
+	.strict();
+const GuidanceLoadSchema = z
+	.object({
+		sequence: z.number().int().safe().nonnegative(),
+		sessionIndex: z.number().int().safe().nonnegative(),
+		agent: TextSchema,
+		id: TextSchema.nullable(),
+		rawOutput: z.string().max(4 * 1024 * 1024),
+		utf8Bytes: z.number().int().safe().nonnegative(),
+	})
+	.strict();
+
+export const RetainedScenarioEvidenceSchema = z
+	.object({
+		schemaVersion: z.literal(1),
+		attempt: z
+			.object({
+				attemptId: TextSchema,
+				cellId: TextSchema,
+				caseId: TextSchema,
+				repetition: z.number().int().safe().nonnegative(),
+				model: RequestedModelSchema,
+			})
+			.strict(),
+		actors: z.array(RetainedActorSchema).max(64),
+		guidanceLoads: z.array(GuidanceLoadSchema).max(4096),
+		gradeInput: ScenarioGradeInputSchema,
+		usage: z
+			.object({
+				durationMs: z.number().int().safe().nonnegative(),
+				outputTokens: z.number().int().safe().nonnegative(),
+				costUsd: z.number().finite().nonnegative().nullable(),
+			})
+			.strict(),
+	})
+	.strict();
+
+export type ScenarioGradeInput = Pick<
+	Outcome,
+	"flowCalls" | "allCalls" | "session" | "archives" | "finalText"
+>;
+export type RetainedScenarioGradeInput = z.infer<
+	typeof ScenarioGradeInputSchema
+>;
+export type RetainedScenarioEvidence = z.infer<
+	typeof RetainedScenarioEvidenceSchema
+>;
+
+export function pseudonymousEvalId(id: string): string {
+	return `id_${canonicalSha256("flow-eval-redacted-id-v1", id).slice("sha256:".length, "sha256:".length + 16)}`;
+}
+
+export function pseudonymizeEvalIds<T>(value: T): T {
+	return mapStrings(value, (text) =>
+		text.replace(
+			/\b(?:ses_[A-Za-z0-9]+|(?:session|review):[A-Za-z0-9-]+)\b/g,
+			pseudonymousEvalId,
+		),
+	) as T;
+}
+
+export function actorsWithSessions<
+	Actor extends { readonly sessionIds: readonly string[] },
+>(actors: readonly Actor[]): Actor[] {
+	return actors.filter((actor) => actor.sessionIds.length > 0);
+}
+
+export function scenarioGradeInput(
+	outcome: Outcome,
+): RetainedScenarioGradeInput {
+	return ScenarioGradeInputSchema.parse({
+		schemaVersion: 1,
+		flowCalls: outcome.flowCalls,
+		allCalls: outcome.allCalls,
+		session: outcome.session,
+		archives: outcome.archives,
+		finalText: outcome.finalText,
+	});
+}

evals/harness.ts

@@ -33,6 +33,7 @@ import {
 	preservePrimaryFailure,
 	providerFailure,
 } from "./failure-origin.js";
+import type { ScenarioGradeInput } from "./grader-input.js";
 import {
 	extractObservedActor,
 	guidanceLoad,
@@ -292,7 +293,7 @@ export type Scenario = {
 	 */
 	readonly mayEscalate?: boolean;
 	/** Returns a list of failures. Empty means the scenario passed. */
-	readonly check: (outcome: Outcome) => readonly string[];
+	readonly check: (outcome: ScenarioGradeInput) => readonly string[];
 };
 
 /**
@@ -810,7 +811,7 @@ export async function runQueues<Job, Result>(
  * both for a human judging whether asking was right and for a check that reads
  * whether the blocker was named at all.
  */
-export function askedQuestions(outcome: Outcome): string[] {
+export function askedQuestions(outcome: Pick<Outcome, "allCalls">): string[] {
 	return outcome.allCalls
 		.filter((call) => call.tool === "question")
 		.map((call) => JSON.stringify(call.input));

evals/provenance.ts

@@ -6,6 +6,7 @@ import { promisify } from "node:util";
 import { createFileSourceIdentityProvider } from "../src/infrastructure/fs/source-identity.js";
 import { canonicalJson, canonicalSha256 } from "./canonical-json.js";
 import { normalizeRecorded, REDACTED } from "./cassette.js";
+import { pseudonymizeEvalIds } from "./grader-input.js";
 import type {
 	ArtifactIdentity,
 	EvaluatorIdentity,
@@ -345,7 +346,9 @@ export function redactTranscript(input: {
 	readonly projectPath: string;
 }): RedactedTranscript {
 	const text = canonicalJson(
-		redactSensitiveFields(normalizeRecorded(input.value, input.projectPath)),
+		pseudonymizeEvalIds(
+			redactSensitiveFields(normalizeRecorded(input.value, input.projectPath)),
+		),
 	);
 	return { text, sha256: sha256(new TextEncoder().encode(text)) };
 }