Rederive qualification at release time

.agents/plans/03-assurance-hardening/decisions.tsv

@@ -57,3 +57,7 @@ ts	phase	decision	why	evidence	result
 2026-08-28T05:21:31Z	phase-7	proved the complete qualifier CLI path	placeholder bundle tests did not exercise the canonical 76-cell campaign and exact canary wiring	real packed plugin; eight passing cassette replays expanded to 76 cells; evidence-derived canary; real CLI	positive end-to-end bundle seals with 12 fixed roles, 76 attempt/transcript pairs, and complete source closure
 2026-08-28T05:30:13Z	phase-7	closed the final outcome and independence gaps	second review found empty reviewer observations, unretained usage, repeatable host evidence, extra sidecars, and weaker tar text scanning	filtered actor observations; schema-bound attempt identity and usage; unique manager pseudonyms; completion rederivation; strict root closure	all budget-relevant usage, product outcomes, provenance, and per-cell independence now reproduce from retained evidence
 2026-08-28T05:30:13Z	phase-7	completed immutable bundle verification	the corrected implementation needed whole-product, replay, live-host, and multi-model review evidence	bun run check; bun run replay; pinned OpenCode smoke; three final reviewers	599 pass, 1 intentional skip, 13 of 13 replays, live smoke pass, all reviewers green
+2026-08-28T05:41:49Z	phase-7	merged sealed regradable qualification bundles	the exact 23-file head passed CI, real 76-cell CLI qualification, and isolated shipping verification	PR 55; merge 4e7e2bf	merged to main
+2026-08-28T05:41:49Z	phase-8	made the sealed bundle release authority	digest records could be internally consistent without proving their verdict	regradeQualificationBundle; assertQualificationBundle; release workflow	verifier reopens all objects, regrades 76 attempts, rederives canary, provenance, usage, authority, and decision; forged decision and missing source bundles fail
+2026-08-28T05:55:21Z	phase-8	fixed all actionable release-authority review findings	ignored fixed roles, unsafe reader artifacts, unbounded references, stale campaigns, ambiguous bundles, swallowed corruption, and script cycles weakened the gate	fixed-role equality; read-time tar scan; total cap; seven-day policy; unique SHA-emitting selection; injected decision authority	current release source must match the sealed closure and exactly one clean fresh bundle must reproduce
+2026-08-28T05:55:21Z	phase-8	completed independent release regrading verification	forged decisions, omitted sources, contradictory roles/manifests, stale campaigns, digest-only records, and corrupted siblings needed executable rejection	positive 76-cell CLI plus adversarial reseals; full check; replay; live smoke; final reviews	598 pass, 1 intentional skip, 13 of 13 replays, live smoke pass, no production blockers

.github/workflows/release.yml

@@ -121,7 +121,7 @@ jobs:
           set -euo pipefail
           bun pm pack --destination .
 
-      - name: Verify exact VERIFIED V2 artifact decision and fresh canary
+      - name: Verify independently regraded qualification bundle and fresh canary
         shell: bash
         run: |
           set -euo pipefail

docs/release-qualification.md

@@ -63,8 +63,8 @@ direction. The cadence follows from that:
   allowed; removals and renames are not.
 - **No release** without a sealed V2 qualification bundle and fresh canary.
   The bundle retains every attempt, transcript, grader source, and exact artifact
-  needed to reproduce its decision. A `CHANGELOG` entry states the schema impact
-  explicitly.
+  needed to reproduce its decision. Release metadata independently regrades those
+  bytes before publication. A `CHANGELOG` entry states the schema impact explicitly.
 - **Patch releases** for defects and host-compatibility fixes, which is what the
   weekly OpenCode compatibility smoke exists to catch early.
 - **Deprecate before removing.** A surface that is going away is announced in one

evals/qualification-bundle.ts

@@ -701,6 +701,9 @@ export async function readQualificationBundle(path: string): Promise<{
 			bytes: new Uint8Array(),
 		})),
 	);
+	const totalBytes = manifest.files.reduce((sum, file) => sum + file.bytes, 0);
+	if (totalBytes > MAX_BUNDLE_BYTES)
+		throw new Error("Qualification bundle exceeds its total byte limit.");
 	const objectNames = (await readdir(join(path, "objects"))).sort();
 	const expectedNames = [
 		...new Set(
@@ -725,6 +728,9 @@ export async function readQualificationBundle(path: string): Promise<{
 			return { ref, bytes };
 		}),
 	);
+	const artifact = files.find(({ ref }) => ref.role === "artifact");
+	if (!artifact) throw new Error("Qualification bundle artifact is missing.");
+	await assertSafeArtifact(artifact.bytes);
 	if (
 		!(await sameDirectoryIdentity(bundleIdentity)) ||
 		!(await sameDirectoryIdentity(objectsIdentity))

evals/qualification-regrade.ts

@@ -0,0 +1,343 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+	type CanaryRecord,
+	deriveCanaryResult,
+	parseCanaryRecord,
+} from "../scripts/eval-canary.js";
+import type { ReleaseDecision, ReleaseExpectedProvenance } from "./analysis.js";
+import { canonicalJson, canonicalSha256 } from "./canonical-json.js";
+import type { ValidatedCaseCatalog } from "./catalog.js";
+import {
+	deriveConformanceOutcome,
+	retainedInstructions,
+	retainedReportActors,
+} from "./conformance-evidence.js";
+import { RetainedScenarioEvidenceSchema } from "./grader-input.js";
+import {
+	inspectArtifact,
+	instructionDelivery,
+	samePackedArtifact,
+} from "./provenance.js";
+import { readQualificationBundle } from "./qualification-bundle.js";
+import {
+	RELEASE_ANALYSIS_SHA256,
+	RELEASE_MAX_CAMPAIGN_AGE_MS,
+	RELEASE_POLICY_SHA256,
+	releaseGraderBundle,
+} from "./release-policy.js";
+import type { ValidatedReport } from "./report.js";
+import { SCENARIOS } from "./scenarios.js";
+
+type BundleFile = Awaited<
+	ReturnType<typeof readQualificationBundle>
+>["files"][number];
+type RegradedDecision = Readonly<{
+	verdict: "VERIFIED" | "NOT VERIFIED" | "INCONCLUSIVE";
+	artifact: Exclude<
+		ValidatedReport["attempts"][number]["artifact"],
+		{ kind: string }
+	>;
+	canarySha256: string | null;
+	analyzerSha256: string;
+	[key: string]: unknown;
+}>;
+export type QualificationRegradeAuthority = Readonly<{
+	qualify(input: {
+		reportInput: unknown;
+		catalogInput: unknown;
+		artifact: RegradedDecision["artifact"];
+		canary?: CanaryRecord | null;
+	}): {
+		report: ValidatedReport;
+		catalog: ValidatedCaseCatalog;
+		expected: ReleaseExpectedProvenance;
+		decision: ReleaseDecision;
+		canary: CanaryRecord | null;
+	};
+	decisionRecord(input: {
+		report: ValidatedReport;
+		catalog: ValidatedCaseCatalog;
+		expected: ReleaseExpectedProvenance;
+		decision: ReleaseDecision;
+		canarySha256?: string | null;
+	}): RegradedDecision;
+}>;
+
+function json(file: BundleFile): unknown {
+	return JSON.parse(file.bytes.toString("utf8"));
+}
+
+function one(
+	files: readonly BundleFile[],
+	role: BundleFile["ref"]["role"],
+	id?: string,
+): BundleFile {
+	const matches = files.filter(
+		(file) => file.ref.role === role && file.ref.id === id,
+	);
+	if (matches.length !== 1)
+		throw new Error(`Qualification bundle role ${role} is not unique.`);
+	const match = matches[0];
+	if (!match) throw new Error(`Qualification bundle role ${role} is missing.`);
+	return match;
+}
+
+function digest(bytes: Uint8Array): string {
+	return `sha256:${new Bun.CryptoHasher("sha256").update(bytes).digest("hex")}`;
+}
+
+function verifyCanary(
+	files: readonly BundleFile[],
+	artifact: ValidatedReport["attempts"][number]["artifact"],
+): string {
+	if ("kind" in artifact)
+		throw new Error("Qualification artifact is unavailable.");
+	const parsed = parseCanaryRecord(json(one(files, "canary-record")));
+	if (!parsed.ok) throw new Error(parsed.issues.join("; "));
+	const record = parsed.value;
+	const evidence = [
+		[record.artifacts.installation, one(files, "canary-installation")],
+		[record.artifacts.session, one(files, "canary-session")],
+		[record.artifacts.transcript, one(files, "canary-transcript")],
+	] as const;
+	for (const [ref, file] of evidence) {
+		if (
+			!ref ||
+			ref.bytes !== file.bytes.byteLength ||
+			ref.sha256 !== digest(file.bytes)
+		)
+			throw new Error("Bundled canary evidence differs from its record.");
+	}
+	if (
+		!samePackedArtifact(record.artifact, artifact) ||
+		record.status !== "passed"
+	)
+		throw new Error("Bundled canary does not match the release artifact.");
+	const derived = deriveCanaryResult({
+		packageVersion: record.artifact.packageVersion,
+		artifactSha256: record.artifactSha256,
+		tarballSha256: record.artifact.tarballSha256,
+		preparedSha256: record.preparedSha256,
+		pluginEntrySha256: record.pluginEntrySha256,
+		installation: json(evidence[0][1]),
+		session: json(evidence[1][1]),
+		transcript: json(evidence[2][1]),
+	});
+	if (
+		derived.status !== record.status ||
+		canonicalJson(derived.checks) !== canonicalJson(record.checks) ||
+		canonicalJson(derived.actors) !== canonicalJson(record.actors) ||
+		derived.hostConfigSha256 !== record.hostConfigSha256
+	)
+		throw new Error("Bundled canary claims do not reproduce from evidence.");
+	return record.recordSha256;
+}
+
+function regradeAttempts(
+	report: ValidatedReport,
+	files: readonly BundleFile[],
+): void {
+	const sessions = new Set<string>();
+	for (const attempt of report.attempts) {
+		const retained = json(one(files, "attempt", attempt.attemptId));
+		if (canonicalJson(retained) !== canonicalJson(attempt))
+			throw new Error(
+				`Bundled attempt ${attempt.attemptId} differs from report.`,
+			);
+		if (!attempt.transcript)
+			throw new Error(
+				`Bundled attempt ${attempt.attemptId} has no transcript.`,
+			);
+		const transcript = one(files, "transcript", attempt.attemptId);
+		if (digest(transcript.bytes) !== attempt.transcript.sha256)
+			throw new Error(
+				`Bundled transcript ${attempt.attemptId} has the wrong digest.`,
+			);
+		const evidence = RetainedScenarioEvidenceSchema.parse(json(transcript));
+		const scenario = SCENARIOS.find(({ id }) => id === attempt.caseId);
+		const manager = attempt.actors.find(({ role }) => role === "manager");
+		if (!scenario || !manager || attempt.outcome.kind !== "product")
+			throw new Error(
+				`Bundled attempt ${attempt.attemptId} is not regradable.`,
+			);
+		if (
+			canonicalJson(evidence.attempt) !==
+				canonicalJson({
+					attemptId: attempt.attemptId,
+					cellId: attempt.cellId,
+					caseId: attempt.caseId,
+					repetition: attempt.repetition,
+					model: manager.requestedModel,
+				}) ||
+			canonicalJson(evidence.usage) !== canonicalJson(attempt.usage)
+		)
+			throw new Error(`Bundled attempt ${attempt.attemptId} binding differs.`);
+		const actors = retainedReportActors(evidence);
+		const retainedManager = actors.find(({ role }) => role === "manager");
+		if (!retainedManager)
+			throw new Error("Bundled attempt has no manager evidence.");
+		for (const id of retainedManager.sessionIds) {
+			if (sessions.has(id))
+				throw new Error("Bundled attempts reuse manager evidence.");
+			sessions.add(id);
+		}
+		const outcome = deriveConformanceOutcome({
+			evidence,
+			check: scenario.check,
+			scenarioId: attempt.caseId,
+			model: `${evidence.attempt.model.routeProvider}/${evidence.attempt.model.model}`,
+			attempt: attempt.repetition + 1,
+		});
+		const commands = scenario.steps.map((step, sequence) =>
+			instructionDelivery({
+				source: "command",
+				name: step.command,
+				sequence,
+				text: `/${step.command} ${step.arguments}`.trim(),
+			}),
+		);
+		const guidance = retainedInstructions(evidence).map(
+			(instruction, sequence) => ({
+				...instruction,
+				sequence: commands.length + sequence,
+			}),
+		);
+		if (
+			canonicalJson(outcome) !== canonicalJson(attempt.outcome) ||
+			canonicalJson(actors) !== canonicalJson(attempt.actors) ||
+			canonicalJson([...commands, ...guidance]) !==
+				canonicalJson(attempt.instructions)
+		)
+			throw new Error(`Bundled attempt ${attempt.attemptId} grade differs.`);
+	}
+	const cost = report.attempts.some((attempt) => attempt.usage.costUsd === null)
+		? null
+		: report.attempts.reduce(
+				(sum, attempt) => sum + (attempt.usage.costUsd ?? 0),
+				0,
+			);
+	const observed = {
+		attempts: report.attempts.length,
+		outputTokens: report.attempts.reduce(
+			(sum, attempt) => sum + attempt.usage.outputTokens,
+			0,
+		),
+		costUsd: cost,
+		wallClockMs: Math.max(
+			Date.parse(report.completion.finishedAt) -
+				Date.parse(report.completion.startedAt),
+			...report.attempts.map((attempt) => attempt.usage.durationMs),
+		),
+	};
+	if (canonicalJson(observed) !== canonicalJson(report.completion.observed))
+		throw new Error("Bundled completion usage does not reproduce.");
+}
+
+export async function regradeQualificationBundle(input: {
+	readonly path: string;
+	readonly repositoryRoot: string;
+	readonly authority: QualificationRegradeAuthority;
+	readonly now?: Date;
+}): Promise<{
+	readonly decision: RegradedDecision;
+	readonly bundleSha256: string;
+}> {
+	const bundle = await readQualificationBundle(input.path);
+	const files = bundle.files;
+	const reportInput = json(one(files, "report"));
+	const catalogInput = json(one(files, "catalog"));
+	const planStored = json(one(files, "plan"));
+	const completionStored = json(one(files, "completion"));
+	const policy = json(one(files, "policy")) as Record<string, unknown>;
+	const expectedStored = json(one(files, "expected-provenance"));
+	const decisionStored = json(one(files, "decision"));
+	const bundledGrader = {
+		files: files
+			.filter(({ ref }) => ref.role === "authority-source")
+			.map(({ ref, bytes }) => ({
+				path: ref.id,
+				sha256: canonicalSha256(
+					"flow-release-grader-file-v1",
+					bytes.toString("utf8"),
+				),
+			}))
+			.sort((left, right) =>
+				String(left.path).localeCompare(String(right.path)),
+			),
+	};
+	if (
+		policy.policySha256 !== RELEASE_POLICY_SHA256 ||
+		policy.analysisSha256 !== RELEASE_ANALYSIS_SHA256 ||
+		canonicalJson(policy.graderBundle) !== canonicalJson(bundledGrader) ||
+		canonicalJson(policy.graderBundle) !==
+			canonicalJson(releaseGraderBundle(input.repositoryRoot))
+	)
+		throw new Error("Bundled release authority does not match the verifier.");
+	const temporary = await mkdtemp(join(tmpdir(), "flow-bundle-regrade-"));
+	const artifactPath = join(temporary, "artifact.tgz");
+	try {
+		await writeFile(artifactPath, one(files, "artifact").bytes);
+		const artifact = await inspectArtifact({
+			repositoryRoot: input.repositoryRoot,
+			tarballPath: artifactPath,
+		});
+		const prelim = input.authority.qualify({
+			reportInput,
+			catalogInput,
+			artifact,
+		});
+		const firstAttempt = prelim.report.attempts.at(0);
+		if (!firstAttempt) throw new Error("Bundled report has no attempts.");
+		const canarySha256 = verifyCanary(files, firstAttempt.artifact);
+		const parsedCanary = parseCanaryRecord(json(one(files, "canary-record")));
+		if (!parsedCanary.ok) throw new Error(parsedCanary.issues.join("; "));
+		const result = input.authority.qualify({
+			reportInput,
+			catalogInput,
+			artifact,
+			canary: parsedCanary.value,
+		});
+		const now = (input.now ?? new Date()).getTime();
+		const campaignFinished = Date.parse(result.report.completion.finishedAt);
+		const canaryRecorded = Date.parse(parsedCanary.value.recordedAt);
+		if (
+			campaignFinished > canaryRecorded ||
+			canaryRecorded > now ||
+			campaignFinished > now ||
+			now - campaignFinished > RELEASE_MAX_CAMPAIGN_AGE_MS
+		)
+			throw new Error("Bundled campaign and canary freshness is invalid.");
+		if (
+			canonicalJson(planStored) !== canonicalJson(result.report.plan) ||
+			canonicalJson(completionStored) !==
+				canonicalJson(result.report.completion)
+		)
+			throw new Error("Bundled plan or completion differs from the report.");
+		regradeAttempts(result.report, files);
+		if (canonicalJson(result.expected) !== canonicalJson(expectedStored))
+			throw new Error("Bundled expected provenance does not reproduce.");
+		const decision = input.authority.decisionRecord({
+			...result,
+			canarySha256,
+		});
+		if (
+			decision.verdict !== "VERIFIED" ||
+			canonicalJson(decision) !== canonicalJson(decisionStored)
+		)
+			throw new Error("Bundled decision does not reproduce.");
+		if (
+			bundle.manifest.reportId !== result.report.reportId ||
+			bundle.manifest.packageVersion !== artifact.packageVersion ||
+			bundle.manifest.verdict !== decision.verdict ||
+			policy.analyzerSha256 !== decision.analyzerSha256
+		)
+			throw new Error(
+				"Bundled manifest or analyzer identity does not reproduce.",
+			);
+		return { decision, bundleSha256: bundle.manifest.bundleSha256 };
+	} finally {
+		await rm(temporary, { recursive: true, force: true });
+	}
+}