From 42de3ce30d97d7d0a2dcc4fe10ed38aa143c4e8c Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Fri, 28 Aug 2026 07:56:53 +0200 Subject: [PATCH] Rederive qualification at release time --- .../03-assurance-hardening/decisions.tsv | 4 + .github/workflows/release.yml | 2 +- docs/release-qualification.md | 4 +- evals/qualification-bundle.ts | 6 + evals/qualification-regrade.ts | 343 ++++++++++++++++++ evals/qualification/README.md | 5 +- evals/release-policy.ts | 7 +- scripts/qualify-release.ts | 37 +- scripts/release-metadata.ts | 164 ++++----- tests/documentation-contract.test.ts | 2 +- tests/qualification-cli.test.ts | 157 +++++++- tests/release-metadata.test.ts | 22 +- tests/release-qualification.test.ts | 26 +- 13 files changed, 624 insertions(+), 155 deletions(-) create mode 100644 evals/qualification-regrade.ts diff --git a/.agents/plans/03-assurance-hardening/decisions.tsv b/.agents/plans/03-assurance-hardening/decisions.tsv index 9c41940..d32a7c3 100644 --- a/.agents/plans/03-assurance-hardening/decisions.tsv +++ b/.agents/plans/03-assurance-hardening/decisions.tsv @@ -57,3 +57,7 @@ ts phase decision why evidence result 2026-08-28T05:21:31Z phase-7 proved the complete qualifier CLI path placeholder bundle tests did not exercise the canonical 76-cell campaign and exact canary wiring real packed plugin; eight passing cassette replays expanded to 76 cells; evidence-derived canary; real CLI positive end-to-end bundle seals with 12 fixed roles, 76 attempt/transcript pairs, and complete source closure 2026-08-28T05:30:13Z phase-7 closed the final outcome and independence gaps second review found empty reviewer observations, unretained usage, repeatable host evidence, extra sidecars, and weaker tar text scanning filtered actor observations; schema-bound attempt identity and usage; unique manager pseudonyms; completion rederivation; strict root closure all budget-relevant usage, product outcomes, provenance, and per-cell independence now reproduce from retained evidence 2026-08-28T05:30:13Z phase-7 completed immutable bundle verification the corrected implementation needed whole-product, replay, live-host, and multi-model review evidence bun run check; bun run replay; pinned OpenCode smoke; three final reviewers 599 pass, 1 intentional skip, 13 of 13 replays, live smoke pass, all reviewers green +2026-08-28T05:41:49Z phase-7 merged sealed regradable qualification bundles the exact 23-file head passed CI, real 76-cell CLI qualification, and isolated shipping verification PR 55; merge 4e7e2bf merged to main +2026-08-28T05:41:49Z phase-8 made the sealed bundle release authority digest records could be internally consistent without proving their verdict regradeQualificationBundle; assertQualificationBundle; release workflow verifier reopens all objects, regrades 76 attempts, rederives canary, provenance, usage, authority, and decision; forged decision and missing source bundles fail +2026-08-28T05:55:21Z phase-8 fixed all actionable release-authority review findings ignored fixed roles, unsafe reader artifacts, unbounded references, stale campaigns, ambiguous bundles, swallowed corruption, and script cycles weakened the gate fixed-role equality; read-time tar scan; total cap; seven-day policy; unique SHA-emitting selection; injected decision authority current release source must match the sealed closure and exactly one clean fresh bundle must reproduce +2026-08-28T05:55:21Z phase-8 completed independent release regrading verification forged decisions, omitted sources, contradictory roles/manifests, stale campaigns, digest-only records, and corrupted siblings needed executable rejection positive 76-cell CLI plus adversarial reseals; full check; replay; live smoke; final reviews 598 pass, 1 intentional skip, 13 of 13 replays, live smoke pass, no production blockers diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d5363f..0a25d88 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,7 +121,7 @@ jobs: set -euo pipefail bun pm pack --destination . - - name: Verify exact VERIFIED V2 artifact decision and fresh canary + - name: Verify independently regraded qualification bundle and fresh canary shell: bash run: | set -euo pipefail diff --git a/docs/release-qualification.md b/docs/release-qualification.md index fa3c623..1f064e5 100644 --- a/docs/release-qualification.md +++ b/docs/release-qualification.md @@ -63,8 +63,8 @@ direction. The cadence follows from that: allowed; removals and renames are not. - **No release** without a sealed V2 qualification bundle and fresh canary. The bundle retains every attempt, transcript, grader source, and exact artifact - needed to reproduce its decision. A `CHANGELOG` entry states the schema impact - explicitly. + needed to reproduce its decision. Release metadata independently regrades those + bytes before publication. A `CHANGELOG` entry states the schema impact explicitly. - **Patch releases** for defects and host-compatibility fixes, which is what the weekly OpenCode compatibility smoke exists to catch early. - **Deprecate before removing.** A surface that is going away is announced in one diff --git a/evals/qualification-bundle.ts b/evals/qualification-bundle.ts index 2c594a4..7c30f13 100644 --- a/evals/qualification-bundle.ts +++ b/evals/qualification-bundle.ts @@ -701,6 +701,9 @@ export async function readQualificationBundle(path: string): Promise<{ bytes: new Uint8Array(), })), ); + const totalBytes = manifest.files.reduce((sum, file) => sum + file.bytes, 0); + if (totalBytes > MAX_BUNDLE_BYTES) + throw new Error("Qualification bundle exceeds its total byte limit."); const objectNames = (await readdir(join(path, "objects"))).sort(); const expectedNames = [ ...new Set( @@ -725,6 +728,9 @@ export async function readQualificationBundle(path: string): Promise<{ return { ref, bytes }; }), ); + const artifact = files.find(({ ref }) => ref.role === "artifact"); + if (!artifact) throw new Error("Qualification bundle artifact is missing."); + await assertSafeArtifact(artifact.bytes); if ( !(await sameDirectoryIdentity(bundleIdentity)) || !(await sameDirectoryIdentity(objectsIdentity)) diff --git a/evals/qualification-regrade.ts b/evals/qualification-regrade.ts new file mode 100644 index 0000000..c463717 --- /dev/null +++ b/evals/qualification-regrade.ts @@ -0,0 +1,343 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type CanaryRecord, + deriveCanaryResult, + parseCanaryRecord, +} from "../scripts/eval-canary.js"; +import type { ReleaseDecision, ReleaseExpectedProvenance } from "./analysis.js"; +import { canonicalJson, canonicalSha256 } from "./canonical-json.js"; +import type { ValidatedCaseCatalog } from "./catalog.js"; +import { + deriveConformanceOutcome, + retainedInstructions, + retainedReportActors, +} from "./conformance-evidence.js"; +import { RetainedScenarioEvidenceSchema } from "./grader-input.js"; +import { + inspectArtifact, + instructionDelivery, + samePackedArtifact, +} from "./provenance.js"; +import { readQualificationBundle } from "./qualification-bundle.js"; +import { + RELEASE_ANALYSIS_SHA256, + RELEASE_MAX_CAMPAIGN_AGE_MS, + RELEASE_POLICY_SHA256, + releaseGraderBundle, +} from "./release-policy.js"; +import type { ValidatedReport } from "./report.js"; +import { SCENARIOS } from "./scenarios.js"; + +type BundleFile = Awaited< + ReturnType +>["files"][number]; +type RegradedDecision = Readonly<{ + verdict: "VERIFIED" | "NOT VERIFIED" | "INCONCLUSIVE"; + artifact: Exclude< + ValidatedReport["attempts"][number]["artifact"], + { kind: string } + >; + canarySha256: string | null; + analyzerSha256: string; + [key: string]: unknown; +}>; +export type QualificationRegradeAuthority = Readonly<{ + qualify(input: { + reportInput: unknown; + catalogInput: unknown; + artifact: RegradedDecision["artifact"]; + canary?: CanaryRecord | null; + }): { + report: ValidatedReport; + catalog: ValidatedCaseCatalog; + expected: ReleaseExpectedProvenance; + decision: ReleaseDecision; + canary: CanaryRecord | null; + }; + decisionRecord(input: { + report: ValidatedReport; + catalog: ValidatedCaseCatalog; + expected: ReleaseExpectedProvenance; + decision: ReleaseDecision; + canarySha256?: string | null; + }): RegradedDecision; +}>; + +function json(file: BundleFile): unknown { + return JSON.parse(file.bytes.toString("utf8")); +} + +function one( + files: readonly BundleFile[], + role: BundleFile["ref"]["role"], + id?: string, +): BundleFile { + const matches = files.filter( + (file) => file.ref.role === role && file.ref.id === id, + ); + if (matches.length !== 1) + throw new Error(`Qualification bundle role ${role} is not unique.`); + const match = matches[0]; + if (!match) throw new Error(`Qualification bundle role ${role} is missing.`); + return match; +} + +function digest(bytes: Uint8Array): string { + return `sha256:${new Bun.CryptoHasher("sha256").update(bytes).digest("hex")}`; +} + +function verifyCanary( + files: readonly BundleFile[], + artifact: ValidatedReport["attempts"][number]["artifact"], +): string { + if ("kind" in artifact) + throw new Error("Qualification artifact is unavailable."); + const parsed = parseCanaryRecord(json(one(files, "canary-record"))); + if (!parsed.ok) throw new Error(parsed.issues.join("; ")); + const record = parsed.value; + const evidence = [ + [record.artifacts.installation, one(files, "canary-installation")], + [record.artifacts.session, one(files, "canary-session")], + [record.artifacts.transcript, one(files, "canary-transcript")], + ] as const; + for (const [ref, file] of evidence) { + if ( + !ref || + ref.bytes !== file.bytes.byteLength || + ref.sha256 !== digest(file.bytes) + ) + throw new Error("Bundled canary evidence differs from its record."); + } + if ( + !samePackedArtifact(record.artifact, artifact) || + record.status !== "passed" + ) + throw new Error("Bundled canary does not match the release artifact."); + const derived = deriveCanaryResult({ + packageVersion: record.artifact.packageVersion, + artifactSha256: record.artifactSha256, + tarballSha256: record.artifact.tarballSha256, + preparedSha256: record.preparedSha256, + pluginEntrySha256: record.pluginEntrySha256, + installation: json(evidence[0][1]), + session: json(evidence[1][1]), + transcript: json(evidence[2][1]), + }); + if ( + derived.status !== record.status || + canonicalJson(derived.checks) !== canonicalJson(record.checks) || + canonicalJson(derived.actors) !== canonicalJson(record.actors) || + derived.hostConfigSha256 !== record.hostConfigSha256 + ) + throw new Error("Bundled canary claims do not reproduce from evidence."); + return record.recordSha256; +} + +function regradeAttempts( + report: ValidatedReport, + files: readonly BundleFile[], +): void { + const sessions = new Set(); + for (const attempt of report.attempts) { + const retained = json(one(files, "attempt", attempt.attemptId)); + if (canonicalJson(retained) !== canonicalJson(attempt)) + throw new Error( + `Bundled attempt ${attempt.attemptId} differs from report.`, + ); + if (!attempt.transcript) + throw new Error( + `Bundled attempt ${attempt.attemptId} has no transcript.`, + ); + const transcript = one(files, "transcript", attempt.attemptId); + if (digest(transcript.bytes) !== attempt.transcript.sha256) + throw new Error( + `Bundled transcript ${attempt.attemptId} has the wrong digest.`, + ); + const evidence = RetainedScenarioEvidenceSchema.parse(json(transcript)); + const scenario = SCENARIOS.find(({ id }) => id === attempt.caseId); + const manager = attempt.actors.find(({ role }) => role === "manager"); + if (!scenario || !manager || attempt.outcome.kind !== "product") + throw new Error( + `Bundled attempt ${attempt.attemptId} is not regradable.`, + ); + if ( + canonicalJson(evidence.attempt) !== + canonicalJson({ + attemptId: attempt.attemptId, + cellId: attempt.cellId, + caseId: attempt.caseId, + repetition: attempt.repetition, + model: manager.requestedModel, + }) || + canonicalJson(evidence.usage) !== canonicalJson(attempt.usage) + ) + throw new Error(`Bundled attempt ${attempt.attemptId} binding differs.`); + const actors = retainedReportActors(evidence); + const retainedManager = actors.find(({ role }) => role === "manager"); + if (!retainedManager) + throw new Error("Bundled attempt has no manager evidence."); + for (const id of retainedManager.sessionIds) { + if (sessions.has(id)) + throw new Error("Bundled attempts reuse manager evidence."); + sessions.add(id); + } + const outcome = deriveConformanceOutcome({ + evidence, + check: scenario.check, + scenarioId: attempt.caseId, + model: `${evidence.attempt.model.routeProvider}/${evidence.attempt.model.model}`, + attempt: attempt.repetition + 1, + }); + const commands = scenario.steps.map((step, sequence) => + instructionDelivery({ + source: "command", + name: step.command, + sequence, + text: `/${step.command} ${step.arguments}`.trim(), + }), + ); + const guidance = retainedInstructions(evidence).map( + (instruction, sequence) => ({ + ...instruction, + sequence: commands.length + sequence, + }), + ); + if ( + canonicalJson(outcome) !== canonicalJson(attempt.outcome) || + canonicalJson(actors) !== canonicalJson(attempt.actors) || + canonicalJson([...commands, ...guidance]) !== + canonicalJson(attempt.instructions) + ) + throw new Error(`Bundled attempt ${attempt.attemptId} grade differs.`); + } + const cost = report.attempts.some((attempt) => attempt.usage.costUsd === null) + ? null + : report.attempts.reduce( + (sum, attempt) => sum + (attempt.usage.costUsd ?? 0), + 0, + ); + const observed = { + attempts: report.attempts.length, + outputTokens: report.attempts.reduce( + (sum, attempt) => sum + attempt.usage.outputTokens, + 0, + ), + costUsd: cost, + wallClockMs: Math.max( + Date.parse(report.completion.finishedAt) - + Date.parse(report.completion.startedAt), + ...report.attempts.map((attempt) => attempt.usage.durationMs), + ), + }; + if (canonicalJson(observed) !== canonicalJson(report.completion.observed)) + throw new Error("Bundled completion usage does not reproduce."); +} + +export async function regradeQualificationBundle(input: { + readonly path: string; + readonly repositoryRoot: string; + readonly authority: QualificationRegradeAuthority; + readonly now?: Date; +}): Promise<{ + readonly decision: RegradedDecision; + readonly bundleSha256: string; +}> { + const bundle = await readQualificationBundle(input.path); + const files = bundle.files; + const reportInput = json(one(files, "report")); + const catalogInput = json(one(files, "catalog")); + const planStored = json(one(files, "plan")); + const completionStored = json(one(files, "completion")); + const policy = json(one(files, "policy")) as Record; + const expectedStored = json(one(files, "expected-provenance")); + const decisionStored = json(one(files, "decision")); + const bundledGrader = { + files: files + .filter(({ ref }) => ref.role === "authority-source") + .map(({ ref, bytes }) => ({ + path: ref.id, + sha256: canonicalSha256( + "flow-release-grader-file-v1", + bytes.toString("utf8"), + ), + })) + .sort((left, right) => + String(left.path).localeCompare(String(right.path)), + ), + }; + if ( + policy.policySha256 !== RELEASE_POLICY_SHA256 || + policy.analysisSha256 !== RELEASE_ANALYSIS_SHA256 || + canonicalJson(policy.graderBundle) !== canonicalJson(bundledGrader) || + canonicalJson(policy.graderBundle) !== + canonicalJson(releaseGraderBundle(input.repositoryRoot)) + ) + throw new Error("Bundled release authority does not match the verifier."); + const temporary = await mkdtemp(join(tmpdir(), "flow-bundle-regrade-")); + const artifactPath = join(temporary, "artifact.tgz"); + try { + await writeFile(artifactPath, one(files, "artifact").bytes); + const artifact = await inspectArtifact({ + repositoryRoot: input.repositoryRoot, + tarballPath: artifactPath, + }); + const prelim = input.authority.qualify({ + reportInput, + catalogInput, + artifact, + }); + const firstAttempt = prelim.report.attempts.at(0); + if (!firstAttempt) throw new Error("Bundled report has no attempts."); + const canarySha256 = verifyCanary(files, firstAttempt.artifact); + const parsedCanary = parseCanaryRecord(json(one(files, "canary-record"))); + if (!parsedCanary.ok) throw new Error(parsedCanary.issues.join("; ")); + const result = input.authority.qualify({ + reportInput, + catalogInput, + artifact, + canary: parsedCanary.value, + }); + const now = (input.now ?? new Date()).getTime(); + const campaignFinished = Date.parse(result.report.completion.finishedAt); + const canaryRecorded = Date.parse(parsedCanary.value.recordedAt); + if ( + campaignFinished > canaryRecorded || + canaryRecorded > now || + campaignFinished > now || + now - campaignFinished > RELEASE_MAX_CAMPAIGN_AGE_MS + ) + throw new Error("Bundled campaign and canary freshness is invalid."); + if ( + canonicalJson(planStored) !== canonicalJson(result.report.plan) || + canonicalJson(completionStored) !== + canonicalJson(result.report.completion) + ) + throw new Error("Bundled plan or completion differs from the report."); + regradeAttempts(result.report, files); + if (canonicalJson(result.expected) !== canonicalJson(expectedStored)) + throw new Error("Bundled expected provenance does not reproduce."); + const decision = input.authority.decisionRecord({ + ...result, + canarySha256, + }); + if ( + decision.verdict !== "VERIFIED" || + canonicalJson(decision) !== canonicalJson(decisionStored) + ) + throw new Error("Bundled decision does not reproduce."); + if ( + bundle.manifest.reportId !== result.report.reportId || + bundle.manifest.packageVersion !== artifact.packageVersion || + bundle.manifest.verdict !== decision.verdict || + policy.analyzerSha256 !== decision.analyzerSha256 + ) + throw new Error( + "Bundled manifest or analyzer identity does not reproduce.", + ); + return { decision, bundleSha256: bundle.manifest.bundleSha256 }; + } finally { + await rm(temporary, { recursive: true, force: true }); + } +} diff --git a/evals/qualification/README.md b/evals/qualification/README.md index ac2299a..9933cf4 100644 --- a/evals/qualification/README.md +++ b/evals/qualification/README.md @@ -9,5 +9,6 @@ its evidence, derived decision, and the complete grader source closure. The seal written last. An interrupted directory is not qualification evidence; an identical retry completes or replays it, while conflicting bytes are refused. -Bundle creation is not release authorization. The bundle retains the inputs that -release verification will independently regrade and rederive. +Bundle creation is not release authorization. Release verification reopens every +object, regrades each attempt, and rederives the canary, provenance, usage, and +decision before accepting the bundle. diff --git a/evals/release-policy.ts b/evals/release-policy.ts index 22caa9b..104e931 100644 --- a/evals/release-policy.ts +++ b/evals/release-policy.ts @@ -103,6 +103,7 @@ export const RELEASE_ANALYSIS_SHA256 = canonicalSha256("flow-v2-analysis-v1", { kind: "rate", primaryOutcome: "conformance-pass", }); +export const RELEASE_MAX_CAMPAIGN_AGE_MS = 7 * 24 * 60 * 60 * 1_000; export const RELEASE_HOST_POLICY = { opencodeVersion: "1.18.6", @@ -283,7 +284,11 @@ export function releaseCaseCatalogSha256( export function releaseGraderSourceBundle(repositoryRoot: string) { const root = resolve(repositoryRoot); - const pending = ["evals/run.ts", "scripts/qualify-release.ts"]; + const pending = [ + "evals/run.ts", + "scripts/qualify-release.ts", + "evals/qualification-regrade.ts", + ]; const files = new Map(); const transpiler = new Bun.Transpiler({ loader: "ts" }); while (pending.length > 0) { diff --git a/scripts/qualify-release.ts b/scripts/qualify-release.ts index fdb9288..a67aa32 100644 --- a/scripts/qualify-release.ts +++ b/scripts/qualify-release.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, dirname, join } from "node:path"; import { @@ -21,6 +21,7 @@ import { evaluatorIdentity, inspectArtifact, instructionDelivery, + samePackedArtifact, } from "../evals/provenance.js"; import { listStableQualificationDirectory, @@ -58,7 +59,6 @@ import { parseCanaryRecord, canaryRecordIssue as verifyCanaryRecord, } from "./eval-canary.js"; -import { canaryRecordIssue } from "./release-metadata.js"; export type DecisionRecord = { readonly schemaVersion: 1; @@ -275,12 +275,12 @@ export function qualifyV2(input: { } } if (input.canary) { - const issue = canaryRecordIssue( - input.artifact.packageVersion, - input.canary, - input.artifact, - ); - if (issue) throw new Error(issue); + if ( + input.canary.status !== "passed" || + input.canary.artifact.packageVersion !== input.artifact.packageVersion || + !samePackedArtifact(input.canary.artifact, input.artifact) + ) + throw new Error("Canary does not match the exact qualifying artifact."); } return { report: parsed.value, @@ -362,27 +362,6 @@ export function decisionRecordFor(input: { }; } -export async function writeDecisionRecord(input: { - readonly record: DecisionRecord; - readonly directory: string; -}): Promise { - await mkdir(input.directory, { recursive: true }); - const suffix = input.record.canarySha256 - ? `-canary-${input.record.canarySha256.slice("sha256:".length, "sha256:".length + 12)}` - : ""; - const path = join(input.directory, `${input.record.reportId}${suffix}.json`); - const bytes = canonicalJson(input.record); - try { - await writeFile(path, bytes, { encoding: "utf8", flag: "wx" }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - if ((await readFile(path, "utf8")) !== bytes) { - throw new Error(`Immutable decision record conflicts: ${path}`); - } - } - return path; -} - const USAGE = "Usage: bun run qualify -- --campaign-dir --canary [--bundles-dir ]"; diff --git a/scripts/release-metadata.ts b/scripts/release-metadata.ts index 4f3fc0d..9445eca 100644 --- a/scripts/release-metadata.ts +++ b/scripts/release-metadata.ts @@ -1,4 +1,4 @@ -import { readFile, writeFile } from "node:fs/promises"; +import { readdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { canonicalSha256 } from "../evals/canonical-json.js"; import { @@ -6,6 +6,8 @@ import { inspectArtifact, samePackedArtifact, } from "../evals/provenance.js"; +import { QualificationBundleManifestSchema } from "../evals/qualification-bundle.js"; +import { regradeQualificationBundle } from "../evals/qualification-regrade.js"; import { RELEASE_POLICY_SHA256, releaseCatalog, @@ -27,6 +29,7 @@ import { parseCanaryRecord, canaryRecordIssue as verifyCanaryRecord, } from "./eval-canary.js"; +import { decisionRecordFor, qualifyV2 } from "./qualify-release.js"; export function canaryRecordIssue( version: string, @@ -274,44 +277,82 @@ export function qualificationRecordIssue( return null; } -export async function assertQualificationRecord( - version: string, - directory = join("evals", "decisions"), - expectedArtifact?: ArtifactIdentity, -): Promise { - let records: unknown[] = []; +export async function assertQualificationBundle(input: { + readonly version: string; + readonly directory?: string; + readonly expectedArtifact?: ArtifactIdentity; + readonly expectedCanarySha256?: string; + readonly now?: Date; +}): Promise<{ readonly bundleSha256: string }> { + const directory = + input.directory ?? join("evals", "qualification", "bundles"); + let names: string[] = []; try { - const { readdir } = await import("node:fs/promises"); - records = await Promise.all( - (await readdir(directory)) - .filter((name) => name.endsWith(".json")) - .map(async (name) => - JSON.parse(await readFile(join(directory, name), "utf8")), - ), - ); + names = (await readdir(directory, { withFileTypes: true })) + .filter((entry) => entry.isDirectory() && !entry.isSymbolicLink()) + .map((entry) => entry.name); } catch { - records = []; + names = []; } - if ( - !records.some( - (record) => - qualificationRecordIssue(version, record, expectedArtifact) === null, - ) - ) { + const matches: Array<{ readonly bundleSha256: string }> = []; + for (const name of names) { + let manifest: ReturnType; + try { + manifest = QualificationBundleManifestSchema.parse( + JSON.parse( + await readFile(join(directory, name, "bundle.json"), "utf8"), + ), + ); + } catch (error) { + throw new Error(`Qualification bundle ${name} has an invalid seal.`, { + cause: error, + }); + } + if (manifest.packageVersion !== input.version) continue; + try { + const result = await regradeQualificationBundle({ + path: join(directory, name), + repositoryRoot: join(import.meta.dir, ".."), + authority: { qualify: qualifyV2, decisionRecord: decisionRecordFor }, + ...(input.now ? { now: input.now } : {}), + }); + if ( + result.decision.artifact.packageVersion === input.version && + (!input.expectedArtifact || + samePackedArtifact( + result.decision.artifact, + input.expectedArtifact, + )) && + (input.expectedCanarySha256 === undefined || + result.decision.canarySha256 === input.expectedCanarySha256) + ) + matches.push({ bundleSha256: result.bundleSha256 }); + } catch (error) { + throw new Error( + `Qualification bundle ${name} for ${input.version} did not regrade cleanly.`, + { cause: error }, + ); + } + } + const match = matches[0]; + if (matches.length === 1 && match) return match; + if (matches.length > 1) throw new Error( - `Release ${version} cannot proceed: no exact VERIFIED v2 decision record exists. Run \`bun run qualify -- --campaign-dir --canary \` and commit the sealed bundle.`, + `Release ${input.version} cannot proceed: multiple sealed qualification bundles match the release.`, ); - } + throw new Error( + `Release ${input.version} cannot proceed: no sealed qualification bundle independently regrades to the exact VERIFIED decision.`, + ); } export async function assertStrictReleaseEvidence(input: { readonly version: string; - readonly decisionsDirectory?: string; + readonly bundlesDirectory?: string; readonly canaryPath: string; readonly expectedArtifact: ArtifactIdentity; readonly tag?: string; readonly now?: Date; -}): Promise { +}): Promise<{ readonly bundleSha256: string }> { const tag = input.tag ?? `v${input.version}`; const canary = JSON.parse( await readFile(input.canaryPath, "utf8"), @@ -333,66 +374,14 @@ export async function assertStrictReleaseEvidence(input: { }); if (evidenceIssue) throw new Error(evidenceIssue); const canaryHash = (canary as CanaryRecord).recordSha256; - let records: unknown[] = []; - try { - const { readdir } = await import("node:fs/promises"); - records = await Promise.all( - (await readdir(input.decisionsDirectory ?? join("evals", "decisions"))) - .filter((name) => name.endsWith(".json")) - .map(async (name) => - JSON.parse( - await readFile( - join( - input.decisionsDirectory ?? join("evals", "decisions"), - name, - ), - "utf8", - ), - ), - ), - ); - } catch { - records = []; - } - const match = records.find((record) => { - const entry = record as { - readonly reportSha256?: unknown; - readonly artifactSha256?: unknown; - readonly evaluatorSha256?: unknown; - readonly catalogSha256?: unknown; - readonly policySha256?: unknown; - readonly actorSha256?: unknown; - readonly analyzerSha256?: unknown; - readonly expectedProvenanceSha256?: unknown; - readonly canarySha256?: unknown; - readonly decisionInputSha256?: unknown; - }; - const decisionInputSha256 = canonicalSha256("flow-decision-input-v1", { - reportSha256: entry.reportSha256, - artifactSha256: entry.artifactSha256, - evaluatorSha256: entry.evaluatorSha256, - catalogSha256: entry.catalogSha256, - policySha256: entry.policySha256, - actorSha256: entry.actorSha256, - analyzerSha256: entry.analyzerSha256, - expectedProvenanceSha256: entry.expectedProvenanceSha256, - canarySha256: canaryHash, - }); - return ( - entry.canarySha256 === canaryHash && - entry.decisionInputSha256 === decisionInputSha256 && - qualificationRecordIssue( - input.version, - record, - input.expectedArtifact, - canaryHash, - ) === null - ); + return assertQualificationBundle({ + version: input.version, + directory: + input.bundlesDirectory ?? join("evals", "qualification", "bundles"), + expectedArtifact: input.expectedArtifact, + expectedCanarySha256: canaryHash, + ...(input.now ? { now: input.now } : {}), }); - if (!match) - throw new Error( - `Release ${input.version} cannot proceed: no exact VERIFIED canary-bound v2 decision record exists.`, - ); } function optionValue( @@ -459,12 +448,15 @@ async function main(args: readonly string[]): Promise { throw new Error("Strict tag release metadata requires --canary."); if (!expectedArtifact) throw new Error("Strict tag release metadata requires an artifact."); - await assertStrictReleaseEvidence({ + const qualification = await assertStrictReleaseEvidence({ version: packageMetadata.version, tag, canaryPath, expectedArtifact, }); + process.stdout.write( + `Qualification bundle verified: ${qualification.bundleSha256}\n`, + ); } else if (artifactPath) { process.stdout.write( `INCONCLUSIVE: rebuilt artifact ${packageMetadata.version} has no strict tag evidence.\n`, diff --git a/tests/documentation-contract.test.ts b/tests/documentation-contract.test.ts index 1c162b0..b60a986 100644 --- a/tests/documentation-contract.test.ts +++ b/tests/documentation-contract.test.ts @@ -509,7 +509,7 @@ describe("Flow documentation contract", () => { expect(release).toMatch(/tag="v\$\{version\}"/); expect(release).toMatch(/--target "\$\{GITHUB_SHA\}"/); expect(release).toContain( - "Verify exact VERIFIED V2 artifact decision and fresh canary", + "Verify independently regraded qualification bundle and fresh canary", ); expect(release).toContain("bun run eval:canary -- verify"); expect(release).toContain("--mode dry-run"); diff --git a/tests/qualification-cli.test.ts b/tests/qualification-cli.test.ts index 811070c..61bb58f 100644 --- a/tests/qualification-cli.test.ts +++ b/tests/qualification-cli.test.ts @@ -22,7 +22,11 @@ import { inspectArtifact, instructionDelivery, } from "../evals/provenance.js"; -import { readQualificationBundle } from "../evals/qualification-bundle.js"; +import { + readQualificationBundle, + writeQualificationBundle, +} from "../evals/qualification-bundle.js"; +import { regradeQualificationBundle } from "../evals/qualification-regrade.js"; import { releaseCatalog, releaseGraderBundle, @@ -35,6 +39,8 @@ import { campaignPlanFor, releaseScenarios } from "../evals/run.js"; import { SCENARIOS } from "../evals/scenarios.js"; import packageJson from "../package.json" with { type: "json" }; import { prepareCanary, recordCanary } from "../scripts/eval-canary.js"; +import { decisionRecordFor, qualifyV2 } from "../scripts/qualify-release.js"; +import { assertQualificationBundle } from "../scripts/release-metadata.js"; import { assuranceProjection } from "../src/application/delivery.js"; import { SessionSchema } from "../src/application/schema.js"; import { operationInputDigest } from "../src/domain/operation.js"; @@ -196,6 +202,10 @@ function canaryTranscript(input: { test("qualifies and seals a complete exact-artifact campaign through the CLI", async () => { const repositoryRoot = join(import.meta.dir, ".."); + const regradeAuthority = { + qualify: qualifyV2, + decisionRecord: decisionRecordFor, + }; const temporary = await mkdtemp(join(tmpdir(), "flow-qualification-cli-")); try { const artifactPath = await packPlugin( @@ -326,11 +336,14 @@ test("qualifies and seals a complete exact-artifact campaign through the CLI", a }); } + const recordedAt = new Date(); + const verificationNow = new Date(recordedAt.getTime() + 60 * 60 * 1_000); + const staleNow = new Date(recordedAt.getTime() + 8 * 24 * 60 * 60 * 1_000); const completion = { status: "complete" as const, cause: "fixed-target" as const, - startedAt: "2026-08-28T00:00:00.000Z", - finishedAt: "2026-08-28T00:00:01.000Z", + startedAt: new Date(recordedAt.getTime() - 2_000).toISOString(), + finishedAt: new Date(recordedAt.getTime() - 1_000).toISOString(), activatedReserveCellIds: [], observed: { attempts: plan.cells.length, @@ -348,7 +361,6 @@ test("qualifies and seals a complete exact-artifact campaign through the CLI", a const preparedDirectory = join(temporary, "prepared-canary"); await mkdir(preparedDirectory, { recursive: true }); - const recordedAt = new Date(); const prepared = await prepareCanary({ repositoryRoot, artifactPath, @@ -425,6 +437,143 @@ test("qualifies and seals a complete exact-artifact campaign through the CLI", a .files.map(({ path }) => path) .sort(), ); + const regraded = await regradeQualificationBundle({ + path: bundlePath, + repositoryRoot, + authority: regradeAuthority, + now: verificationNow, + }); + expect(regraded.decision.verdict).toBe("VERIFIED"); + await expect( + regradeQualificationBundle({ + path: bundlePath, + repositoryRoot, + authority: regradeAuthority, + now: staleNow, + }), + ).rejects.toThrow(/freshness/); + const releaseAuthority = await assertQualificationBundle({ + version: artifact.packageVersion, + directory: bundlesDirectory, + expectedArtifact: artifact, + expectedCanarySha256: canary.record.recordSha256, + now: verificationNow, + }); + expect(releaseAuthority.bundleSha256).toBe(bundle.manifest.bundleSha256); + const sealedFiles = bundle.files.map(({ ref, bytes }) => ({ + role: ref.role, + ...(ref.id ? { id: ref.id } : {}), + mediaType: ref.mediaType, + bytes, + })); + const forgedFiles = sealedFiles.map((file) => ({ + ...file, + bytes: + file.role === "decision" + ? Buffer.from( + canonicalJson({ + ...(JSON.parse(file.bytes.toString("utf8")) as object), + reasons: ["forged digest-only decision"], + }), + ) + : file.bytes, + })); + const forged = await writeQualificationBundle({ + input: { + reportId: bundle.manifest.reportId, + packageVersion: bundle.manifest.packageVersion, + verdict: bundle.manifest.verdict, + files: forgedFiles, + }, + outputRoot: bundlesDirectory, + }); + await expect( + regradeQualificationBundle({ + path: forged.path, + repositoryRoot, + authority: regradeAuthority, + now: verificationNow, + }), + ).rejects.toThrow(/decision does not reproduce/); + await expect( + assertQualificationBundle({ + version: artifact.packageVersion, + directory: bundlesDirectory, + expectedArtifact: artifact, + expectedCanarySha256: canary.record.recordSha256, + now: verificationNow, + }), + ).rejects.toThrow(/did not regrade cleanly/); + const missingSource = await writeQualificationBundle({ + input: { + reportId: bundle.manifest.reportId, + packageVersion: bundle.manifest.packageVersion, + verdict: bundle.manifest.verdict, + files: sealedFiles.filter( + (file, index) => + file.role !== "authority-source" || + index !== + sealedFiles.findIndex( + (candidate) => candidate.role === "authority-source", + ), + ), + }, + outputRoot: join(temporary, "missing-source-bundles"), + }); + await expect( + regradeQualificationBundle({ + path: missingSource.path, + repositoryRoot, + authority: regradeAuthority, + now: verificationNow, + }), + ).rejects.toThrow(/authority does not match/); + const contradictoryPlan = await writeQualificationBundle({ + input: { + reportId: bundle.manifest.reportId, + packageVersion: bundle.manifest.packageVersion, + verdict: bundle.manifest.verdict, + files: sealedFiles.map((file) => + file.role === "plan" + ? { + ...file, + bytes: Buffer.from( + canonicalJson({ + ...(JSON.parse(file.bytes.toString("utf8")) as object), + planId: "contradictory-plan", + }), + ), + } + : file, + ), + }, + outputRoot: join(temporary, "contradictory-plan-bundles"), + }); + await expect( + regradeQualificationBundle({ + path: contradictoryPlan.path, + repositoryRoot, + authority: regradeAuthority, + now: verificationNow, + }), + ).rejects.toThrow(/plan or completion differs/); + const contradictoryManifest = await writeQualificationBundle({ + input: { + reportId: "contradictory-report-id", + packageVersion: bundle.manifest.packageVersion, + verdict: bundle.manifest.verdict, + files: sealedFiles, + }, + outputRoot: join(temporary, "contradictory-manifest-bundles"), + }); + await expect( + regradeQualificationBundle({ + path: contradictoryManifest.path, + repositoryRoot, + authority: regradeAuthority, + now: verificationNow, + }), + ).rejects.toThrow(/manifest or analyzer identity/); } finally { await rm(temporary, { recursive: true, force: true }); } diff --git a/tests/release-metadata.test.ts b/tests/release-metadata.test.ts index f0f8c8c..677f4f1 100644 --- a/tests/release-metadata.test.ts +++ b/tests/release-metadata.test.ts @@ -23,7 +23,7 @@ import { deriveCanaryResult, } from "../scripts/eval-canary.js"; import { - assertQualificationRecord, + assertQualificationBundle, assertStrictReleaseEvidence, canaryRecordIssue, isMajorRelease, @@ -404,20 +404,20 @@ describe("release metadata", () => { const directory = await recordDirectory(); for (const version of ["7.0.0", "7.1.0", "7.0.1"]) { await expect( - assertQualificationRecord(version, directory), - ).rejects.toThrow(/no exact VERIFIED v2 decision record exists/); + assertQualificationBundle({ version, directory }), + ).rejects.toThrow(/no sealed qualification bundle/); } }); - test("accepts only an exact VERIFIED v2 record and refuses mismatches", async () => { + test("refuses a digest-only decision as release authority", async () => { const directory = await recordDirectory(); await writeFile( join(directory, "report.json"), JSON.stringify(decisionRecord("7.0.0")), ); await expect( - assertQualificationRecord("7.0.0", directory), - ).resolves.toBeUndefined(); + assertQualificationBundle({ version: "7.0.0", directory }), + ).rejects.toThrow(/no sealed qualification bundle/); expect(qualificationRecordIssue("8.0.0", null)).toMatch( /no qualification record exists for 8\.0\.0/, @@ -475,7 +475,7 @@ describe("release metadata", () => { ).toMatch(/missing v2 decision digests/); }); - test("requires a fresh passed exact-artifact canary for non-major strict evidence", async () => { + test("requires a regradable bundle beyond a fresh exact-artifact canary", async () => { const decisions = await recordDirectory(); const canaries = await recordDirectory(); const version = "8.1.1"; @@ -499,12 +499,12 @@ describe("release metadata", () => { await expect( assertStrictReleaseEvidence({ version, - decisionsDirectory: decisions, + bundlesDirectory: decisions, canaryPath: join(canaries, `${version}.json`), expectedArtifact: expected, now: CANARY_NOW, }), - ).resolves.toBeUndefined(); + ).rejects.toThrow(/no sealed qualification bundle/); }); test("rejects stale, failed, incomplete, and artifact-mismatched canaries", () => { @@ -558,11 +558,11 @@ describe("release metadata", () => { await expect( assertStrictReleaseEvidence({ version, - decisionsDirectory: decisions, + bundlesDirectory: decisions, canaryPath: join(canaries, `${version}.json`), expectedArtifact: expected, now: CANARY_NOW, }), - ).rejects.toThrow(/canary-bound/); + ).rejects.toThrow(/no sealed qualification bundle/); }); }); diff --git a/tests/release-qualification.test.ts b/tests/release-qualification.test.ts index 66e41dd..6ae6d53 100644 --- a/tests/release-qualification.test.ts +++ b/tests/release-qualification.test.ts @@ -22,7 +22,6 @@ import { assertCampaignEvidenceLayout, decisionRecordFor, qualifyV2, - writeDecisionRecord, } from "../scripts/qualify-release.js"; const digest = (letter: string) => `sha256:${letter.repeat(64)}`; @@ -305,6 +304,10 @@ describe("repository-owned v2 qualification", () => { join(root, "scripts", "qualify-release.ts"), "export {};\n", ); + await writeFile( + join(root, "evals", "qualification-regrade.ts"), + "export {};\n", + ); await writeFile( join(root, "evals", "grade.ts"), "export const grade = () => 1;\n", @@ -335,6 +338,10 @@ describe("repository-owned v2 qualification", () => { join(root, "scripts", "qualify-release.ts"), "export {};\n", ); + await writeFile( + join(root, "evals", "qualification-regrade.ts"), + "export {};\n", + ); expect(() => releaseGraderBundle(root)).toThrow(/non-literal import/); } finally { await rm(root, { recursive: true, force: true }); @@ -416,23 +423,6 @@ describe("repository-owned v2 qualification", () => { ).toThrow("Invalid v2 report"); }); - test("writes decision records immutably", async () => { - const directory = await mkdtemp(join(tmpdir(), "flow-decision-")); - try { - const result = qualifyV2({ - reportInput: releaseReport(), - catalogInput: releaseCatalog(), - artifact: ARTIFACT, - }); - const record = decisionRecordFor(result); - const first = await writeDecisionRecord({ record, directory }); - const replay = await writeDecisionRecord({ record, directory }); - expect(replay).toBe(first); - } finally { - await rm(directory, { recursive: true, force: true }); - } - }); - test("requires explicit report, catalog, and artifact paths in the CLI", async () => { const process = Bun.spawn(["bun", "run", "scripts/qualify-release.ts"], { cwd: new URL("..", import.meta.url).pathname,