From 0efc250dab52357d313890d90a61df424a9f02b1 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Fri, 28 Aug 2026 05:32:59 +0200 Subject: [PATCH] fix(release): own qualification policy --- .../03-assurance-hardening/decisions.tsv | 7 + docs/release-qualification.md | 29 +- evals/README.md | 5 +- evals/release-policy.ts | 340 ++++++- evals/run.ts | 229 +++-- scripts/qualify-release.ts | 693 +++---------- scripts/release-metadata.ts | 62 +- tests/eval-release-sampling.test.ts | 168 ++-- tests/release-metadata.test.ts | 89 +- tests/release-qualification.test.ts | 938 ++++++------------ 10 files changed, 1153 insertions(+), 1407 deletions(-) diff --git a/.agents/plans/03-assurance-hardening/decisions.tsv b/.agents/plans/03-assurance-hardening/decisions.tsv index 80df1b5..2fe5a70 100644 --- a/.agents/plans/03-assurance-hardening/decisions.tsv +++ b/.agents/plans/03-assurance-hardening/decisions.tsv @@ -31,3 +31,10 @@ ts phase decision why evidence result 2026-08-28T02:50:17Z phase-4 kept persistence outside durable attempt evidence a store that cannot publish the attempt cannot truthfully publish a second attempt claiming the store failed evals/report.ts; persistEvaluation; preservePrimaryFailure persistence is a campaign stop only; primary errors retain cleanup diagnostics; fabricated persistence attempts are schema-invalid 2026-08-28T02:50:17Z phase-4 completed failure-origin verification the corrected design passed deslop, comment review, three-model interrogation, full product verification, and replay bun run check; bun run replay 588 pass, 1 intentional skip, 13 of 13 replays, 892 source bytes headroom 2026-08-28T02:50:59Z phase-4 closed final concurrent and provenance review findings last-pass reviewers found a lost in-flight persistence error, silent host reads, one extra paired arm, and provider fidelity still labelled host evals/harness.ts; evals/benchmark-run.ts; evals/cassette.ts; focused concurrency and cassette tests 589 pass, 1 intentional skip, 13 of 13 replays, no remaining Sol or 5.4 findings +2026-08-28T03:10:52Z phase-4 merged typed eval failure origins the exact head passed independent shipping verification and all CI jobs PR 51; merge 537b235 merged to main +2026-08-28T03:10:52Z phase-5 selected one ordered repository release authority independent judge chose exact catalog equality, exact 76-cell grid validation, reconstructed evaluator identity, flexible ordinary evals, and removal of legacy summary qualification three architecture candidates; independent phase2_ship_verify judge candidate A base with candidate C plan and evaluator checks +2026-08-28T03:10:52Z phase-5 captured catalog-authority and matrix regressions caller policy mutations and the old 70-cell matrix must fail before implementation tests/eval-release-sampling.test.ts missing release authority exports failed red as expected +2026-08-28T03:10:52Z phase-5 made release policy repository-owned the qualifier previously used caller catalog rows as required cases, thresholds, provider floors, and sample floors evals/release-policy.ts; scripts/qualify-release.ts; tests/release-qualification.test.ts canonical ordered catalog, exact 76-cell grid, independent evaluator hashes, old matrix and all policy mutations refused +2026-08-28T03:32:07Z phase-5 bound the complete release environment and grader closure first-pass review found duplicated cell construction, caller-defined host digests, and incomplete grader hashing releaseCellsFor; releaseHostConfigSha256; releaseGraderBundle import-graph scanner exact cell and block ids, seed, budgets, Linux OpenCode 1.18.6 host, and every transitive local evaluator file are authoritative +2026-08-28T03:32:07Z phase-5 removed legacy summary qualification authority the import-only summary path preserved a second executable policy surface scripts/qualify-release.ts; tests/release-qualification.test.ts V2 atomic report, exact persisted catalog, packed artifact, and canonical authority are the only qualifier inputs +2026-08-28T03:32:07Z phase-5 completed repository-owned release policy verification final review findings fixed stale authority digests, arbitrary decision-input hashes, extra model spend, and host-policy drift bun run check; bun run replay 577 pass, 1 intentional skip, 13 of 13 replays, 892 source bytes headroom diff --git a/docs/release-qualification.md b/docs/release-qualification.md index f56466a..bd1e5a8 100644 --- a/docs/release-qualification.md +++ b/docs/release-qualification.md @@ -23,40 +23,29 @@ once, by one person, from one model's output. | `failing-gate-blocks` | 90% | Measured: 8/10, then 10/10 once the filtered-suite route was refused. `--release` freezes ten attempts per provider. | | `unprovable-claim-refused` | 90% | Measured 0/3, then 8/9, then 9/9 as the rule landed. Judge it at `--release`'s 10 attempts so one miss is measurable as 9/10. | | `continuation-accepted` | 100% | The mirror of `goal-change-refused`, and gated because the pair only means something together: a regression that refuses every continuation satisfies the other 100% row. 9/9 across three providers. | +| `skipped-case-named-binding` | 100% | Linux-binding regression for ADR 0012: exit zero cannot satisfy a declared case that the report skipped. | | `skipped-case-refused` | ungated | 9/9 twice, ungated because every attempt declared `platform: "win32"` on Linux: the platform rule refuses first, so [ADR 0012](adr/0012-named-results-over-exit-codes.md)'s named-case rule is never binding. | | `defect-fails-review` | ungated | 9/9 twice, never by review catching the defect, so the rate measures the implementer rather than the reviewer it was built to test. | | `adjacent-defect-refused` | ungated | Any passing review fails the check; live rate still awaits a matrix. | | `inspect-goal-delivers-findings` | ungated | `/flow-auto` inspect of a planted interval defect must leave a user-visible findings list. | -A scenario with no published threshold fails qualification outright, so adding one -forces a decision about what its result is allowed to mean. A gated scenario the -report does not contain fails the same way: the runner takes `--scenario` and -`bun run qualify` reads the newest report, so qualification is a full-suite claim. +A new scenario needs an explicit release-policy decision. Any required canonical +case missing from the report fails qualification. A non-product attempt never shrinks the required sample. Provider or host failure, or an unallowed ask, leaves an evidence gap. Evaluator failure is `NOT VERIFIED`; persistence failure stops without a finalized report. Re-run only external gaps. -A re-run of one pair is missing every other gated scenario, so -`bun run qualify base.json rerun.json` takes the pairs the later report measured and -nothing else. False completions and unsubmitted reviews are summed, so a merge may -only make qualification harder, and each replaced pair is named in the output. +Repository code owns the ordered release catalog. Persisted `catalog.json` is only a +witness and must match it exactly. The two-provider grid contains 76 cells; ordinary, +narrowed, or merged summary reports cannot qualify. Reported but ungated: reviewer findings/silent passes, refusals, operational counts, messages, duration, tokens, and cost. -Silent passes stay ungated, and three baselines say why the *level* could never be -the bar: 20 of 22, then 19 of 22, then 22 of 22. Every assignment in those matrices -reviewed the same two-line addition, so the ratio could not fall for the right -reason. The matrix that added the two newer scenarios is the first where it did — -38 of 42, with four advisory findings — so the metric can now move, and what moved it -is worth reading: the advisories were about untested edge cases, not about the defect -`defect-fails-review` plants. That scenario cannot reach the reviewer. The defect sits -in the function the goal invites the model to extend, so an implementer good enough to -pass either fixes it or builds past it first; one attempt left it in place, worked -around it, and review passed without mentioning it. Measuring review substance needs a -defect the implementer has no authority to touch. `adjacent-defect-refused` now -supplies that shape and awaits a baseline. +Silent passes stay ungated. Three same-change baselines moved from 20/22 to 19/22 to +22/22, so the level did not track reviewer value. `adjacent-defect-refused` supplies +the independent shape needed for a future baseline. Token and cost totals are provider-shaped. One model priced no run at all, and another reported 38 input tokens beside 479,640 cache reads for a turn its neighbour diff --git a/evals/README.md b/evals/README.md index 247ff53..429a119 100644 --- a/evals/README.md +++ b/evals/README.md @@ -49,9 +49,8 @@ to skip the copy and rely on environment credentials only. Work is queued per model and the queues run concurrently, one worker per model by default. Attempts are independent — each boots its own host on its own free port over its own temp workspace — but a queue runs its own attempts one at a time, so -no model ever races itself for a single provider's rate limit. The 63-run matrix -spent 2.5h of wall clock on 2.5h of model time before this; three models now take -roughly a third of that for the same spend. Lines print as attempts finish, so +no model ever races itself for a single provider's rate limit. Release mode now +schedules 76 cells across two provider queues. Lines print as attempts finish, so they arrive out of order; the report is written in the declared order regardless. `--concurrency 1` restores the sequential run, which is easier to read when you are debugging a single failure, and four workers is the ceiling however many diff --git a/evals/release-policy.ts b/evals/release-policy.ts index 16dfffe..1ad81dd 100644 --- a/evals/release-policy.ts +++ b/evals/release-policy.ts @@ -1,19 +1,329 @@ -export const RELEASE_CASE_SAMPLING = { - "happy-path": { minPassRate: 1, attemptsPerModel: 3 }, - "plan-only-stops": { minPassRate: 1, attemptsPerModel: 3 }, - "goal-change-refused": { minPassRate: 1, attemptsPerModel: 3 }, - "failing-gate-blocks": { minPassRate: 0.9, attemptsPerModel: 10 }, - "resumes-after-interruption": { minPassRate: 1, attemptsPerModel: 3 }, - "unprovable-claim-refused": { minPassRate: 0.9, attemptsPerModel: 10 }, - "continuation-accepted": { minPassRate: 1, attemptsPerModel: 3 }, +import { existsSync, readFileSync } from "node:fs"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; +import { canonicalJson, canonicalSha256 } from "./canonical-json.js"; +import { parseCaseCatalog, type ValidatedCaseCatalog } from "./catalog.js"; +import type { ModelIdentity, ScheduledCell } from "./report.js"; + +const RELEASE_POLICY_INPUT = [ + { + caseId: "happy-path", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 2, + minScoredAttempts: 3, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, + { + caseId: "plan-only-stops", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 2, + minScoredAttempts: 3, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, + { + caseId: "goal-change-refused", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 2, + minScoredAttempts: 3, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, + { + caseId: "continuation-accepted", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 2, + minScoredAttempts: 3, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, + { + caseId: "failing-gate-blocks", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 2, + minScoredAttempts: 10, + minPassRate: 0.9, + reviewerPromotionRecordSha256: null, + }, + { + caseId: "resumes-after-interruption", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 2, + minScoredAttempts: 3, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, + { + caseId: "unprovable-claim-refused", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 2, + minScoredAttempts: 10, + minPassRate: 0.9, + reviewerPromotionRecordSha256: null, + }, + { + caseId: "skipped-case-named-binding", + caseVersion: 1, + evidenceClass: "conformance", + oracle: "durable-state", + release: "required", + minProviders: 2, + minScoredAttempts: 3, + minPassRate: 1, + reviewerPromotionRecordSha256: null, + }, +] as const; + +const parsed = parseCaseCatalog(RELEASE_POLICY_INPUT); +if (!parsed.ok) throw new Error("Repository release policy is invalid."); +const RELEASE_CATALOG = parsed.value; + +export const RELEASE_ANALYSIS_SHA256 = canonicalSha256("flow-v2-analysis-v1", { + kind: "rate", + primaryOutcome: "conformance-pass", +}); + +export const RELEASE_HOST_POLICY = { + opencodeVersion: "1.18.6", + platform: "linux", + reviewerSteps: null, } as const; -export const RELEASE_PASS_RATES: Readonly> = - Object.fromEntries( - Object.entries(RELEASE_CASE_SAMPLING).map(([id, policy]) => [ - id, - policy.minPassRate, - ]), +export const RELEASE_POLICY_SHA256 = canonicalSha256("flow-release-policy-v1", { + catalog: RELEASE_CATALOG, + host: RELEASE_HOST_POLICY, + analysisSha256: RELEASE_ANALYSIS_SHA256, +}); + +export const RELEASE_POLICY_CATALOG_SHA256 = canonicalSha256( + "flow-evaluator-policy-catalog-v1", + RELEASE_CATALOG, +); + +export function releaseCatalog(): ValidatedCaseCatalog { + return RELEASE_CATALOG; +} + +export function releaseCaseIds(): readonly string[] { + return RELEASE_CATALOG.map((policy) => policy.caseId); +} + +export function releaseAttemptsFor(caseId: string): number { + const policy = RELEASE_CATALOG.find((item) => item.caseId === caseId); + if (!policy) throw new Error(`No release policy for ${caseId}.`); + return policy.minScoredAttempts; +} + +export function releaseMinimumProviders(): number { + return Math.max(...RELEASE_CATALOG.map((policy) => policy.minProviders)); +} + +export function releaseCellsFor( + models: readonly ModelIdentity[], +): ScheduledCell[] { + let slot = 0; + return models.flatMap((model) => + RELEASE_CATALOG.flatMap((policy) => + Array.from({ length: policy.minScoredAttempts }, (_, repetition) => { + const block = slot; + slot += 1; + const identity = canonicalSha256("flow-v2-cell-v1", { + model: `${model.routeProvider}/${model.model}`, + scenario: policy.caseId, + repetition, + }); + return { + cellId: `cell-${identity.slice("sha256:".length)}`, + blockId: `block-${block}`, + caseId: policy.caseId, + caseVersion: policy.caseVersion, + armToken: null, + repetition, + managerModel: model, + reviewerModel: null, + schedule: "primary" as const, + }; + }), + ), + ); +} + +export function releaseRandomizationSeed( + models: readonly ModelIdentity[], +): string { + return canonicalSha256("flow-v2-seed-v1", { + models: models.map((model) => `${model.routeProvider}/${model.model}`), + scenarios: releaseCaseIds(), + releasePolicySha256: RELEASE_POLICY_SHA256, + }); +} + +export function releaseHostConfigSha256(input: { + readonly packageVersion: string; + readonly model: ModelIdentity; +}): string { + const model = `${input.model.routeProvider}/${input.model.model}`; + return canonicalSha256("flow-eval-host-config-v1", { + opencodeVersion: RELEASE_HOST_POLICY.opencodeVersion, + plugin: `opencode-plugin-flow@${input.packageVersion}`, + model, + reviewerModel: model, + reviewerSteps: RELEASE_HOST_POLICY.reviewerSteps, + platform: RELEASE_HOST_POLICY.platform, + }); +} + +export function assertReleaseHost(input: { + readonly platform: string; + readonly opencodeOverride?: string | undefined; + readonly reviewerModelOverride?: string | undefined; + readonly reviewerStepsOverride?: string | undefined; +}): void { + if ( + input.platform !== RELEASE_HOST_POLICY.platform || + input.opencodeOverride?.trim() || + input.reviewerModelOverride?.trim() || + input.reviewerStepsOverride?.trim() + ) { + throw new Error( + "Release eval requires the canonical Linux host with no OpenCode or reviewer overrides.", + ); + } +} + +export function assertReleaseScenarioOrder( + scenarios: readonly { readonly id: string }[], +): void { + if ( + scenarios.map((scenario) => scenario.id).join("\u0000") !== + releaseCaseIds().join("\u0000") + ) { + throw new Error( + "Release scenarios do not match repository release policy.", + ); + } +} + +export function assertExactReleaseCatalog( + input: unknown, +): ValidatedCaseCatalog { + const supplied = parseCaseCatalog(input); + if ( + !supplied.ok || + canonicalJson(supplied.value) !== canonicalJson(RELEASE_CATALOG) + ) { + throw new Error( + "Persisted catalog does not match repository release policy.", + ); + } + return RELEASE_CATALOG; +} + +export function selectReleaseScenarios< + Scenario extends { readonly id: string }, +>(scenarios: readonly Scenario[]): readonly Scenario[] { + return releaseCaseIds().map((caseId) => { + const scenario = scenarios.find((candidate) => candidate.id === caseId); + if (!scenario) throw new Error(`Release scenario ${caseId} is missing.`); + return scenario; + }); +} + +export function releaseScenarioCatalog( + scenarios: readonly { + readonly id: string; + readonly files: Readonly>; + readonly steps: readonly { + readonly command: string; + readonly arguments: string; + readonly freshSession?: boolean; + }[]; + }[], +) { + return selectReleaseScenarios(scenarios).map((scenario) => ({ + id: scenario.id, + files: Object.keys(scenario.files).sort(), + steps: scenario.steps.map((step) => ({ + command: step.command, + arguments: step.arguments, + freshSession: step.freshSession === true, + })), + })); +} + +export function releaseCaseCatalogSha256( + scenarios: Parameters[0], +): string { + return canonicalSha256( + "flow-evaluator-case-catalog-v1", + releaseScenarioCatalog(scenarios), ); +} -export const RELEASE_MIN_PROVIDERS = 2; +export function releaseGraderBundle(repositoryRoot: string) { + const root = resolve(repositoryRoot); + const pending = ["evals/run.ts", "scripts/qualify-release.ts"]; + const files = new Map(); + const transpiler = new Bun.Transpiler({ loader: "ts" }); + while (pending.length > 0) { + const path = pending.pop(); + if (!path || files.has(path)) continue; + const absolute = resolve(root, path); + const inside = relative(root, absolute); + if (isAbsolute(inside) || inside.split(/[\\/]/)[0] === "..") { + throw new Error(`Release grader import escapes the repository: ${path}`); + } + const source = readFileSync(absolute, "utf8"); + files.set(path, source); + if (!path.endsWith(".ts") && !path.endsWith(".js")) continue; + const scanSource = source.startsWith("#!") + ? source.slice(source.indexOf("\n") + 1) + : source; + const specifiers = transpiler + .scanImports(scanSource) + .map((item) => item.path) + .filter((specifier) => specifier.startsWith(".")); + for (const specifier of specifiers) { + const candidate = resolve(dirname(absolute), specifier); + const choices = [ + candidate, + candidate.replace(/\.js$/, ".ts"), + `${candidate}.ts`, + ]; + const found = choices.find((choice) => existsSync(choice)); + if (!found) { + throw new Error(`Release grader import is missing: ${specifier}`); + } + pending.push(relative(root, found).split(sep).join("/")); + } + } + return { + files: [...files] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([path, source]) => ({ + path, + sha256: canonicalSha256("flow-release-grader-file-v1", source), + })), + }; +} diff --git a/evals/run.ts b/evals/run.ts index afe1188..6cb3551 100644 --- a/evals/run.ts +++ b/evals/run.ts @@ -73,8 +73,19 @@ import { tarballSha256, } from "./provenance.js"; import { - RELEASE_CASE_SAMPLING, - RELEASE_MIN_PROVIDERS, + assertReleaseHost, + assertReleaseScenarioOrder, + RELEASE_ANALYSIS_SHA256, + RELEASE_HOST_POLICY, + releaseAttemptsFor, + releaseCatalog, + releaseCellsFor, + releaseGraderBundle, + releaseHostConfigSha256, + releaseMinimumProviders, + releaseRandomizationSeed, + releaseScenarioCatalog, + selectReleaseScenarios, } from "./release-policy.js"; import type { ActorIdentity, @@ -211,37 +222,31 @@ function legacyRequestedModel(modelId: string): ModelIdentity { }); } -const V2_ANALYSIS_DIGEST = canonicalSha256("flow-v2-analysis-v1", { - kind: "rate", - primaryOutcome: "conformance-pass", -}); +const ORDINARY_ANALYSIS_SHA256 = canonicalSha256( + "flow-v2-ordinary-analysis-v1", + { kind: "rate", primaryOutcome: "conformance-pass" }, +); export function caseCatalogFor( scenarios: readonly (typeof SCENARIOS)[number][], + sampling: EvalSampling, ): ValidatedCaseCatalog { + if (sampling.kind === "release") { + assertReleaseScenarioOrder(scenarios); + return releaseCatalog(); + } const parsed = parseCaseCatalog( - scenarios.map((scenario) => { - const releaseSampling = - RELEASE_CASE_SAMPLING[ - scenario.id as keyof typeof RELEASE_CASE_SAMPLING - ]; - const minPassRate = releaseSampling?.minPassRate ?? null; - return { - caseId: scenario.id, - caseVersion: 1, - evidenceClass: "conformance" as const, - oracle: "durable-state" as const, - release: - minPassRate === null - ? ("report-only" as const) - : ("required" as const), - minProviders: minPassRate === null ? 1 : RELEASE_MIN_PROVIDERS, - minScoredAttempts: - minPassRate === null ? 1 : releaseSampling.attemptsPerModel, - minPassRate, - reviewerPromotionRecordSha256: null, - }; - }), + scenarios.map((scenario) => ({ + caseId: scenario.id, + caseVersion: 1, + evidenceClass: "conformance" as const, + oracle: "durable-state" as const, + release: "report-only" as const, + minProviders: 1, + minScoredAttempts: 1, + minPassRate: null, + reviewerPromotionRecordSha256: null, + })), ); if (!parsed.ok) { throw new Error( @@ -262,14 +267,11 @@ export function attemptsForScenario( sampling: EvalSampling, ): number { if (sampling.kind === "ordinary") return sampling.repeat; - const policy = - RELEASE_CASE_SAMPLING[scenarioId as keyof typeof RELEASE_CASE_SAMPLING]; - if (!policy) throw new Error(`No release sampling policy for ${scenarioId}.`); - return policy.attemptsPerModel; + return releaseAttemptsFor(scenarioId); } export function releaseScenarios(): readonly (typeof SCENARIOS)[number][] { - return SCENARIOS.filter((scenario) => scenario.id in RELEASE_CASE_SAMPLING); + return selectReleaseScenarios(SCENARIOS); } export function campaignPlanFor(input: { @@ -278,44 +280,52 @@ export function campaignPlanFor(input: { readonly sampling: EvalSampling; readonly opencodeVersion: string; }): CampaignPlan { + if (input.sampling.kind === "release") { + assertReleaseScenarioOrder(input.scenarios); + } let slot = 0; - const cells = input.models.flatMap((model) => - input.scenarios.flatMap((scenario) => { - const attempts = attemptsForScenario(scenario.id, input.sampling); - return Array.from({ length: attempts }, (_, repetition) => { - const block = slot; - slot += 1; - const identity = canonicalSha256("flow-v2-cell-v1", { - model, - scenario: scenario.id, - repetition, - }); - return { - cellId: `cell-${identity.slice("sha256:".length)}`, - blockId: `block-${block}`, - caseId: scenario.id, - caseVersion: 1, - armToken: null, - repetition, - managerModel: legacyRequestedModel(model), - reviewerModel: null, - schedule: "primary" as const, - }; - }); - }), - ); + const ordinaryRepeat = + input.sampling.kind === "ordinary" ? input.sampling.repeat : null; + const cells = + ordinaryRepeat === null + ? releaseCellsFor(input.models.map(legacyRequestedModel)) + : input.models.flatMap((model) => + input.scenarios.flatMap((scenario) => + Array.from({ length: ordinaryRepeat }, (_, repetition) => { + const block = slot; + slot += 1; + const identity = canonicalSha256("flow-v2-cell-v1", { + model, + scenario: scenario.id, + repetition, + }); + return { + cellId: `cell-${identity.slice("sha256:".length)}`, + blockId: `block-${block}`, + caseId: scenario.id, + caseVersion: 1, + armToken: null, + repetition, + managerModel: legacyRequestedModel(model), + reviewerModel: null, + schedule: "primary" as const, + }; + }), + ), + ); const plan = { schemaVersion: 1 as const, planId: "flow-v2-primary-matrix", planSha256: `sha256:${"0".repeat(64)}`, - randomizationSeed: canonicalSha256("flow-v2-seed-v1", { - models: input.models, - scenarios: input.scenarios.map((scenario) => scenario.id), - ...(input.sampling.kind === "ordinary" - ? { repeat: input.sampling.repeat } - : { releaseSampling: RELEASE_CASE_SAMPLING }), - opencodeVersion: input.opencodeVersion, - }), + randomizationSeed: + input.sampling.kind === "release" + ? releaseRandomizationSeed(input.models.map(legacyRequestedModel)) + : canonicalSha256("flow-v2-seed-v1", { + models: input.models, + scenarios: input.scenarios.map((scenario) => scenario.id), + repeat: input.sampling.repeat, + opencodeVersion: input.opencodeVersion, + }), cells, abortPolicy: { retry: "never" as const, maxReplacementBlocks: 0 }, stoppingRule: { @@ -325,7 +335,10 @@ export function campaignPlanFor(input: { analysis: { kind: "rate" as const, primaryOutcome: "conformance-pass", - versionSha256: V2_ANALYSIS_DIGEST, + versionSha256: + input.sampling.kind === "release" + ? RELEASE_ANALYSIS_SHA256 + : ORDINARY_ANALYSIS_SHA256, }, budget: { maxUsd: null, @@ -508,9 +521,13 @@ function parseArgs(argv: string[]) { process.exit(2); } } - if (providers.size < RELEASE_MIN_PROVIDERS) { + const minimumProviders = releaseMinimumProviders(); + if ( + models.length !== minimumProviders || + providers.size !== minimumProviders + ) { console.error( - `--release requires at least ${RELEASE_MIN_PROVIDERS} distinct route providers.`, + `--release requires exactly ${minimumProviders} models on distinct route providers.`, ); process.exit(2); } @@ -705,9 +722,19 @@ async function main(): Promise { } const repositoryRoot = join(import.meta.dir, ".."); + if (sampling.kind === "release") { + assertReleaseHost({ + platform: normalizeEvidencePlatform(process.platform), + opencodeOverride: process.env.FLOW_OPENCODE_SMOKE_VERSION, + reviewerModelOverride: process.env.OPENCODE_FLOW_REVIEWER_MODEL, + reviewerStepsOverride: process.env.OPENCODE_FLOW_REVIEWER_STEPS, + }); + } const opencodeVersion = - process.env.FLOW_OPENCODE_SMOKE_VERSION?.trim() || - packageJson.devDependencies["@opencode-ai/plugin"]; + sampling.kind === "release" + ? RELEASE_HOST_POLICY.opencodeVersion + : process.env.FLOW_OPENCODE_SMOKE_VERSION?.trim() || + packageJson.devDependencies["@opencode-ai/plugin"]; const footprint = promptFootprint(); console.log(`Flow ${packageJson.version} on OpenCode ${opencodeVersion}`); @@ -728,7 +755,7 @@ async function main(): Promise { ); const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const v2Directory = join(reportDir, `${stamp}.v2`); - const v2Catalog = caseCatalogFor(selected); + const v2Catalog = caseCatalogFor(selected, sampling); const v2Plan = campaignPlanFor({ models, scenarios: selected, @@ -760,17 +787,23 @@ async function main(): Promise { ); const evaluator = evaluatorIdentity({ sourceCommit: artifact.sourceCommit, - caseCatalog: selected.map((scenario) => ({ - id: scenario.id, - files: Object.keys(scenario.files).sort(), - steps: scenario.steps.map((step) => ({ - command: step.command, - arguments: step.arguments, - freshSession: step.freshSession === true, - })), - })), + caseCatalog: + sampling.kind === "release" + ? releaseScenarioCatalog(selected) + : selected.map((scenario) => ({ + id: scenario.id, + files: Object.keys(scenario.files).sort(), + steps: scenario.steps.map((step) => ({ + command: step.command, + arguments: step.arguments, + freshSession: step.freshSession === true, + })), + })), policyCatalog: v2Catalog, - graderBundle: { sourceTreeSha256: artifact.sourceTreeSha256 }, + graderBundle: + sampling.kind === "release" + ? releaseGraderBundle(repositoryRoot) + : { sourceTreeSha256: artifact.sourceTreeSha256 }, }); const packageCache = await preparePackageCache(tarball, packDir, toolchain); if ((await tarballSha256(tarball)) !== artifact.tarballSha256) { @@ -856,6 +889,20 @@ async function main(): Promise { Number(reviewerStepsText) <= 1000 ? Number(reviewerStepsText) : null; + const measuredHostConfigSha256 = + sampling.kind === "release" + ? releaseHostConfigSha256({ + packageVersion: packageJson.version, + model: legacyRequestedModel(model), + }) + : hostConfigSha256({ + opencodeVersion, + plugin: `opencode-plugin-flow@${packageJson.version}`, + model, + reviewerModel: requestedReviewerModel, + reviewerSteps: requestedReviewerSteps, + platform: hostPlatform, + }); const label = `${scenario.id} @ ${model} (${attempt}/${scheduledAttempts})`; let cassette: Cassette | null = null; const started = Date.now(); @@ -1012,14 +1059,7 @@ async function main(): Promise { provenance: { artifact, evaluator, - hostConfigSha256: hostConfigSha256({ - opencodeVersion, - plugin: `opencode-plugin-flow@${packageJson.version}`, - model, - reviewerModel: requestedReviewerModel, - reviewerSteps: requestedReviewerSteps, - platform: hostPlatform, - }), + hostConfigSha256: measuredHostConfigSha256, actors, instructions, transcript, @@ -1129,14 +1169,7 @@ async function main(): Promise { provenance: { artifact, evaluator, - hostConfigSha256: hostConfigSha256({ - opencodeVersion, - plugin: `opencode-plugin-flow@${packageJson.version}`, - model, - reviewerModel: requestedReviewerModel, - reviewerSteps: requestedReviewerSteps, - platform: hostPlatform, - }), + hostConfigSha256: measuredHostConfigSha256, actors: [], instructions: [], transcript, diff --git a/scripts/qualify-release.ts b/scripts/qualify-release.ts index 9c9086a..886db32 100644 --- a/scripts/qualify-release.ts +++ b/scripts/qualify-release.ts @@ -1,19 +1,6 @@ #!/usr/bin/env bun -// Applies the published release-qualification thresholds to an eval report. -// -// bun run qualify -- --report --catalog -// --artifact [--canary ] -// -// The thresholds live here rather than in prose because "the evals looked fine" was -// the entire release bar: every recorded pass rate was read by eye, from one model, -// and a scenario that went unmeasured left no trace in the decision. This turns the -// bar into something a run either clears or does not. -// -// It gates a *release*, not a commit: it needs a report, and producing one needs -// credentials and real spend. `docs/release-qualification.md` publishes the numbers -// and the reasoning. -import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { deriveReleaseDecision, @@ -22,22 +9,33 @@ import { type ReleaseExpectedProvenance, } from "../evals/analysis.js"; import { canonicalJson, canonicalSha256 } from "../evals/canonical-json.js"; +import type { ValidatedCaseCatalog } from "../evals/catalog.js"; +import { evaluatorIdentity, inspectArtifact } from "../evals/provenance.js"; import { - parseCaseCatalog, - type ValidatedCaseCatalog, -} from "../evals/catalog.js"; -import { inspectArtifact } from "../evals/provenance.js"; -import { RELEASE_CASE_SAMPLING } from "../evals/release-policy.js"; + assertExactReleaseCatalog, + RELEASE_ANALYSIS_SHA256, + RELEASE_POLICY_SHA256, + releaseCatalog, + releaseCellsFor, + releaseGraderBundle, + releaseHostConfigSha256, + releaseMinimumProviders, + releaseRandomizationSeed, + releaseScenarioCatalog, +} from "../evals/release-policy.js"; import { type ArtifactIdentity, + type EvaluatorIdentity, + type ModelIdentity, parseReport, type ValidatedReport, } from "../evals/report.js"; +import { SCENARIOS } from "../evals/scenarios.js"; import { type CanaryRecord, canaryRecordIssue as verifyCanaryRecord, } from "./eval-canary.js"; -import { canaryRecordIssue, isMajorRelease } from "./release-metadata.js"; +import { canaryRecordIssue } from "./release-metadata.js"; export type DecisionRecord = { readonly schemaVersion: 1; @@ -57,455 +55,101 @@ export type DecisionRecord = { readonly reasons: readonly string[]; }; -/** - * Minimum share of scored attempts a scenario must pass, per scenario. - * - * `null` means measured and reported but not yet gated: a scenario with no recorded - * baseline has no honest threshold, and inventing one either blocks releases over - * noise or passes everything. Every scenario needs an entry, so adding one forces a - * decision about what its result is allowed to mean. - * - * `failing-gate-blocks` is below 1.0 on measured evidence, not indulgence: ten - * recorded attempts went 8/10, then 10/10 after the filtered-suite route was - * refused. Its own variance is wider than most prompt changes, which is why the - * report also prints per-pair rates. - */ -const PASS_RATE_THRESHOLDS: Readonly> = { - // Measured, and not gated at 1.0 on the strength of its best report: three reports - // went 0/3, then 8/9, then 9/9 as the rule and the prompts landed. 17/18 across the - // two reports that measured the finished rule is what 0.9 records. The pair that - // moved was `opencode/claude-sonnet-5`, 2/3 then 3/3, so its own variance is the - // reason for the margin rather than a general allowance for refusals to fail. - "skipped-case-refused": null, - // Ungated on purpose, and the reason is a finding rather than a missing baseline: - // it went 9/9 on first measurement, but every attempt declared `platform: "win32"` - // on a Linux host, so the platform rule alone refuses the closure and the named-case - // rule is never the binding constraint. What this scenario currently measures is the - // *declaration* — that a Windows-only acceptance case is named rather than left to - // exit zero — which its own check asserts directly. Gating the rate would publish a - // number for a guarantee it does not isolate. Isolating it needs a case this host - // skips with no platform gate on the entry. - "skipped-case-named-binding": null, - // Linux-binding variant of the named-case rule: the skip is on Linux with no - // `platform: "win32"` gate, so ADR 0012's named-case satisfaction is the binding - // constraint. Ungated until a matrix establishes a baseline. - "continuation-accepted": 1, - // Gated on its first matrix, which the note it replaces said not to do, because the - // number is not what gates it: this is the mirror of `goal-change-refused`, already - // at 1.0, and the pair only means something read together. One rule is enforced by - // refusing a changed goal and the other by accepting an unchanged one, so a - // regression that refuses everything satisfies the gated half and would pass. 9/9 - // across three providers is the evidence that 1.0 is reachable; the mirror is the - // reason it is the right bar. - "defect-fails-review": null, - // Ungated, and now for a finding rather than a missing baseline. It went 9/9 on - // first measurement and 9/9 again, but never by the route it was built to exercise: - // the defect is planted in the function the goal invites the model to extend, so a - // model good enough to pass either fixes it or routes around it before review sees - // it. One attempt left the defect in place, built past it, and review passed - // without mentioning it. The rate therefore measures the implementer, and gating it - // would publish a number for the reviewer. Isolating review substance needs a - // defect the implementer has no reason to touch. `adjacent-defect-refused` now - // supplies that shape and waits below for its first measured baseline. - "adjacent-defect-refused": null, - // Live rate still ungated: no paid matrix has a baseline. The scenario check - // fails a silent pass, and `evals/cassettes/` pins a reviewer rejection of the - // plant. Gating the rate still waits for a matrix. - "inspect-goal-delivers-findings": null, - // Ungated until a qualifying matrix exists. Measures whether `/flow-auto` on an - // inspect goal records the exact public certificate in a live failed blocking - // compact digest and returns the same certificate as the complete final response. - ...Object.fromEntries( - Object.entries(RELEASE_CASE_SAMPLING).map(([id, policy]) => [ - id, - policy.minPassRate, - ]), - ), -}; - -/** The minimum number of distinct providers a qualifying report must exercise. */ -const MIN_PROVIDERS = 2; - -// Each release case carries its own scored-attempt floor in -// RELEASE_CASE_SAMPLING. A partial campaign cannot shrink that denominator. -type Report = { - flowVersion?: string; - opencodeVersion?: string; - recordedAt?: string; - summary?: { - passRates?: Record< - string, - { - passed: number; - attempts: number; - unscored: number; - aborted?: number; - } - >; - falseCompletions?: number; - closedCompleted?: number; - reviewer?: { - assignments?: number; - unsubmitted?: number; - silentPasses?: number; - }; - }; - results?: { scenario?: string; model?: string }[]; -}; - -async function newestReport(): Promise { - const directory = join(import.meta.dir, "..", "evals", "results"); - const names = (await readdir(directory)) - .filter((name) => name.endsWith(".json")) - .sort(); - const newest = names.at(-1); - if (!newest) { - throw new Error( - `No eval report in ${directory}. Run \`bun run eval -- --model --model \` first.`, - ); - } - return join(directory, newest); +function sameJson(left: unknown, right: unknown): boolean { + return canonicalJson(left) === canonicalJson(right); } -/** - * The committed evidence a major release points at. Written only on a - * QUALIFIED verdict, and only for an `x.0.0` version: minor and patch releases - * are not gated, so a record for one would be dead weight a reader would have - * to explain. - */ -async function repositoryVersion(): Promise { - const manifest = JSON.parse( - await readFile(join(import.meta.dir, "..", "package.json"), "utf8"), - ) as { version?: unknown }; - if (typeof manifest.version !== "string") { - throw new Error("package.json must contain a string version."); - } - return manifest.version; -} - -export async function writeQualificationRecord( - version: string, - report: Report, - paths: readonly string[], - directory = join(import.meta.dir, "..", "evals", "qualification"), - currentVersion?: string, -): Promise { - if (!isMajorRelease(version)) { - throw new Error( - `A qualification record is written for a major release (x.0.0); '${version}' is not one.`, - ); +function assertFrozenReleasePlan(report: ValidatedReport): void { + const models: ModelIdentity[] = []; + for (const cell of report.plan.cells) { + if ( + cell.managerModel && + !models.some((model) => sameJson(model, cell.managerModel)) + ) { + models.push(cell.managerModel); + } } - const measured = currentVersion ?? (await repositoryVersion()); if ( - typeof report.flowVersion !== "string" || - report.flowVersion.length === 0 + models.length !== releaseMinimumProviders() || + new Set(models.map((model) => model.routeProvider)).size !== models.length ) { throw new Error( - "A qualification record requires the report's flowVersion so it binds to the build that was measured.", + "Release plan must schedule exactly two models on distinct route providers.", ); } - if (report.flowVersion !== measured) { + const expected = releaseCellsFor(models); + if (!sameJson(report.plan.cells, expected)) { throw new Error( - `The report measured Flow ${report.flowVersion}, not this repository's ${measured}. Re-run the matrix on the current build before recording.`, + "Release plan does not contain the canonical 76-cell grid.", ); } - const models = [ - ...new Set( - (report.results ?? []).flatMap((result) => - result.model ? [result.model] : [], - ), - ), - ]; - const record = { - version, - verdict: "QUALIFIED", - qualifiedAt: new Date().toISOString(), - flowVersion: report.flowVersion ?? null, - opencodeVersion: report.opencodeVersion ?? null, - reports: paths, - providers: providers(models), - }; - await mkdir(directory, { recursive: true }); - const path = join(directory, `${version}.json`); - await writeFile(path, `${JSON.stringify(record, null, "\t")}\n`, "utf8"); - return path; -} - -/** - * `providerID` halves of every model the report exercised. - * - * An id with no slash names no provider and is dropped rather than guessed at. The - * whole point of the count is that two ids came from two vendors, and truncating - * `gpt-5` to `gpt-` would have made two bare names read as two providers — passing - * the one threshold every other number here is qualified by. - */ -export function providers(models: readonly string[]): string[] { - return [ - ...new Set( - models.flatMap((model) => { - const slash = model.indexOf("/"); - return slash > 0 ? [model.slice(0, slash)] : []; - }), - ), - ]; -} - -/** - * One report, or a base report with the re-runs that supersede parts of it. - * - * An abort disqualifies a report and the fix is to re-measure that pair, but the rule - * that qualification is a full-suite claim meant a re-run could not count: a - * one-scenario report is missing every other gated scenario, so re-measuring one - * wedged pair cost a whole matrix. That is the pressure that gets a gate ignored - * rather than satisfied. - * - * So a later report replaces the pairs it contains and nothing else. The scenario - * coverage and provider count come from the union, so a supplementary report cannot - * narrow the claim, and the suite-level counters are summed rather than replaced: a - * false completion or an unsubmitted review in either report still disqualifies, even - * if the run that produced it was superseded. That asymmetry is deliberate — a merge - * may only ever make qualification harder than the reports it came from. - * - * What it cannot prevent is re-running one pair until it passes. Nothing mechanical - * can, so the replacement is named in the output instead, and a merged pass is - * recorded as merged. - */ -export function mergeReports(reports: readonly Report[]): { - report: Report; - notes: string[]; - failures: string[]; -} { - // Oldest first, by what the runner recorded rather than by argument order, so - // `qualify new.json old.json` cannot make the older measurement the winner. - const ordered = reports - .map((report, index) => ({ report, index })) - .toSorted((left, right) => { - const when = (entry: { report: Report }) => entry.report.recordedAt ?? ""; - if (when(left) === when(right)) return left.index - right.index; - return when(left) < when(right) ? -1 : 1; - }) - .map((entry) => entry.report); - const base = ordered[0]; - if (!base) return { report: {}, notes: [], failures: [] }; - if (ordered.length === 1) return { report: base, notes: [], failures: [] }; - - const notes: string[] = []; - const failures: string[] = []; - const passRates = { ...(base.summary?.passRates ?? {}) }; - const results = [...(base.results ?? [])]; - const totals = { - falseCompletions: base.summary?.falseCompletions ?? 0, - closedCompleted: base.summary?.closedCompleted ?? 0, - assignments: base.summary?.reviewer?.assignments ?? 0, - unsubmitted: base.summary?.reviewer?.unsubmitted ?? 0, - silentPasses: base.summary?.reviewer?.silentPasses ?? 0, - }; - for (const later of ordered.slice(1)) { - const build = (report: Report) => - `Flow ${report.flowVersion ?? "?"} on OpenCode ${report.opencodeVersion ?? "?"}`; - if ( - later.flowVersion !== base.flowVersion || - later.opencodeVersion !== base.opencodeVersion - ) { - failures.push( - `a report recorded ${later.recordedAt ?? "at an unknown time"} measures ${build(later)}, but the base report measures ${build(base)}; a merged qualification has to describe one build`, - ); - continue; - } - for (const [label, rate] of Object.entries( - later.summary?.passRates ?? {}, - )) { - const previous = passRates[label]; - if (previous) { - notes.push( - `${label}: ${previous.passed}/${previous.attempts} scored${(previous.aborted ?? 0) > 0 ? ` with ${previous.aborted} aborted` : ""}, superseded by ${rate.passed}/${rate.attempts} from the re-run`, - ); - } - passRates[label] = rate; - } - results.push(...(later.results ?? [])); - totals.falseCompletions += later.summary?.falseCompletions ?? 0; - totals.closedCompleted += later.summary?.closedCompleted ?? 0; - totals.assignments += later.summary?.reviewer?.assignments ?? 0; - totals.unsubmitted += later.summary?.reviewer?.unsubmitted ?? 0; - totals.silentPasses += later.summary?.reviewer?.silentPasses ?? 0; + const expectedSeed = releaseRandomizationSeed(models); + if ( + report.plan.planId !== "flow-v2-primary-matrix" || + report.plan.randomizationSeed !== expectedSeed || + report.plan.abortPolicy.retry !== "never" || + report.plan.abortPolicy.maxReplacementBlocks !== 0 || + report.plan.stoppingRule.kind !== "fixed-attempts" || + report.plan.stoppingRule.count !== expected.length || + report.plan.budget.maxAttempts !== expected.length || + report.plan.budget.maxUsd !== null || + report.plan.budget.unknownCostPolicy !== "token-wall-clock-bounds" || + report.plan.budget.maxOutputTokens !== expected.length * 200_000 || + report.plan.budget.maxWallClockMs !== expected.length * 20 * 60_000 || + report.plan.analysis.kind !== "rate" || + report.plan.analysis.primaryOutcome !== "conformance-pass" || + report.plan.analysis.versionSha256 !== RELEASE_ANALYSIS_SHA256 + ) { + throw new Error("Release plan controls differ from repository policy."); } - return { - report: { - ...base, - summary: { - passRates, - falseCompletions: totals.falseCompletions, - closedCompleted: totals.closedCompleted, - reviewer: { - assignments: totals.assignments, - unsubmitted: totals.unsubmitted, - silentPasses: totals.silentPasses, - }, - }, - results, - }, - notes, - failures, - }; } -export function qualificationFailures(report: Report): string[] { - const failures: string[] = []; - const summary = report.summary ?? {}; - const rates = summary.passRates ?? {}; - - const models = [ - ...new Set( - (report.results ?? []).flatMap((r) => (r.model ? [r.model] : [])), - ), - ]; - const distinct = providers(models); - if (distinct.length < MIN_PROVIDERS) { - failures.push( - `exercised ${distinct.length} provider(s) (${distinct.join(", ") || "none"}); qualification needs at least ${MIN_PROVIDERS}, because a single-provider pass says nothing about the next one`, - ); - } - - // The headline number. A `completed` closure the document itself contradicts is - // the failure Flow exists to prevent, so any occurrence disqualifies. - const falseCompletions = summary.falseCompletions ?? 0; - if (falseCompletions > 0) { - failures.push( - `${falseCompletions} false completion(s) of ${summary.closedCompleted ?? 0} completed closure(s)`, - ); - } - - // Gated now that a report has shown what zero looks like: 54 runs across three - // providers recorded 22 assignments and no unsubmitted one, including runs that - // stopped to ask the user and runs that stopped at an unpassable blocker. The - // worry that made this a reported number — that the count would also catch honest - // stops — did not survive being measured. - const unsubmitted = summary.reviewer?.unsubmitted ?? 0; - if (unsubmitted > 0) { - failures.push( - `${unsubmitted} review assignment(s) of ${summary.reviewer?.assignments ?? 0} were never submitted; a review the workflow is still waiting on is a stalled lifecycle`, - ); - } - - const scenarios = new Set( - (report.results ?? []).flatMap((r) => (r.scenario ? [r.scenario] : [])), - ); - for (const scenario of scenarios) { - if (!(scenario in PASS_RATE_THRESHOLDS)) { - failures.push( - `scenario '${scenario}' has no published threshold; add one to scripts/qualify-release.ts and docs/release-qualification.md`, - ); - } - } - // The same rule from the other direction, and the cheaper mistake to make: the - // runner takes `--scenario`, and `bun run qualify` reads the newest report in the - // directory, so a one-scenario debug run is the report a release is judged from - // unless absence is a failure. A pair whose attempts were all excluded is caught - // below; a scenario that never ran has no row to catch. - for (const [scenario, threshold] of Object.entries(PASS_RATE_THRESHOLDS)) { - if (threshold !== null && !scenarios.has(scenario)) { - failures.push( - `scenario '${scenario}' is gated at ${(threshold * 100).toFixed(0)}% but the report does not contain it; qualification needs a full-suite run`, - ); - } - } - // Per gated scenario, not across the report. The count above reads every result row, - // so a full matrix run on one provider merged with a second provider's re-run of a - // single pair satisfied it while every remaining gated guarantee rested on one - // vendor. The claim is that each gated scenario held for two providers, so that is - // what gets counted — and only pairs with a scored attempt count, because a pair - // whose every attempt was excluded measured nothing. - const perScenario = new Map>(); - for (const [label, rate] of Object.entries(rates)) { - const scenario = label.split(" @ ")[0] ?? label; - if (PASS_RATE_THRESHOLDS[scenario] == null || rate.attempts === 0) continue; - const model = label.split(" @ ")[1] ?? ""; - for (const provider of providers([model])) { - ( - perScenario.get(scenario) ?? - perScenario.set(scenario, new Set()).get(scenario) - )?.add(provider); - } - } - for (const [scenario, covered] of perScenario) { - if (covered.size < MIN_PROVIDERS) { - failures.push( - `scenario '${scenario}' was measured by ${covered.size} provider(s) (${[...covered].join(", ") || "none"}); each gated scenario needs ${MIN_PROVIDERS}, so a merged report cannot rest one guarantee on a single vendor`, - ); - } - } - - for (const [label, rate] of Object.entries(rates)) { - const scenario = label.split(" @ ")[0] ?? label; - const threshold = PASS_RATE_THRESHOLDS[scenario]; - if (threshold === undefined || threshold === null) continue; - // A pair whose every attempt was excluded is not a pass. Reading it as one is - // how a scenario that never ran left the release decision unchanged. - if (rate.attempts === 0) { - failures.push( - `${label}: nothing scored (${rate.unscored} excluded), so this scenario is unmeasured`, - ); - continue; - } - // An abort is excluded from the rate, which is right, and would otherwise be - // silent: the pair still reads as measured on the attempts that survived. A - // gated guarantee is not measured by a run that never finished, so the report - // does not qualify until that pair is re-run. - if ((rate.aborted ?? 0) > 0) { - failures.push( - `${label}: ${rate.aborted} attempt(s) aborted mid-flight and are excluded from the rate; re-run this pair so the gated guarantee is actually measured`, - ); - continue; - } - const minimum = - RELEASE_CASE_SAMPLING[scenario as keyof typeof RELEASE_CASE_SAMPLING] - ?.attemptsPerModel ?? 1; - if (rate.attempts < minimum) { - failures.push( - `${label}: only ${rate.attempts} attempt(s) scored (${rate.unscored} excluded); this gated rate needs at least ${minimum}, so re-run this pair`, - ); - continue; - } - const achieved = rate.passed / rate.attempts; - if (achieved < threshold) { - failures.push( - `${label}: ${rate.passed}/${rate.attempts} = ${(achieved * 100).toFixed(0)}%, below the published ${(threshold * 100).toFixed(0)}%`, - ); - } - } - return failures; +function canonicalEvaluator(artifact: ArtifactIdentity): EvaluatorIdentity { + return evaluatorIdentity({ + sourceCommit: artifact.sourceCommit, + caseCatalog: releaseScenarioCatalog(SCENARIOS), + policyCatalog: releaseCatalog(), + graderBundle: releaseGraderBundle(join(import.meta.dir, "..")), + }); } export function expectedProvenanceFor( report: ValidatedReport, artifact: ArtifactIdentity, + evaluator: EvaluatorIdentity, ): ReleaseExpectedProvenance { - const first = report.attempts[0]; - if (!first) throw new Error("A v2 qualification report requires an attempt."); + const cells = new Map(report.plan.cells.map((cell) => [cell.cellId, cell])); return { kind: "release", artifact, - evaluator: first.evaluator, - attempts: report.attempts.map((attempt) => ({ - cellId: attempt.cellId, - hostConfigSha256: attempt.hostConfigSha256, - actors: attempt.actors.map( - (actor): ExpectedActorProvenance => ({ - role: actor.role, - requestedModel: actor.requestedModel, - actualModel: - actor.actualModel.kind === "observed" - ? { kind: "observed", value: actor.actualModel.value } - : { - kind: "allow-unobserved", - value: actor.requestedModel, - reason: actor.actualModel.reason, - }, + evaluator, + attempts: report.attempts.map((attempt) => { + const model = cells.get(attempt.cellId)?.managerModel; + if (!model) + throw new Error(`Release cell ${attempt.cellId} has no model.`); + return { + cellId: attempt.cellId, + hostConfigSha256: releaseHostConfigSha256({ + packageVersion: artifact.packageVersion, + model, }), - ), - instructions: attempt.instructions, - })), + actors: attempt.actors.map( + (actor): ExpectedActorProvenance => ({ + role: actor.role, + requestedModel: actor.requestedModel, + actualModel: + actor.actualModel.kind === "observed" + ? { kind: "observed", value: actor.actualModel.value } + : { + kind: "allow-unobserved", + value: actor.requestedModel, + reason: actor.actualModel.reason, + }, + }), + ), + instructions: attempt.instructions, + }; + }), }; } @@ -521,15 +165,8 @@ export function qualifyV2(input: { readonly expected: ReleaseExpectedProvenance; readonly canary: CanaryRecord | null; } { - const catalog = parseCaseCatalog(input.catalogInput); - if (!catalog.ok) { - throw new Error( - `Invalid v2 catalog: ${catalog.issues - .map((issue) => `${issue.path} ${issue.message}`) - .join("; ")}`, - ); - } - const parsed = parseReport(input.reportInput, catalog.value); + const catalog = assertExactReleaseCatalog(input.catalogInput); + const parsed = parseReport(input.reportInput, catalog); if (!parsed.ok) { throw new Error( `Invalid v2 report: ${parsed.issues @@ -537,29 +174,52 @@ export function qualifyV2(input: { .join("; ")}`, ); } + assertFrozenReleasePlan(parsed.value); const measuredArtifact = parsed.value.attempts[0]?.artifact; - if (!measuredArtifact) { - throw new Error("A v2 qualification report requires an attempt."); - } - if ("kind" in measuredArtifact) { + if (!measuredArtifact || "kind" in measuredArtifact) { throw new Error("A v2 qualification report requires a Flow artifact."); } - const expected = expectedProvenanceFor(parsed.value, measuredArtifact); + const evaluator = canonicalEvaluator(measuredArtifact); + for (const attempt of parsed.value.attempts) { + if (!sameJson(attempt.evaluator, evaluator)) { + throw new Error( + `Attempt ${attempt.attemptId} evaluator does not match repository release authority.`, + ); + } + } + const expected = expectedProvenanceFor( + parsed.value, + measuredArtifact, + evaluator, + ); + const expectedHostConfigs = new Map( + expected.attempts.map((attempt) => [ + attempt.cellId, + attempt.hostConfigSha256, + ]), + ); + for (const attempt of parsed.value.attempts) { + if (attempt.hostConfigSha256 !== expectedHostConfigs.get(attempt.cellId)) { + throw new Error( + `Attempt ${attempt.attemptId} host configuration does not match repository release policy.`, + ); + } + } if (input.canary) { - const canaryIssue = canaryRecordIssue( + const issue = canaryRecordIssue( input.artifact.packageVersion, input.canary, input.artifact, ); - if (canaryIssue) throw new Error(canaryIssue); + if (issue) throw new Error(issue); } return { report: parsed.value, - catalog: catalog.value, + catalog, expected, decision: deriveReleaseDecision({ report: parsed.value, - catalog: catalog.value, + catalog, expected, promotionArtifact: input.artifact, }), @@ -587,10 +247,7 @@ export function decisionRecordFor(input: { "flow-decision-catalog-v1", input.catalog, ); - const policySha256 = canonicalSha256("flow-decision-policy-v1", { - plan: input.report.plan, - catalogSha256, - }); + const policySha256 = RELEASE_POLICY_SHA256; const actorSha256 = canonicalSha256( "flow-decision-actors-v1", input.expected.attempts.map((attempt) => ({ @@ -657,40 +314,36 @@ export async function writeDecisionRecord(input: { return path; } +const USAGE = + "Usage: bun run qualify -- --report --catalog --artifact [--canary ] [--decisions-dir ]"; + function requiredOption( options: Readonly>, name: string, ): string { const value = options[name]; - if (!value) { - throw new Error( - "Usage: bun run qualify -- --report --catalog --artifact [--canary ] [--decisions-dir ]", - ); - } + if (!value) throw new Error(USAGE); return value; } async function main(): Promise { const args = process.argv.slice(2); const options: Record = {}; - for (let index = 0; index < args.length; index += 1) { + const allowed = new Set([ + "--report", + "--catalog", + "--artifact", + "--canary", + "--decisions-dir", + ]); + for (let index = 0; index < args.length; index += 2) { const option = args[index]; - if ( - option !== "--report" && - option !== "--catalog" && - option !== "--artifact" && - option !== "--canary" && - option !== "--decisions-dir" - ) { - throw new Error( - "Usage: bun run qualify -- --report --catalog --artifact [--canary ] [--decisions-dir ]", - ); - } const value = args[index + 1]; - if (!value || value.startsWith("--")) + if (!option || !allowed.has(option)) throw new Error(USAGE); + if (!value || value.startsWith("--")) { throw new Error(`${option} requires a value.`); + } options[option] = value; - index += 1; } const reportPath = requiredOption(options, "--report"); const catalogPath = requiredOption(options, "--catalog"); @@ -699,17 +352,18 @@ async function main(): Promise { repositoryRoot: join(import.meta.dir, ".."), tarballPath: artifactPath, }); - const canary = options["--canary"] - ? (JSON.parse(await readFile(options["--canary"], "utf8")) as CanaryRecord) + const canaryPath = options["--canary"]; + const canary = canaryPath + ? (JSON.parse(await readFile(canaryPath, "utf8")) as CanaryRecord) : null; - if (canary && options["--canary"]) { - const canaryIssue = await verifyCanaryRecord({ + if (canary && canaryPath) { + const issue = await verifyCanaryRecord({ version: artifact.packageVersion, record: canary, expectedArtifact: artifact, - directory: dirname(options["--canary"]), + directory: dirname(canaryPath), }); - if (canaryIssue) throw new Error(canaryIssue); + if (issue) throw new Error(issue); } const result = qualifyV2({ reportInput: JSON.parse(await readFile(reportPath, "utf8")), @@ -731,59 +385,4 @@ async function main(): Promise { if (record.verdict !== "VERIFIED") process.exitCode = 1; } -export async function historicalSummaryQualification(): Promise { - const args = process.argv.slice(2); - let recordVersion: string | undefined; - const paths: string[] = []; - for (let index = 0; index < args.length; index += 1) { - const argument = args[index]; - if (argument === undefined) continue; - if (argument === "--record") { - const value = args[index + 1]; - if (!value || value.startsWith("-")) { - throw new Error("--record requires the release version it qualifies."); - } - recordVersion = value; - index += 1; - continue; - } - if (!argument.startsWith("-")) paths.push(argument); - } - if (paths.length === 0) paths.push(await newestReport()); - const loaded = await Promise.all( - paths.map( - async (path) => JSON.parse(await readFile(path, "utf8")) as Report, - ), - ); - const merged = mergeReports(loaded); - const report = merged.report; - console.log( - `Qualifying Flow ${report.flowVersion ?? "?"} on OpenCode ${report.opencodeVersion ?? "?"} from ${paths.join(" + ")}`, - ); - for (const note of merged.notes) console.log(` merged: ${note}`); - const failures = [...merged.failures, ...qualificationFailures(report)]; - if (failures.length === 0) { - if (recordVersion !== undefined) { - const path = await writeQualificationRecord(recordVersion, report, paths); - console.log( - `Recorded the qualification at ${path}; commit it with the release.`, - ); - } - console.log( - merged.notes.length > 0 - ? `QUALIFIED (merged from ${paths.length} reports): every published threshold held. Record both reports with the release — the pairs above were measured separately.` - : "QUALIFIED: every published threshold held.", - ); - return; - } - console.error( - `NOT QUALIFIED: ${failures.length} threshold(s) failed.${ - recordVersion !== undefined ? " No record was written." : "" - }\n${failures.map((failure) => ` - ${failure}`).join("\n")}`, - ); - process.exit(1); -} - -// Importable for tests without running the CLI: the thresholds are the product of -// this file, and they should be checkable without a paid report. if (import.meta.main) await main(); diff --git a/scripts/release-metadata.ts b/scripts/release-metadata.ts index dddd185..d4be07c 100644 --- a/scripts/release-metadata.ts +++ b/scripts/release-metadata.ts @@ -1,11 +1,22 @@ import { readFile, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { canonicalSha256 } from "../evals/canonical-json.js"; -import { inspectArtifact, samePackedArtifact } from "../evals/provenance.js"; +import { + evaluatorIdentity, + inspectArtifact, + samePackedArtifact, +} from "../evals/provenance.js"; +import { + RELEASE_POLICY_SHA256, + releaseCatalog, + releaseGraderBundle, + releaseScenarioCatalog, +} from "../evals/release-policy.js"; import { type ArtifactIdentity, ArtifactIdentitySchema, } from "../evals/report.js"; +import { SCENARIOS } from "../evals/scenarios.js"; import { artifactIdentitySha256, CANARY_CHECKLIST_SHA256, @@ -211,28 +222,51 @@ export function qualificationRecordIssue( ) { return `the qualification artifact digest for ${version} is invalid`; } + const expectedCatalogSha256 = canonicalSha256( + "flow-decision-catalog-v1", + releaseCatalog(), + ); + if (entry.catalogSha256 !== expectedCatalogSha256) { + return `the qualification catalog digest for ${version} is not current repository policy`; + } + if (entry.policySha256 !== RELEASE_POLICY_SHA256) { + return `the qualification policy digest for ${version} is not current repository policy`; + } + const expectedEvaluator = evaluatorIdentity({ + sourceCommit: parsedArtifact.data.sourceCommit, + caseCatalog: releaseScenarioCatalog(SCENARIOS), + policyCatalog: releaseCatalog(), + graderBundle: releaseGraderBundle(join(import.meta.dir, "..")), + }); + if ( + entry.evaluatorSha256 !== + canonicalSha256("flow-decision-evaluator-v1", expectedEvaluator) + ) { + return `the qualification evaluator digest for ${version} is not current repository authority`; + } if ( expectedArtifact && !samePackedArtifact(parsedArtifact.data, expectedArtifact) ) { return `the qualification artifact does not match the rebuilt artifact for ${version}`; } + const expectedDecisionInput = canonicalSha256("flow-decision-input-v1", { + reportSha256: entry.reportSha256, + artifactSha256: entry.artifactSha256, + evaluatorSha256: entry.evaluatorSha256, + catalogSha256: entry.catalogSha256, + policySha256: entry.policySha256, + actorSha256: entry.actorSha256, + analyzerSha256: entry.analyzerSha256, + expectedProvenanceSha256: entry.expectedProvenanceSha256, + canarySha256: entry.canarySha256 ?? null, + }); + if (entry.decisionInputSha256 !== expectedDecisionInput) { + return `the qualification decision input digest for ${version} is invalid`; + } if (expectedCanarySha256 !== undefined) { if (entry.canarySha256 !== expectedCanarySha256) return `the qualification record for ${version} is not bound to the exact canary`; - const expectedDecisionInput = canonicalSha256("flow-decision-input-v1", { - reportSha256: entry.reportSha256, - artifactSha256: entry.artifactSha256, - evaluatorSha256: entry.evaluatorSha256, - catalogSha256: entry.catalogSha256, - policySha256: entry.policySha256, - actorSha256: entry.actorSha256, - analyzerSha256: entry.analyzerSha256, - expectedProvenanceSha256: entry.expectedProvenanceSha256, - canarySha256: entry.canarySha256, - }); - if (entry.decisionInputSha256 !== expectedDecisionInput) - return `the qualification decision input does not bind the exact canary for ${version}`; } return null; } diff --git a/tests/eval-release-sampling.test.ts b/tests/eval-release-sampling.test.ts index 88311e8..9eef42a 100644 --- a/tests/eval-release-sampling.test.ts +++ b/tests/eval-release-sampling.test.ts @@ -1,5 +1,11 @@ import { describe, expect, test } from "bun:test"; -import { RELEASE_CASE_SAMPLING } from "../evals/release-policy.js"; +import { + assertExactReleaseCatalog, + assertReleaseHost, + releaseAttemptsFor, + releaseCaseIds, + releaseCatalog, +} from "../evals/release-policy.js"; import { campaignPlanFor, caseCatalogFor, @@ -11,32 +17,48 @@ import { SCENARIOS } from "../evals/scenarios.js"; describe("release eval sampling", () => { test("gives 90 percent cases ten attempts and 100 percent cases three", () => { - for (const policy of Object.values(RELEASE_CASE_SAMPLING)) { - expect(policy.attemptsPerModel).toBe(policy.minPassRate === 0.9 ? 10 : 3); + for (const policy of releaseCatalog()) { + expect(releaseAttemptsFor(policy.caseId)).toBe( + policy.minPassRate === 0.9 ? 10 : 3, + ); } }); - test("freezes mixed release counts without changing ordinary repeats", () => { + test("rejects any persisted catalog drift from repository policy", () => { + const canonical = releaseCatalog(); + const mutations: unknown[] = [ + canonical.slice(1), + [...canonical].reverse(), + canonical.map((row, index) => + index === 0 ? { ...row, minPassRate: 0.9 } : row, + ), + canonical.map((row, index) => + index === 0 ? { ...row, minProviders: 1 } : row, + ), + canonical.map((row, index) => + index === 0 ? { ...row, minScoredAttempts: 1 } : row, + ), + ]; + for (const mutation of mutations) { + expect(() => assertExactReleaseCatalog(mutation)).toThrow( + "does not match repository release policy", + ); + } + expect(assertExactReleaseCatalog(canonical)).toEqual(canonical); + }); + + test("rejects narrowed release plans without changing ordinary repeats", () => { const scenarios = SCENARIOS.filter((scenario) => ["happy-path", "unprovable-claim-refused"].includes(scenario.id), ); - const release = campaignPlanFor({ - models: ["xai/grok-4.6"], - scenarios, - sampling: { kind: "release" }, - opencodeVersion: "1.18.6", - }); - expect(release.cells).toHaveLength(13); - expect( - release.cells.filter((cell) => cell.caseId === "happy-path"), - ).toHaveLength(3); - expect( - release.cells.filter( - (cell) => cell.caseId === "unprovable-claim-refused", - ), - ).toHaveLength(10); - expect(release.stoppingRule.count).toBe(13); - expect(release.budget.maxAttempts).toBe(13); + expect(() => + campaignPlanFor({ + models: ["xai/grok-4.6"], + scenarios, + sampling: { kind: "release" }, + opencodeVersion: "1.18.6", + }), + ).toThrow("Release scenarios do not match repository release policy"); const ordinarySampling: EvalSampling = { kind: "ordinary", repeat: 2 }; const ordinary = campaignPlanFor({ @@ -53,31 +75,31 @@ describe("release eval sampling", () => { "sha256:647ef4f649e660cae94154da571c2508a1a4d0a1ad8171cc9ba654c6bfa8b0bf", ); expect(ordinary.planSha256).toBe( - "sha256:b1b9aca59f2c74f6a74b86e42924183d320fdd929225a11d7f25fa00ffcb101c", + "sha256:59ddde9655ac5aca872104603e8674504046db7de88e5f16f819a53af19532d5", ); }); - test("schedules exactly the required 70-cell two-provider release", () => { + test("schedules exactly the required 76-cell two-provider release", () => { const scenarios = releaseScenarios(); expect(scenarios.map((scenario) => scenario.id).sort()).toEqual( - Object.keys(RELEASE_CASE_SAMPLING).sort(), + [...releaseCaseIds()].sort(), ); + expect(releaseCaseIds()).toContain("skipped-case-named-binding"); const plan = campaignPlanFor({ models: ["xai/grok-4.6", "openai/gpt-5.6-sol"], scenarios, sampling: { kind: "release" }, opencodeVersion: "1.18.6", }); - expect(plan.cells).toHaveLength(70); - expect(plan.stoppingRule.count).toBe(70); - expect(plan.budget.maxAttempts).toBe(70); - for (const policy of caseCatalogFor(scenarios)) { - expect(policy.minScoredAttempts).toBe( - RELEASE_CASE_SAMPLING[ - policy.caseId as keyof typeof RELEASE_CASE_SAMPLING - ].attemptsPerModel, - ); - } + expect(plan.cells).toHaveLength(76); + expect(plan.stoppingRule.count).toBe(76); + expect(plan.budget.maxAttempts).toBe(76); + expect(caseCatalogFor(scenarios, { kind: "release" })).toEqual( + releaseCatalog(), + ); + expect( + plan.cells.filter((cell) => cell.caseId === "skipped-case-named-binding"), + ).toHaveLength(6); const jobs = jobsFor(["xai/grok-4.6", "openai/gpt-5.6-sol"], scenarios, { kind: "release", }).flat(); @@ -88,6 +110,36 @@ describe("release eval sampling", () => { ); }); + test("keeps ordinary scenario catalogs report-only", () => { + const scenarios = SCENARIOS.filter((scenario) => + ["happy-path", "skipped-case-named-binding"].includes(scenario.id), + ); + for (const policy of caseCatalogFor(scenarios, { + kind: "ordinary", + repeat: 2, + })) { + expect(policy.release).toBe("report-only"); + expect(policy.minScoredAttempts).toBe(1); + expect(policy.minPassRate).toBeNull(); + } + }); + + test("pins release host and rejects every runtime override", () => { + expect(() => assertReleaseHost({ platform: "darwin" })).toThrow( + "canonical Linux host", + ); + for (const override of [ + { opencodeOverride: "1.18.7" }, + { reviewerModelOverride: "xai/other" }, + { reviewerStepsOverride: "20" }, + ]) { + expect(() => + assertReleaseHost({ platform: "linux", ...override }), + ).toThrow("no OpenCode or reviewer overrides"); + } + expect(() => assertReleaseHost({ platform: "linux" })).not.toThrow(); + }); + test("rejects release sampling overrides", async () => { for (const override of [ ["--repeat", "3"], @@ -120,28 +172,32 @@ describe("release eval sampling", () => { } }); - test("rejects a release with fewer than two route providers", async () => { - const child = Bun.spawn( - [ - "bun", - "run", - "evals/run.ts", - "--model", - "xai/not-a-real-model-a", - "--model", - "xai/not-a-real-model-b", - "--release", - ], - { cwd: new URL("..", import.meta.url).pathname, stderr: "pipe" }, - ); - const [exitCode, stderr] = await Promise.all([ - child.exited, - new Response(child.stderr).text(), - ]); - expect(exitCode).toBe(2); - expect(stderr).toContain( - "--release requires at least 2 distinct route providers", - ); + test("rejects every release model grid except two distinct providers", async () => { + for (const models of [ + ["xai/a"], + ["xai/a", "xai/b"], + ["xai/a", "openai/b", "anthropic/c"], + ["xai/a", "openai/b", "xai/a"], + ]) { + const child = Bun.spawn( + [ + "bun", + "run", + "evals/run.ts", + ...models.flatMap((model) => ["--model", model]), + "--release", + ], + { cwd: new URL("..", import.meta.url).pathname, stderr: "pipe" }, + ); + const [exitCode, stderr] = await Promise.all([ + child.exited, + new Response(child.stderr).text(), + ]); + expect(exitCode).toBe(2); + expect(stderr).toContain( + "--release requires exactly 2 models on distinct route providers", + ); + } }); test("does not consume a flag as a model value", async () => { diff --git a/tests/release-metadata.test.ts b/tests/release-metadata.test.ts index b1b2dad..7b79c72 100644 --- a/tests/release-metadata.test.ts +++ b/tests/release-metadata.test.ts @@ -4,6 +4,14 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { canonicalSha256 } from "../evals/canonical-json.js"; +import { evaluatorIdentity } from "../evals/provenance.js"; +import { + RELEASE_POLICY_SHA256, + releaseCatalog, + releaseGraderBundle, + releaseScenarioCatalog, +} from "../evals/release-policy.js"; +import { SCENARIOS } from "../evals/scenarios.js"; import type { CanaryRecord } from "../scripts/eval-canary.js"; import { artifactIdentitySha256, @@ -49,25 +57,50 @@ const artifact = (packageVersion: string) => ({ tarballSha256: digest("b"), unpackedManifestSha256: digest("c"), }); -const decisionRecord = (packageVersion: string, verdict = "VERIFIED") => ({ - schemaVersion: 1, - reportId: "report", - verdict, - artifact: artifact(packageVersion), - reportSha256: digest("d"), - artifactSha256: canonicalSha256( - "flow-decision-artifact-v1", - artifact(packageVersion), - ), - evaluatorSha256: digest("f"), - catalogSha256: digest("9"), - policySha256: digest("0"), - actorSha256: digest("1"), - analyzerSha256: digest("2"), - expectedProvenanceSha256: digest("3"), - decisionInputSha256: digest("4"), - canarySha256: null, -}); +const decisionRecord = (packageVersion: string, verdict = "VERIFIED") => { + const measuredArtifact = artifact(packageVersion); + const evaluator = evaluatorIdentity({ + sourceCommit: measuredArtifact.sourceCommit, + caseCatalog: releaseScenarioCatalog(SCENARIOS), + policyCatalog: releaseCatalog(), + graderBundle: releaseGraderBundle(join(import.meta.dir, "..")), + }); + const base = { + schemaVersion: 1, + reportId: "report", + verdict, + artifact: measuredArtifact, + reportSha256: digest("d"), + artifactSha256: canonicalSha256( + "flow-decision-artifact-v1", + measuredArtifact, + ), + evaluatorSha256: canonicalSha256("flow-decision-evaluator-v1", evaluator), + catalogSha256: canonicalSha256( + "flow-decision-catalog-v1", + releaseCatalog(), + ), + policySha256: RELEASE_POLICY_SHA256, + actorSha256: digest("1"), + analyzerSha256: digest("2"), + expectedProvenanceSha256: digest("3"), + canarySha256: null, + }; + return { + ...base, + decisionInputSha256: canonicalSha256("flow-decision-input-v1", { + reportSha256: base.reportSha256, + artifactSha256: base.artifactSha256, + evaluatorSha256: base.evaluatorSha256, + catalogSha256: base.catalogSha256, + policySha256: base.policySha256, + actorSha256: base.actorSha256, + analyzerSha256: base.analyzerSha256, + expectedProvenanceSha256: base.expectedProvenanceSha256, + canarySha256: null, + }), + }; +}; function canaryBoundDecision(packageVersion: string, canarySha256: string) { const base = decisionRecord(packageVersion); return { @@ -248,6 +281,24 @@ describe("release metadata", () => { artifactSha256: digest("e"), }), ).toMatch(/artifact digest/); + for (const field of [ + "catalogSha256", + "policySha256", + "evaluatorSha256", + ] as const) { + expect( + qualificationRecordIssue("8.0.0", { + ...decisionRecord("8.0.0"), + [field]: digest("e"), + }), + ).toMatch(/not current repository/); + } + expect( + qualificationRecordIssue("8.0.0", { + ...decisionRecord("8.0.0"), + decisionInputSha256: digest("e"), + }), + ).toMatch(/decision input digest/); expect( qualificationRecordIssue("8.0.0", decisionRecord("8.0.0"), { ...artifact("8.0.0"), diff --git a/tests/release-qualification.test.ts b/tests/release-qualification.test.ts index 9aa59c0..3c50d6b 100644 --- a/tests/release-qualification.test.ts +++ b/tests/release-qualification.test.ts @@ -1,712 +1,380 @@ -import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { evaluatorIdentity } from "../evals/provenance.js"; import { - RELEASE_CASE_SAMPLING, - RELEASE_PASS_RATES, + RELEASE_POLICY_CATALOG_SHA256, + releaseCaseCatalogSha256, + releaseCatalog, + releaseGraderBundle, + releaseHostConfigSha256, + releaseScenarioCatalog, } from "../evals/release-policy.js"; import { campaignPlanSha256 } from "../evals/report.js"; +import { campaignPlanFor, releaseScenarios } from "../evals/run.js"; import { SCENARIOS } from "../evals/scenarios.js"; import { decisionRecordFor, - mergeReports, - providers, - qualificationFailures, qualifyV2, - writeQualificationRecord, + writeDecisionRecord, } from "../scripts/qualify-release.js"; -// Producing a real report costs credentials and money, so what is proven here is -// the decision made *from* one: which reports qualify a release and which do not. -// Three of these encode failures that a by-eye reading of a report actually let -// through -- one provider, a scenario nothing scored, and a completed closure the -// document contradicted. - -/** - * Every scenario the published thresholds gate. - * - * A qualifying report has to contain all of them, so the default report here is a - * full-suite one; a test that narrows `scenarios` is describing a partial run. - */ -const GATED = [ - "happy-path", - "plan-only-stops", - "goal-change-refused", - "continuation-accepted", - "failing-gate-blocks", - "resumes-after-interruption", - "unprovable-claim-refused", -]; - const digest = (letter: string) => `sha256:${letter.repeat(64)}`; - -const V2_CATALOG = [ - { - caseId: "v2-case", - caseVersion: 1, - evidenceClass: "conformance", - oracle: "durable-state", - release: "required", - minProviders: 1, - minScoredAttempts: 1, - minPassRate: 1, - reviewerPromotionRecordSha256: null, - }, -]; - -const V2_ARTIFACT = { - packageVersion: "8.1.1", - sourceCommit: "commit", +const MODELS = ["openai/gpt-test", "xai/grok-test"]; +const ARTIFACT = { + packageVersion: "8.1.2", + sourceCommit: "measured-commit", sourceTreeSha256: digest("a"), tarballSha256: digest("b"), unpackedManifestSha256: digest("c"), }; -function v2Report(stopped = false) { - const model = { - routeProvider: "openai", - gateway: null, - family: "gpt", - model: "test", - revision: null, - }; - const plan = { - schemaVersion: 1 as const, - planId: "v2-plan", - planSha256: digest("d"), - randomizationSeed: "seed", - cells: [ - { - cellId: "cell", - blockId: "block", - caseId: "v2-case", - caseVersion: 1, - armToken: null, - repetition: 0, - managerModel: model, - reviewerModel: null, - schedule: "primary" as const, - }, - ], - abortPolicy: { retry: "never" as const, maxReplacementBlocks: 0 }, - stoppingRule: { kind: "fixed-attempts" as const, count: 1 }, - analysis: { - kind: "rate" as const, - primaryOutcome: "pass", - versionSha256: digest("e"), - }, - budget: { - maxUsd: 1, - unknownCostPolicy: "stop" as const, - maxOutputTokens: 10, - maxWallClockMs: 10_000, - maxAttempts: 1, - }, - }; - plan.planSha256 = campaignPlanSha256(plan); +function releaseReport(stopped = false) { + const scenarios = releaseScenarios(); + const plan = campaignPlanFor({ + models: MODELS, + scenarios, + sampling: { kind: "release" }, + opencodeVersion: "1.18.6", + }); + const evaluator = evaluatorIdentity({ + sourceCommit: ARTIFACT.sourceCommit, + caseCatalog: releaseScenarioCatalog(scenarios), + policyCatalog: releaseCatalog(), + graderBundle: releaseGraderBundle(join(import.meta.dir, "..")), + }); + const attempts = plan.cells.map((cell, index) => { + const failed = stopped && index === 0; + const model = cell.managerModel; + if (!model) throw new Error("Release fixture requires a manager model."); + return { + schemaVersion: 2 as const, + attemptId: `attempt-${cell.cellId}`, + cellId: cell.cellId, + blockId: cell.blockId, + caseId: cell.caseId, + caseVersion: cell.caseVersion, + armToken: null, + repetition: cell.repetition, + artifact: ARTIFACT, + evaluator, + hostConfigSha256: releaseHostConfigSha256({ + packageVersion: ARTIFACT.packageVersion, + model, + }), + actors: failed + ? [] + : [ + { + role: "manager" as const, + requestedModel: model, + actualModel: { kind: "observed" as const, value: model }, + sessionIds: [`session-${index}`], + }, + ], + instructions: failed + ? [] + : [ + { + source: "command" as const, + name: "flow-auto", + sequence: 0, + sha256: digest("e"), + bytes: 1, + }, + ], + transcript: failed + ? null + : { + sha256: digest("f"), + artifact: `transcripts/${index}.json`, + }, + outcome: failed + ? { + kind: "failure" as const, + origin: "host" as const, + code: "host-down", + retryable: true, + } + : { + kind: "product" as const, + passed: true, + endedBy: "quiet" as const, + issues: [], + evidence: { + kind: "conformance" as const, + falseCompletion: false, + unsubmittedReviews: 0, + facts: { fixture: true }, + }, + }, + usage: { durationMs: 1, outputTokens: 1, costUsd: 0 }, + }; + }); return { schemaVersion: 2 as const, - reportId: stopped ? "v2-stopped" : "v2-verified", + reportId: stopped ? "release-stopped" : "release-verified", plan, - attempts: [ - { - schemaVersion: 2 as const, - attemptId: "attempt", - cellId: "cell", - blockId: "block", - caseId: "v2-case", - caseVersion: 1, - armToken: null, - repetition: 0, - artifact: V2_ARTIFACT, - evaluator: { - sourceCommit: "evaluator", - caseCatalogSha256: digest("f"), - policyCatalogSha256: digest("0"), - graderBundleSha256: digest("1"), - }, - hostConfigSha256: digest("2"), - actors: stopped - ? [] - : [ - { - role: "manager" as const, - requestedModel: model, - actualModel: { kind: "observed" as const, value: model }, - sessionIds: ["session"], - }, - ], - instructions: stopped - ? [] - : [ - { - source: "command" as const, - name: "eval", - sequence: 0, - sha256: digest("3"), - bytes: 1, - }, - ], - transcript: stopped - ? null - : { sha256: digest("4"), artifact: "attempt.json" }, - outcome: stopped - ? { - kind: "failure" as const, - origin: "host" as const, - code: "down", - retryable: true, - } - : { - kind: "product" as const, - passed: true, - endedBy: "quiet" as const, - issues: [], - evidence: { - kind: "conformance" as const, - falseCompletion: false, - unsubmittedReviews: 0, - facts: {}, - }, - }, - usage: { durationMs: 1, outputTokens: 1, costUsd: 0 }, - }, - ], + attempts, completion: { status: stopped ? ("stopped" as const) : ("complete" as const), cause: stopped ? ("host" as const) : ("fixed-target" as const), - startedAt: "2026-08-25T00:00:00.000Z", - finishedAt: "2026-08-25T00:00:01.000Z", + startedAt: "2026-08-28T00:00:00.000Z", + finishedAt: "2026-08-28T00:01:00.000Z", activatedReserveCellIds: [], observed: { - attempts: 1, - outputTokens: 1, + attempts: attempts.length, + outputTokens: attempts.length, costUsd: 0, - wallClockMs: 1_000, + wallClockMs: 60_000, }, }, allocationCommitmentSha256: null, }; } -function report(overrides: { - models?: string[]; - scenarios?: string[]; - rates?: Record< - string, - { - passed: number; - attempts: number; - unscored: number; - aborted?: number; - } - >; - falseCompletions?: number; - unsubmitted?: number; - recordedAt?: string; - flowVersion?: string; -}) { - const models = overrides.models ?? [ - "anthropic/claude-opus-5", - "openai/gpt-5.6", - ]; - const scenarios = overrides.scenarios ?? GATED; - return { - flowVersion: overrides.flowVersion ?? "7.0.2", - opencodeVersion: "1.18.6", - recordedAt: overrides.recordedAt ?? "2026-07-28T03:20:43.936Z", - summary: { - passRates: - overrides.rates ?? - Object.fromEntries( - scenarios.flatMap((scenario) => - models.map((model) => [ - `${scenario} @ ${model}`, - { - passed: - RELEASE_CASE_SAMPLING[ - scenario as keyof typeof RELEASE_CASE_SAMPLING - ]?.attemptsPerModel ?? 3, - attempts: - RELEASE_CASE_SAMPLING[ - scenario as keyof typeof RELEASE_CASE_SAMPLING - ]?.attemptsPerModel ?? 3, - unscored: 0, - }, - ]), - ), - ), - falseCompletions: overrides.falseCompletions ?? 0, - closedCompleted: 6, - reviewer: { assignments: 6, unsubmitted: overrides.unsubmitted ?? 0 }, - }, - results: scenarios.flatMap((scenario) => - models.map((model) => ({ scenario, model })), - ), - }; -} - -describe("release qualification", () => { - test("shares the v2 required-case policy with the live runner", () => { - expect(Object.keys(RELEASE_PASS_RATES).sort()).toEqual([...GATED].sort()); - }); - test("qualifies a clean two-provider report", () => { - expect(qualificationFailures(report({}))).toEqual([]); - }); - - test("refuses a single-provider report", () => { - // Every recorded report before the matrix existed was this shape, and read as - // evidence about Flow rather than about one provider. - const failures = qualificationFailures( - report({ models: ["anthropic/claude-opus-5"] }), - ); - expect(failures[0]).toContain("at least 2"); - // And once per gated scenario, because the report-wide count is not the claim: - // each gated guarantee is the thing that has to have held for two vendors. - expect( - failures.filter((failure) => failure.includes("was measured by")), - ).toHaveLength(GATED.length); - }); +describe("repository-owned v2 qualification", () => { + test("derives all verdicts from the canonical 76-cell policy", () => { + const verified = qualifyV2({ + reportInput: releaseReport(), + catalogInput: releaseCatalog(), + artifact: ARTIFACT, + }); + expect(verified.decision.verdict).toBe("VERIFIED"); + expect(verified.report.plan.cells).toHaveLength(76); + expect(decisionRecordFor(verified)).toEqual(decisionRecordFor(verified)); - test("splits a gateway model id on its first slash only", () => { - expect(providers(["openrouter/openai/gpt-5.6", "anthropic/x"])).toEqual([ - "openrouter", - "anthropic", - ]); - }); + const notVerified = qualifyV2({ + reportInput: releaseReport(), + catalogInput: releaseCatalog(), + artifact: { ...ARTIFACT, tarballSha256: digest("9") }, + }); + expect(notVerified.decision.verdict).toBe("NOT VERIFIED"); - test("names no provider for a model id that names none", () => { - // Truncating at a missing slash invented one provider per bare id, so two bare - // names cleared the two-provider threshold everything else here is qualified - // by. An unattributable id is dropped, and dropping both leaves zero. - expect(providers(["gpt-5", "claude-opus-5"])).toEqual([]); - expect(providers(["/leading", "anthropic/x"])).toEqual(["anthropic"]); - const failures = qualificationFailures( - report({ models: ["gpt-5", "claude-opus-5"] }), - ); - expect(failures.join()).toContain("exercised 0 provider(s)"); + const inconclusive = qualifyV2({ + reportInput: releaseReport(true), + catalogInput: releaseCatalog(), + artifact: ARTIFACT, + }); + expect(inconclusive.decision.verdict).toBe("INCONCLUSIVE"); }); - test("refuses a report that never ran a gated scenario", () => { - // The runner takes `--scenario` and `bun run qualify` reads the newest report, - // so a one-scenario debug run was the newest report and qualified a release - // with every other gated scenario never run. Absence had no row to fail on. - const failures = qualificationFailures( - report({ scenarios: ["happy-path"] }), - ); - expect(failures).toHaveLength(GATED.length - 1); - for (const failure of failures) { - expect(failure).toContain("does not contain it"); + test("rejects every caller attempt to weaken or reorder policy", () => { + const canonical = releaseCatalog(); + const mutations: unknown[] = [ + canonical.slice(1), + [...canonical].reverse(), + canonical.map((row, index) => + index === 0 ? { ...row, release: "report-only" } : row, + ), + canonical.map((row, index) => + index === 0 ? { ...row, minPassRate: 0.9 } : row, + ), + canonical.map((row, index) => + index === 0 ? { ...row, minProviders: 1 } : row, + ), + ]; + for (const catalogInput of mutations) { + expect(() => + qualifyV2({ + reportInput: releaseReport(), + catalogInput, + artifact: ARTIFACT, + }), + ).toThrow("does not match repository release policy"); } }); - test("refuses any false completion", () => { - expect( - qualificationFailures(report({ falseCompletions: 1 })).join(), - ).toContain("1 false completion"); - }); - - test("refuses a report with an unsubmitted review assignment", () => { - // Held back from the gate until a report showed what zero looks like: 54 runs - // across three providers submitted every one of 22 assignments, including runs - // that stopped to ask and runs that stopped at an unpassable blocker. - const failures = qualificationFailures(report({ unsubmitted: 2 })); - expect(failures).toHaveLength(1); - expect(failures[0]).toContain("never submitted"); + test("rejects every self-consistent but noncanonical evaluator identity", () => { + for (const field of [ + "caseCatalogSha256", + "policyCatalogSha256", + "graderBundleSha256", + ] as const) { + const report = releaseReport(); + for (const attempt of report.attempts) { + attempt.evaluator = { ...attempt.evaluator, [field]: digest("9") }; + } + expect(() => + qualifyV2({ + reportInput: report, + catalogInput: releaseCatalog(), + artifact: ARTIFACT, + }), + ).toThrow("does not match repository release authority"); + } }); - test("refuses a gated rate scored on fewer attempts than the floor", () => { - // Only the numerator was ever checked. An excluded attempt shrank a measured - // pair's denominator to 2, and the pair cleared a 100% threshold on the two - // that remained -- while the excluded attempt was the one that behaved. - const failures = qualificationFailures( - report({ - rates: { - "happy-path @ anthropic/claude-opus-5": { - passed: 2, - attempts: 2, - unscored: 1, - }, - "happy-path @ openai/gpt-5.6": { - passed: 3, - attempts: 3, - unscored: 0, - }, - }, + test("rejects caller-defined release host configuration", () => { + const report = releaseReport(); + for (const attempt of report.attempts) { + attempt.hostConfigSha256 = digest("9"); + } + expect(() => + qualifyV2({ + reportInput: report, + catalogInput: releaseCatalog(), + artifact: ARTIFACT, }), - ); - expect(failures).toHaveLength(1); - expect(failures[0]).toContain("only 2 attempt(s) scored"); + ).toThrow("host configuration does not match repository release policy"); }); - test("refuses a pair whose attempt aborted mid-flight", () => { - // The measured defect this closes from both ends: a wedged attempt used to be - // scored as a failure, which produced the only NOT QUALIFIED line in a report - // on a guarantee that never ran. Excluding it silently would have inverted the - // error -- the pair would read as measured on what survived -- so the exclusion - // is itself a qualification failure until the pair is re-run. - const failures = qualificationFailures( - report({ - rates: { - "failing-gate-blocks @ anthropic/claude-opus-5": { - passed: 2, - attempts: 2, - unscored: 0, - aborted: 1, - }, - "failing-gate-blocks @ openai/gpt-5.6": { - passed: 10, - attempts: 10, - unscored: 0, - aborted: 0, - }, - }, - }), + test("runner evidence carries independently reconstructed authority hashes", () => { + const evaluator = releaseReport().attempts[0]?.evaluator; + const graderFiles = releaseGraderBundle( + join(import.meta.dir, ".."), + ).files.map((file) => file.path); + expect(graderFiles).toEqual( + expect.arrayContaining([ + "evals/catalog.ts", + "evals/failure-origin.ts", + "evals/provenance.ts", + "evals/report-store.ts", + "evals/report.ts", + "evals/run.ts", + "scripts/qualify-release.ts", + ]), ); - expect(failures).toHaveLength(1); - expect(failures[0]).toContain("aborted mid-flight"); - }); - - test("refuses a scenario nothing scored", () => { - const failures = qualificationFailures( - report({ - rates: { - "happy-path @ anthropic/claude-opus-5": { - passed: 0, - attempts: 0, - unscored: 3, - }, - "happy-path @ openai/gpt-5.6": { - passed: 3, - attempts: 3, - unscored: 0, - }, - }, - }), + expect(evaluator?.policyCatalogSha256).toBe(RELEASE_POLICY_CATALOG_SHA256); + expect(evaluator?.caseCatalogSha256).toBe( + releaseCaseCatalogSha256(SCENARIOS), ); - expect(failures.join()).toContain("unmeasured"); }); - test("holds each scenario to its own published rate", () => { - // The gate scenario's threshold is below 1.0 on measured evidence; the others - // are not, and one failure of them is a release blocker. - expect( - qualificationFailures( - report({ - rates: { - "failing-gate-blocks @ anthropic/claude-opus-5": { - passed: 9, - attempts: 10, - unscored: 0, - }, - "failing-gate-blocks @ openai/gpt-5.6": { - passed: 10, - attempts: 10, - unscored: 0, - }, - }, - }), - ), - ).toEqual([]); - expect( - qualificationFailures( - report({ - rates: { - "happy-path @ anthropic/claude-opus-5": { - passed: 9, - attempts: 10, - unscored: 0, - }, - "happy-path @ openai/gpt-5.6": { - passed: 10, - attempts: 10, - unscored: 0, - }, - }, - }), - ).join(), - ).toContain("below the published 100%"); + test("grader authority changes with any transitive evaluator source", async () => { + const root = await mkdtemp(join(tmpdir(), "flow-grader-bundle-")); + try { + await mkdir(join(root, "evals"), { recursive: true }); + await mkdir(join(root, "scripts"), { recursive: true }); + await writeFile( + join(root, "evals", "run.ts"), + 'import { grade } from "./grade.js"; grade();', + ); + await writeFile( + join(root, "scripts", "qualify-release.ts"), + "export {};\n", + ); + await writeFile( + join(root, "evals", "grade.ts"), + "export const grade = () => 1;\n", + ); + const before = releaseGraderBundle(root); + await writeFile( + join(root, "evals", "grade.ts"), + "export const grade = () => 2;\n", + ); + const after = releaseGraderBundle(root); + expect(after).not.toEqual(before); + expect(after.files.map((file) => file.path)).toContain("evals/grade.ts"); + } finally { + await rm(root, { recursive: true, force: true }); + } }); - test("requires a published threshold for every scenario the suite ships", () => { - // A new scenario has no honest threshold until it has a baseline, so it must be - // entered as ungated rather than silently scoring nothing. - const failures = qualificationFailures( - report({ scenarios: SCENARIOS.map((scenario) => scenario.id) }), + test("rejects the old 70-cell plan even after its hash is recomputed", () => { + const report = releaseReport(); + report.plan.cells = report.plan.cells.filter( + (cell) => cell.caseId !== "skipped-case-named-binding", ); - expect( - failures.filter((failure) => failure.includes("no published threshold")), - ).toEqual([]); + report.attempts = report.attempts.filter( + (attempt) => attempt.caseId !== "skipped-case-named-binding", + ); + report.plan.stoppingRule.count = report.plan.cells.length; + report.plan.budget.maxAttempts = report.plan.cells.length; + report.plan.planSha256 = campaignPlanSha256(report.plan); + report.completion.observed.attempts = report.attempts.length; + report.completion.observed.outputTokens = report.attempts.length; + expect(() => + qualifyV2({ + reportInput: report, + catalogInput: releaseCatalog(), + artifact: ARTIFACT, + }), + ).toThrow("canonical 76-cell grid"); }); -}); -// Merging exists because re-measuring one wedged pair cost a whole matrix, and a gate -// that expensive to satisfy gets argued with instead. These pin the direction of the -// asymmetry: a merge may make qualification harder, never easier. -describe("merging a re-run into a matrix", () => { - const WEDGED = { - "failing-gate-blocks @ anthropic/claude-opus-5": { - passed: 2, - attempts: 2, - unscored: 0, - aborted: 1, - }, - }; - const RERUN = { - "failing-gate-blocks @ anthropic/claude-opus-5": { - passed: 10, - attempts: 10, - unscored: 0, - aborted: 0, - }, - }; - - function base(rates: typeof WEDGED = WEDGED) { - const full = report({}); - return { - ...full, - summary: { - ...full.summary, - passRates: { ...full.summary.passRates, ...rates }, + test("rejects self-consistent release-control drift", () => { + for (const mutate of [ + (report: ReturnType) => { + if (report.plan.analysis.kind !== "rate") { + throw new Error("Release fixture requires rate analysis."); + } + report.plan.analysis.primaryOutcome = "caller-selected"; }, - }; - } - - function rerun(overrides: Parameters[0] = {}) { - return { - ...report({ - scenarios: ["failing-gate-blocks"], - models: ["anthropic/claude-opus-5"], - recordedAt: "2026-07-28T09:00:00.000Z", - ...overrides, - }), - summary: { - passRates: RERUN, - falseCompletions: 0, - closedCompleted: 0, - reviewer: { assignments: 0, unsubmitted: 0 }, + (report: ReturnType) => { + report.plan.budget.maxOutputTokens += 1; }, - }; - } - - test("qualifies the matrix once the wedged pair is re-measured", () => { - expect(qualificationFailures(base()).join()).toContain( - "aborted mid-flight", - ); - const merged = mergeReports([base(), rerun()]); - expect(merged.failures).toEqual([]); - expect(qualificationFailures(merged.report)).toEqual([]); - expect(merged.notes.join()).toContain("superseded by 10/10"); - }); - - test("takes the newer measurement whichever order the reports are given", () => { - const forwards = mergeReports([base(), rerun()]).report; - const backwards = mergeReports([rerun(), base()]).report; - expect(backwards.summary?.passRates).toEqual(forwards.summary?.passRates); - expect(qualificationFailures(backwards)).toEqual([]); - }); - - test("keeps the coverage of the full run, so a re-run cannot narrow the claim", () => { - // The re-run alone is a one-scenario report and fails on every gated scenario it - // does not contain. Merged, it inherits the matrix's coverage and nothing else. - expect(qualificationFailures(rerun()).join()).toContain( - "does not contain it", - ); - const merged = mergeReports([base(), rerun()]).report; - expect((merged.results ?? []).map((result) => result.scenario)).toEqual( - expect.arrayContaining(GATED), - ); - }); - - test("refuses a merge that rests a gated scenario on one vendor", () => { - // The hole the merge opened: provider count came from the union of result rows, - // so a full matrix on one vendor plus another vendor's re-run of a single pair - // cleared the two-provider bar while every other gated guarantee rested on one. - const single = report({ models: ["anthropic/claude-opus-5"] }); - const merged = mergeReports([ - { - ...single, - summary: { - ...single.summary, - passRates: { ...single.summary.passRates, ...WEDGED }, - }, + (report: ReturnType) => { + report.plan.randomizationSeed = "caller-selected"; }, - rerun({ models: ["openai/gpt-5.6"] }), - ]); - expect(merged.failures).toEqual([]); - const failures = qualificationFailures(merged.report); - expect(failures.join()).toContain("single vendor"); - // The union still reads as two providers, which is exactly why the report-wide - // count could not be the check. - expect(failures.some((failure) => failure.includes(`at least ${2}`))).toBe( - false, - ); - }); - - test("refuses to merge a report from another build", () => { - const { failures } = mergeReports([ - base(), - rerun({ flowVersion: "7.1.0" }), - ]); - expect(failures.join()).toContain("one build"); - }); - - test("still counts a false completion from the superseded run", () => { - // The direction that matters. A pair's *rate* is replaced, but the failure Flow - // exists to prevent is summed, so re-running cannot launder one away. - const merged = mergeReports([ - { ...base(), summary: { ...base().summary, falseCompletions: 1 } }, - rerun(), - ]); - expect(qualificationFailures(merged.report).join()).toContain( - "false completion", - ); - }); -}); - -describe("qualification records", () => { - const temporary: string[] = []; - afterEach(async () => { - await Promise.all( - temporary - .splice(0) - .map((directory) => rm(directory, { recursive: true, force: true })), - ); - }); - - test("writes a QUALIFIED record for a major version", async () => { - const directory = await mkdtemp(join(tmpdir(), "flow-qualify-record-")); - temporary.push(directory); - const path = await writeQualificationRecord( - "9.0.0", - report({}), - ["evals/results/a.json"], - directory, - "7.0.2", - ); - const record = JSON.parse(await readFile(path, "utf8")); - expect(record).toMatchObject({ - version: "9.0.0", - verdict: "QUALIFIED", - flowVersion: "7.0.2", - opencodeVersion: "1.18.6", - reports: ["evals/results/a.json"], - providers: ["anthropic", "openai"], - }); - expect(typeof record.qualifiedAt).toBe("string"); - }); - - test("refuses to record a non-major version", async () => { - const directory = await mkdtemp(join(tmpdir(), "flow-qualify-record-")); - temporary.push(directory); - await expect( - writeQualificationRecord("9.1.0", report({}), [], directory, "7.0.2"), - ).rejects.toThrow(/major release/); - }); - - test("refuses a report that measured a different Flow build", async () => { - const directory = await mkdtemp(join(tmpdir(), "flow-qualify-record-")); - temporary.push(directory); - await expect( - writeQualificationRecord( - "9.0.0", - report({ flowVersion: "8.1.0" }), - [], - directory, - "8.1.1", - ), - ).rejects.toThrow(/8\.1\.0, not this repository's 8\.1\.1/); - }); - - test("refuses a report with no flowVersion", async () => { - const directory = await mkdtemp(join(tmpdir(), "flow-qualify-record-")); - temporary.push(directory); - const { flowVersion: _omitted, ...measured } = report({}); - await expect( - writeQualificationRecord("9.0.0", measured, [], directory, "8.1.1"), - ).rejects.toThrow(/requires the report's flowVersion/); - }); -}); - -describe("v2 qualification cutover", () => { - test("derives and records all three decision verdicts from explicit atomic inputs", () => { - const verified = qualifyV2({ - reportInput: v2Report(), - catalogInput: V2_CATALOG, - artifact: V2_ARTIFACT, - }); - expect(verified.decision.verdict).toBe("VERIFIED"); - const first = decisionRecordFor(verified); - expect(first.verdict).toBe("VERIFIED"); - expect(first).toEqual(decisionRecordFor(verified)); - expect(first).toMatchObject({ - reportSha256: expect.stringMatching(/^sha256:/), - artifactSha256: expect.stringMatching(/^sha256:/), - evaluatorSha256: expect.stringMatching(/^sha256:/), - policySha256: expect.stringMatching(/^sha256:/), - actorSha256: expect.stringMatching(/^sha256:/), - analyzerSha256: expect.stringMatching(/^sha256:/), - expectedProvenanceSha256: expect.stringMatching(/^sha256:/), - decisionInputSha256: expect.stringMatching(/^sha256:/), - canarySha256: null, - }); - const canaryBound = decisionRecordFor({ - ...verified, - canarySha256: digest("9"), - }); - expect(canaryBound.canarySha256).toBe(digest("9")); - expect(canaryBound.decisionInputSha256).not.toBe(first.decisionInputSha256); - - const notVerified = qualifyV2({ - reportInput: v2Report(), - catalogInput: V2_CATALOG, - artifact: { ...V2_ARTIFACT, tarballSha256: digest("9") }, - }); - expect(notVerified.decision.verdict).toBe("NOT VERIFIED"); - - const evidenceOnlyDrift = qualifyV2({ - reportInput: v2Report(), - catalogInput: V2_CATALOG, - artifact: { - ...V2_ARTIFACT, - sourceCommit: "tag-commit-after-evidence", - sourceTreeSha256: digest("8"), + (report: ReturnType) => { + const firstCell = report.plan.cells[0]; + const firstAttempt = report.attempts[0]; + if (!firstCell || !firstAttempt) { + throw new Error("Release fixture requires a first cell."); + } + report.plan.cells[0] = { + ...firstCell, + cellId: "cell-caller-selected", + }; + report.attempts[0] = { + ...firstAttempt, + cellId: "cell-caller-selected", + }; }, - }); - expect(evidenceOnlyDrift.decision.verdict).toBe("VERIFIED"); - expect(evidenceOnlyDrift.expected.artifact).toEqual(V2_ARTIFACT); - - const inconclusive = qualifyV2({ - reportInput: v2Report(true), - catalogInput: V2_CATALOG, - artifact: V2_ARTIFACT, - }); - expect(inconclusive.decision.verdict).toBe("INCONCLUSIVE"); + ]) { + const report = releaseReport(); + mutate(report); + report.plan.planSha256 = campaignPlanSha256(report.plan); + expect(() => + qualifyV2({ + reportInput: report, + catalogInput: releaseCatalog(), + artifact: ARTIFACT, + }), + ).toThrow(/Release plan/); + } }); - test("rejects legacy summary-only input rather than converting it", () => { + test("rejects legacy summary-only input", () => { expect(() => qualifyV2({ - reportInput: report({}), - catalogInput: V2_CATALOG, - artifact: V2_ARTIFACT, + reportInput: { summary: { passRates: {} } }, + catalogInput: releaseCatalog(), + artifact: ARTIFACT, }), ).toThrow("Invalid v2 report"); }); + test("writes decision records immutably", async () => { + const directory = await mkdtemp(join(tmpdir(), "flow-decision-")); + try { + const result = qualifyV2({ + reportInput: releaseReport(), + catalogInput: releaseCatalog(), + artifact: ARTIFACT, + }); + const record = decisionRecordFor(result); + const first = await writeDecisionRecord({ record, directory }); + const replay = await writeDecisionRecord({ record, directory }); + expect(replay).toBe(first); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + test("requires explicit report, catalog, and artifact paths in the CLI", async () => { const process = Bun.spawn(["bun", "run", "scripts/qualify-release.ts"], { cwd: new URL("..", import.meta.url).pathname, stdout: "pipe", stderr: "pipe", }); - const [stdout, stderr, exitCode] = await Promise.all([ - new Response(process.stdout).text(), + const [output, exitCode] = await Promise.all([ new Response(process.stderr).text(), process.exited, ]); expect(exitCode).not.toBe(0); - expect(`${stdout}${stderr}`).toContain("Usage: bun run qualify"); + expect(output).toContain("Usage: bun run qualify"); }); });