diff --git a/.agents/plans/03-assurance-hardening/decisions.tsv b/.agents/plans/03-assurance-hardening/decisions.tsv index 2fe5a70..1e3432b 100644 --- a/.agents/plans/03-assurance-hardening/decisions.tsv +++ b/.agents/plans/03-assurance-hardening/decisions.tsv @@ -38,3 +38,12 @@ ts phase decision why evidence result 2026-08-28T03:32:07Z phase-5 bound the complete release environment and grader closure first-pass review found duplicated cell construction, caller-defined host digests, and incomplete grader hashing releaseCellsFor; releaseHostConfigSha256; releaseGraderBundle import-graph scanner exact cell and block ids, seed, budgets, Linux OpenCode 1.18.6 host, and every transitive local evaluator file are authoritative 2026-08-28T03:32:07Z phase-5 removed legacy summary qualification authority the import-only summary path preserved a second executable policy surface scripts/qualify-release.ts; tests/release-qualification.test.ts V2 atomic report, exact persisted catalog, packed artifact, and canonical authority are the only qualifier inputs 2026-08-28T03:32:07Z phase-5 completed repository-owned release policy verification final review findings fixed stale authority digests, arbitrary decision-input hashes, extra model spend, and host-policy drift bun run check; bun run replay 577 pass, 1 intentional skip, 13 of 13 replays, 892 source bytes headroom +2026-08-28T04:01:46Z phase-5 merged repository-owned release policy the exact PR head passed independent shipping verification and every required check PR 52; merge cda1234 merged to main +2026-08-28T04:01:46Z phase-6 selected a minimal evidence-derived canary pipeline the independent architecture judge required exact preparation and installation binding, evidence-derived claims, deterministic actor pseudonyms, and a retained host artifact three architecture candidates; independent judge candidate A base with candidate B installation binding and candidate C evidence-first facts +2026-08-28T04:01:46Z phase-6 captured the committed canary contradiction before implementation the 8.1.2 record claimed every check passed while its retained close concluded completion-unsupported tests/eval-canary.test.ts; focused red typecheck missing derivation API failed red as expected +2026-08-28T04:01:46Z phase-6 derived canary claims from retained executable evidence caller status, checks, actors, and host identity were untrusted assertions; installation also needed exact byte proof scripts/eval-canary.ts; focused canary and release tests status and checks now rederive from measured tarball/plugin bytes plus sanitized session/transcript structure; future legacy records refused +2026-08-28T04:22:39Z phase-6 fixed all valid first-wave interrogate findings minimal record-shaped sessions, recursive transcript scans, path reads, and file-only installation proof could still admit synthetic evidence SessionSchema and assuranceProjection; OpenCode message/part parser; readWorkspaceTestReport; runtimeIdentity retained evidence is domain-valid, same-handle bounded, transcript-spoof resistant, and tied to the loaded plugin entry digest +2026-08-28T04:22:39Z phase-6 invalidated the caller-attested 8.1.2 canary grandfathering the known contradictory record would preserve the authority bypass this phase removes CanaryRecordSchema; scripts/release-metadata.ts; release metadata tests all accepted canaries now require derivation and installation evidence; the historical format is rejected +2026-08-28T04:22:39Z phase-6 bound reviewer independence through host lineage separate reviewer messages and task calls did not prove they represented the same child session and model task parentSessionId, sessionId, model metadata; negative mismatch test reviewer dispatch passes only when task lineage matches the observed manager and reviewer actors +2026-08-28T04:22:39Z phase-6 completed corrected whole-product verification the review fixes changed runtime status output, evidence parsing, and release metadata boundaries bun run check; bun run replay; pinned OpenCode live smoke 585 pass, 1 intentional skip, 13 of 13 replays, live OpenCode 1.18.6 pass, 339 source bytes headroom +2026-08-28T04:23:41Z phase-6 closed the final transcript and lineage review findings flat synthetic transcripts and unlinked reviewer observations remained broader than the real OpenCode evidence boundary root export messages plus tool parts; parent-child-model lineage match; final three-model review 586 pass, 1 intentional skip, 13 of 13 replays, pinned live smoke pass; all final reviewers green diff --git a/scripts/eval-canary.ts b/scripts/eval-canary.ts index 7708240..a5eae97 100644 --- a/scripts/eval-canary.ts +++ b/scripts/eval-canary.ts @@ -7,11 +7,10 @@ import { mkdir, open, readFile, - stat, unlink, writeFile, } from "node:fs/promises"; -import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import { z } from "zod"; import { canonicalJson, canonicalSha256 } from "../evals/canonical-json.js"; import { @@ -26,8 +25,14 @@ import { } from "../evals/provenance.js"; import type { ActorIdentity, ArtifactIdentity } from "../evals/report.js"; import { reportArtifactForCanary } from "../evals/report-artifact.js"; +import { assuranceProjection } from "../src/application/delivery.js"; +import { SessionSchema } from "../src/application/schema.js"; +import { MAX_TEST_REPORT_BYTES } from "../src/domain/limits.js"; +import { operationInputDigest } from "../src/domain/operation.js"; +import { readWorkspaceTestReport } from "../src/infrastructure/fs/workspace-validation.js"; export const CANARY_CHECKLIST_VERSION = "phase9-canary-v1"; +export const CANARY_DERIVATION_VERSION = "canary-evidence-v1"; export const CANARY_CHECK_IDS = [ "installs-packed-artifact", "loads-flow-tools", @@ -75,7 +80,10 @@ const ActorIdentitySchema = z .strict(); const RedactedActorIdentitySchema = ActorIdentitySchema.refine( (actor) => - actor.sessionIds.every((sessionId) => sessionId === ""), + actor.sessionIds.every( + (sessionId) => + sessionId === "" || /^id_[a-f0-9]{16}$/.test(sessionId), + ), "Canary actor session ids must be redacted.", ); const ArtifactIdentitySchema = z @@ -87,6 +95,16 @@ const ArtifactIdentitySchema = z unpackedManifestSha256: DigestSchema, }) .strict(); +const InstallationEvidenceSchema = z + .object({ + schemaVersion: z.literal(1), + preparedSha256: DigestSchema, + artifactSha256: DigestSchema, + tarballSha256: DigestSchema, + pluginEntrySha256: DigestSchema, + installedPluginSha256: DigestSchema, + }) + .strict(); const ChecksSchema = z .object({ "installs-packed-artifact": z.boolean(), @@ -97,6 +115,442 @@ const ChecksSchema = z "closes-with-delivery": z.boolean(), }) .strict(); + +type Checks = z.infer; + +function record(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function array(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function records(values: readonly unknown[]): Record[] { + return values.flatMap((value) => { + const entry = record(value); + return entry ? [entry] : []; + }); +} + +function parsedJson(value: unknown): unknown { + if (typeof value !== "string") return value; + try { + return JSON.parse(value); + } catch { + return value; + } +} + +type ObservedCall = Readonly<{ + tool: string; + status: string; + input: Record; + output: unknown; + metadata: Record; +}>; + +type TranscriptShape = Readonly<{ + supported: boolean; + entries: readonly Record[]; +}>; + +function transcriptShape(value: unknown): TranscriptShape { + const root = record(value); + if (!root || !Array.isArray(root.messages)) { + return { supported: false, entries: [] }; + } + const messages = records(root.messages); + return { + supported: + messages.length === root.messages.length && + messages.every( + (message) => record(message.info) && Array.isArray(message.parts), + ), + entries: [root, ...messages], + }; +} + +function observedCall(entry: Record): ObservedCall | null { + if (typeof entry.tool !== "string") return null; + const state = record(entry.state); + return { + tool: entry.tool, + status: + (typeof entry.status === "string" ? entry.status : null) ?? + (typeof state?.status === "string" ? state.status : "unknown"), + input: record(entry.input) ?? record(state?.input) ?? {}, + output: parsedJson( + entry.output ?? state?.output ?? entry.rawOutput ?? state?.error, + ), + metadata: record(entry.metadata) ?? record(state?.metadata) ?? {}, + }; +} + +function observedCalls( + entries: readonly Record[], +): ObservedCall[] { + return entries.flatMap((entry) => + records(array(entry.parts)).flatMap((part) => { + if (part.type !== "tool") return []; + const call = observedCall(part); + return call ? [call] : []; + }), + ); +} + +function completed(calls: readonly ObservedCall[], tool: string): boolean { + return calls.some( + (call) => + call.tool === tool && + (call.status === "completed" || call.status === "ok"), + ); +} + +function loadedPluginMatches( + calls: readonly ObservedCall[], + packageVersion: string, + pluginEntrySha256: string, +): boolean { + return calls.some((call) => { + if (call.tool !== "flow_status" || call.status !== "completed") + return false; + const output = record(call.output); + const workflowData = record(output?.workflowData); + const identity = record(workflowData?.runtimeIdentity); + return ( + identity?.packageVersion === packageVersion && + identity.pluginEntrySha256 === pluginEntrySha256 + ); + }); +} + +function completionSupportedFromDelivery( + calls: readonly ObservedCall[], + expected: ReturnType, +): boolean { + return calls.some((call) => { + if (call.tool !== "flow_session_close" || call.status !== "completed") { + return false; + } + const output = record(call.output); + const workflowData = record(output?.workflowData); + const delivery = record(workflowData?.delivery); + const assurance = record(delivery?.assurance); + const checks = array(assurance?.checks).map(record).filter(Boolean); + return ( + assurance?.conclusion === "completion-supported" && + expected.conclusion === "completion-supported" && + checks.length === expected.checks.length && + expected.checks.every((expectedCheck) => + checks.some( + (check) => + check?.id === expectedCheck.id && + check.status === expectedCheck.status, + ), + ) + ); + }); +} + +function modelIdentity(value: Record): { + readonly provider: string; + readonly model: string; +} | null { + const nested = record(value.model); + const provider = + typeof value.providerID === "string" + ? value.providerID + : typeof nested?.providerID === "string" + ? nested.providerID + : null; + const model = + typeof value.modelID === "string" + ? value.modelID + : typeof nested?.modelID === "string" + ? nested.modelID + : typeof nested?.id === "string" + ? nested.id + : null; + return provider && model ? { provider, model } : null; +} + +function derivedActors( + entries: readonly Record[], + calls: readonly ObservedCall[], +): { readonly actors: readonly ActorIdentity[]; readonly complete: boolean } { + const actors = new Map(); + const lineages: Array<{ + readonly parent: string; + readonly child: string; + readonly identity: { readonly provider: string; readonly model: string }; + }> = []; + let consistent = true; + const add = ( + role: "manager" | "reviewer", + identity: { readonly provider: string; readonly model: string }, + sessionId: string, + ): void => { + const model = { + routeProvider: identity.provider, + gateway: null, + family: identity.model, + model: identity.model, + revision: null, + }; + const prior = actors.get(role); + if (prior && canonicalJson(prior.requestedModel) !== canonicalJson(model)) { + consistent = false; + return; + } + actors.set(role, { + role, + requestedModel: model, + actualModel: { kind: "observed", value: model }, + sessionIds: [...new Set([...(prior?.sessionIds ?? []), sessionId])], + }); + }; + for (const entry of entries) { + const info = record(entry.info); + const identity = info ? modelIdentity(info) : null; + if (info?.role === "assistant" && identity) { + add( + info.agent === "flow-reviewer" ? "reviewer" : "manager", + identity, + typeof info.sessionID === "string" ? info.sessionID : "", + ); + } + } + for (const call of calls) { + if ( + call.tool !== "task" || + call.status !== "completed" || + call.input.subagent_type !== "flow-reviewer" + ) + continue; + const identity = modelIdentity(call.metadata); + const sessionId = call.metadata.sessionId; + const parentSessionId = call.metadata.parentSessionId; + if ( + identity && + typeof sessionId === "string" && + typeof parentSessionId === "string" + ) { + const observedReviewer = actors.get("reviewer"); + if ( + observedReviewer && + !observedReviewer.sessionIds.includes(sessionId) + ) { + consistent = false; + continue; + } + add("reviewer", identity, sessionId); + lineages.push({ parent: parentSessionId, child: sessionId, identity }); + } + } + const manager = actors.get("manager"); + const reviewer = actors.get("reviewer"); + const distinct = + manager !== undefined && + reviewer !== undefined && + manager.sessionIds.every((id) => !reviewer.sessionIds.includes(id)); + const linked = + manager !== undefined && + reviewer !== undefined && + lineages.some( + (lineage) => + manager.sessionIds.includes(lineage.parent) && + reviewer.sessionIds.includes(lineage.child) && + reviewer.requestedModel.routeProvider === lineage.identity.provider && + reviewer.requestedModel.model === lineage.identity.model, + ); + return { + actors: [...actors.values()], + complete: consistent && distinct && linked, + }; +} + +function observedHost(entries: readonly Record[]): { + readonly versions: readonly string[]; + readonly preparedFixture: boolean; +} { + const versions = new Set(); + let preparedFixture = false; + for (const entry of entries) { + const info = record(entry.info); + if (typeof info?.version === "string") versions.add(info.version); + if ( + info?.directory === "" || + info?.path === "" + ) { + preparedFixture = true; + } + } + return { versions: [...versions].sort(), preparedFixture }; +} + +function parsedSession( + value: unknown, +): + | { readonly ok: true; readonly value: z.infer } + | { readonly ok: false } { + const direct = SessionSchema.safeParse(value); + if (direct.success) return { ok: true, value: direct.data }; + const candidate = structuredClone(value); + const session = record(candidate); + const closure = record(session?.closure); + if (!session || !closure || !Array.isArray(session.operations)) { + return { ok: false }; + } + const operation = session.operations + .map(record) + .find((entry) => entry?.id === closure.operationId); + if ( + !operation || + typeof closure.operationId !== "string" || + typeof closure.recordedRevision !== "number" || + typeof session.id !== "string" || + (closure.kind !== "completed" && + closure.kind !== "deferred" && + closure.kind !== "abandoned") || + typeof closure.summary !== "string" + ) { + return { ok: false }; + } + operation.inputDigest = operationInputDigest({ + operationId: closure.operationId, + expectedRevision: closure.recordedRevision - 1, + sessionId: session.id, + kind: closure.kind, + summary: closure.summary, + }); + const repaired = SessionSchema.safeParse(session); + return repaired.success ? { ok: true, value: repaired.data } : { ok: false }; +} + +function assuranceSatisfied( + assurance: ReturnType, + id: string, +): boolean { + return assurance.checks.some( + (check) => check.id === id && check.status === "satisfied", + ); +} + +export function deriveCanaryResult(input: { + readonly packageVersion: string; + readonly artifactSha256: string; + readonly tarballSha256: string; + readonly preparedSha256: string; + readonly pluginEntrySha256: string; + readonly installation: unknown | null; + readonly session: unknown | null; + readonly transcript: unknown | null; +}): Readonly<{ + status: "passed" | "failed" | "incomplete"; + checks: Checks; + actors: readonly ActorIdentity[]; + hostConfigSha256: string; +}> { + const transcript = transcriptShape(input.transcript); + const calls = observedCalls(transcript.entries); + const installation = InstallationEvidenceSchema.safeParse(input.installation); + const session = parsedSession(input.session); + const assurance = + session.ok && session.value.closure + ? assuranceProjection(session.value) + : null; + const actors = derivedActors(transcript.entries, calls); + const host = observedHost(transcript.entries); + const hasCompletedFlowCall = calls.some( + (call) => call.tool.startsWith("flow_") && call.status === "completed", + ); + const loadedPlugin = loadedPluginMatches( + calls, + input.packageVersion, + input.pluginEntrySha256, + ); + const checks: Checks = { + "installs-packed-artifact": + installation.success && + installation.data.preparedSha256 === input.preparedSha256 && + installation.data.artifactSha256 === input.artifactSha256 && + installation.data.tarballSha256 === input.tarballSha256 && + installation.data.pluginEntrySha256 === input.pluginEntrySha256 && + installation.data.installedPluginSha256 === input.pluginEntrySha256 && + host.preparedFixture && + loadedPlugin && + hasCompletedFlowCall, + "loads-flow-tools": + host.preparedFixture && + host.versions.length === 1 && + hasCompletedFlowCall && + loadedPlugin, + "saves-plan": + session.ok && + session.value.approval === "approved" && + session.value.plan !== null && + session.value.operations.some( + (operation) => operation.kind === "plan-save", + ) && + completed(calls, "flow_plan_save"), + "captures-validation": + assurance !== null && + assuranceSatisfied(assurance, "accepted-validation") && + assuranceSatisfied(assurance, "canonical-gate") && + assuranceSatisfied(assurance, "declared-evidence") && + completed(calls, "flow_validation_start"), + "dispatches-reviewer": + assurance !== null && + assuranceSatisfied(assurance, "recorded-completion") && + actors.complete && + completed(calls, "flow_review_start") && + calls.some( + (call) => + call.tool === "task" && + call.status === "completed" && + call.input.subagent_type === "flow-reviewer", + ), + "closes-with-delivery": + assurance !== null && + assurance.conclusion === "completion-supported" && + completionSupportedFromDelivery(calls, assurance), + }; + const missing = + input.installation === null || + input.session === null || + input.transcript === null || + !installation.success || + !session.ok || + !transcript.supported; + const status = missing + ? "incomplete" + : Object.values(checks).every(Boolean) + ? "passed" + : "failed"; + return { + status, + checks, + actors: actors.actors, + hostConfigSha256: canonicalSha256("flow-canary-host-config-v2", { + artifactSha256: input.artifactSha256, + installation: installation.success ? installation.data : null, + actors: actors.actors, + host, + platforms: [ + ...new Set( + (session.ok ? session.value.runs : []) + .flatMap((run) => run.validations) + .flatMap((validation) => + validation.hostPlatform ? [validation.hostPlatform] : [], + ), + ), + ].sort(), + }), + }; +} const PackageMetadataSchema = z .object({ dependencies: z.object({ zod: TextSchema }).passthrough(), @@ -109,10 +563,9 @@ const EvidenceRefSchema = z .object({ path: TextSchema, sha256: DigestSchema, - bytes: z.number().int().safe().nonnegative(), + bytes: z.number().int().safe().nonnegative().max(MAX_TEST_REPORT_BYTES), }) .strict(); - export const PreparedCanarySchema = z .object({ schemaVersion: z.literal(1), @@ -132,6 +585,9 @@ export type PreparedCanary = z.infer; export const CanaryRecordSchema = z .object({ schemaVersion: z.literal(1), + derivationVersion: z.literal(CANARY_DERIVATION_VERSION), + preparedSha256: DigestSchema, + pluginEntrySha256: DigestSchema, status: z.enum(["passed", "failed", "incomplete"]), artifact: ArtifactIdentitySchema, artifactSha256: DigestSchema, @@ -146,6 +602,7 @@ export const CanaryRecordSchema = z actors: z.array(RedactedActorIdentitySchema), artifacts: z .object({ + installation: EvidenceRefSchema, session: EvidenceRefSchema.nullable(), transcript: EvidenceRefSchema.nullable(), }) @@ -172,6 +629,8 @@ export const CanaryRecordSchema = z if ( record.status === "passed" && (record.actors.length === 0 || + !record.actors.some((actor) => actor.role === "manager") || + !record.actors.some((actor) => actor.role === "reviewer") || record.artifacts.session === null || record.artifacts.transcript === null) ) { @@ -432,22 +891,55 @@ function redactEvidence(value: unknown, projectPath: string): unknown { return mapStrings(normalized, (text) => scrubSecrets(text).replace( /\b(?:ses_[A-Za-z0-9]+|(?:session|review):[A-Za-z0-9-]+)\b/g, - "", + (id) => + `id_${canonicalSha256("flow-canary-redacted-id-v1", id).slice("sha256:".length, "sha256:".length + 16)}`, ), ); } +async function measureInstallation( + prepared: PreparedCanary, + directory: string, +): Promise> { + const retainedPreparation = parsePreparedCanary( + JSON.parse(await readFile(join(directory, "prepared.json"), "utf8")), + ); + if (canonicalJson(retainedPreparation) !== canonicalJson(prepared)) { + throw new Error( + "Retained canary preparation does not match the recorder input.", + ); + } + const [artifactBytes, installedPlugin] = await Promise.all([ + readFile(join(directory, prepared.artifactFile)), + readFile(join(directory, "fixture", ".opencode", "plugins", "flow.js")), + ]); + if (sha256(artifactBytes) !== prepared.artifact.tarballSha256) { + throw new Error("Prepared canary tarball bytes do not match the artifact."); + } + const installedPluginSha256 = sha256(installedPlugin); + if (installedPluginSha256 !== prepared.pluginEntrySha256) { + throw new Error( + "Prepared canary installed plugin bytes do not match the artifact.", + ); + } + return { + schemaVersion: 1, + preparedSha256: prepared.sha256, + artifactSha256: prepared.artifactSha256, + tarballSha256: prepared.artifact.tarballSha256, + pluginEntrySha256: prepared.pluginEntrySha256, + installedPluginSha256, + }; +} + async function writeEvidence(input: { readonly repositoryRoot: string; readonly version: string; - readonly kind: "session" | "transcript"; + readonly kind: "installation" | "session" | "transcript"; readonly value: unknown | null; - readonly projectPath: string; }): Promise | null> { if (input.value === null) return null; - const bytes = Buffer.from( - canonicalJson(redactEvidence(input.value, input.projectPath)), - ); + const bytes = Buffer.from(canonicalJson(input.value)); const artifact = `artifacts/${input.version}-${input.kind}.json`; await writeImmutable( join(input.repositoryRoot, "evals", "canary", artifact), @@ -459,54 +951,69 @@ async function writeEvidence(input: { export async function recordCanary(input: { readonly repositoryRoot: string; readonly prepared: PreparedCanary; - readonly status: "passed" | "failed" | "incomplete"; + readonly preparedDirectory: string; readonly operator: string; - readonly hostConfig: unknown; - readonly actors: readonly ActorIdentity[]; - readonly checks: z.infer; - readonly projectPath: string; readonly session: unknown | null; readonly transcript: unknown | null; readonly recordedAt?: Date; }): Promise<{ readonly path: string; readonly record: CanaryRecord }> { const prepared = parsePreparedCanary(input.prepared); const recordedAt = input.recordedAt ?? new Date(); - const parsedChecks = ChecksSchema.parse(input.checks); - const parsedActors = z.array(ActorIdentitySchema).parse(input.actors); - const checkValues = Object.values(parsedChecks); - if (input.status === "passed" && !checkValues.every(Boolean)) { - throw new Error("Passed canaries require every check."); - } - if (input.status === "failed" && !checkValues.some((value) => !value)) { - throw new Error("Failed canaries require a failed check."); - } + const installationValue = await measureInstallation( + prepared, + input.preparedDirectory, + ); if ( - input.status === "passed" && - (parsedActors.length === 0 || - input.session === null || - input.transcript === null) + input.session !== null && + !SessionSchema.safeParse(input.session).success ) { throw new Error( - "Passed canaries require actors, session, and transcript evidence.", + "Canary session evidence is not a valid Session v5 document.", ); } + const fixturePath = resolve(input.preparedDirectory, "fixture"); + const redactedSession = + input.session === null ? null : redactEvidence(input.session, fixturePath); + const redactedTranscript = + input.transcript === null + ? null + : redactEvidence(input.transcript, fixturePath); + const derived = deriveCanaryResult({ + packageVersion: prepared.artifact.packageVersion, + artifactSha256: prepared.artifactSha256, + tarballSha256: prepared.artifact.tarballSha256, + preparedSha256: prepared.sha256, + pluginEntrySha256: prepared.pluginEntrySha256, + installation: installationValue, + session: redactedSession, + transcript: redactedTranscript, + }); const session = await writeEvidence({ repositoryRoot: input.repositoryRoot, version: prepared.artifact.packageVersion, kind: "session", - value: input.session, - projectPath: input.projectPath, + value: redactedSession, + }); + const installation = await writeEvidence({ + repositoryRoot: input.repositoryRoot, + version: prepared.artifact.packageVersion, + kind: "installation", + value: installationValue, }); + if (!installation) + throw new Error("Canary installation evidence is missing."); const transcript = await writeEvidence({ repositoryRoot: input.repositoryRoot, version: prepared.artifact.packageVersion, kind: "transcript", - value: input.transcript, - projectPath: input.projectPath, + value: redactedTranscript, }); const base: Omit = { schemaVersion: 1 as const, - status: input.status, + derivationVersion: CANARY_DERIVATION_VERSION, + preparedSha256: prepared.sha256, + pluginEntrySha256: prepared.pluginEntrySha256, + status: derived.status, artifact: prepared.artifact, artifactSha256: prepared.artifactSha256, releaseTag: prepared.releaseTag, @@ -515,16 +1022,10 @@ export async function recordCanary(input: { expiresAt: new Date(recordedAt.getTime() + CANARY_MAX_AGE_MS).toISOString(), checklistVersion: CANARY_CHECKLIST_VERSION, checklistSha256: CANARY_CHECKLIST_SHA256, - checks: parsedChecks, - hostConfigSha256: canonicalSha256( - "flow-canary-host-config-v1", - input.hostConfig, - ), - actors: parsedActors.map((actor) => ({ - ...actor, - sessionIds: actor.sessionIds.map(() => ""), - })), - artifacts: { session, transcript }, + checks: derived.checks, + hostConfigSha256: derived.hostConfigSha256, + actors: [...derived.actors], + artifacts: { installation, session, transcript }, }; const record: CanaryRecord = { ...base, @@ -542,33 +1043,30 @@ export async function recordCanary(input: { return { path, record }; } -function insideCanaryDirectory(directory: string, path: string): string | null { - if (isAbsolute(path)) return null; - const root = resolve(directory); - const target = resolve(join(root, path)); - const within = relative(root, target); - return within && !within.startsWith("..") && !isAbsolute(within) - ? target - : null; -} - -async function evidenceIssue( +async function evidenceValue( directory: string, ref: z.infer | null, -): Promise { - if (!ref) return "Canary evidence artifact is missing."; - const target = insideCanaryDirectory(directory, ref.path); - if (!target) return "Canary evidence path escapes its directory."; +): Promise<{ readonly issue: string | null; readonly value: unknown | null }> { + if (!ref) + return { issue: "Canary evidence artifact is missing.", value: null }; try { - const bytes = await readFile(target); - const info = await stat(target); - return bytes.byteLength === ref.bytes && - sha256(bytes) === ref.sha256 && - info.isFile() - ? null - : "Canary evidence digest or size does not match."; + const retained = await readWorkspaceTestReport(directory, ref.path); + if (!retained) + return { + issue: "Canary evidence artifact is unreadable or unstable.", + value: null, + }; + const bytes = Buffer.from(retained.text); + const issue = + bytes.byteLength === ref.bytes && sha256(bytes) === ref.sha256 + ? null + : "Canary evidence digest or size does not match."; + return { + issue, + value: issue ? null : JSON.parse(bytes.toString("utf8")), + }; } catch { - return "Canary evidence artifact is unreadable."; + return { issue: "Canary evidence artifact is unreadable.", value: null }; } } @@ -590,12 +1088,40 @@ export async function canaryRecordIssue(input: { const now = (input.now ?? new Date()).getTime(); if (Date.parse(record.recordedAt) > now) return "Canary is future-dated."; if (Date.parse(record.expiresAt) <= now) return "Canary is expired."; - const sessionIssue = await evidenceIssue( + const session = await evidenceValue( input.directory, record.artifacts.session, ); - if (sessionIssue) return sessionIssue; - return evidenceIssue(input.directory, record.artifacts.transcript); + if (session.issue) return session.issue; + const transcript = await evidenceValue( + input.directory, + record.artifacts.transcript, + ); + if (transcript.issue) return transcript.issue; + const installation = await evidenceValue( + input.directory, + record.artifacts.installation, + ); + if (installation.issue) return installation.issue; + const derived = deriveCanaryResult({ + packageVersion: record.artifact.packageVersion, + artifactSha256: record.artifactSha256, + tarballSha256: record.artifact.tarballSha256, + preparedSha256: record.preparedSha256, + pluginEntrySha256: record.pluginEntrySha256, + installation: installation.value, + session: session.value, + transcript: transcript.value, + }); + if ( + derived.status !== record.status || + canonicalJson(derived.checks) !== canonicalJson(record.checks) || + canonicalJson(derived.actors) !== canonicalJson(record.actors) || + derived.hostConfigSha256 !== record.hostConfigSha256 + ) { + return "Canary derived claims do not match retained evidence."; + } + return null; } export async function verifyCanary(input: { @@ -689,22 +1215,12 @@ async function main(args: readonly string[]): Promise { return; } if (command === "record") { - const status = required(args, "--status"); - if (status !== "passed" && status !== "failed" && status !== "incomplete") - throw new Error("--status must be passed, failed, or incomplete."); + const preparedPath = required(args, "--prepared"); const result = await recordCanary({ repositoryRoot, - prepared: PreparedCanarySchema.parse( - await json(required(args, "--prepared")), - ), - status, + prepared: PreparedCanarySchema.parse(await json(preparedPath)), + preparedDirectory: dirname(resolve(preparedPath)), operator: required(args, "--operator"), - hostConfig: await json(required(args, "--host-config")), - actors: z - .array(ActorIdentitySchema) - .parse(await json(required(args, "--actors"))), - checks: ChecksSchema.parse(await json(required(args, "--checks"))), - projectPath: required(args, "--project-path"), session: option(args, "--session") ? await json(required(args, "--session")) : null, diff --git a/scripts/release-metadata.ts b/scripts/release-metadata.ts index d4be07c..0e379bc 100644 --- a/scripts/release-metadata.ts +++ b/scripts/release-metadata.ts @@ -21,6 +21,7 @@ import { artifactIdentitySha256, CANARY_CHECKLIST_SHA256, CANARY_CHECKLIST_VERSION, + CANARY_DERIVATION_VERSION, type CanaryRecord, canaryRecordSha256, parseCanaryRecord, @@ -39,6 +40,8 @@ export function canaryRecordIssue( const entry = record as Partial; if (entry.schemaVersion !== 1 || entry.status !== "passed") return `the canary for ${version} is not a passed v1 record`; + if (entry.derivationVersion !== CANARY_DERIVATION_VERSION) + return `the canary record for ${version} is not evidence-derived`; if (entry.releaseTag !== expectedTag) return `the canary tag ${String(entry.releaseTag)} does not match ${expectedTag}`; const parsed = parseCanaryRecord(record); diff --git a/src/platform/opencode/plugin.ts b/src/platform/opencode/plugin.ts index 2da558f..60747bc 100644 --- a/src/platform/opencode/plugin.ts +++ b/src/platform/opencode/plugin.ts @@ -1,3 +1,6 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; import { dataNote } from "../../application/flow-response.js"; import { FLOW_CORE_COMMANDS } from "../../config-shared.js"; import { createWorkspaceFlowService } from "../../infrastructure/fs/workspace-flow-service.js"; @@ -245,6 +248,9 @@ function guardTools( const FlowPlugin: Plugin = async (ctx) => { const log = createFlowLog(ctx); const version = resolveFlowPluginVersion(); + const pluginEntrySha256 = `sha256:${createHash("sha256") + .update(await readFile(fileURLToPath(import.meta.url))) + .digest("hex")}`; const runtimeGuard = registerFlowPluginInstance( ctx.worktree ?? ctx.directory, { @@ -297,6 +303,7 @@ const FlowPlugin: Plugin = async (ctx) => { prepareValidation: prepareWorkspaceValidation, autoTimingSnapshot: () => autoDrive.timingSnapshot(), autoContinuationSupport: () => autoDrive.continuationSupport(), + runtimeIdentity: { packageVersion: version, pluginEntrySha256 }, }); return { config: createConfigHook(ctx, { diff --git a/src/platform/opencode/tools.ts b/src/platform/opencode/tools.ts index b541d51..a4525d4 100644 --- a/src/platform/opencode/tools.ts +++ b/src/platform/opencode/tools.ts @@ -44,6 +44,9 @@ type ToolOptions = Readonly<{ }>; autoTimingSnapshot?: (() => AutoTimingSnapshot | null) | undefined; autoContinuationSupport?: (() => AutoContinuationSupport) | undefined; + runtimeIdentity?: + | Readonly<{ packageVersion: string; pluginEntrySha256: string }> + | undefined; }>; function json(value: unknown): string { @@ -78,6 +81,11 @@ function withAutoContext( view?: string, ): FlowToolResponse { let workflowData = response.workflowData; + if (options.runtimeIdentity) + workflowData = { + ...workflowData, + runtimeIdentity: options.runtimeIdentity, + }; const timing = view === "detail" ? bestEffort(() => options.autoTimingSnapshot?.()) diff --git a/tests/distribution-and-surface.test.ts b/tests/distribution-and-surface.test.ts index 2a1cc65..93a357b 100644 --- a/tests/distribution-and-surface.test.ts +++ b/tests/distribution-and-surface.test.ts @@ -1,5 +1,13 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdir, mkdtemp, rm, unlink, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { + mkdir, + mkdtemp, + readFile, + rm, + unlink, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { ToolContext } from "@opencode-ai/plugin"; @@ -238,6 +246,25 @@ describe("Flow distribution surface", () => { const workspace = await createTestWorkspace("flow-surface-"); const hooks = await loadPlugin(workspace); expect(Object.keys(hooks.tool ?? {}).sort()).toEqual([...TOOL_NAMES]); + const status = JSON.parse( + String( + await hooks.tool?.flow_status?.execute( + { request: { view: "compact" } }, + { + agent: "build", + directory: workspace, + worktree: workspace, + sessionID: "runtime-identity", + } as ToolContext, + ), + ), + ) as { workflowData: { runtimeIdentity: { pluginEntrySha256: string } } }; + const pluginBytes = await readFile( + join(import.meta.dir, "../src/platform/opencode/plugin.ts"), + ); + expect(status.workflowData.runtimeIdentity.pluginEntrySha256).toBe( + `sha256:${createHash("sha256").update(pluginBytes).digest("hex")}`, + ); }); test("isolates worker permissions while keeping manager and reviewer dispatch separate", () => { diff --git a/tests/eval-canary.test.ts b/tests/eval-canary.test.ts index 7f1b5c5..1243675 100644 --- a/tests/eval-canary.test.ts +++ b/tests/eval-canary.test.ts @@ -1,21 +1,36 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { artifactIdentitySha256, CANARY_CHECKLIST_SHA256, CANARY_CHECKLIST_VERSION, + CANARY_DERIVATION_VERSION, CANARY_MAX_AGE_MS, type CanaryRecord, canaryRecordIssue, canaryRecordSha256, + deriveCanaryResult, type PreparedCanary, parseCanaryRecord, prepareCanary, preparedCanarySha256, recordCanary, } from "../scripts/eval-canary.js"; +import { assuranceProjection } from "../src/application/delivery.js"; +import { SessionSchema } from "../src/application/schema.js"; +import { MAX_TEST_REPORT_BYTES } from "../src/domain/limits.js"; +import { operationInputDigest } from "../src/domain/operation.js"; const temporary: string[] = []; afterEach(async () => { @@ -27,6 +42,8 @@ afterEach(async () => { }); const digest = (letter: string) => `sha256:${letter.repeat(64)}`; +const sha256 = (value: string) => + `sha256:${createHash("sha256").update(value).digest("hex")}`; const artifact = { packageVersion: "1.2.3", sourceCommit: "commit", @@ -58,6 +75,148 @@ const actor = { sessionIds: ["ses_secret"], }; +function canarySession() { + const session = JSON.parse( + readFileSync( + join(import.meta.dir, "../evals/canary/artifacts/8.1.2-session.json"), + "utf8", + ), + ) as { + id: string; + closure: { + kind: "completed"; + summary: string; + operationId: string; + recordedRevision: number; + }; + operations: Array<{ id: string; inputDigest: string }>; + runs: Array<{ + validations: Array<{ + observedAssertions?: Array<{ status: string }>; + }>; + }>; + }; + for (const run of session.runs) { + for (const validation of run.validations) { + for (const assertion of validation.observedAssertions ?? []) { + assertion.status = "passed"; + } + } + } + const closureOperation = session.operations.find( + (operation) => operation.id === session.closure.operationId, + ); + if (!closureOperation) + throw new Error("Canary closure operation is missing."); + closureOperation.inputDigest = operationInputDigest({ + operationId: session.closure.operationId, + expectedRevision: session.closure.recordedRevision - 1, + sessionId: session.id, + kind: session.closure.kind, + summary: session.closure.summary, + }); + return SessionSchema.parse(session); +} + +function canaryTranscript( + conclusion = "completion-supported", + directory = "", + packageVersion = "1.2.3", + pluginEntrySha256 = digest("d"), +) { + const calls = [ + "flow_status", + "flow_plan_save", + "flow_validation_start", + "flow_review_start", + ].map((tool) => ({ + type: "tool", + tool, + state: { + status: "completed", + input: {}, + output: + tool === "flow_status" + ? { + workflowData: { + runtimeIdentity: { packageVersion, pluginEntrySha256 }, + }, + } + : {}, + }, + })); + return { + info: { directory, version: "1.18.6" }, + messages: [ + { + info: { + role: "assistant", + agent: "build", + providerID: "provider", + modelID: "model", + sessionID: "ses_manager", + }, + parts: [ + ...calls, + { + type: "tool", + tool: "task", + state: { + status: "completed", + input: { subagent_type: "flow-reviewer" }, + output: {}, + metadata: { + model: { providerID: "provider", modelID: "model" }, + parentSessionId: "ses_manager", + sessionId: "ses_reviewer", + }, + }, + }, + { + type: "tool", + tool: "flow_session_close", + state: { + status: "completed", + input: {}, + output: { + workflowData: { + delivery: { + assurance: { + conclusion, + checks: assuranceProjection(canarySession()).checks, + }, + }, + }, + }, + }, + }, + ], + }, + { + info: { + role: "assistant", + agent: "flow-reviewer", + providerID: "provider", + modelID: "model", + sessionID: "ses_reviewer", + }, + parts: [], + }, + ], + }; +} + +function installation(value: PreparedCanary) { + return { + schemaVersion: 1, + preparedSha256: value.sha256, + artifactSha256: value.artifactSha256, + tarballSha256: value.artifact.tarballSha256, + pluginEntrySha256: value.pluginEntrySha256, + installedPluginSha256: value.pluginEntrySha256, + }; +} + function prepared(): PreparedCanary { const base: Omit = { schemaVersion: 1 as const, @@ -73,6 +232,44 @@ function prepared(): PreparedCanary { return { ...base, sha256: preparedCanarySha256(base) }; } +async function writePreparedFixture(root: string): Promise<{ + readonly directory: string; + readonly prepared: PreparedCanary; +}> { + const directory = join(root, "prepared"); + await mkdir(join(directory, "fixture/.opencode/plugins"), { + recursive: true, + }); + const artifactBytes = "packed artifact"; + const pluginBytes = "export const FlowPlugin = true;"; + const initial = prepared(); + const artifactValue = { + ...initial.artifact, + tarballSha256: sha256(artifactBytes), + }; + const { sha256: _initialSha256, ...initialBase } = initial; + const base: Omit = { + ...initialBase, + artifact: artifactValue, + artifactSha256: artifactIdentitySha256(artifactValue), + pluginEntrySha256: sha256(pluginBytes), + }; + const preparedValue = { + ...base, + sha256: preparedCanarySha256(base), + }; + await writeFile( + join(directory, "prepared.json"), + JSON.stringify(preparedValue), + ); + await writeFile(join(directory, "artifact.tgz"), artifactBytes); + await writeFile( + join(directory, "fixture/.opencode/plugins/flow.js"), + pluginBytes, + ); + return { directory, prepared: preparedValue }; +} + function record( input: { readonly status?: "passed" | "failed" | "incomplete"; @@ -86,6 +283,9 @@ function record( const artifactValue = input.artifactValue ?? artifact; const base: Omit = { schemaVersion: 1 as const, + derivationVersion: CANARY_DERIVATION_VERSION, + preparedSha256: prepared().sha256, + pluginEntrySha256: prepared().pluginEntrySha256, status: input.status ?? ("passed" as const), artifact: artifactValue, artifactSha256: artifactIdentitySha256(artifactValue), @@ -99,8 +299,16 @@ function record( checklistSha256: CANARY_CHECKLIST_SHA256, checks: input.checks ?? checks, hostConfigSha256: digest("e"), - actors: [{ ...actor, sessionIds: [""] }], + actors: [ + { ...actor, sessionIds: [""] }, + { ...actor, role: "reviewer", sessionIds: [""] }, + ], artifacts: { + installation: { + path: "artifacts/1.2.3-installation.json", + sha256: digest("8"), + bytes: 1, + }, session: { path: "artifacts/1.2.3-session.json", sha256: digest("f"), @@ -117,6 +325,207 @@ function record( } describe("canary record boundary", () => { + test("derives the committed unsupported completion as failed", async () => { + const repositoryRoot = join(import.meta.dir, ".."); + const session = JSON.parse( + await readFile( + join(repositoryRoot, "evals/canary/artifacts/8.1.2-session.json"), + "utf8", + ), + ); + const transcript = JSON.parse( + await readFile( + join(repositoryRoot, "evals/canary/artifacts/8.1.2-transcript.json"), + "utf8", + ), + ); + const derived = deriveCanaryResult({ + packageVersion: prepared().artifact.packageVersion, + artifactSha256: digest("a"), + tarballSha256: prepared().artifact.tarballSha256, + preparedSha256: prepared().sha256, + pluginEntrySha256: prepared().pluginEntrySha256, + installation: installation(prepared()), + session, + transcript, + }); + expect(derived.status).toBe("failed"); + expect(derived.checks["closes-with-delivery"]).toBe(false); + expect(derived.actors.map(({ role }) => role).sort()).toEqual([ + "manager", + "reviewer", + ]); + }); + + test("derives every passed claim from complete structural evidence", () => { + const value = prepared(); + const derived = deriveCanaryResult({ + packageVersion: value.artifact.packageVersion, + artifactSha256: value.artifactSha256, + tarballSha256: value.artifact.tarballSha256, + preparedSha256: value.sha256, + pluginEntrySha256: value.pluginEntrySha256, + installation: installation(value), + session: canarySession(), + transcript: canaryTranscript(), + }); + expect(derived.status).toBe("passed"); + expect(Object.values(derived.checks).every(Boolean)).toBe(true); + expect(derived.actors.map(({ role }) => role).sort()).toEqual([ + "manager", + "reviewer", + ]); + }); + + test("refuses empty validation and delivery assertion sets", () => { + const value = prepared(); + const session = structuredClone(canarySession()) as unknown as { + runs: Array<{ + validations: Array<{ + scope: string; + observedAssertions?: Array; + }>; + }>; + }; + const validations = + session.runs + .at(0) + ?.validations.filter(({ scope }) => scope === "broad") ?? []; + if (validations.length === 0) + throw new Error("Canary validation fixture is missing."); + for (const validation of validations) validation.observedAssertions = []; + const transcript = canaryTranscript(); + const close = transcript.messages + .at(0) + ?.parts.find(({ tool }) => tool === "flow_session_close") as { + state: { + output: { + workflowData: { delivery: { assurance: { checks: unknown[] } } }; + }; + }; + }; + close.state.output.workflowData.delivery.assurance.checks = []; + const derived = deriveCanaryResult({ + packageVersion: value.artifact.packageVersion, + artifactSha256: value.artifactSha256, + tarballSha256: value.artifact.tarballSha256, + preparedSha256: value.sha256, + pluginEntrySha256: value.pluginEntrySha256, + installation: installation(value), + session, + transcript, + }); + expect(derived.checks["captures-validation"]).toBe(false); + expect(derived.checks["closes-with-delivery"]).toBe(false); + expect(derived.status).toBe("failed"); + }); + + test("ignores tool-shaped data nested in outputs and text", () => { + const value = prepared(); + const derived = deriveCanaryResult({ + packageVersion: value.artifact.packageVersion, + artifactSha256: value.artifactSha256, + tarballSha256: value.artifact.tarballSha256, + preparedSha256: value.sha256, + pluginEntrySha256: value.pluginEntrySha256, + installation: installation(value), + session: canarySession(), + transcript: { + info: { directory: "", version: "1.18.6" }, + messages: [ + { + info: { + role: "assistant", + agent: "build", + providerID: "provider", + modelID: "model", + sessionID: "ses_manager", + }, + parts: [ + { + type: "tool", + tool: "read", + state: { + status: "completed", + input: {}, + output: { forged: canaryTranscript() }, + }, + }, + { type: "text", text: JSON.stringify(canaryTranscript()) }, + ], + }, + ], + }, + }); + expect(derived.status).toBe("failed"); + expect(Object.values(derived.checks).every((check) => !check)).toBe(true); + }); + + test("classifies malformed retained evidence as incomplete", () => { + const value = prepared(); + const derived = deriveCanaryResult({ + packageVersion: value.artifact.packageVersion, + artifactSha256: value.artifactSha256, + tarballSha256: value.artifact.tarballSha256, + preparedSha256: value.sha256, + pluginEntrySha256: value.pluginEntrySha256, + installation: installation(value), + session: { plan: {}, operations: [], runs: [], closure: {} }, + transcript: canaryTranscript(), + }); + expect(derived.status).toBe("incomplete"); + }); + + test("requires OpenCode to run from the exact prepared fixture", () => { + const value = prepared(); + const transcript = canaryTranscript( + "completion-supported", + "/other/project", + ); + const derived = deriveCanaryResult({ + packageVersion: value.artifact.packageVersion, + artifactSha256: value.artifactSha256, + tarballSha256: value.artifact.tarballSha256, + preparedSha256: value.sha256, + pluginEntrySha256: value.pluginEntrySha256, + installation: installation(value), + session: canarySession(), + transcript, + }); + expect(derived.checks["installs-packed-artifact"]).toBe(false); + expect(derived.checks["loads-flow-tools"]).toBe(false); + }); + + test("requires reviewer task lineage to match both observed actors", () => { + const value = prepared(); + const transcript = structuredClone(canaryTranscript()) as unknown as { + messages: Array<{ + parts: Array<{ + tool?: string; + state?: { metadata?: { sessionId?: string } }; + }>; + }>; + }; + const task = transcript.messages + .at(0) + ?.parts.find(({ tool }) => tool === "task"); + if (!task?.state?.metadata) + throw new Error("Canary reviewer task fixture is missing."); + task.state.metadata.sessionId = "ses_unrelated"; + const derived = deriveCanaryResult({ + packageVersion: value.artifact.packageVersion, + artifactSha256: value.artifactSha256, + tarballSha256: value.artifact.tarballSha256, + preparedSha256: value.sha256, + pluginEntrySha256: value.pluginEntrySha256, + installation: installation(value), + session: canarySession(), + transcript, + }); + expect(derived.checks["dispatches-reviewer"]).toBe(false); + expect(derived.status).toBe("failed"); + }); + test("accepts strict passed, failed, and incomplete records", () => { expect(parseCanaryRecord(record()).ok).toBe(true); expect( @@ -152,6 +561,18 @@ describe("canary record boundary", () => { ]) { expect(parseCanaryRecord(changed).ok).toBe(false); } + const oversized = record(); + const oversizedBase = { + ...oversized, + artifacts: { + ...oversized.artifacts, + session: { + ...oversized.artifacts.session, + bytes: MAX_TEST_REPORT_BYTES + 1, + }, + }, + }; + expect(parseCanaryRecord(oversizedBase).ok).toBe(false); }); }); @@ -159,6 +580,10 @@ async function evidenceDirectory(value: CanaryRecord): Promise { const directory = await mkdtemp(join(tmpdir(), "flow-canary-evidence-")); temporary.push(directory); await mkdir(join(directory, "artifacts"), { recursive: true }); + await writeFile( + join(directory, value.artifacts.installation?.path ?? ""), + "z", + ); await writeFile(join(directory, value.artifacts.session?.path ?? ""), "x"); await writeFile(join(directory, value.artifacts.transcript?.path ?? ""), "y"); return directory; @@ -219,50 +644,93 @@ describe("canary release verification", () => { }), ).toMatch(/digest or size/); }); + + test("rejects symlinked retained evidence", async () => { + const valid = record(); + const directory = await mkdtemp(join(tmpdir(), "flow-canary-evidence-")); + const outside = await mkdtemp(join(tmpdir(), "flow-canary-outside-")); + temporary.push(directory, outside); + await mkdir(join(directory, "artifacts"), { recursive: true }); + await writeFile(join(outside, "session.json"), "x"); + await symlink( + join(outside, "session.json"), + join(directory, valid.artifacts.session?.path ?? ""), + ); + expect( + await canaryRecordIssue({ + version: "1.2.3", + record: valid, + expectedArtifact: artifact, + directory, + now: new Date("2026-08-25T01:00:00.000Z"), + }), + ).toMatch(/unreadable or unstable/); + }); }); describe("canary recording", () => { test("redacts evidence and allows only byte-identical replay", async () => { const root = await mkdtemp(join(tmpdir(), "flow-canary-record-")); temporary.push(root); + const fixture = await writePreparedFixture(root); const input = { repositoryRoot: root, - prepared: prepared(), - status: "passed" as const, + prepared: fixture.prepared, + preparedDirectory: fixture.directory, operator: "maintainer", - hostConfig: { opencode: "1.18.6" }, - actors: [actor], - checks, - projectPath: "/secret/project", - session: { - id: "ses_secret", - path: "/secret/project", - apiKey: "sk-proj-1234567890123456", - }, + session: canarySession(), transcript: { - session: "session:1234-abcd", - text: "Bearer abcdefghijklmnop", + ...canaryTranscript( + "completion-supported", + join(fixture.directory, "fixture"), + fixture.prepared.artifact.packageVersion, + fixture.prepared.pluginEntrySha256, + ), + secret: { + session: "session:1234-abcd", + path: join(fixture.directory, "fixture"), + text: "Bearer abcdefghijklmnop sk-proj-1234567890123456", + }, }, recordedAt: new Date("2026-08-25T00:00:00.000Z"), }; const first = await recordCanary(input); const second = await recordCanary(input); expect(second.record).toEqual(first.record); + const actorIds = first.record.actors.flatMap( + ({ sessionIds }) => sessionIds, + ); + expect(actorIds.every((id) => /^id_[a-f0-9]{16}$/.test(id))).toBe(true); + expect(new Set(actorIds).size).toBe(2); expect( await canaryRecordIssue({ version: "1.2.3", record: first.record, - expectedArtifact: artifact, + expectedArtifact: first.record.artifact, directory: join(root, "evals", "canary"), now: new Date("2026-08-25T01:00:00.000Z"), }), ).toBeNull(); + const { recordSha256: _recordSha256, ...recordBase } = first.record; + const tamperedBase = { ...recordBase, hostConfigSha256: digest("0") }; + expect( + await canaryRecordIssue({ + version: "1.2.3", + record: { + ...tamperedBase, + recordSha256: canaryRecordSha256(tamperedBase), + }, + expectedArtifact: first.record.artifact, + directory: join(root, "evals/canary"), + now: new Date("2026-08-25T01:00:00.000Z"), + }), + ).toMatch(/derived claims/); expect( await canaryRecordIssue({ version: "1.2.3", record: first.record, expectedArtifact: { - ...artifact, + ...first.record.artifact, sourceCommit: "tag-commit-after-evidence", sourceTreeSha256: digest("8"), }, @@ -281,33 +749,36 @@ describe("canary recording", () => { ), await readFile(first.path, "utf8"), ].join("\n"); - expect(stored).not.toContain("/secret/project"); - expect(stored).not.toContain("ses_secret"); + expect(stored).not.toContain(fixture.directory); expect(stored).not.toContain("1234-abcd"); expect(stored).not.toContain("sk-proj-"); expect(stored).not.toContain("Bearer abcdef"); await expect(recordCanary({ ...input, operator: "other" })).rejects.toThrow( "conflicts", ); + await writeFile( + join(fixture.directory, "fixture/.opencode/plugins/flow.js"), + "mutated plugin", + ); + await expect(recordCanary(input)).rejects.toThrow(/installed plugin/i); }); - test("passed recording requires actors and both artifacts", async () => { + test("missing evidence derives an incomplete record", async () => { const root = await mkdtemp(join(tmpdir(), "flow-canary-record-")); temporary.push(root); - await expect( - recordCanary({ - repositoryRoot: root, - prepared: prepared(), - status: "passed", - operator: "maintainer", - hostConfig: {}, - actors: [], - checks, - projectPath: root, - session: null, - transcript: null, - }), - ).rejects.toThrow(/require actors/); + const fixture = await writePreparedFixture(root); + const result = await recordCanary({ + repositoryRoot: root, + prepared: fixture.prepared, + preparedDirectory: fixture.directory, + operator: "maintainer", + session: null, + transcript: null, + }); + expect(result.record.status).toBe("incomplete"); + expect(Object.values(result.record.checks).every((value) => !value)).toBe( + true, + ); }); }); diff --git a/tests/release-metadata.test.ts b/tests/release-metadata.test.ts index 7b79c72..f0f8c8c 100644 --- a/tests/release-metadata.test.ts +++ b/tests/release-metadata.test.ts @@ -1,9 +1,10 @@ import { afterEach, describe, expect, test } from "bun:test"; import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { canonicalSha256 } from "../evals/canonical-json.js"; +import { canonicalJson, canonicalSha256 } from "../evals/canonical-json.js"; import { evaluatorIdentity } from "../evals/provenance.js"; import { RELEASE_POLICY_SHA256, @@ -17,7 +18,9 @@ import { artifactIdentitySha256, CANARY_CHECKLIST_SHA256, CANARY_CHECKLIST_VERSION, + CANARY_DERIVATION_VERSION, canaryRecordSha256, + deriveCanaryResult, } from "../scripts/eval-canary.js"; import { assertQualificationRecord, @@ -28,6 +31,9 @@ import { releaseNotesForVersion, validateReleaseMetadata, } from "../scripts/release-metadata.js"; +import { assuranceProjection } from "../src/application/delivery.js"; +import { SessionSchema } from "../src/application/schema.js"; +import { operationInputDigest } from "../src/domain/operation.js"; const temporary: string[] = []; @@ -50,6 +56,129 @@ const CANARY_NOW = new Date("2026-08-26T00:00:00.000Z"); const digest = (letter: string) => `sha256:${letter.repeat(64)}`; const bytesDigest = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; +const canarySession = (() => { + const session = JSON.parse( + readFileSync( + join(import.meta.dir, "../evals/canary/artifacts/8.1.2-session.json"), + "utf8", + ), + ) as { + id: string; + closure: { + kind: "completed"; + summary: string; + operationId: string; + recordedRevision: number; + }; + operations: Array<{ id: string; inputDigest: string }>; + runs: Array<{ + validations: Array<{ + observedAssertions?: Array<{ status: string }>; + }>; + }>; + }; + for (const run of session.runs) { + for (const validation of run.validations) { + for (const assertion of validation.observedAssertions ?? []) { + assertion.status = "passed"; + } + } + } + const operation = session.operations.find( + (candidate) => candidate.id === session.closure.operationId, + ); + if (!operation) throw new Error("Canary closure operation is missing."); + operation.inputDigest = operationInputDigest({ + operationId: session.closure.operationId, + expectedRevision: session.closure.recordedRevision - 1, + sessionId: session.id, + kind: session.closure.kind, + summary: session.closure.summary, + }); + return SessionSchema.parse(session); +})(); +const canaryTranscript = (packageVersion: string) => ({ + info: { directory: "", version: "1.18.6" }, + messages: [ + { + info: { + role: "assistant", + agent: "build", + providerID: "openai", + modelID: "test", + sessionID: "id_aaaaaaaaaaaaaaaa", + }, + parts: [ + ...[ + "flow_status", + "flow_plan_save", + "flow_validation_start", + "flow_review_start", + ].map((tool) => ({ + type: "tool", + tool, + state: { + status: "completed", + input: {}, + output: + tool === "flow_status" + ? { + workflowData: { + runtimeIdentity: { + packageVersion, + pluginEntrySha256: digest("5"), + }, + }, + } + : {}, + }, + })), + { + type: "tool", + tool: "task", + state: { + status: "completed", + input: { subagent_type: "flow-reviewer" }, + output: {}, + metadata: { + model: { providerID: "openai", modelID: "test" }, + parentSessionId: "id_aaaaaaaaaaaaaaaa", + sessionId: "id_bbbbbbbbbbbbbbbb", + }, + }, + }, + { + type: "tool", + tool: "flow_session_close", + state: { + status: "completed", + input: {}, + output: { + workflowData: { + delivery: { + assurance: { + conclusion: "completion-supported", + checks: assuranceProjection(canarySession).checks, + }, + }, + }, + }, + }, + }, + ], + }, + { + info: { + role: "assistant", + agent: "flow-reviewer", + providerID: "openai", + modelID: "test", + sessionID: "id_bbbbbbbbbbbbbbbb", + }, + parts: [], + }, + ], +}); const artifact = (packageVersion: string) => ({ packageVersion, sourceCommit: "commit", @@ -123,59 +252,62 @@ function canaryRecord( packageVersion: string, overrides: Record = {}, ) { + const measuredArtifact = artifact(packageVersion); + const preparedSha256 = digest("4"); + const pluginEntrySha256 = digest("5"); + const installation = { + schemaVersion: 1 as const, + preparedSha256, + artifactSha256: artifactIdentitySha256(measuredArtifact), + tarballSha256: measuredArtifact.tarballSha256, + pluginEntrySha256, + installedPluginSha256: pluginEntrySha256, + }; + const derived = deriveCanaryResult({ + packageVersion, + artifactSha256: artifactIdentitySha256(measuredArtifact), + tarballSha256: measuredArtifact.tarballSha256, + preparedSha256, + pluginEntrySha256, + installation, + session: canarySession, + transcript: canaryTranscript(packageVersion), + }); + const installationJson = canonicalJson(installation); + const sessionJson = canonicalJson(canarySession); + const transcriptJson = canonicalJson(canaryTranscript(packageVersion)); const base: Omit = { schemaVersion: 1 as const, + derivationVersion: CANARY_DERIVATION_VERSION, + preparedSha256, + pluginEntrySha256, releaseTag: `v${packageVersion}`, - status: "passed" as const, - artifact: artifact(packageVersion), + status: derived.status, + artifact: measuredArtifact, checklistVersion: CANARY_CHECKLIST_VERSION, checklistSha256: CANARY_CHECKLIST_SHA256, - artifactSha256: artifactIdentitySha256(artifact(packageVersion)), - checks: { - "installs-packed-artifact": true, - "loads-flow-tools": true, - "saves-plan": true, - "captures-validation": true, - "dispatches-reviewer": true, - "closes-with-delivery": true, - }, + artifactSha256: artifactIdentitySha256(measuredArtifact), + checks: derived.checks, operator: "maintainer@example.com", recordedAt: "2026-08-25T00:00:00.000Z", expiresAt: "2026-08-28T00:00:00.000Z", - hostConfigSha256: digest("6"), - actors: [ - { - role: "manager" as const, - requestedModel: { - routeProvider: "openai", - gateway: null, - family: "gpt", - model: "test", - revision: null, - }, - actualModel: { - kind: "observed" as const, - value: { - routeProvider: "openai", - gateway: null, - family: "gpt", - model: "test", - revision: null, - }, - }, - sessionIds: [""], - }, - ], + hostConfigSha256: derived.hostConfigSha256, + actors: [...derived.actors], artifacts: { + installation: { + path: "artifacts/installation.json", + sha256: bytesDigest(installationJson), + bytes: Buffer.byteLength(installationJson), + }, session: { path: "artifacts/session.json", - sha256: bytesDigest("session"), - bytes: 7, + sha256: bytesDigest(sessionJson), + bytes: Buffer.byteLength(sessionJson), }, transcript: { path: "artifacts/transcript.json", - sha256: bytesDigest("transcript"), - bytes: 10, + sha256: bytesDigest(transcriptJson), + bytes: Buffer.byteLength(transcriptJson), }, }, ...overrides, @@ -185,6 +317,36 @@ function canaryRecord( recordSha256: canaryRecordSha256(base), }; } + +async function writeCanaryEvidence( + directory: string, + packageVersion: string, +): Promise { + const measuredArtifact = artifact(packageVersion); + const installation = { + schemaVersion: 1, + preparedSha256: digest("4"), + artifactSha256: artifactIdentitySha256(measuredArtifact), + tarballSha256: measuredArtifact.tarballSha256, + pluginEntrySha256: digest("5"), + installedPluginSha256: digest("5"), + }; + await mkdir(join(directory, "artifacts"), { recursive: true }); + await Promise.all([ + writeFile( + join(directory, "artifacts", "installation.json"), + canonicalJson(installation), + ), + writeFile( + join(directory, "artifacts", "session.json"), + canonicalJson(canarySession), + ), + writeFile( + join(directory, "artifacts", "transcript.json"), + canonicalJson(canaryTranscript(packageVersion)), + ), + ]); +} const exactChangelog = [ "# Changelog", "", @@ -324,12 +486,7 @@ describe("release metadata", () => { sourceTreeSha256: digest("8"), }; const canary = canaryRecord(version); - await mkdir(join(canaries, "artifacts"), { recursive: true }); - await writeFile(join(canaries, "artifacts", "session.json"), "session"); - await writeFile( - join(canaries, "artifacts", "transcript.json"), - "transcript", - ); + await writeCanaryEvidence(canaries, version); await writeFile(join(canaries, `${version}.json`), JSON.stringify(canary)); await writeFile( join(decisions, "report-canary.json"), @@ -365,6 +522,19 @@ describe("release metadata", () => { } }); + test("rejects the historical caller-attested canary format", () => { + const legacy = { ...canaryRecord("8.1.2"), derivationVersion: undefined }; + expect( + canaryRecordIssue( + "8.1.2", + legacy, + artifact("8.1.2"), + "v8.1.2", + CANARY_NOW, + ), + ).toMatch(/not evidence-derived/); + }); + test("does not accept a null or grafted canary decision", async () => { const decisions = await recordDirectory(); const canaries = await recordDirectory(); @@ -372,12 +542,7 @@ describe("release metadata", () => { const expected = artifact(version); const canary = canaryRecord(version); await writeFile(join(canaries, `${version}.json`), JSON.stringify(canary)); - await mkdir(join(canaries, "artifacts"), { recursive: true }); - await writeFile(join(canaries, "artifacts", "session.json"), "session"); - await writeFile( - join(canaries, "artifacts", "transcript.json"), - "transcript", - ); + await writeCanaryEvidence(canaries, version); await writeFile( join(decisions, "null.json"), JSON.stringify(decisionRecord(version)),