diff --git a/.agents/plans/03-assurance-hardening/decisions.tsv b/.agents/plans/03-assurance-hardening/decisions.tsv index 1e3432b..e41d087 100644 --- a/.agents/plans/03-assurance-hardening/decisions.tsv +++ b/.agents/plans/03-assurance-hardening/decisions.tsv @@ -47,3 +47,5 @@ ts phase decision why evidence result 2026-08-28T04:22:39Z phase-6 bound reviewer independence through host lineage separate reviewer messages and task calls did not prove they represented the same child session and model task parentSessionId, sessionId, model metadata; negative mismatch test reviewer dispatch passes only when task lineage matches the observed manager and reviewer actors 2026-08-28T04:22:39Z phase-6 completed corrected whole-product verification the review fixes changed runtime status output, evidence parsing, and release metadata boundaries bun run check; bun run replay; pinned OpenCode live smoke 585 pass, 1 intentional skip, 13 of 13 replays, live OpenCode 1.18.6 pass, 339 source bytes headroom 2026-08-28T04:23:41Z phase-6 closed the final transcript and lineage review findings flat synthetic transcripts and unlinked reviewer observations remained broader than the real OpenCode evidence boundary root export messages plus tool parts; parent-child-model lineage match; final three-model review 586 pass, 1 intentional skip, 13 of 13 replays, pinned live smoke pass; all final reviewers green +2026-08-28T04:32:34Z phase-6 merged evidence-derived canary qualification the exact PR head passed two CI runs and isolated shipping verification PR 53; merge 269bd855 merged to main +2026-08-28T04:32:34Z phase-6-lineage captured and closed cross-session transcript splicing late review showed an authentic runtime status could be pooled with lifecycle calls from another manager session ObservedCall session lineage; focused red splice test all canary proof calls now come from the one manager session named by the reviewer task lineage diff --git a/scripts/eval-canary.ts b/scripts/eval-canary.ts index a5eae97..d33c8ea 100644 --- a/scripts/eval-canary.ts +++ b/scripts/eval-canary.ts @@ -147,6 +147,7 @@ function parsedJson(value: unknown): unknown { type ObservedCall = Readonly<{ tool: string; status: string; + sessionId: string | null; input: Record; output: unknown; metadata: Record; @@ -173,11 +174,16 @@ function transcriptShape(value: unknown): TranscriptShape { }; } -function observedCall(entry: Record): ObservedCall | null { +function observedCall( + entry: Record, + messageSessionId: string | null, +): ObservedCall | null { if (typeof entry.tool !== "string") return null; const state = record(entry.state); return { tool: entry.tool, + sessionId: + typeof entry.sessionID === "string" ? entry.sessionID : messageSessionId, status: (typeof entry.status === "string" ? entry.status : null) ?? (typeof state?.status === "string" ? state.status : "unknown"), @@ -195,7 +201,11 @@ function observedCalls( return entries.flatMap((entry) => records(array(entry.parts)).flatMap((part) => { if (part.type !== "tool") return []; - const call = observedCall(part); + const info = record(entry.info); + const call = observedCall( + part, + typeof info?.sessionID === "string" ? info.sessionID : null, + ); return call ? [call] : []; }), ); @@ -280,7 +290,11 @@ function modelIdentity(value: Record): { function derivedActors( entries: readonly Record[], calls: readonly ObservedCall[], -): { readonly actors: readonly ActorIdentity[]; readonly complete: boolean } { +): { + readonly actors: readonly ActorIdentity[]; + readonly complete: boolean; + readonly managerSessionId: string | null; +} { const actors = new Map(); const lineages: Array<{ readonly parent: string; @@ -336,7 +350,8 @@ function derivedActors( if ( identity && typeof sessionId === "string" && - typeof parentSessionId === "string" + typeof parentSessionId === "string" && + call.sessionId === parentSessionId ) { const observedReviewer = actors.get("reviewer"); if ( @@ -356,19 +371,28 @@ function derivedActors( manager !== undefined && reviewer !== undefined && manager.sessionIds.every((id) => !reviewer.sessionIds.includes(id)); - const linked = - manager !== undefined && - reviewer !== undefined && - lineages.some( - (lineage) => - manager.sessionIds.includes(lineage.parent) && - reviewer.sessionIds.includes(lineage.child) && - reviewer.requestedModel.routeProvider === lineage.identity.provider && - reviewer.requestedModel.model === lineage.identity.model, - ); + const linkedLineages = + manager && reviewer + ? lineages.filter( + (lineage) => + manager.sessionIds.includes(lineage.parent) && + reviewer.sessionIds.includes(lineage.child) && + reviewer.requestedModel.routeProvider === + lineage.identity.provider && + reviewer.requestedModel.model === lineage.identity.model, + ) + : []; + const linkedPairs = [ + ...new Set( + linkedLineages.map((lineage) => `${lineage.parent}\0${lineage.child}`), + ), + ]; + const managerSessionId = + linkedPairs.length === 1 ? (linkedPairs[0]?.split("\0")[0] ?? null) : null; return { actors: [...actors.values()], - complete: consistent && distinct && linked, + complete: consistent && distinct && managerSessionId !== null, + managerSessionId, }; } @@ -463,12 +487,15 @@ export function deriveCanaryResult(input: { ? assuranceProjection(session.value) : null; const actors = derivedActors(transcript.entries, calls); + const managerCalls = actors.managerSessionId + ? calls.filter((call) => call.sessionId === actors.managerSessionId) + : []; const host = observedHost(transcript.entries); - const hasCompletedFlowCall = calls.some( + const hasCompletedFlowCall = managerCalls.some( (call) => call.tool.startsWith("flow_") && call.status === "completed", ); const loadedPlugin = loadedPluginMatches( - calls, + managerCalls, input.packageVersion, input.pluginEntrySha256, ); @@ -495,19 +522,19 @@ export function deriveCanaryResult(input: { session.value.operations.some( (operation) => operation.kind === "plan-save", ) && - completed(calls, "flow_plan_save"), + completed(managerCalls, "flow_plan_save"), "captures-validation": assurance !== null && assuranceSatisfied(assurance, "accepted-validation") && assuranceSatisfied(assurance, "canonical-gate") && assuranceSatisfied(assurance, "declared-evidence") && - completed(calls, "flow_validation_start"), + completed(managerCalls, "flow_validation_start"), "dispatches-reviewer": assurance !== null && assuranceSatisfied(assurance, "recorded-completion") && actors.complete && - completed(calls, "flow_review_start") && - calls.some( + completed(managerCalls, "flow_review_start") && + managerCalls.some( (call) => call.tool === "task" && call.status === "completed" && @@ -516,7 +543,7 @@ export function deriveCanaryResult(input: { "closes-with-delivery": assurance !== null && assurance.conclusion === "completion-supported" && - completionSupportedFromDelivery(calls, assurance), + completionSupportedFromDelivery(managerCalls, assurance), }; const missing = input.installation === null || diff --git a/tests/eval-canary.test.ts b/tests/eval-canary.test.ts index 1243675..10c6ba1 100644 --- a/tests/eval-canary.test.ts +++ b/tests/eval-canary.test.ts @@ -526,6 +526,41 @@ describe("canary record boundary", () => { expect(derived.status).toBe("failed"); }); + test("rejects lifecycle evidence spliced across manager sessions", () => { + const value = prepared(); + const transcript = canaryTranscript(); + const manager = transcript.messages.at(0); + if (!manager) throw new Error("Canary manager fixture is missing."); + const statusIndex = manager.parts.findIndex( + ({ tool }) => tool === "flow_status", + ); + const status = manager.parts.splice(statusIndex, 1).at(0); + if (!status) throw new Error("Canary status fixture is missing."); + transcript.messages.push({ + info: { + role: "assistant", + agent: "build", + providerID: "provider", + modelID: "model", + sessionID: "ses_other_manager", + }, + parts: [status], + }); + const derived = deriveCanaryResult({ + packageVersion: value.artifact.packageVersion, + artifactSha256: value.artifactSha256, + tarballSha256: value.artifact.tarballSha256, + preparedSha256: value.sha256, + pluginEntrySha256: value.pluginEntrySha256, + installation: installation(value), + session: canarySession(), + transcript, + }); + expect(derived.checks["installs-packed-artifact"]).toBe(false); + expect(derived.checks["loads-flow-tools"]).toBe(false); + expect(derived.status).toBe("failed"); + }); + test("accepts strict passed, failed, and incomplete records", () => { expect(parseCanaryRecord(record()).ok).toBe(true); expect(