Skip to content

Bind canary proof to one manager session #54

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vriesd merged 1 commit into from
Aug 28, 2026
Merged

Bind canary proof to one manager session #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .agents/plans/03-assurance-hardening/decisions.tsv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,5 @@ ts phase decision why evidence result
2026-08-28T04:22:39Z phase-6 bound reviewer independence through host lineage separate reviewer messages and task calls did not prove they represented the same child session and model task parentSessionId, sessionId, model metadata; negative mismatch test reviewer dispatch passes only when task lineage matches the observed manager and reviewer actors
2026-08-28T04:22:39Z phase-6 completed corrected whole-product verification the review fixes changed runtime status output, evidence parsing, and release metadata boundaries bun run check; bun run replay; pinned OpenCode live smoke 585 pass, 1 intentional skip, 13 of 13 replays, live OpenCode 1.18.6 pass, 339 source bytes headroom
2026-08-28T04:23:41Z phase-6 closed the final transcript and lineage review findings flat synthetic transcripts and unlinked reviewer observations remained broader than the real OpenCode evidence boundary root export messages plus tool parts; parent-child-model lineage match; final three-model review 586 pass, 1 intentional skip, 13 of 13 replays, pinned live smoke pass; all final reviewers green
2026-08-28T04:32:34Z phase-6 merged evidence-derived canary qualification the exact PR head passed two CI runs and isolated shipping verification PR 53; merge 269bd855 merged to main
2026-08-28T04:32:34Z phase-6-lineage captured and closed cross-session transcript splicing late review showed an authentic runtime status could be pooled with lifecycle calls from another manager session ObservedCall session lineage; focused red splice test all canary proof calls now come from the one manager session named by the reviewer task lineage
71 changes: 49 additions & 22 deletions scripts/eval-canary.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ function parsedJson(value: unknown): unknown {
type ObservedCall = Readonly<{
tool: string;
status: string;
sessionId: string | null;
input: Record<string, unknown>;
output: unknown;
metadata: Record<string, unknown>;
Expand All @@ -173,11 +174,16 @@ function transcriptShape(value: unknown): TranscriptShape {
};
}

function observedCall(entry: Record<string, unknown>): ObservedCall | null {
function observedCall(
entry: Record<string, unknown>,
messageSessionId: string | null,
): ObservedCall | null {
if (typeof entry.tool !== "string") return null;
const state = record(entry.state);
return {
tool: entry.tool,
sessionId:
typeof entry.sessionID === "string" ? entry.sessionID : messageSessionId,
status:
(typeof entry.status === "string" ? entry.status : null) ??
(typeof state?.status === "string" ? state.status : "unknown"),
Expand All @@ -195,7 +201,11 @@ function observedCalls(
return entries.flatMap((entry) =>
records(array(entry.parts)).flatMap((part) => {
if (part.type !== "tool") return [];
const call = observedCall(part);
const info = record(entry.info);
const call = observedCall(
part,
typeof info?.sessionID === "string" ? info.sessionID : null,
);
return call ? [call] : [];
}),
);
Expand Down Expand Up @@ -280,7 +290,11 @@ function modelIdentity(value: Record<string, unknown>): {
function derivedActors(
entries: readonly Record<string, unknown>[],
calls: readonly ObservedCall[],
): { readonly actors: readonly ActorIdentity[]; readonly complete: boolean } {
): {
readonly actors: readonly ActorIdentity[];
readonly complete: boolean;
readonly managerSessionId: string | null;
} {
const actors = new Map<string, ActorIdentity>();
const lineages: Array<{
readonly parent: string;
Expand Down Expand Up @@ -336,7 +350,8 @@ function derivedActors(
if (
identity &&
typeof sessionId === "string" &&
typeof parentSessionId === "string"
typeof parentSessionId === "string" &&
call.sessionId === parentSessionId
) {
const observedReviewer = actors.get("reviewer");
if (
Expand All @@ -356,19 +371,28 @@ function derivedActors(
manager !== undefined &&
reviewer !== undefined &&
manager.sessionIds.every((id) => !reviewer.sessionIds.includes(id));
const linked =
manager !== undefined &&
reviewer !== undefined &&
lineages.some(
(lineage) =>
manager.sessionIds.includes(lineage.parent) &&
reviewer.sessionIds.includes(lineage.child) &&
reviewer.requestedModel.routeProvider === lineage.identity.provider &&
reviewer.requestedModel.model === lineage.identity.model,
);
const linkedLineages =
manager && reviewer
? lineages.filter(
(lineage) =>
manager.sessionIds.includes(lineage.parent) &&
reviewer.sessionIds.includes(lineage.child) &&
reviewer.requestedModel.routeProvider ===
lineage.identity.provider &&
reviewer.requestedModel.model === lineage.identity.model,
)
: [];
const linkedPairs = [
...new Set(
linkedLineages.map((lineage) => `${lineage.parent}\0${lineage.child}`),
),
];
const managerSessionId =
linkedPairs.length === 1 ? (linkedPairs[0]?.split("\0")[0] ?? null) : null;
Comment on lines +390 to +391

@chatgpt-codex-connector chatgpt-codex-connector Bot Aug 28, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Select the unique parent rather than a unique review pair

When a canary legitimately completes two reviewer dispatches in the same manager session—for example, the first review reports a finding and a second review passes after repair—each task has a different child session, so linkedPairs.length is 2 even though both pairs have the same parent. This sets managerSessionId to null, empties managerCalls, and incorrectly fails every transcript-derived check. Determine uniqueness from the lineage parent IDs while allowing multiple reviewer children belonging to that one parent.

Useful? React with 👍 / 👎.

return {
actors: [...actors.values()],
complete: consistent && distinct && linked,
complete: consistent && distinct && managerSessionId !== null,
managerSessionId,
};
}

Expand Down Expand Up @@ -463,12 +487,15 @@ export function deriveCanaryResult(input: {
? assuranceProjection(session.value)
: null;
const actors = derivedActors(transcript.entries, calls);
const managerCalls = actors.managerSessionId
? calls.filter((call) => call.sessionId === actors.managerSessionId)
: [];
const host = observedHost(transcript.entries);
const hasCompletedFlowCall = calls.some(
const hasCompletedFlowCall = managerCalls.some(
(call) => call.tool.startsWith("flow_") && call.status === "completed",
);
const loadedPlugin = loadedPluginMatches(
calls,
managerCalls,
input.packageVersion,
input.pluginEntrySha256,
);
Expand All @@ -495,19 +522,19 @@ export function deriveCanaryResult(input: {
session.value.operations.some(
(operation) => operation.kind === "plan-save",
) &&
completed(calls, "flow_plan_save"),
completed(managerCalls, "flow_plan_save"),
"captures-validation":
assurance !== null &&
assuranceSatisfied(assurance, "accepted-validation") &&
assuranceSatisfied(assurance, "canonical-gate") &&
assuranceSatisfied(assurance, "declared-evidence") &&
completed(calls, "flow_validation_start"),
completed(managerCalls, "flow_validation_start"),
"dispatches-reviewer":
assurance !== null &&
assuranceSatisfied(assurance, "recorded-completion") &&
actors.complete &&
completed(calls, "flow_review_start") &&
calls.some(
completed(managerCalls, "flow_review_start") &&
managerCalls.some(
(call) =>
call.tool === "task" &&
call.status === "completed" &&
Expand All @@ -516,7 +543,7 @@ export function deriveCanaryResult(input: {
"closes-with-delivery":
assurance !== null &&
assurance.conclusion === "completion-supported" &&
completionSupportedFromDelivery(calls, assurance),
completionSupportedFromDelivery(managerCalls, assurance),
};
const missing =
input.installation === null ||
Expand Down
35 changes: 35 additions & 0 deletions tests/eval-canary.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,41 @@ describe("canary record boundary", () => {
expect(derived.status).toBe("failed");
});

test("rejects lifecycle evidence spliced across manager sessions", () => {
const value = prepared();
const transcript = canaryTranscript();
const manager = transcript.messages.at(0);
if (!manager) throw new Error("Canary manager fixture is missing.");
const statusIndex = manager.parts.findIndex(
({ tool }) => tool === "flow_status",
);
const status = manager.parts.splice(statusIndex, 1).at(0);
if (!status) throw new Error("Canary status fixture is missing.");
transcript.messages.push({
info: {
role: "assistant",
agent: "build",
providerID: "provider",
modelID: "model",
sessionID: "ses_other_manager",
},
parts: [status],
});
const derived = deriveCanaryResult({
packageVersion: value.artifact.packageVersion,
artifactSha256: value.artifactSha256,
tarballSha256: value.artifact.tarballSha256,
preparedSha256: value.sha256,
pluginEntrySha256: value.pluginEntrySha256,
installation: installation(value),
session: canarySession(),
transcript,
});
expect(derived.checks["installs-packed-artifact"]).toBe(false);
expect(derived.checks["loads-flow-tools"]).toBe(false);
expect(derived.status).toBe("failed");
});

test("accepts strict passed, failed, and incomplete records", () => {
expect(parseCanaryRecord(record()).ok).toBe(true);
expect(
Expand Down