-
Notifications
You must be signed in to change notification settings - Fork 258
[Fix] Task history can disappear when users restart after completion #1452
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Draft
zoomote
wants to merge
4
commits into
main
Choose a base branch
from
fix/model-completion-persistence-176wyz2fscjpe
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Draft
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
ffa64eb
test(formal): model completion persistence ordering
roomote 68fb953
test(task): reproduce completion persistence race
roomote e3a191d
fix(task): persist history before completion
roomote 096c469
test(e2e): require restored completion turn
roomote File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| module CompletionPersistence | ||
|
|
||
| abstract sig CompletionPolicy {} | ||
| one sig CurrentPolicy, DurableFirstPolicy extends CompletionPolicy {} | ||
|
|
||
| one sig Config { | ||
| policy: one CompletionPolicy | ||
| } | ||
|
|
||
| one sig Marker {} | ||
|
|
||
| one sig Lifecycle { | ||
| var historyWriteStarted: lone Marker, | ||
| var historyDurable: lone Marker, | ||
| var completionAccepted: lone Marker, | ||
| var completionEmitted: lone Marker, | ||
| var hostStopped: lone Marker | ||
| } | ||
|
|
||
| pred init { | ||
| no Lifecycle.historyWriteStarted | ||
| no Lifecycle.historyDurable | ||
| no Lifecycle.completionAccepted | ||
| no Lifecycle.completionEmitted | ||
| no Lifecycle.hostStopped | ||
| } | ||
|
|
||
| pred startHistoryWrite { | ||
| no Lifecycle.historyWriteStarted | ||
| no Lifecycle.hostStopped | ||
| Lifecycle.historyWriteStarted' = Marker | ||
| Lifecycle.historyDurable' = Lifecycle.historyDurable | ||
| Lifecycle.completionAccepted' = Lifecycle.completionAccepted | ||
| Lifecycle.completionEmitted' = Lifecycle.completionEmitted | ||
| Lifecycle.hostStopped' = Lifecycle.hostStopped | ||
| } | ||
|
|
||
| pred finishHistoryWrite { | ||
| some Lifecycle.historyWriteStarted | ||
| no Lifecycle.historyDurable | ||
| no Lifecycle.hostStopped | ||
| Lifecycle.historyWriteStarted' = Lifecycle.historyWriteStarted | ||
| Lifecycle.historyDurable' = Marker | ||
| Lifecycle.completionAccepted' = Lifecycle.completionAccepted | ||
| Lifecycle.completionEmitted' = Lifecycle.completionEmitted | ||
| Lifecycle.hostStopped' = Lifecycle.hostStopped | ||
| } | ||
|
|
||
| pred acceptCompletion { | ||
| no Lifecycle.completionAccepted | ||
| no Lifecycle.hostStopped | ||
| Lifecycle.historyWriteStarted' = Lifecycle.historyWriteStarted | ||
| Lifecycle.historyDurable' = Lifecycle.historyDurable | ||
| Lifecycle.completionAccepted' = Marker | ||
| Lifecycle.completionEmitted' = Lifecycle.completionEmitted | ||
| Lifecycle.hostStopped' = Lifecycle.hostStopped | ||
| } | ||
|
|
||
| pred emitCompletion { | ||
| some Lifecycle.historyWriteStarted | ||
| some Lifecycle.completionAccepted | ||
| no Lifecycle.completionEmitted | ||
| no Lifecycle.hostStopped | ||
| Config.policy = DurableFirstPolicy implies some Lifecycle.historyDurable | ||
| Lifecycle.historyWriteStarted' = Lifecycle.historyWriteStarted | ||
| Lifecycle.historyDurable' = Lifecycle.historyDurable | ||
| Lifecycle.completionAccepted' = Lifecycle.completionAccepted | ||
| Lifecycle.completionEmitted' = Marker | ||
| Lifecycle.hostStopped' = Lifecycle.hostStopped | ||
| } | ||
|
|
||
| pred stopHost { | ||
| some Lifecycle.completionEmitted | ||
| no Lifecycle.hostStopped | ||
| Lifecycle.historyWriteStarted' = Lifecycle.historyWriteStarted | ||
| Lifecycle.historyDurable' = Lifecycle.historyDurable | ||
| Lifecycle.completionAccepted' = Lifecycle.completionAccepted | ||
| Lifecycle.completionEmitted' = Lifecycle.completionEmitted | ||
| Lifecycle.hostStopped' = Marker | ||
| } | ||
|
|
||
| pred stutter { | ||
| Lifecycle.historyWriteStarted' = Lifecycle.historyWriteStarted | ||
| Lifecycle.historyDurable' = Lifecycle.historyDurable | ||
| Lifecycle.completionAccepted' = Lifecycle.completionAccepted | ||
| Lifecycle.completionEmitted' = Lifecycle.completionEmitted | ||
| Lifecycle.hostStopped' = Lifecycle.hostStopped | ||
| } | ||
|
|
||
| fact traces { | ||
| init | ||
| always ( | ||
| startHistoryWrite or | ||
| finishHistoryWrite or | ||
| acceptCompletion or | ||
| emitCompletion or | ||
| stopHost or | ||
| stutter | ||
| ) | ||
| } | ||
|
|
||
| pred DurableFirstHappyPath { | ||
| Config.policy = DurableFirstPolicy | ||
| eventually ( | ||
| some Lifecycle.hostStopped and | ||
| some Lifecycle.completionEmitted and | ||
| some Lifecycle.historyDurable | ||
| ) | ||
| } | ||
|
|
||
| assert CurrentCompletionIsDurable { | ||
| Config.policy = CurrentPolicy implies | ||
| always (some Lifecycle.completionEmitted implies some Lifecycle.historyDurable) | ||
| } | ||
|
|
||
| assert CurrentShutdownPreservesHistory { | ||
| Config.policy = CurrentPolicy implies | ||
| always ( | ||
| some Lifecycle.hostStopped and some Lifecycle.completionEmitted implies | ||
| some Lifecycle.historyDurable | ||
| ) | ||
| } | ||
|
|
||
| assert DurableFirstCompletionIsDurable { | ||
| Config.policy = DurableFirstPolicy implies | ||
| always (some Lifecycle.completionEmitted implies some Lifecycle.historyDurable) | ||
| } | ||
|
|
||
| assert DurableFirstShutdownPreservesHistory { | ||
| Config.policy = DurableFirstPolicy implies | ||
| always ( | ||
| some Lifecycle.hostStopped and some Lifecycle.completionEmitted implies | ||
| some Lifecycle.historyDurable | ||
| ) | ||
| } | ||
|
|
||
| check CurrentCompletionIsDurable for 6 but 6 steps | ||
| check CurrentShutdownPreservesHistory for 6 but 6 steps | ||
| run DurableFirstHappyPath for 6 but 6 steps | ||
| check DurableFirstCompletionIsDurable for 6 but 8 steps | ||
| check DurableFirstShutdownPreservesHistory for 6 but 8 steps |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| # Completion persistence model | ||
|
|
||
| `CompletionPersistence.als` models the narrow lifecycle behind the restart-persistence E2E failure: | ||
|
|
||
| - the streamed assistant history write starts; | ||
| - completion is accepted and `TaskCompleted` is emitted; | ||
| - the history write becomes durable; | ||
| - the extension host stops after observing completion. | ||
|
|
||
| The model compares two event contracts: | ||
|
|
||
| - `CurrentPolicy` permits `TaskCompleted` once completion is accepted and a history write has started; | ||
| - `DurableFirstPolicy` additionally requires the history write to be durable before completion is emitted. | ||
|
|
||
| The current-policy assertions search for a hypothesized, contract-permitted bad shape: the host sees completion and stops while API history is still not durable. Here, durable means that the required history version is visible to a fresh extension host; the model does not claim power-loss durability or filesystem `fsync` semantics. The durability-gated assertions check that completion and shutdown cannot expose that state. | ||
|
|
||
| The model is intentionally small. It establishes the missing ordering invariant but does not prove that the CI failure followed this exact trace or that every concrete runtime path maps to the abstract current-policy transition. Unrestricted stuttering also means this is a bounded safety model: it does not guarantee write completion, retries, or eventual task completion when persistence keeps failing. | ||
|
|
||
| ## Deterministic production regression | ||
|
|
||
| `src/core/task/__tests__/Task.persistence.spec.ts` blocks the real `saveApiMessages` boundary on a deferred promise and accepts completion on the same `Task`. It confirms that `TaskCompleted` remains pending while the write is unresolved, then emits after the write succeeds. A second case exhausts the bounded persistence retries and confirms that the failure is reported without emitting `TaskCompleted`. | ||
|
|
||
| The test maps to the model as follows: | ||
|
|
||
| - the captured `saveApiMessages` call for the assistant `attempt_completion` turn is `startHistoryWrite`; | ||
| - the unresolved deferred save is `not historyDurable`; | ||
| - accepting the matching completion call is `acceptCompletion`; | ||
| - resolving the deferred is `finishHistoryWrite`; | ||
| - observing `TaskCompleted` afterward is `emitCompletion`. | ||
|
|
||
| An indefinitely delayed write keeps completion pending rather than weakening the public event contract. A failed initial write is retried with the existing bounded retry policy; if all retries fail, the completion handler reports the persistence error and does not emit `TaskCompleted`. | ||
|
|
||
| ## Code mapping | ||
|
|
||
| - `startHistoryWrite` and `finishHistoryWrite` represent `Task.saveApiConversationHistory()` entering and completing its durable file write. | ||
| - `acceptCompletion` and `emitCompletion` represent completion approval followed by `AttemptCompletionTool.emitPublicTaskCompleted()`. | ||
| - `stopHost` represents the restart E2E (or a real extension shutdown) acting on the public completion event. | ||
| - `DurableFirstPolicy` represents the production contract: the public completion boundary is not crossed until the required API history write succeeds. | ||
|
|
||
| ## Run Alloy 6 | ||
|
|
||
| Download the pinned Alloy release, verify it, and execute all commands: | ||
|
|
||
| ```bash | ||
| cd .github/alloy | ||
| curl -fsSL https://github.com/AlloyTools/org.alloytools.alloy/releases/download/v6.2.0/org.alloytools.alloy.dist.jar -o alloy.jar | ||
| printf '%s %s\n' '6b8c1cb5bc93bedfc7c61435c4e1ab6e688a242dc702a394628d9a9801edb78d' alloy.jar | sha256sum --check | ||
| java -jar alloy.jar exec -c '*' -t text -o - CompletionPersistence.als | ||
| ``` | ||
|
|
||
| Expected results: | ||
|
|
||
| - both `Current...` checks produce counterexamples where completion precedes durable history, including a trace that stops the host in that state; | ||
| - `DurableFirstHappyPath` is satisfiable, so the stronger guard does not prevent completion; | ||
| - both `DurableFirst...` assertions have no counterexample within the configured bounds. | ||
|
|
||
| The JAR is a local analysis tool and must not be committed. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🗄️ Data Integrity & Integration | 🟡 Minor | ⚡ Quick win
Assert the restored turn roles, not only the history length.
conversationLength >= 2does not prove that the fresh host restored both the user turn and the accepted assistant completion turn. Assert the roles or expected content of the restored entries, or use a helper whose contract guarantees those exact turns. Otherwise, a regression that restores two unrelated entries can pass this test.As per coding guidelines, regression tests must use behavior-focused assertions for persistence and boundary cases.
🤖 Prompt for AI Agents
Source: Coding guidelines