From b97f2704a9ff6c7c6401d4dbd8d0b05b8fa9a155 Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 22:43:31 +0900 Subject: [PATCH 01/14] chore(eval): remove legacy observation surfaces Why: - The bounded 20-task experiment does not require v1 compatibility, report scaffolding, or module-specific schemas. - Keeping those contracts would preserve unsupported product surface and inflate the recorder implementation. What: - Remove the v0.1 schema, fixtures, verifier, report template, distributed protocol documents, and toolkit manifest. - Leave only repository metadata before adding the clean-slate recorder. Validation: - git diff --cached --check - No runtime tests apply to this deletion-only commit --- .gitignore | 11 - CHARTER.md | 61 --- PRIVACY.md | 73 --- README.md | 130 ----- fixtures/invalid/raw/blank-line.jsonl | 1 - fixtures/invalid/raw/duplicate-nested.jsonl | 1 - .../invalid/raw/duplicate-top-level.jsonl | 1 - fixtures/invalid/raw/malformed.jsonl | 1 - fixtures/invalid/raw/nonfinite.jsonl | 1 - .../observation-count-over-maximum.json | 50 -- .../schema/observation-forbidden-notes.json | 66 --- .../observation-invalid-bounded-time.json | 69 --- .../schema/observation-invalid-date.json | 65 --- .../schema/observation-invalid-latency.json | 78 --- .../observation-invalid-measured-time.json | 68 --- .../schema/observation-invalid-uuid.json | 65 --- .../schema/observation-negative-count.json | 65 --- .../observation-revision-1-supersedes.json | 65 --- .../observation-revision-2-no-supersedes.json | 65 --- .../observation-seal-unused-metrics.json | 74 --- ...observation-seal-used-missing-version.json | 80 --- .../observation-spec-unused-metrics.json | 75 --- ...observation-spec-used-missing-version.json | 83 --- .../observation-ward-unused-metrics.json | 75 --- ...observation-ward-used-missing-version.json | 78 --- fixtures/valid/observations.jsonl | 6 - protocol.md | 163 ------ requirements-dev.txt | 2 - schemas/observation-v1.schema.json | 477 ------------------ scripts/verify.sh | 439 ---------------- templates/decision-report-v1.md | 94 ---- toolkit-module.json | 29 -- 32 files changed, 2611 deletions(-) delete mode 100644 CHARTER.md delete mode 100644 PRIVACY.md delete mode 100644 README.md delete mode 100644 fixtures/invalid/raw/blank-line.jsonl delete mode 100644 fixtures/invalid/raw/duplicate-nested.jsonl delete mode 100644 fixtures/invalid/raw/duplicate-top-level.jsonl delete mode 100644 fixtures/invalid/raw/malformed.jsonl delete mode 100644 fixtures/invalid/raw/nonfinite.jsonl delete mode 100644 fixtures/invalid/schema/observation-count-over-maximum.json delete mode 100644 fixtures/invalid/schema/observation-forbidden-notes.json delete mode 100644 fixtures/invalid/schema/observation-invalid-bounded-time.json delete mode 100644 fixtures/invalid/schema/observation-invalid-date.json delete mode 100644 fixtures/invalid/schema/observation-invalid-latency.json delete mode 100644 fixtures/invalid/schema/observation-invalid-measured-time.json delete mode 100644 fixtures/invalid/schema/observation-invalid-uuid.json delete mode 100644 fixtures/invalid/schema/observation-negative-count.json delete mode 100644 fixtures/invalid/schema/observation-revision-1-supersedes.json delete mode 100644 fixtures/invalid/schema/observation-revision-2-no-supersedes.json delete mode 100644 fixtures/invalid/schema/observation-seal-unused-metrics.json delete mode 100644 fixtures/invalid/schema/observation-seal-used-missing-version.json delete mode 100644 fixtures/invalid/schema/observation-spec-unused-metrics.json delete mode 100644 fixtures/invalid/schema/observation-spec-used-missing-version.json delete mode 100644 fixtures/invalid/schema/observation-ward-unused-metrics.json delete mode 100644 fixtures/invalid/schema/observation-ward-used-missing-version.json delete mode 100644 fixtures/valid/observations.jsonl delete mode 100644 protocol.md delete mode 100644 requirements-dev.txt delete mode 100644 schemas/observation-v1.schema.json delete mode 100755 scripts/verify.sh delete mode 100644 templates/decision-report-v1.md delete mode 100644 toolkit-module.json diff --git a/.gitignore b/.gitignore index 05fb9f9..3d79cfd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,5 @@ .DS_Store .venv/ -.local/ - -# Raw or generated evaluation data must not be tracked. -/observations.jsonl -/reports/private/ -/reports/generated/ -/reports/drafts/ - -# Development and editor artifacts. -__pycache__/ -*.py[cod] *.tmp *.swp coverage.out diff --git a/CHARTER.md b/CHARTER.md deleted file mode 100644 index ae827d6..0000000 --- a/CHARTER.md +++ /dev/null @@ -1,61 +0,0 @@ -# Eval Charter - -## Responsibility - -Eval measures whether each Agent Toolkit module creates more value than cost on -real tasks. - -It provides a bounded post-task observation and reporting contract for module -effects, defects, cost, and user friction. - -## Position - -The Native Agent, user, or CI owns task execution, module choice, composition, -and Eval invocation. Eval begins only after that caller identifies a terminal -task outcome and decides measurement is useful. - -Eval's one intended operational side effect is an authorized append to the -external private observation log. It does not mutate the user's completed task -or any Toolkit module state. - -## Invariants - -- A task remains executable and completable without Eval. -- The Native Agent, user, or CI explicitly invokes Eval; Eval never - self-activates. -- Observation requires a terminal outcome and Host write authority. -- Eval never invokes Spec, Ward, Seal, an Agent, or CI. -- Eval owns no workflow transition, execution order, retry, repair, or release - gate. -- No module automatically invokes Eval, and Eval does not automatically invoke - another module. -- Task effects remain separate from used-only module metrics so used and unused - cohorts can be compared without inventing module behavior. -- Unknown and unassessed values remain `null`. -- Observation and report contracts are versioned and provider-neutral. -- Raw observations remain outside source repositories under current-user-only - access. -- One observation file has one active writer. -- Reports disclose sample sizes, missing values, and observational limitations. -- Product and release decisions remain human-owned. - -## Fixed non-goals - -Eval is not: - -- Agent execution or orchestration; -- module execution, retry, or repair; -- a Plugin, Skill, Hook, or automatic task-completion listener; -- a central Toolkit runtime or shared lifecycle state; -- automatic release approval or promotion; -- a real-time telemetry platform or dashboard-first product; -- repository context, Agent Memory, RAG, or prompt injection; -- prompt, transcript, Chain-of-Thought, command, source-code, secret, or raw - per-event user behavior collection; -- proof that an observational difference is causal. - -## Maturity boundary - -Contract verification proves only that the scaffold is internally consistent. -A cumulative report proves only that the evidence workflow operated. Neither -state alone proves that a module is valuable or ready for release. diff --git a/PRIVACY.md b/PRIVACY.md deleted file mode 100644 index 0e16414..0000000 --- a/PRIVACY.md +++ /dev/null @@ -1,73 +0,0 @@ -# Eval Privacy Contract - -Eval collects only bounded structured facts needed to measure module value and -cost. The observation contract has no free-text field. - -## Prohibited observation data - -Never store any of the following in an observation: - -- repository, organization, customer, product, project, or internal system - names; -- actual file or directory paths; -- task descriptions, business content, source code, patches, or artifacts; -- prompts, transcripts, Chain-of-Thought, or raw Agent output; -- commands, command arguments, URLs, hostnames, or infrastructure identifiers; -- credentials, tokens, keys, secrets, personal data, or raw/per-event user - behavior traces; -- free-text notes, explanations, rationales, copied errors, or task narratives. - -Unknown properties are rejected. `task_id` and each `observation_id` are -independently generated random UUIDv4 values and must never be derived from -work content. `supersedes` is either `null` for revision 1 or the immediately -preceding `observation_id`; it is never generated from work content. Repository -identity is not recorded, even as a hash. - -Module versions and model identifiers contain only bounded public identifiers. -The schema constrains their syntax; it does not prove that an identifier is -public. A model identifier is `null` unless public availability is known, and -whenever the value is unavailable or sensitive. A used module requires its -exact public version; if that version cannot be recorded safely, do not append -the observation and do not misstate the module as unused. - -## External private storage - -Raw observations stay outside every source checkout at the path defined in -[protocol.md](protocol.md). A set `XDG_STATE_HOME` must be absolute. Only an -unset `XDG_STATE_HOME` falls back to `$HOME/.local/state`, and `HOME` must be -absolute. Resolve the path physically and reject any location inside a Git -worktree. - -On POSIX-like hosts, the state directory has mode `0700` and -`observations.jsonl` has mode `0600`. Other hosts use equivalent -current-user-only access controls. The file must be a regular, non-symlink file -owned by the current user where the Host exposes those concepts. - -The caller must hold Host write authority and be designated as the sole active -writer for that file by external Host policy or coordination. Multiple allowed -invocation owners do not imply simultaneous writers. If sole-writer status -cannot be established, do not append. Eval v0.1 does not lock, encrypt, -transmit, retain, delete, synchronize, or merge the file automatically. The -user owns backup and retention policy. - -## Reports - -Private reports remain outside source checkouts. They may contain aggregate -counts but never raw rows, task IDs, observation IDs, task narratives, or other -prohibited content. - -A report copied into the repository or otherwise published must: - -1. disclose real and excluded synthetic counts, superseded rows, missing values, - and cohort sizes; -2. omit raw observations, individual rows, and private identifiers; -3. suppress every public cohort cell with `n < 5` using `<5`; -4. suppress complementary cells when another displayed value could reveal a - hidden count; -5. use only aggregate, non-sensitive language; -6. label sampling, missing-data, measurement, and causal limitations; -7. confirm that it records a human decision and triggers no workflow or release - action. - -Publication requires an explicit manual privacy review. Schema or repository -verification is necessary but does not constitute that review. diff --git a/README.md b/README.md deleted file mode 100644 index 7b74970..0000000 --- a/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# Eval - -Eval measures whether each Agent Toolkit module creates more value than cost on -real tasks. - -Eval is a post-task artifact protocol in the Evaluation plane. It is not a -Plugin, Skill, Hook, runtime, telemetry service, or workflow controller. - -## Status - -Eval v0.1 currently provides a charter, observation protocol, JSON Schema, -synthetic fixtures, report template, provider-neutral module manifest, and -development verification. This is a contract scaffold, not a completed -Evaluation MVP: the repository contains no real observations or cumulative -decision report. - -## Position and invocation - -The Native Agent, user, or CI owns the task, chooses modules, and decides -whether Eval is useful after the task reaches a terminal outcome. - -```text -Native Agent / User / CI - | - | uses Spec, Ward, Seal, or none as needed - v - terminal task outcome - | - | selects Eval when measurement is useful - v - post-task observation - | - v - aggregate report and human decision -``` - -Caller-owned invocation is not self-activation. Eval does not detect task -completion, schedule itself, modify the completed task, invoke another module, -choose workflow order, repair work, or apply a release decision. Appending an -observation requires the caller to have Host write authority for the external -private data file. A standing user or Host policy may provide that authority; -Eval adds no per-observation approval workflow. - -The provider-neutral discovery contract is -[`toolkit-module.json`](toolkit-module.json). A Toolkit registry may link to -that contract, but it must not execute Eval or turn it into a central runtime. - -## Observation model - -One user objective is one real task. Retries, resumptions, recovery attempts, -and OS-specific reruns remain part of that task. Synthetic observations test -the contract and never count as real-task evidence. - -Each observation records bounded facts about: - -- task type, terminal outcome, Agent, model, and primary host OS; -- which Spec, Ward, and Seal versions were used; -- task effects that can be assessed whether a module was used or not; -- module-specific decisions, defects, cost, and friction for modules that were - used. - -Unknown or unassessed values remain `null`; they are never guessed as false or -zero. Time is either measured, a bounded estimate, or `null`. - -Read [CHARTER.md](CHARTER.md), [protocol.md](protocol.md), and -[PRIVACY.md](PRIVACY.md) before appending data. - -## Private data and single writer - -Raw observations stay outside every source checkout: - -- `$XDG_STATE_HOME/jgoneit/eval/v1/observations.jsonl` when - `XDG_STATE_HOME` is set to an absolute path; -- `$HOME/.local/state/jgoneit/eval/v1/observations.jsonl` only when - `XDG_STATE_HOME` is unset and `HOME` is absolute. - -Fail closed for relative state roots or any resolved path inside a Git -worktree. On POSIX-like hosts, the state directory has mode `0700` and the file -has mode `0600`; other hosts use equivalent current-user-only access controls. - -Eval v0.1 permits exactly one active writer for an observation file. If the -caller cannot establish exclusive single-writer access, it does not append. -There is no daemon, lock service, concurrent merge protocol, or shared Toolkit -state. - -## Development verification - -Eval has no Python or Go product implementation. Repository verification uses a -pinned development-only JSON Schema validator: - -```sh -python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt -scripts/verify.sh -``` - -This verifies schemas, synthetic fixtures, privacy canaries, report structure, -and repository boundaries. It is not an installed Eval command and does not -read, validate, or mutate the user's private observation file. The designated -writer and report reviewer remain responsible for protocol-level date, bound, -count, and revision-chain relationships. - -## Reports and decisions - -Analysis uses the latest valid revision for each real task, discloses missing -values and sample sizes, and treats used-versus-unused results as observational -rather than causal. A private cumulative report remains outside source -checkouts. A sanitized aggregate may be published only after manual privacy -review; public cohorts with `n < 5` and inferable complementary cells are -suppressed. - -Eval supplies evidence, not a universal threshold. Retention, modification, -promotion, removal, and additional experiments remain human decisions. - -## Completion boundary - -Passing `scripts/verify.sh` establishes only that the contract scaffold is -internally consistent. The Evaluation MVP additionally requires real Task -observations and one privacy-reviewed cumulative report. That milestone proves -the evidence workflow operated; it does not prove that a module caused an -outcome or met a release threshold. - -## Future automation boundary - -Only repeated malformed records, missing versions, inconsistent aggregation, -multiple writers, or repeated reporting work can justify a future Go CLI. The -only candidates are `eval validate`, `eval summarize`, and `eval compare`. - -`eval run-agent`, module execution, orchestration, repair, automatic promotion, -and self-triggered recording remain prohibited. diff --git a/fixtures/invalid/raw/blank-line.jsonl b/fixtures/invalid/raw/blank-line.jsonl deleted file mode 100644 index 8b13789..0000000 --- a/fixtures/invalid/raw/blank-line.jsonl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/fixtures/invalid/raw/duplicate-nested.jsonl b/fixtures/invalid/raw/duplicate-nested.jsonl deleted file mode 100644 index 683712a..0000000 --- a/fixtures/invalid/raw/duplicate-nested.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"modules":{"seal":{"used":false,"used":true}}} diff --git a/fixtures/invalid/raw/duplicate-top-level.jsonl b/fixtures/invalid/raw/duplicate-top-level.jsonl deleted file mode 100644 index 23d969d..0000000 --- a/fixtures/invalid/raw/duplicate-top-level.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"schema_version":"eval-observation/v1","schema_version":"eval-observation/v1"} diff --git a/fixtures/invalid/raw/malformed.jsonl b/fixtures/invalid/raw/malformed.jsonl deleted file mode 100644 index 9240a21..0000000 --- a/fixtures/invalid/raw/malformed.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"schema_version":"eval-observation/v1" diff --git a/fixtures/invalid/raw/nonfinite.jsonl b/fixtures/invalid/raw/nonfinite.jsonl deleted file mode 100644 index 129f8bb..0000000 --- a/fixtures/invalid/raw/nonfinite.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"revision":NaN} diff --git a/fixtures/invalid/schema/observation-count-over-maximum.json b/fixtures/invalid/schema/observation-count-over-maximum.json deleted file mode 100644 index 156431b..0000000 --- a/fixtures/invalid/schema/observation-count-over-maximum.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000099", - "task_id": "10000000-0000-4000-8000-000000000099", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": {"used": false, "version": null}, - "ward": {"used": false, "version": null}, - "spec": {"used": false, "version": null} - }, - "outcome": { - "status": "completed", - "user_interventions": 1000001, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": {"terminal_completion_invalidated": false}, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": {"seal": null, "ward": null, "spec": null} -} diff --git a/fixtures/invalid/schema/observation-forbidden-notes.json b/fixtures/invalid/schema/observation-forbidden-notes.json deleted file mode 100644 index bbcda63..0000000 --- a/fixtures/invalid/schema/observation-forbidden-notes.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": null - }, - "notes": "forbidden" -} diff --git a/fixtures/invalid/schema/observation-invalid-bounded-time.json b/fixtures/invalid/schema/observation-invalid-bounded-time.json deleted file mode 100644 index 36373a8..0000000 --- a/fixtures/invalid/schema/observation-invalid-bounded-time.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": { - "method": "bounded-estimate", - "lower_seconds": -1, - "upper_seconds": 30 - }, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-invalid-date.json b/fixtures/invalid/schema/observation-invalid-date.json deleted file mode 100644 index 0beba5a..0000000 --- a/fixtures/invalid/schema/observation-invalid-date.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-02-30", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-invalid-latency.json b/fixtures/invalid/schema/observation-invalid-latency.json deleted file mode 100644 index 7a3408d..0000000 --- a/fixtures/invalid/schema/observation-invalid-latency.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000003", - "task_id": "10000000-0000-4000-8000-000000000003", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-04", - "recorded_on": "2026-01-04", - "population": "synthetic", - "task_type": "security", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "linux", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": true, - "version": "synthetic-ward-v1" - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": { - "method": "measured", - "seconds": 10 - }, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": { - "normal_workflow_false_deny": false, - "added_model_visible_prompt": false, - "required_disablement": false, - "defer_mutated_output_or_audit": false, - "safe_recovery_after_deny": true, - "hook_latency": { - "median_ms": 12, - "sample_count": 0 - } - }, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-invalid-measured-time.json b/fixtures/invalid/schema/observation-invalid-measured-time.json deleted file mode 100644 index 5d0a0c8..0000000 --- a/fixtures/invalid/schema/observation-invalid-measured-time.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": { - "method": "measured", - "seconds": -1 - }, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-invalid-uuid.json b/fixtures/invalid/schema/observation-invalid-uuid.json deleted file mode 100644 index 7e505e0..0000000 --- a/fixtures/invalid/schema/observation-invalid-uuid.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "not-a-uuid", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-negative-count.json b/fixtures/invalid/schema/observation-negative-count.json deleted file mode 100644 index 102e143..0000000 --- a/fixtures/invalid/schema/observation-negative-count.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": -1, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-revision-1-supersedes.json b/fixtures/invalid/schema/observation-revision-1-supersedes.json deleted file mode 100644 index 2181753..0000000 --- a/fixtures/invalid/schema/observation-revision-1-supersedes.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": "30000000-0000-4000-8000-000000000001", - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-revision-2-no-supersedes.json b/fixtures/invalid/schema/observation-revision-2-no-supersedes.json deleted file mode 100644 index 3ea636c..0000000 --- a/fixtures/invalid/schema/observation-revision-2-no-supersedes.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 2, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-seal-unused-metrics.json b/fixtures/invalid/schema/observation-seal-unused-metrics.json deleted file mode 100644 index a29d34b..0000000 --- a/fixtures/invalid/schema/observation-seal-unused-metrics.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": { - "completion_decision": "accepted", - "false_acceptance": false, - "evidence_corruption_bypass": false, - "source_binding_bypass": false, - "false_source_mismatch": false, - "task_authoring_time": null, - "completion_refusal_understood": null, - "added_user_interventions": 0 - }, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-seal-used-missing-version.json b/fixtures/invalid/schema/observation-seal-used-missing-version.json deleted file mode 100644 index d4384af..0000000 --- a/fixtures/invalid/schema/observation-seal-used-missing-version.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000004", - "task_id": "10000000-0000-4000-8000-000000000004", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-05", - "recorded_on": "2026-01-05", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": true, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": { - "method": "measured", - "seconds": 60 - }, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": { - "completion_decision": "accepted", - "false_acceptance": false, - "evidence_corruption_bypass": false, - "source_binding_bypass": false, - "false_source_mismatch": false, - "task_authoring_time": { - "method": "measured", - "seconds": 60 - }, - "completion_refusal_understood": null, - "added_user_interventions": 0 - }, - "ward": null, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-spec-unused-metrics.json b/fixtures/invalid/schema/observation-spec-unused-metrics.json deleted file mode 100644 index e9dc897..0000000 --- a/fixtures/invalid/schema/observation-spec-unused-metrics.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": { - "artifact_created": true, - "spec_implementation_divergence": false, - "unnecessary_artifacts": 0, - "drafting_time": null, - "readable_by_user": true, - "directly_usable_by_agent": true, - "appropriately_sized": true, - "important_decisions_distinguished": true, - "would_reuse": true - } - } -} diff --git a/fixtures/invalid/schema/observation-spec-used-missing-version.json b/fixtures/invalid/schema/observation-spec-used-missing-version.json deleted file mode 100644 index a722f3a..0000000 --- a/fixtures/invalid/schema/observation-spec-used-missing-version.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000002", - "task_id": "10000000-0000-4000-8000-000000000002", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-03", - "recorded_on": "2026-01-03", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": true, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 1, - "module_interaction_time": { - "method": "bounded-estimate", - "lower_seconds": 120, - "upper_seconds": 180 - }, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 1, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": null, - "spec": { - "artifact_created": true, - "spec_implementation_divergence": false, - "unnecessary_artifacts": 0, - "drafting_time": { - "method": "bounded-estimate", - "lower_seconds": 120, - "upper_seconds": 180 - }, - "readable_by_user": true, - "directly_usable_by_agent": true, - "appropriately_sized": true, - "important_decisions_distinguished": true, - "would_reuse": true - } - } -} diff --git a/fixtures/invalid/schema/observation-ward-unused-metrics.json b/fixtures/invalid/schema/observation-ward-unused-metrics.json deleted file mode 100644 index 50c25a7..0000000 --- a/fixtures/invalid/schema/observation-ward-unused-metrics.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000001", - "task_id": "10000000-0000-4000-8000-000000000001", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-02", - "recorded_on": "2026-01-02", - "population": "synthetic", - "task_type": "backend-feature", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "darwin", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": false, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": null, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": { - "normal_workflow_false_deny": false, - "added_model_visible_prompt": false, - "required_disablement": false, - "defer_mutated_output_or_audit": false, - "safe_recovery_after_deny": true, - "hook_latency": { - "median_ms": 12, - "sample_count": 4 - } - }, - "spec": null - } -} diff --git a/fixtures/invalid/schema/observation-ward-used-missing-version.json b/fixtures/invalid/schema/observation-ward-used-missing-version.json deleted file mode 100644 index b6b8a5e..0000000 --- a/fixtures/invalid/schema/observation-ward-used-missing-version.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "schema_version": "eval-observation/v1", - "observation_id": "20000000-0000-4000-8000-000000000003", - "task_id": "10000000-0000-4000-8000-000000000003", - "revision": 1, - "supersedes": null, - "terminal_on": "2026-01-04", - "recorded_on": "2026-01-04", - "population": "synthetic", - "task_type": "security", - "agent": "codex", - "model": "synthetic-model-v1", - "host_os": "linux", - "modules": { - "seal": { - "used": false, - "version": null - }, - "ward": { - "used": true, - "version": null - }, - "spec": { - "used": false, - "version": null - } - }, - "outcome": { - "status": "completed", - "user_interventions": 0, - "module_interaction_time": { - "method": "measured", - "seconds": 10 - }, - "rework_required": false - }, - "task_effects": { - "completion": { - "terminal_completion_invalidated": false - }, - "security": { - "destructive_escape": false, - "protected_secret_escape": false - }, - "requirements": { - "late_material_decisions": { - "total": 0, - "api": 0, - "data_model": 0, - "user_behavior": 0, - "authentication_authorization": 0, - "consistency_rules": 0, - "scope": 0 - }, - "api_contract_changes_after_implementation": 0, - "data_model_changes_after_implementation": 0, - "requirements_misunderstanding_rework": false, - "user_confirmation_questions": 0, - "agent_requirements_questions": 0, - "acceptance_criteria_gap_delayed_defect": false - } - }, - "module_metrics": { - "seal": null, - "ward": { - "normal_workflow_false_deny": false, - "added_model_visible_prompt": false, - "required_disablement": false, - "defer_mutated_output_or_audit": false, - "safe_recovery_after_deny": true, - "hook_latency": { - "median_ms": 12, - "sample_count": 4 - } - }, - "spec": null - } -} diff --git a/fixtures/valid/observations.jsonl b/fixtures/valid/observations.jsonl deleted file mode 100644 index 34b4630..0000000 --- a/fixtures/valid/observations.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"schema_version":"eval-observation/v1","observation_id":"20000000-0000-4000-8000-000000000001","task_id":"10000000-0000-4000-8000-000000000001","revision":1,"supersedes":null,"terminal_on":"2026-01-02","recorded_on":"2026-01-02","population":"synthetic","task_type":"backend-feature","agent":"codex","model":"synthetic-model-v1","host_os":"darwin","modules":{"seal":{"used":false,"version":null},"ward":{"used":false,"version":null},"spec":{"used":false,"version":null}},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":false},"task_effects":{"completion":{"terminal_completion_invalidated":false},"security":{"destructive_escape":false,"protected_secret_escape":false},"requirements":{"late_material_decisions":{"total":0,"api":0,"data_model":0,"user_behavior":0,"authentication_authorization":0,"consistency_rules":0,"scope":0},"api_contract_changes_after_implementation":0,"data_model_changes_after_implementation":0,"requirements_misunderstanding_rework":false,"user_confirmation_questions":0,"agent_requirements_questions":0,"acceptance_criteria_gap_delayed_defect":false}},"module_metrics":{"seal":null,"ward":null,"spec":null}} -{"schema_version":"eval-observation/v1","observation_id":"20000000-0000-4000-8000-000000000002","task_id":"10000000-0000-4000-8000-000000000002","revision":1,"supersedes":null,"terminal_on":"2026-01-03","recorded_on":"2026-01-03","population":"synthetic","task_type":"backend-feature","agent":"codex","model":"synthetic-model-v1","host_os":"darwin","modules":{"seal":{"used":false,"version":null},"ward":{"used":false,"version":null},"spec":{"used":true,"version":"synthetic-spec-v1"}},"outcome":{"status":"completed","user_interventions":1,"module_interaction_time":{"method":"bounded-estimate","lower_seconds":120,"upper_seconds":180},"rework_required":false},"task_effects":{"completion":{"terminal_completion_invalidated":false},"security":{"destructive_escape":false,"protected_secret_escape":false},"requirements":{"late_material_decisions":{"total":0,"api":0,"data_model":0,"user_behavior":0,"authentication_authorization":0,"consistency_rules":0,"scope":0},"api_contract_changes_after_implementation":0,"data_model_changes_after_implementation":0,"requirements_misunderstanding_rework":false,"user_confirmation_questions":1,"agent_requirements_questions":0,"acceptance_criteria_gap_delayed_defect":false}},"module_metrics":{"seal":null,"ward":null,"spec":{"artifact_created":true,"spec_implementation_divergence":false,"unnecessary_artifacts":0,"drafting_time":{"method":"bounded-estimate","lower_seconds":120,"upper_seconds":180},"readable_by_user":true,"directly_usable_by_agent":true,"appropriately_sized":true,"important_decisions_distinguished":true,"would_reuse":true}}} -{"schema_version":"eval-observation/v1","observation_id":"20000000-0000-4000-8000-000000000003","task_id":"10000000-0000-4000-8000-000000000003","revision":1,"supersedes":null,"terminal_on":"2026-01-04","recorded_on":"2026-01-04","population":"synthetic","task_type":"security","agent":"codex","model":"synthetic-model-v1","host_os":"linux","modules":{"seal":{"used":false,"version":null},"ward":{"used":true,"version":"synthetic-ward-v1"},"spec":{"used":false,"version":null}},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":{"method":"measured","seconds":10},"rework_required":false},"task_effects":{"completion":{"terminal_completion_invalidated":false},"security":{"destructive_escape":false,"protected_secret_escape":false},"requirements":{"late_material_decisions":{"total":0,"api":0,"data_model":0,"user_behavior":0,"authentication_authorization":0,"consistency_rules":0,"scope":0},"api_contract_changes_after_implementation":0,"data_model_changes_after_implementation":0,"requirements_misunderstanding_rework":false,"user_confirmation_questions":0,"agent_requirements_questions":0,"acceptance_criteria_gap_delayed_defect":false}},"module_metrics":{"seal":null,"ward":{"normal_workflow_false_deny":false,"added_model_visible_prompt":false,"required_disablement":false,"defer_mutated_output_or_audit":false,"safe_recovery_after_deny":true,"hook_latency":{"median_ms":12,"sample_count":4}},"spec":null}} -{"schema_version":"eval-observation/v1","observation_id":"20000000-0000-4000-8000-000000000004","task_id":"10000000-0000-4000-8000-000000000004","revision":1,"supersedes":null,"terminal_on":"2026-01-05","recorded_on":"2026-01-05","population":"synthetic","task_type":"backend-feature","agent":"codex","model":"synthetic-model-v1","host_os":"darwin","modules":{"seal":{"used":true,"version":"synthetic-seal-v1"},"ward":{"used":false,"version":null},"spec":{"used":false,"version":null}},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":{"method":"measured","seconds":60},"rework_required":false},"task_effects":{"completion":{"terminal_completion_invalidated":false},"security":{"destructive_escape":false,"protected_secret_escape":false},"requirements":{"late_material_decisions":{"total":0,"api":0,"data_model":0,"user_behavior":0,"authentication_authorization":0,"consistency_rules":0,"scope":0},"api_contract_changes_after_implementation":0,"data_model_changes_after_implementation":0,"requirements_misunderstanding_rework":false,"user_confirmation_questions":0,"agent_requirements_questions":0,"acceptance_criteria_gap_delayed_defect":false}},"module_metrics":{"seal":{"completion_decision":"accepted","false_acceptance":false,"evidence_corruption_bypass":false,"source_binding_bypass":false,"false_source_mismatch":false,"task_authoring_time":{"method":"measured","seconds":60},"completion_refusal_understood":null,"added_user_interventions":0},"ward":null,"spec":null}} -{"schema_version":"eval-observation/v1","observation_id":"20000000-0000-4000-8000-000000000005","task_id":"10000000-0000-4000-8000-000000000005","revision":1,"supersedes":null,"terminal_on":"2026-01-06","recorded_on":"2026-01-06","population":"synthetic","task_type":"backend-feature","agent":"codex","model":"synthetic-model-v1","host_os":"darwin","modules":{"seal":{"used":true,"version":"synthetic-seal-v1"},"ward":{"used":true,"version":"synthetic-ward-v1"},"spec":{"used":true,"version":"synthetic-spec-v1"}},"outcome":{"status":"completed","user_interventions":1,"module_interaction_time":{"method":"measured","seconds":300},"rework_required":false},"task_effects":{"completion":{"terminal_completion_invalidated":false},"security":{"destructive_escape":false,"protected_secret_escape":false},"requirements":{"late_material_decisions":{"total":0,"api":0,"data_model":0,"user_behavior":0,"authentication_authorization":0,"consistency_rules":0,"scope":0},"api_contract_changes_after_implementation":0,"data_model_changes_after_implementation":0,"requirements_misunderstanding_rework":false,"user_confirmation_questions":1,"agent_requirements_questions":0,"acceptance_criteria_gap_delayed_defect":false}},"module_metrics":{"seal":{"completion_decision":"accepted","false_acceptance":false,"evidence_corruption_bypass":false,"source_binding_bypass":false,"false_source_mismatch":false,"task_authoring_time":{"method":"measured","seconds":60},"completion_refusal_understood":null,"added_user_interventions":1},"ward":{"normal_workflow_false_deny":false,"added_model_visible_prompt":false,"required_disablement":false,"defer_mutated_output_or_audit":false,"safe_recovery_after_deny":true,"hook_latency":{"median_ms":12,"sample_count":4}},"spec":{"artifact_created":true,"spec_implementation_divergence":false,"unnecessary_artifacts":0,"drafting_time":{"method":"measured","seconds":240},"readable_by_user":true,"directly_usable_by_agent":true,"appropriately_sized":true,"important_decisions_distinguished":true,"would_reuse":true}}} -{"schema_version":"eval-observation/v1","observation_id":"20000000-0000-4000-8000-000000000006","task_id":"10000000-0000-4000-8000-000000000001","revision":2,"supersedes":"20000000-0000-4000-8000-000000000001","terminal_on":"2026-01-02","recorded_on":"2026-01-16","population":"synthetic","task_type":"backend-feature","agent":"codex","model":"synthetic-model-v1","host_os":"darwin","modules":{"seal":{"used":false,"version":null},"ward":{"used":false,"version":null},"spec":{"used":false,"version":null}},"outcome":{"status":"completed","user_interventions":0,"module_interaction_time":null,"rework_required":true},"task_effects":{"completion":{"terminal_completion_invalidated":false},"security":{"destructive_escape":false,"protected_secret_escape":false},"requirements":{"late_material_decisions":{"total":0,"api":0,"data_model":0,"user_behavior":0,"authentication_authorization":0,"consistency_rules":0,"scope":0},"api_contract_changes_after_implementation":0,"data_model_changes_after_implementation":0,"requirements_misunderstanding_rework":false,"user_confirmation_questions":0,"agent_requirements_questions":0,"acceptance_criteria_gap_delayed_defect":false}},"module_metrics":{"seal":null,"ward":null,"spec":null}} diff --git a/protocol.md b/protocol.md deleted file mode 100644 index 3b5f8e2..0000000 --- a/protocol.md +++ /dev/null @@ -1,163 +0,0 @@ -# Eval Observation Protocol v1 - -## 1. Task unit - -One real task is one user objective that reaches a terminal outcome. - -- Retries, resumptions, recovery attempts, and OS-specific reruns remain part - of the same task. -- A genuinely new objective receives a new random `task_id`. -- Terminal status is `completed`, `failed`, or `abandoned`. -- Synthetic fixtures use `population: "synthetic"` and never count as real-task - evidence. - -## 2. Invocation boundary - -The Native Agent, user, or CI may invoke Eval when all of these conditions hold: - -1. a terminal task outcome exists; -2. the caller decides post-task measurement is useful; and -3. the caller has Host authority to append to the external private observation - file. - -This is caller-owned module invocation, not Eval self-activation. Eval does not -watch for task completion, run during the task, mutate task artifacts, invoke a -module, retry or repair work, choose workflow order, or apply a product or -release decision. - -If the authority or single-writer condition is not satisfied, do not append. -Absence of an observation does not change the task outcome. - -A standing user or Host policy that authorizes this external state write -satisfies the authority condition. Eval does not introduce a per-observation -confirmation or a separate approval workflow. - -## 3. Observation values - -Each JSONL row conforms to `schemas/observation-v1.schema.json` and records: - -- schema version, random observation and task IDs, append-only revision data, - `terminal_on`, and `recorded_on`; -- real or synthetic population, task type, Agent, model, and primary host OS; -- whether Spec, Ward, and Seal were used and their exact public versions; -- common terminal outcome, user intervention, module interaction time, and - rework facts; -- `task_effects` assessable whether a module was used or not; -- `module_metrics` for the behavior and friction of modules that were used. - -`terminal_on` is the date the terminal outcome was established. `recorded_on` -is the append date and must not precede `terminal_on`. - -Module use means material use in the task, not installation or availability: - -- Spec was used when a versioned Spec artifact shaped planning, - implementation, or verification. -- Ward was used when that version's protection or decision boundary was active - for at least part of the task. -- Seal was used when that version evaluated completion Evidence or materially - governed the terminal completion attempt. - -`used: true` requires an exact public version and that module's metric object. -`used: false` requires `version: null` and that module's metric entry to be -`null`. If a used module's exact version cannot be recorded as a public -identifier, do not append the observation; never record that module as unused. - -A boolean is true or false only when assessed. A count is zero only when it was -measured and no event occurred. Otherwise the value is `null`. Reports never -combine `null` with false or zero. Counts are limited to `0..1,000,000`. - -Late Material Decision `total` is the number of distinct decisions. Its six -category counts are multi-label: do not sum them to derive `total`, and each -known category count must not exceed a known `total`. - -General time is one of: - -- `null`; -- `{"method":"measured","seconds":N}`; or -- `{"method":"bounded-estimate","lower_seconds":N,"upper_seconds":N}`. - -General time values are limited to `0..31,536,000` seconds, and a bounded -estimate must have `lower_seconds <= upper_seconds`. - -Ward Hook latency is `null` or -`{"median_ms":N,"sample_count":N}`. Reports keep measured and bounded -estimates separate and do not invent a point estimate from a range. - -When both values are known, Seal `added_user_interventions` must not exceed -the outcome's total `user_interventions`. - -The JSON Schema enforces row shape, bounded scalar values, and module coupling. -Cross-value and cross-row relationships stated in this protocol are checked by -the designated writer and again during manual report preparation. Eval v0.1 -does not include an installed or external-state data validator. - -## 4. Raw storage and single writer - -Append exactly one compact JSON object per non-empty line to: - -- `$XDG_STATE_HOME/jgoneit/eval/v1/observations.jsonl` when - `XDG_STATE_HOME` is set and absolute; or -- `$HOME/.local/state/jgoneit/eval/v1/observations.jsonl` only when - `XDG_STATE_HOME` is unset and `HOME` is absolute. - -Fail closed for a relative state root. Resolve the path physically and reject a -location inside any Git worktree. On POSIX-like hosts, use directory mode -`0700`, file mode `0600`, and `umask 077`; on other hosts use equivalent -current-user-only access controls. - -There is exactly one active writer per observation file. External Host policy -or coordination designates that writer; allowing Native Agent, user, and CI as -invocation owners does not permit simultaneous writers. The writer completes -one append before another begins. Eval v0.1 has no lock service, concurrent -writer protocol, or automatic merge. Raw observations are never stored in this -repository. - -## 5. Identifiers and corrections - -- Generate `task_id` and `observation_id` independently as canonical lowercase - random UUIDv4 values. Never hash work content to create an ID. -- The first record for a task has `revision: 1` and `supersedes: null`. -- A correction appends a new observation with the same `task_id`, the next - contiguous revision, a new `observation_id`, and `supersedes` equal to the - immediately preceding observation ID. -- Never edit or delete an earlier JSONL row as a correction. -- A valid single-writer chain is physically ordered and linear, with no - duplicate ID, self-reference, fork, forward link, or multiple initial rows. - -Analysis uses only the latest record from each valid task chain and discloses -invalid or superseded rows. - -## 6. Analysis - -Aggregate latest-valid real observations by task type, module use, and public -module version. Report: - -- real tasks included and synthetic rows excluded; -- terminal outcomes, false-positive and false-negative indicators, missed - defects, intervention, rework, and time; -- task effects for used and unused cohorts; -- used-only module decisions and friction; -- known and unavailable counts for every rate or aggregate; -- repeated metric categories and measurement limitations. - -Do not present `null` as false or zero. Do not compare a cohort without its -sample size. Caller-selected observations are not a randomized experiment, so -used-versus-unused differences are observational and do not establish -causality. - -## 7. Decision report - -Use `templates/decision-report-v1.md`. A report may support a human decision to -retain, promote to stable, keep experimental, modify, remove, stop, or run -another experiment. - -Eval never applies that decision, mutates a release, starts implementation, -repairs work, or invokes another module. - -## 8. Completion boundary - -Repository verification establishes only that the contract scaffold is -internally consistent. The Evaluation MVP additionally requires real Task -observations and one privacy-reviewed cumulative report. A report demonstrates -operation of the evidence workflow, not causal proof or automatic release -authority. diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 43ead75..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Development validation only; Eval has no Python product implementation. -check-jsonschema==0.37.4 diff --git a/schemas/observation-v1.schema.json b/schemas/observation-v1.schema.json deleted file mode 100644 index 2ba071f..0000000 --- a/schemas/observation-v1.schema.json +++ /dev/null @@ -1,477 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:jgoneit:eval:observation:v1", - "title": "Eval Observation v1", - "description": "A privacy-preserving post-task observation for Spec, Ward, and Seal.", - "type": "object", - "additionalProperties": false, - "required": [ - "schema_version", - "observation_id", - "task_id", - "revision", - "supersedes", - "terminal_on", - "recorded_on", - "population", - "task_type", - "agent", - "model", - "host_os", - "modules", - "outcome", - "task_effects", - "module_metrics" - ], - "properties": { - "schema_version": {"const": "eval-observation/v1"}, - "observation_id": {"$ref": "#/$defs/uuid4"}, - "task_id": {"$ref": "#/$defs/uuid4"}, - "revision": {"type": "integer", "minimum": 1}, - "supersedes": { - "oneOf": [ - {"type": "null"}, - {"$ref": "#/$defs/uuid4"} - ] - }, - "terminal_on": {"$ref": "#/$defs/date"}, - "recorded_on": {"$ref": "#/$defs/date"}, - "population": {"enum": ["real", "synthetic"]}, - "task_type": {"$ref": "#/$defs/taskType"}, - "agent": { - "oneOf": [ - {"type": "null"}, - {"enum": ["codex", "other"]} - ] - }, - "model": { - "oneOf": [ - {"type": "null"}, - {"$ref": "#/$defs/publicIdentifier"} - ] - }, - "host_os": { - "oneOf": [ - {"type": "null"}, - {"enum": ["darwin", "linux", "windows", "other"]} - ] - }, - "modules": {"$ref": "#/$defs/modules"}, - "outcome": {"$ref": "#/$defs/outcome"}, - "task_effects": {"$ref": "#/$defs/taskEffects"}, - "module_metrics": {"$ref": "#/$defs/moduleMetrics"} - }, - "allOf": [ - { - "if": { - "required": ["revision"], - "properties": {"revision": {"const": 1}} - }, - "then": {"properties": {"supersedes": {"type": "null"}}}, - "else": {"properties": {"supersedes": {"$ref": "#/$defs/uuid4"}}} - }, - { - "if": { - "required": ["modules"], - "properties": { - "modules": { - "required": ["seal"], - "properties": { - "seal": { - "required": ["used"], - "properties": {"used": {"const": true}} - } - } - } - } - }, - "then": { - "properties": { - "module_metrics": { - "properties": {"seal": {"$ref": "#/$defs/sealMetrics"}} - } - } - }, - "else": { - "properties": { - "module_metrics": {"properties": {"seal": {"type": "null"}}} - } - } - }, - { - "if": { - "required": ["modules"], - "properties": { - "modules": { - "required": ["ward"], - "properties": { - "ward": { - "required": ["used"], - "properties": {"used": {"const": true}} - } - } - } - } - }, - "then": { - "properties": { - "module_metrics": { - "properties": {"ward": {"$ref": "#/$defs/wardMetrics"}} - } - } - }, - "else": { - "properties": { - "module_metrics": {"properties": {"ward": {"type": "null"}}} - } - } - }, - { - "if": { - "required": ["modules"], - "properties": { - "modules": { - "required": ["spec"], - "properties": { - "spec": { - "required": ["used"], - "properties": {"used": {"const": true}} - } - } - } - } - }, - "then": { - "properties": { - "module_metrics": { - "properties": {"spec": {"$ref": "#/$defs/specMetrics"}} - } - } - }, - "else": { - "properties": { - "module_metrics": {"properties": {"spec": {"type": "null"}}} - } - } - } - ], - "$defs": { - "uuid4": { - "type": "string", - "format": "uuid", - "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" - }, - "date": {"type": "string", "format": "date"}, - "publicIdentifier": { - "type": "string", - "minLength": 1, - "maxLength": 128, - "pattern": "^[A-Za-z0-9][A-Za-z0-9._+-]*$" - }, - "taskType": { - "enum": [ - "backend-feature", - "frontend-feature", - "full-stack-feature", - "bug-fix", - "refactor", - "test", - "documentation", - "security", - "release", - "operations", - "research", - "other" - ] - }, - "nullableBoolean": {"type": ["boolean", "null"]}, - "nullableCount": { - "type": ["integer", "null"], - "minimum": 0, - "maximum": 1000000 - }, - "moduleIdentity": { - "type": "object", - "additionalProperties": false, - "required": ["used", "version"], - "properties": { - "used": {"type": "boolean"}, - "version": { - "oneOf": [ - {"type": "null"}, - {"$ref": "#/$defs/publicIdentifier"} - ] - } - }, - "allOf": [ - { - "if": { - "required": ["used"], - "properties": {"used": {"const": true}} - }, - "then": {"properties": {"version": {"$ref": "#/$defs/publicIdentifier"}}}, - "else": {"properties": {"version": {"type": "null"}}} - } - ] - }, - "modules": { - "type": "object", - "additionalProperties": false, - "required": ["seal", "ward", "spec"], - "properties": { - "seal": {"$ref": "#/$defs/moduleIdentity"}, - "ward": {"$ref": "#/$defs/moduleIdentity"}, - "spec": {"$ref": "#/$defs/moduleIdentity"} - } - }, - "measuredTime": { - "type": "object", - "additionalProperties": false, - "required": ["method", "seconds"], - "properties": { - "method": {"const": "measured"}, - "seconds": {"type": "integer", "minimum": 0, "maximum": 31536000} - } - }, - "boundedEstimatedTime": { - "type": "object", - "additionalProperties": false, - "required": ["method", "lower_seconds", "upper_seconds"], - "properties": { - "method": {"const": "bounded-estimate"}, - "lower_seconds": {"type": "integer", "minimum": 0, "maximum": 31536000}, - "upper_seconds": {"type": "integer", "minimum": 0, "maximum": 31536000} - } - }, - "timeMeasurement": { - "oneOf": [ - {"$ref": "#/$defs/measuredTime"}, - {"$ref": "#/$defs/boundedEstimatedTime"}, - {"type": "null"} - ] - }, - "latencyMeasurement": { - "oneOf": [ - { - "type": "object", - "additionalProperties": false, - "required": ["median_ms", "sample_count"], - "properties": { - "median_ms": {"type": "integer", "minimum": 0, "maximum": 86400000}, - "sample_count": {"type": "integer", "minimum": 1, "maximum": 1000000} - } - }, - {"type": "null"} - ] - }, - "outcome": { - "type": "object", - "additionalProperties": false, - "required": [ - "status", - "user_interventions", - "module_interaction_time", - "rework_required" - ], - "properties": { - "status": {"enum": ["completed", "failed", "abandoned"]}, - "user_interventions": {"$ref": "#/$defs/nullableCount"}, - "module_interaction_time": {"$ref": "#/$defs/timeMeasurement"}, - "rework_required": {"$ref": "#/$defs/nullableBoolean"} - } - }, - "lateMaterialDecisions": { - "type": "object", - "additionalProperties": false, - "required": [ - "total", - "api", - "data_model", - "user_behavior", - "authentication_authorization", - "consistency_rules", - "scope" - ], - "properties": { - "total": {"$ref": "#/$defs/nullableCount"}, - "api": {"$ref": "#/$defs/nullableCount"}, - "data_model": {"$ref": "#/$defs/nullableCount"}, - "user_behavior": {"$ref": "#/$defs/nullableCount"}, - "authentication_authorization": {"$ref": "#/$defs/nullableCount"}, - "consistency_rules": {"$ref": "#/$defs/nullableCount"}, - "scope": {"$ref": "#/$defs/nullableCount"} - } - }, - "taskEffects": { - "type": "object", - "additionalProperties": false, - "required": ["completion", "security", "requirements"], - "properties": { - "completion": { - "type": "object", - "additionalProperties": false, - "required": ["terminal_completion_invalidated"], - "properties": { - "terminal_completion_invalidated": {"$ref": "#/$defs/nullableBoolean"} - } - }, - "security": { - "type": "object", - "additionalProperties": false, - "required": ["destructive_escape", "protected_secret_escape"], - "properties": { - "destructive_escape": {"$ref": "#/$defs/nullableBoolean"}, - "protected_secret_escape": {"$ref": "#/$defs/nullableBoolean"} - } - }, - "requirements": { - "type": "object", - "additionalProperties": false, - "required": [ - "late_material_decisions", - "api_contract_changes_after_implementation", - "data_model_changes_after_implementation", - "requirements_misunderstanding_rework", - "user_confirmation_questions", - "agent_requirements_questions", - "acceptance_criteria_gap_delayed_defect" - ], - "properties": { - "late_material_decisions": {"$ref": "#/$defs/lateMaterialDecisions"}, - "api_contract_changes_after_implementation": {"$ref": "#/$defs/nullableCount"}, - "data_model_changes_after_implementation": {"$ref": "#/$defs/nullableCount"}, - "requirements_misunderstanding_rework": {"$ref": "#/$defs/nullableBoolean"}, - "user_confirmation_questions": {"$ref": "#/$defs/nullableCount"}, - "agent_requirements_questions": {"$ref": "#/$defs/nullableCount"}, - "acceptance_criteria_gap_delayed_defect": {"$ref": "#/$defs/nullableBoolean"} - } - } - } - }, - "sealMetrics": { - "type": "object", - "additionalProperties": false, - "required": [ - "completion_decision", - "false_acceptance", - "evidence_corruption_bypass", - "source_binding_bypass", - "false_source_mismatch", - "task_authoring_time", - "completion_refusal_understood", - "added_user_interventions" - ], - "properties": { - "completion_decision": {"enum": ["accepted", "refused", "not-issued", null]}, - "false_acceptance": {"$ref": "#/$defs/nullableBoolean"}, - "evidence_corruption_bypass": {"$ref": "#/$defs/nullableBoolean"}, - "source_binding_bypass": {"$ref": "#/$defs/nullableBoolean"}, - "false_source_mismatch": {"$ref": "#/$defs/nullableBoolean"}, - "task_authoring_time": {"$ref": "#/$defs/timeMeasurement"}, - "completion_refusal_understood": {"$ref": "#/$defs/nullableBoolean"}, - "added_user_interventions": {"$ref": "#/$defs/nullableCount"} - }, - "allOf": [ - { - "if": { - "required": ["completion_decision"], - "properties": {"completion_decision": {"const": "accepted"}} - }, - "then": { - "properties": {"completion_refusal_understood": {"type": "null"}} - } - }, - { - "if": { - "required": ["completion_decision"], - "properties": {"completion_decision": {"const": "refused"}} - }, - "then": {"properties": {"false_acceptance": {"type": "null"}}} - }, - { - "if": { - "required": ["completion_decision"], - "properties": {"completion_decision": {"enum": ["not-issued", null]}} - }, - "then": { - "properties": { - "false_acceptance": {"type": "null"}, - "completion_refusal_understood": {"type": "null"} - } - } - } - ] - }, - "wardMetrics": { - "type": "object", - "additionalProperties": false, - "required": [ - "normal_workflow_false_deny", - "added_model_visible_prompt", - "required_disablement", - "defer_mutated_output_or_audit", - "safe_recovery_after_deny", - "hook_latency" - ], - "properties": { - "normal_workflow_false_deny": {"$ref": "#/$defs/nullableBoolean"}, - "added_model_visible_prompt": {"$ref": "#/$defs/nullableBoolean"}, - "required_disablement": {"$ref": "#/$defs/nullableBoolean"}, - "defer_mutated_output_or_audit": {"$ref": "#/$defs/nullableBoolean"}, - "safe_recovery_after_deny": {"$ref": "#/$defs/nullableBoolean"}, - "hook_latency": {"$ref": "#/$defs/latencyMeasurement"} - } - }, - "specMetrics": { - "type": "object", - "additionalProperties": false, - "required": [ - "artifact_created", - "spec_implementation_divergence", - "unnecessary_artifacts", - "drafting_time", - "readable_by_user", - "directly_usable_by_agent", - "appropriately_sized", - "important_decisions_distinguished", - "would_reuse" - ], - "properties": { - "artifact_created": {"$ref": "#/$defs/nullableBoolean"}, - "spec_implementation_divergence": {"$ref": "#/$defs/nullableBoolean"}, - "unnecessary_artifacts": {"$ref": "#/$defs/nullableCount"}, - "drafting_time": {"$ref": "#/$defs/timeMeasurement"}, - "readable_by_user": {"$ref": "#/$defs/nullableBoolean"}, - "directly_usable_by_agent": {"$ref": "#/$defs/nullableBoolean"}, - "appropriately_sized": {"$ref": "#/$defs/nullableBoolean"}, - "important_decisions_distinguished": {"$ref": "#/$defs/nullableBoolean"}, - "would_reuse": {"$ref": "#/$defs/nullableBoolean"} - } - }, - "moduleMetrics": { - "type": "object", - "additionalProperties": false, - "required": ["seal", "ward", "spec"], - "properties": { - "seal": { - "oneOf": [ - {"type": "null"}, - {"$ref": "#/$defs/sealMetrics"} - ] - }, - "ward": { - "oneOf": [ - {"type": "null"}, - {"$ref": "#/$defs/wardMetrics"} - ] - }, - "spec": { - "oneOf": [ - {"type": "null"}, - {"$ref": "#/$defs/specMetrics"} - ] - } - } - } - } -} diff --git a/scripts/verify.sh b/scripts/verify.sh deleted file mode 100755 index e9dd4ee..0000000 --- a/scripts/verify.sh +++ /dev/null @@ -1,439 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -script_dir="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(CDPATH= cd -- "$script_dir/.." && pwd)" -schema="$repo_root/schemas/observation-v1.schema.json" - -die() { - echo "error: $*" >&2 - exit 1 -} - -if [[ -x "$repo_root/.venv/bin/check-jsonschema" ]]; then - checker="$repo_root/.venv/bin/check-jsonschema" -elif command -v check-jsonschema >/dev/null 2>&1; then - checker="$(command -v check-jsonschema)" -else - echo "error: check-jsonschema is unavailable" >&2 - echo "run: python3 -m venv .venv && .venv/bin/python -m pip install -r requirements-dev.txt" >&2 - exit 2 -fi - -if [[ -x "$repo_root/.venv/bin/python" ]]; then - python="$repo_root/.venv/bin/python" -elif command -v python3 >/dev/null 2>&1; then - python="$(command -v python3)" -else - echo "error: python3 is unavailable" >&2 - exit 2 -fi - -checker_version="$($checker --version)" -[[ "$checker_version" == "check-jsonschema, version 0.37.4" ]] || { - echo "error: expected check-jsonschema 0.37.4, got: $checker_version" >&2 - exit 2 -} -rg -q '^check-jsonschema==0\.37\.4$' "$repo_root/requirements-dev.txt" || - die "check-jsonschema development pin is missing" -[[ -f "$schema" ]] || die "observation schema is missing" -[[ -x "$repo_root/scripts/verify.sh" ]] || die "verification script is not executable" - -echo "[1/5] validating the observation schema metaschema" -"$checker" --check-metaschema "$schema" - -echo "[2/5] validating the static module contract and synthetic fixtures" -fixture_summary="$($python - "$repo_root" "$checker" <<'PY' -from __future__ import annotations - -import json -from pathlib import Path -import subprocess -import sys -from typing import Any - - -ROOT = Path(sys.argv[1]).resolve() -CHECKER = sys.argv[2] -SCHEMA = ROOT / "schemas/observation-v1.schema.json" - - -class StrictJSONFailure(Exception): - kind = "malformed" - - -class DuplicateKey(StrictJSONFailure): - kind = "duplicate" - - -class NonFiniteNumber(StrictJSONFailure): - kind = "nonfinite" - - -class BlankDocument(StrictJSONFailure): - kind = "blank" - - -class MalformedDocument(StrictJSONFailure): - kind = "malformed" - - -def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise DuplicateKey - result[key] = value - return result - - -def reject_nonfinite(_: str) -> None: - raise NonFiniteNumber - - -def strict_document(raw: bytes) -> dict[str, Any]: - if not raw.strip(): - raise BlankDocument - try: - text = raw.decode("utf-8") - value = json.loads( - text, - object_pairs_hook=reject_duplicates, - parse_constant=reject_nonfinite, - ) - except (DuplicateKey, NonFiniteNumber): - raise - except (UnicodeError, json.JSONDecodeError, RecursionError, ValueError) as error: - raise MalformedDocument from error - if not isinstance(value, dict): - raise MalformedDocument - return value - - -def schema_result(raw: bytes) -> int: - return subprocess.run( - [ - CHECKER, - "-q", - "--force-filetype", - "json", - "--schemafile", - str(SCHEMA), - "-", - ], - input=raw, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ).returncode - - -schema_document = strict_document(SCHEMA.read_bytes()) -if schema_document.get("$schema") != "https://json-schema.org/draft/2020-12/schema": - raise SystemExit("error: observation schema draft changed") -if schema_document.get("additionalProperties") is not False: - raise SystemExit("error: observation schema must close its top-level object") - -expected_observation_fields = [ - "schema_version", - "observation_id", - "task_id", - "revision", - "supersedes", - "terminal_on", - "recorded_on", - "population", - "task_type", - "agent", - "model", - "host_os", - "modules", - "outcome", - "task_effects", - "module_metrics", -] -if schema_document.get("required") != expected_observation_fields: - raise SystemExit("error: observation required fields changed") -if list(schema_document.get("properties", {})) != expected_observation_fields: - raise SystemExit("error: observation top-level fields changed") - - -def assert_closed_objects(value: Any) -> None: - if isinstance(value, dict): - if value.get("type") == "object" and value.get("additionalProperties") is not False: - raise SystemExit("error: observation schema contains an open object") - for nested in value.values(): - assert_closed_objects(nested) - elif isinstance(value, list): - for nested in value: - assert_closed_objects(nested) - - -assert_closed_objects(schema_document) - -manifest_path = ROOT / "toolkit-module.json" -manifest = strict_document(manifest_path.read_bytes()) -expected_manifest = { - "schema_version": "toolkit-module/v1", - "id": "eval", - "name": "Eval", - "description": "Post-task evidence protocol for measuring Agent Toolkit module value and cost on real tasks.", - "plane": "evaluation", - "status": "experimental", - "kind": "artifact-protocol", - "phase": "post-task", - "invocation_owners": ["native-agent", "user", "ci"], - "requires_terminal_outcome": True, - "self_activates": False, - "auto_invokes_modules": False, - "mutates_user_task": False, - "side_effects": ["external-private-observation-append"], - "requires_host_write_authority": True, - "artifacts": { - "charter": "CHARTER.md", - "protocol": "protocol.md", - "observation_schema": "schemas/observation-v1.schema.json", - "report_template": "templates/decision-report-v1.md", - }, -} -if manifest != expected_manifest: - raise SystemExit("error: toolkit-module.json does not match the v1 contract") -for artifact in manifest["artifacts"].values(): - target = (ROOT / artifact).resolve() - try: - target.relative_to(ROOT) - except ValueError as error: - raise SystemExit("error: module artifact escapes the repository") from error - if not target.is_file(): - raise SystemExit(f"error: module artifact is missing: {artifact}") - -required_headings = [ - "## Report metadata", - "## Dataset", - "## Outcomes and cost", - "## Module findings", - "## Used-versus-unused and version comparisons", - "## Limitations", - "## Human decision", - "## Privacy review", -] -template = (ROOT / "templates/decision-report-v1.md").read_text(encoding="utf-8") -actual_headings = [line for line in template.splitlines() if line.startswith("## ")] -if actual_headings != required_headings: - raise SystemExit("error: decision report sections do not match the v1 contract") - -valid_path = ROOT / "fixtures/valid/observations.jsonl" -valid_data = valid_path.read_bytes() -if not valid_data: - raise SystemExit("error: valid observation fixture is empty") -valid_rows = 0 -valid_documents: list[dict[str, Any]] = [] -for line_number, raw in enumerate(valid_data.splitlines(), 1): - try: - row = strict_document(raw) - except StrictJSONFailure as error: - raise SystemExit( - f"error: valid observation line {line_number} is not strict JSON" - ) from error - if row.get("population") != "synthetic": - raise SystemExit( - f"error: valid observation line {line_number} is not synthetic" - ) - if schema_result(raw) != 0: - raise SystemExit( - f"error: valid observation line {line_number} failed schema validation" - ) - valid_documents.append(row) - valid_rows += 1 - -if valid_rows != 6: - raise SystemExit("error: valid observation matrix must contain exactly six rows") -module_vectors = { - tuple(row["modules"][name]["used"] for name in ("seal", "ward", "spec")) - for row in valid_documents -} -expected_vectors = { - (False, False, False), - (False, False, True), - (False, True, False), - (True, False, False), - (True, True, True), -} -if module_vectors != expected_vectors: - raise SystemExit("error: valid observation module-use matrix is incomplete") -time_methods = { - row["outcome"]["module_interaction_time"]["method"] - for row in valid_documents - if row["outcome"]["module_interaction_time"] is not None -} -if time_methods != {"measured", "bounded-estimate"}: - raise SystemExit("error: valid observation time-method matrix is incomplete") - -latest_by_task: dict[str, dict[str, Any]] = {} -for row in valid_documents: - previous = latest_by_task.get(row["task_id"]) - if previous is None: - if row["revision"] != 1 or row["supersedes"] is not None: - raise SystemExit("error: valid observation fixture has a broken root") - elif ( - row["revision"] != previous["revision"] + 1 - or row["supersedes"] != previous["observation_id"] - ): - raise SystemExit("error: valid observation fixture has a broken revision chain") - latest_by_task[row["task_id"]] = row -if not any(row["revision"] > 1 for row in valid_documents): - raise SystemExit("error: valid observation matrix lacks a revision correction") - -schema_invalid = sorted((ROOT / "fixtures/invalid/schema").glob("*.json")) -if not schema_invalid: - raise SystemExit("error: no schema-invalid observation fixtures found") -for path in schema_invalid: - try: - document = strict_document(path.read_bytes()) - except StrictJSONFailure as error: - raise SystemExit(f"error: schema-invalid fixture is not strict JSON: {path.name}") from error - if document.get("schema_version") != "eval-observation/v1": - raise SystemExit(f"error: schema-invalid fixture has wrong contract: {path.name}") - if document.get("population") != "synthetic": - raise SystemExit(f"error: schema-invalid fixture is not synthetic: {path.name}") - result = schema_result(path.read_bytes()) - if result != 1: - raise SystemExit(f"error: expected schema rejection: {path.name}") - -raw_invalid = sorted((ROOT / "fixtures/invalid/raw").glob("*.jsonl")) -if not raw_invalid: - raise SystemExit("error: no raw-invalid fixtures found") -failure_kinds: set[str] = set() -for path in raw_invalid: - data = path.read_bytes() - try: - if not data: - raise BlankDocument - for raw in data.splitlines(): - strict_document(raw) - except StrictJSONFailure as error: - failure_kinds.add(error.kind) - else: - raise SystemExit(f"error: expected strict JSON rejection: {path.name}") -if not {"blank", "duplicate", "nonfinite", "malformed"}.issubset(failure_kinds): - raise SystemExit("error: raw-invalid fixtures do not cover strict JSON failures") - -print( - f"{valid_rows} valid rows; {len(schema_invalid)} schema-invalid and " - f"{len(raw_invalid)} raw-invalid fixtures rejected" -) -PY -)" - -echo "[3/5] checking ignore and minimal-surface rules" -ignored_canaries=( - "observations.jsonl" - ".local/observations.jsonl" - "reports/private/example.md" - "reports/generated/example.md" - "reports/drafts/example.md" - "scratch.tmp" -) -for canary in "${ignored_canaries[@]}"; do - git -C "$repo_root" check-ignore -q --no-index -- "$canary" || - die "private/generated path is not ignored: $canary" -done - -tracked_canaries=( - "toolkit-module.json" - "schemas/observation-v1.schema.json" - "fixtures/valid/observations.jsonl" - "templates/decision-report-v1.md" -) -for canary in "${tracked_canaries[@]}"; do - if git -C "$repo_root" check-ignore -q --no-index -- "$canary"; then - die "contract artifact is unexpectedly ignored: $canary" - fi -done - -obsolete_paths=( - "$repo_root/schemas/study-v1.schema.json" - "$repo_root/fixtures/valid/studies.jsonl" - "$repo_root/fixtures/invalid/dataset" - "$repo_root/scripts/validate-data.sh" -) -for path in "${obsolete_paths[@]}"; do - [[ ! -e "$path" ]] || die "obsolete study/runtime surface exists: ${path#"$repo_root/"}" -done - -while IFS= read -r path; do - die "unexpected minimal-contract file exists: ${path#"$repo_root/"}" -done < <( - find "$repo_root/schemas" -type f ! -path "$schema" -print - find "$repo_root/fixtures/valid" -type f \ - ! -path "$repo_root/fixtures/valid/observations.jsonl" -print - find "$repo_root/scripts" -type f ! -path "$repo_root/scripts/verify.sh" -print -) - -while IFS= read -r path; do - die "prohibited product surface exists: ${path#"$repo_root/"}" -done < <( - find "$repo_root" \ - \( -path "$repo_root/.git" -o -path "$repo_root/.venv" \) -prune -o \ - \( \ - -type d \( -name .codex-plugin -o -name skills -o -name hooks \) -o \ - -type f \( -name SKILL.md -o -name plugin.json -o -name '*.py' -o \ - -name '*.go' -o -name go.mod -o -name go.sum \) \ - \) -print -) - -while IFS= read -r path; do - case "$path" in - "$repo_root/fixtures/valid/observations.jsonl" | \ - "$repo_root"/fixtures/invalid/raw/*.jsonl) ;; - *) die "checkout-local raw observation data exists: ${path#"$repo_root/"}" ;; - esac -done < <( - find "$repo_root" \ - \( -path "$repo_root/.git" -o -path "$repo_root/.venv" \) -prune -o \ - -type f -name '*.jsonl' -print -) - -for path in \ - "$repo_root/.local" \ - "$repo_root/reports/private" \ - "$repo_root/reports/generated" \ - "$repo_root/reports/drafts"; do - [[ ! -e "$path" ]] || die "checkout-local private/generated data exists: ${path#"$repo_root/"}" -done - -echo "[4/5] checking shell syntax and repository whitespace" -script_count=0 -for shell_script in "$repo_root"/scripts/*.sh; do - bash -n "$shell_script" - script_count=$((script_count + 1)) -done -[[ "$script_count" -eq 1 ]] || die "scripts directory must contain only verify.sh" - -"$python" - "$repo_root" <<'PY' -from pathlib import Path -import sys - - -root = Path(sys.argv[1]) -errors = [] -for path in sorted(root.rglob("*")): - if not path.is_file() or ".git" in path.parts or ".venv" in path.parts: - continue - data = path.read_bytes() - relative = path.relative_to(root) - if data and not data.endswith(b"\n"): - errors.append(f"{relative}: missing final newline") - if data.endswith(b"\n\n"): - errors.append(f"{relative}: blank line at end of file") - for line_number, line in enumerate(data.splitlines(), 1): - if line.endswith((b" ", b"\t")): - errors.append(f"{relative}:{line_number}: trailing whitespace") -if errors: - raise SystemExit("\n".join(f"error: {item}" for item in errors)) -PY -git -C "$repo_root" diff --check - -echo "[5/5] verification complete" -echo "Eval verification passed: $fixture_summary." diff --git a/templates/decision-report-v1.md b/templates/decision-report-v1.md deleted file mode 100644 index da2d2af..0000000 --- a/templates/decision-report-v1.md +++ /dev/null @@ -1,94 +0,0 @@ -# Eval Decision Report - -> Observational decision support only. This report must not trigger a workflow, -> implementation, repair, or release action. - -## Report metadata - -| Field | Value | -| --- | --- | -| Reporting period | `` | -| Prepared on | `` | -| Protocol | `Eval Observation Protocol v1` | -| Observation schema | `eval-observation/v1` | -| Publication status | `` | -| Review role | `` | - -## Dataset - -| Population | Count | -| --- | ---: | -| Parsed observation rows | `` | -| Tasks represented | `` | -| Superseded rows excluded | `` | -| Invalid or broken chains excluded | `` | -| Latest valid real-task revisions included | `` | -| Synthetic rows excluded | `` | - -State how observations were selected. Do not list task IDs, observation IDs, -individual rows, repositories, or task narratives. - -## Outcomes and cost - -| Metric | Result | Known n | Unavailable n | Measurement method | -| --- | ---: | ---: | ---: | --- | -| Completed / failed / abandoned | `` | `` | `` | `categorical` | -| User interventions | `` | `` | `` | `count` | -| Module interaction time | `` | `` | `` | `` | -| Rework required | `` | `` | `` | `assessed boolean` | - -Do not convert `null` to false or zero. Do not pool measured time with bounded -estimates or choose a point value from a bounded range. - -## Module findings - -Repeat for each evaluated module. Task effects may compare used and unused -cohorts; used-only module metrics appear only for the used cohort. - -### `` - -| Metric | Version/cohort | Result | Known n | Unavailable n | -| --- | --- | ---: | ---: | ---: | -| `` | `` | `` | `` | `` | - -Summarize repeated defect or friction categories using aggregate language only. - -## Used-versus-unused and version comparisons - -| Comparison | Cohort A n | Cohort B n | Observed difference | Limitation | -| --- | ---: | ---: | --- | --- | -| `` | `` | `` | `` | `` | - -Do not claim that module use caused an observed difference. - -## Limitations - -- `` -- `` -- `` -- `` -- `` - -## Human decision - -| Module | Human decision | Evidence sufficient? | Next step | -| --- | --- | --- | --- | -| Spec | `` | `` | `` | -| Ward | `` | `` | `` | -| Seal | `` | `` | `` | - -The report records a human decision. Eval does not apply it. - -## Privacy review - -- [ ] The publication status is correct; private reports remain external. -- [ ] No raw row, task ID, observation ID, repository, path, prompt, command, - source, secret, transcript, personal data, task narrative, or internal - identifier is included. -- [ ] Public cohort cells with `n < 5` and inferable complementary cells are - suppressed. -- [ ] Synthetic evidence is excluded from real-task conclusions. -- [ ] Every rate and aggregate shows known and unavailable counts. -- [ ] Measured time and bounded estimates remain separate. -- [ ] Sampling, missing-data, measurement, and causal limitations are stated. -- [ ] The report makes no automatic workflow or release decision. diff --git a/toolkit-module.json b/toolkit-module.json deleted file mode 100644 index 203d5ab..0000000 --- a/toolkit-module.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "schema_version": "toolkit-module/v1", - "id": "eval", - "name": "Eval", - "description": "Post-task evidence protocol for measuring Agent Toolkit module value and cost on real tasks.", - "plane": "evaluation", - "status": "experimental", - "kind": "artifact-protocol", - "phase": "post-task", - "invocation_owners": [ - "native-agent", - "user", - "ci" - ], - "requires_terminal_outcome": true, - "self_activates": false, - "auto_invokes_modules": false, - "mutates_user_task": false, - "side_effects": [ - "external-private-observation-append" - ], - "requires_host_write_authority": true, - "artifacts": { - "charter": "CHARTER.md", - "protocol": "protocol.md", - "observation_schema": "schemas/observation-v1.schema.json", - "report_template": "templates/decision-report-v1.md" - } -} From 68a339647dbe6bda2f703e9992c4ee3dc3d59c2c Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 22:56:28 +0900 Subject: [PATCH 02/14] feat(eval): add bounded experiment recorder Why: - Eval needs a small host-owned instrument before product surfaces or analysis are justified. - The experiment must preserve primary task outcomes when input or private storage is unavailable. What: - Add the observe-only evalctl contract and a twenty-slot typed journal. - Add strict duplicate-key and integer decoding with one stored-row schema. - Add rooted private storage, exclusive locking, atomic replacement, and commit-aware durability reporting for Darwin, Linux, and Windows. Validation: - go test ./... - go test -race ./... - go vet ./... - GOOS=darwin/linux/windows cross-build and vet - Manual observe smoke with contiguous slots --- cmd/evalctl/main.go | 12 + go.mod | 5 + go.sum | 2 + internal/cli/cli.go | 178 +++++++++++ internal/experiment/decode.go | 231 ++++++++++++++ internal/experiment/journal.go | 75 +++++ internal/experiment/types.go | 75 +++++ internal/experiment/validate.go | 106 +++++++ internal/state/state.go | 36 +++ internal/store/errors.go | 89 ++++++ internal/store/platform_unix.go | 195 ++++++++++++ internal/store/platform_windows.go | 468 +++++++++++++++++++++++++++++ internal/store/security.go | 158 ++++++++++ internal/store/store.go | 437 +++++++++++++++++++++++++++ internal/version/version.go | 3 + schemas/experiment-v1.schema.json | 79 +++++ 16 files changed, 2149 insertions(+) create mode 100644 cmd/evalctl/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/cli/cli.go create mode 100644 internal/experiment/decode.go create mode 100644 internal/experiment/journal.go create mode 100644 internal/experiment/types.go create mode 100644 internal/experiment/validate.go create mode 100644 internal/state/state.go create mode 100644 internal/store/errors.go create mode 100644 internal/store/platform_unix.go create mode 100644 internal/store/platform_windows.go create mode 100644 internal/store/security.go create mode 100644 internal/store/store.go create mode 100644 internal/version/version.go create mode 100644 schemas/experiment-v1.schema.json diff --git a/cmd/evalctl/main.go b/cmd/evalctl/main.go new file mode 100644 index 0000000..15ad7e5 --- /dev/null +++ b/cmd/evalctl/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "context" + "os" + + "github.com/jgoneit/eval/internal/cli" +) + +func main() { + os.Exit(cli.Run(context.Background(), os.Args[1:], cli.Runtime{})) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0830cff --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/jgoneit/eval + +go 1.25.0 + +require golang.org/x/sys v0.42.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..d2913d5 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/internal/cli/cli.go b/internal/cli/cli.go new file mode 100644 index 0000000..fd753d7 --- /dev/null +++ b/internal/cli/cli.go @@ -0,0 +1,178 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "time" + + "github.com/jgoneit/eval/internal/experiment" + "github.com/jgoneit/eval/internal/state" + "github.com/jgoneit/eval/internal/store" + "github.com/jgoneit/eval/internal/version" +) + +const ( + ExitSuccess = 0 + ExitUsage = 64 +) + +var ( + errInvalidJournal = errors.New("invalid experiment journal") + errJournalFull = errors.New("experiment journal has twenty rows") +) + +type Runtime struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + Getenv func(string) string + Now func() time.Time + StoreOptions store.Options +} + +func Run(ctx context.Context, args []string, runtime Runtime) int { + runtime = defaults(runtime) + if len(args) == 1 && args[0] == "--version" { + _, _ = fmt.Fprintf(runtime.Stdout, "evalctl %s\n", version.Current) + return ExitSuccess + } + if len(args) == 1 && (args[0] == "--help" || args[0] == "-h") { + usage(runtime.Stdout) + return ExitSuccess + } + if len(args) == 0 || args[0] != "observe" { + usage(runtime.Stderr) + return ExitUsage + } + return runObserve(ctx, args[1:], runtime) +} + +func runObserve(ctx context.Context, args []string, runtime Runtime) int { + flags := flag.NewFlagSet("observe", flag.ContinueOnError) + flags.SetOutput(runtime.Stderr) + stateRoot := flags.String("state-root", "", "absolute state root") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return ExitSuccess + } + return ExitUsage + } + if flags.NArg() != 0 { + usage(runtime.Stderr) + return ExitUsage + } + + draft, err := experiment.DecodeDraft(runtime.Stdin) + if err != nil { + writeResult(runtime.Stdout, observeResult{Status: "skipped", Reason: "invalid-observation"}) + return ExitSuccess + } + root, err := state.Root(*stateRoot, runtime.Getenv) + if err != nil { + writeResult(runtime.Stdout, observeResult{Status: "skipped", Reason: "unsafe-state-path"}) + return ExitSuccess + } + journal, err := store.New(root, state.JournalRelativePath, runtime.StoreOptions) + if err != nil { + writeResult(runtime.Stdout, observeResult{Status: "skipped", Reason: store.SafeReason(err)}) + return ExitSuccess + } + + var slot int64 + commit, err := journal.Update(ctx, func(existing []byte) ([]byte, error) { + rows, parseErr := experiment.ParseJournal(bytes.NewReader(existing)) + if parseErr != nil { + return nil, fmt.Errorf("%w: %v", errInvalidJournal, parseErr) + } + if int64(len(rows)) >= experiment.MaxRows { + return nil, errJournalFull + } + slot = int64(len(rows) + 1) + row, rowErr := experiment.NewRow(draft, slot, runtime.Now()) + if rowErr != nil { + return nil, rowErr + } + encoded, rowErr := experiment.MarshalCanonicalRow(row) + if rowErr != nil { + return nil, rowErr + } + prospective := make([]byte, 0, len(existing)+len(encoded)+1) + prospective = append(prospective, existing...) + prospective = append(prospective, encoded...) + prospective = append(prospective, '\n') + return prospective, nil + }) + if commit.Committed { + durability := "confirmed" + if !commit.DurabilityConfirmed { + durability = "unconfirmed" + } + writeResult(runtime.Stdout, observeResult{Status: "recorded", Slot: slot, Durability: durability}) + return ExitSuccess + } + if err != nil { + reason := classifyObserveFailure(err) + writeResult(runtime.Stdout, observeResult{Status: "skipped", Reason: reason}) + return ExitSuccess + } + writeResult(runtime.Stdout, observeResult{Status: "skipped", Reason: "state-io-error"}) + return ExitSuccess +} + +type observeResult struct { + Status string `json:"status"` + Slot int64 `json:"slot,omitempty"` + Durability string `json:"durability,omitempty"` + Reason string `json:"reason,omitempty"` +} + +func classifyObserveFailure(err error) string { + switch { + case errors.Is(err, errInvalidJournal): + return "invalid-state-data" + case errors.Is(err, errJournalFull): + return "journal-full" + default: + var storeErr *store.Error + if errors.As(err, &storeErr) { + return store.SafeReason(err) + } + return "state-io-error" + } +} + +func writeResult(writer io.Writer, result observeResult) { + encoder := json.NewEncoder(writer) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(result) +} + +func usage(writer io.Writer) { + _, _ = fmt.Fprintln(writer, "usage: evalctl --version") + _, _ = fmt.Fprintln(writer, " evalctl observe [--state-root ABS]") +} + +func defaults(runtime Runtime) Runtime { + if runtime.Stdin == nil { + runtime.Stdin = os.Stdin + } + if runtime.Stdout == nil { + runtime.Stdout = os.Stdout + } + if runtime.Stderr == nil { + runtime.Stderr = os.Stderr + } + if runtime.Getenv == nil { + runtime.Getenv = os.Getenv + } + if runtime.Now == nil { + runtime.Now = time.Now + } + return runtime +} diff --git a/internal/experiment/decode.go b/internal/experiment/decode.go new file mode 100644 index 0000000..2d33f55 --- /dev/null +++ b/internal/experiment/decode.go @@ -0,0 +1,231 @@ +package experiment + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" +) + +var ErrInputTooLarge = errors.New("experiment input exceeds 64 KiB") + +// DecodeDraft strictly decodes one host-provided draft. It rejects duplicate +// keys at every depth, trailing JSON values, unknown fields, missing required +// fields, and non-integer JSON representations for integer fields. +func DecodeDraft(r io.Reader) (Draft, error) { + data, err := readLimited(r) + if err != nil { + return Draft{}, err + } + if err := checkStrictJSON(data); err != nil { + return Draft{}, fmt.Errorf("invalid draft JSON: %w", err) + } + if err := checkObjectShape(data, false); err != nil { + return Draft{}, err + } + + var draft Draft + if err := decodeTyped(data, &draft); err != nil { + return Draft{}, fmt.Errorf("invalid draft: %w", err) + } + if err := ValidateDraft(draft); err != nil { + return Draft{}, err + } + return draft, nil +} + +func decodeRow(data []byte) (Row, error) { + if err := checkStrictJSON(data); err != nil { + return Row{}, fmt.Errorf("invalid journal row JSON: %w", err) + } + if err := checkObjectShape(data, true); err != nil { + return Row{}, err + } + + var row Row + if err := decodeTyped(data, &row); err != nil { + return Row{}, fmt.Errorf("invalid journal row: %w", err) + } + if err := ValidateRow(row); err != nil { + return Row{}, err + } + return row, nil +} + +func readLimited(r io.Reader) ([]byte, error) { + if r == nil { + return nil, errors.New("experiment input is nil") + } + data, err := io.ReadAll(io.LimitReader(r, MaxInputBytes+1)) + if err != nil { + return nil, fmt.Errorf("read experiment input: %w", err) + } + if int64(len(data)) > MaxInputBytes { + return nil, ErrInputTooLarge + } + return data, nil +} + +func decodeTyped(data []byte, dst any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + return err + } + return nil +} + +func checkStrictJSON(data []byte) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := scanValue(dec); err != nil { + return err + } + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("trailing JSON value") + } + return fmt.Errorf("trailing JSON data: %w", err) + } + return nil +} + +func scanValue(dec *json.Decoder) error { + tok, err := dec.Token() + if err != nil { + return err + } + delim, ok := tok.(json.Delim) + if !ok { + return nil + } + + switch delim { + case '{': + seen := make(map[string]struct{}) + for dec.More() { + keyToken, err := dec.Token() + if err != nil { + return err + } + key, ok := keyToken.(string) + if !ok { + return errors.New("object key is not a string") + } + if _, exists := seen[key]; exists { + return fmt.Errorf("duplicate object key %q", key) + } + seen[key] = struct{}{} + if err := scanValue(dec); err != nil { + return err + } + } + end, err := dec.Token() + if err != nil { + return err + } + if end != json.Delim('}') { + return errors.New("unterminated JSON object") + } + case '[': + for dec.More() { + if err := scanValue(dec); err != nil { + return err + } + } + end, err := dec.Token() + if err != nil { + return err + } + if end != json.Delim(']') { + return errors.New("unterminated JSON array") + } + default: + return fmt.Errorf("unexpected JSON delimiter %q", delim) + } + return nil +} + +func checkObjectShape(data []byte, row bool) error { + var top map[string]json.RawMessage + if err := json.Unmarshal(data, &top); err != nil { + return fmt.Errorf("JSON root must be an object: %w", err) + } + if top == nil { + return errors.New("JSON root must be an object") + } + + allowed := map[string]struct{}{ + "outcome": {}, "rework_required": {}, "ward": {}, "seal": {}, + } + required := []string{"outcome", "ward", "seal"} + if row { + allowed["schema_version"] = struct{}{} + allowed["slot"] = struct{}{} + allowed["recorded_at"] = struct{}{} + required = append([]string{"schema_version", "slot", "recorded_at"}, required...) + } + label := "draft" + if row { + label = "row" + } + if err := rejectUnknownAndRequire(top, allowed, required, label); err != nil { + return err + } + if raw, ok := top["rework_required"]; ok && bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return errors.New("rework_required must be boolean when present") + } + if err := checkModuleShape("ward", top["ward"]); err != nil { + return err + } + if err := checkModuleShape("seal", top["seal"]); err != nil { + return err + } + return nil +} + +func checkModuleShape(name string, data []byte) error { + var object map[string]json.RawMessage + if err := json.Unmarshal(data, &object); err != nil || object == nil { + return fmt.Errorf("%s must be an object", name) + } + allowed := map[string]struct{}{ + "used": {}, "version": {}, "defects_caught_before_terminal": {}, + "added_user_interventions": {}, "interaction_seconds": {}, + "normal_work_blocked": {}, + } + if err := rejectUnknownAndRequire(object, allowed, []string{"used", "version"}, name); err != nil { + return err + } + for _, field := range []string{ + "defects_caught_before_terminal", + "added_user_interventions", + "interaction_seconds", + "normal_work_blocked", + } { + if raw, ok := object[field]; ok && bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return fmt.Errorf("%s.%s must not be null", name, field) + } + } + return nil +} + +func rejectUnknownAndRequire( + object map[string]json.RawMessage, + allowed map[string]struct{}, + required []string, + label string, +) error { + for key := range object { + if _, ok := allowed[key]; !ok { + return fmt.Errorf("unknown %s field %q", label, key) + } + } + for _, key := range required { + if _, ok := object[key]; !ok { + return fmt.Errorf("missing required %s field %q", label, key) + } + } + return nil +} diff --git a/internal/experiment/journal.go b/internal/experiment/journal.go new file mode 100644 index 0000000..db258ed --- /dev/null +++ b/internal/experiment/journal.go @@ -0,0 +1,75 @@ +package experiment + +import ( + "bytes" + "encoding/json" + "fmt" + "io" +) + +// ParseJournal strictly parses and validates a complete existing JSONL +// journal. An empty journal is valid. Blank rows and more than 20 rows are not. +func ParseJournal(r io.Reader) ([]Row, error) { + data, err := readLimited(r) + if err != nil { + return nil, err + } + if len(data) == 0 { + return nil, nil + } + + lines := bytes.Split(data, []byte{'\n'}) + if len(lines) > 0 && len(lines[len(lines)-1]) == 0 { + lines = lines[:len(lines)-1] + } + if int64(len(lines)) > MaxRows { + return nil, fmt.Errorf("journal contains more than %d rows", MaxRows) + } + + rows := make([]Row, 0, len(lines)) + for index, line := range lines { + if len(bytes.TrimSpace(line)) == 0 { + return nil, fmt.Errorf("journal row %d is blank", index+1) + } + row, err := decodeRow(line) + if err != nil { + return nil, fmt.Errorf("journal row %d: %w", index+1, err) + } + rows = append(rows, row) + } + if err := ValidateJournal(rows); err != nil { + return nil, err + } + return rows, nil +} + +// ValidateJournal requires physical slot order 1..n with no gaps and caps the +// bounded experiment at 20 rows. +func ValidateJournal(rows []Row) error { + if int64(len(rows)) > MaxRows { + return fmt.Errorf("journal contains more than %d rows", MaxRows) + } + for index, row := range rows { + if err := ValidateRow(row); err != nil { + return fmt.Errorf("journal row %d: %w", index+1, err) + } + expected := int64(index + 1) + if row.Slot != expected { + return fmt.Errorf("journal slot must be contiguous: got %d, want %d", row.Slot, expected) + } + } + return nil +} + +// MarshalCanonicalRow validates row and returns compact typed JSON. Struct +// field order defines the stable stored representation. +func MarshalCanonicalRow(row Row) ([]byte, error) { + if err := ValidateRow(row); err != nil { + return nil, err + } + data, err := json.Marshal(row) + if err != nil { + return nil, fmt.Errorf("encode journal row: %w", err) + } + return data, nil +} diff --git a/internal/experiment/types.go b/internal/experiment/types.go new file mode 100644 index 0000000..cadd232 --- /dev/null +++ b/internal/experiment/types.go @@ -0,0 +1,75 @@ +package experiment + +import "time" + +const ( + // SchemaVersion identifies rows produced by the bounded host experiment. + SchemaVersion = "eval-experiment/v1" + // MaxInputBytes is the maximum accepted draft or journal size. + MaxInputBytes int64 = 64 * 1024 + // MaxRows is the fixed number of slots in one experiment journal. + MaxRows int64 = 20 +) + +// Outcome is the terminal state of the primary task. +type Outcome string + +const ( + OutcomeCompleted Outcome = "completed" + OutcomeFailed Outcome = "failed" + OutcomeAbandoned Outcome = "abandoned" +) + +// Module records whether a module was used and the bounded effects already +// known when the task reached a terminal outcome. Version is required in JSON +// but may be null when an exact public version is unavailable. +type Module struct { + Used bool `json:"used"` + Version *string `json:"version"` + DefectsCaughtBeforeTerminal *int64 `json:"defects_caught_before_terminal,omitempty"` + AddedUserInterventions *int64 `json:"added_user_interventions,omitempty"` + InteractionSeconds *int64 `json:"interaction_seconds,omitempty"` + NormalWorkBlocked *bool `json:"normal_work_blocked,omitempty"` +} + +// Draft contains only facts supplied by the host after a task terminates. +// Identity, ordering, and recording time are assigned by the journal owner. +type Draft struct { + Outcome Outcome `json:"outcome"` + ReworkRequired *bool `json:"rework_required,omitempty"` + Ward Module `json:"ward"` + Seal Module `json:"seal"` +} + +// Row is the canonical stored representation of a validated Draft. +type Row struct { + SchemaVersion string `json:"schema_version"` + Slot int64 `json:"slot"` + RecordedAt string `json:"recorded_at"` + Outcome Outcome `json:"outcome"` + ReworkRequired *bool `json:"rework_required,omitempty"` + Ward Module `json:"ward"` + Seal Module `json:"seal"` +} + +// NewRow validates draft and constructs a row for slot at recordedAt. The +// timestamp is normalized to canonical UTC RFC3339 with nanosecond precision. +func NewRow(draft Draft, slot int64, recordedAt time.Time) (Row, error) { + if err := ValidateDraft(draft); err != nil { + return Row{}, err + } + + row := Row{ + SchemaVersion: SchemaVersion, + Slot: slot, + RecordedAt: recordedAt.UTC().Format(time.RFC3339Nano), + Outcome: draft.Outcome, + ReworkRequired: draft.ReworkRequired, + Ward: draft.Ward, + Seal: draft.Seal, + } + if err := ValidateRow(row); err != nil { + return Row{}, err + } + return row, nil +} diff --git a/internal/experiment/validate.go b/internal/experiment/validate.go new file mode 100644 index 0000000..961aabf --- /dev/null +++ b/internal/experiment/validate.go @@ -0,0 +1,106 @@ +package experiment + +import ( + "errors" + "fmt" + "regexp" + "strings" + "time" +) + +var publicVersionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]*$`) + +// ValidateDraft checks the cross-field experiment contract independently of +// JSON decoding so callers constructing typed drafts receive the same rules. +func ValidateDraft(draft Draft) error { + if !draft.Outcome.valid() { + return fmt.Errorf("unsupported outcome %q", draft.Outcome) + } + if err := validateModule("ward", draft.Ward); err != nil { + return err + } + if err := validateModule("seal", draft.Seal); err != nil { + return err + } + return nil +} + +// ValidateRow checks one typed stored row. Journal ordering is validated by +// ValidateJournal. +func ValidateRow(row Row) error { + if row.SchemaVersion != SchemaVersion { + return fmt.Errorf("unsupported schema_version %q", row.SchemaVersion) + } + if row.Slot < 1 || row.Slot > MaxRows { + return fmt.Errorf("slot must be between 1 and %d", MaxRows) + } + if err := validateRecordedAt(row.RecordedAt); err != nil { + return err + } + return ValidateDraft(Draft{ + Outcome: row.Outcome, + ReworkRequired: row.ReworkRequired, + Ward: row.Ward, + Seal: row.Seal, + }) +} + +func (outcome Outcome) valid() bool { + switch outcome { + case OutcomeCompleted, OutcomeFailed, OutcomeAbandoned: + return true + default: + return false + } +} + +func validateModule(name string, module Module) error { + if module.Version != nil { + if len(*module.Version) == 0 || len(*module.Version) > 128 || + !publicVersionPattern.MatchString(*module.Version) { + return fmt.Errorf("%s.version is not a bounded public identifier", name) + } + } + + if !module.Used { + if module.Version != nil { + return fmt.Errorf("%s.version must be null when used is false", name) + } + if hasModuleEffect(module) { + return fmt.Errorf("%s effects are forbidden when used is false", name) + } + return nil + } + + for field, value := range map[string]*int64{ + "defects_caught_before_terminal": module.DefectsCaughtBeforeTerminal, + "added_user_interventions": module.AddedUserInterventions, + "interaction_seconds": module.InteractionSeconds, + } { + if value != nil && *value < 0 { + return fmt.Errorf("%s.%s must be non-negative", name, field) + } + } + return nil +} + +func hasModuleEffect(module Module) bool { + return module.DefectsCaughtBeforeTerminal != nil || + module.AddedUserInterventions != nil || + module.InteractionSeconds != nil || + module.NormalWorkBlocked != nil +} + +func validateRecordedAt(value string) error { + if !strings.HasSuffix(value, "Z") { + return errors.New("recorded_at must use canonical UTC RFC3339") + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return fmt.Errorf("recorded_at must use canonical UTC RFC3339: %w", err) + } + if parsed.Location() != time.UTC || parsed.Format(time.RFC3339Nano) != value { + return errors.New("recorded_at must use canonical UTC RFC3339") + } + return nil +} diff --git a/internal/state/state.go b/internal/state/state.go new file mode 100644 index 0000000..5448012 --- /dev/null +++ b/internal/state/state.go @@ -0,0 +1,36 @@ +package state + +import ( + "fmt" + "os" + "path/filepath" +) + +const JournalRelativePath = "jgoneit/eval-experiment/v1/journal.jsonl" + +// Root resolves the XDG state root without creating it. Store owns creation +// and security checks; this package only defines the public location contract. +func Root(explicit string, getenv func(string) string) (string, error) { + if getenv == nil { + getenv = os.Getenv + } + root := explicit + if root == "" { + root = getenv("XDG_STATE_HOME") + if root == "" { + home := getenv("HOME") + if home == "" || !filepath.IsAbs(home) || filepath.Clean(home) != home { + return "", fmt.Errorf("HOME must be an absolute clean path") + } + root = filepath.Join(home, ".local", "state") + } + } + if !filepath.IsAbs(root) || filepath.Clean(root) != root || filepath.Dir(root) == root { + return "", fmt.Errorf("state root must be an absolute non-root clean path") + } + return root, nil +} + +func JournalPath(root string) string { + return filepath.Join(root, filepath.FromSlash(JournalRelativePath)) +} diff --git a/internal/store/errors.go b/internal/store/errors.go new file mode 100644 index 0000000..1bfcfb1 --- /dev/null +++ b/internal/store/errors.go @@ -0,0 +1,89 @@ +package store + +import ( + "errors" + "fmt" + "os" +) + +type Category string + +const ( + CategoryUnsafePath Category = "unsafe-path" + CategoryPermission Category = "permission" + CategoryLockTimeout Category = "lock-timeout" + CategoryValidation Category = "validation" + CategoryIO Category = "io" +) + +var ( + ErrUnsafePath = errors.New("unsafe state path") + ErrPermission = errors.New("private state permission or ownership violation") + ErrLockTimeout = errors.New("state lock timeout") + ErrValidation = errors.New("state validation failed") + ErrIO = errors.New("state I/O failed") +) + +type Error struct { + Category Category + Op string + Path string + Err error +} + +func (e *Error) Error() string { + if e.Path == "" { + return fmt.Sprintf("store %s: %v", e.Op, e.Err) + } + return fmt.Sprintf("store %s %q: %v", e.Op, e.Path, e.Err) +} + +func (e *Error) Unwrap() error { return e.Err } + +func CategoryOf(err error) Category { + var target *Error + if errors.As(err, &target) { + return target.Category + } + return CategoryIO +} + +func SafeReason(err error) string { + switch CategoryOf(err) { + case CategoryUnsafePath: + return "unsafe-state-path" + case CategoryPermission: + return "state-permission-denied" + case CategoryLockTimeout: + return "state-lock-timeout" + case CategoryValidation: + return "invalid-state-data" + default: + return "state-io-error" + } +} + +func storeError(category Category, op, path string, err error) error { + return &Error{Category: category, Op: op, Path: path, Err: err} +} + +func classifyError(op, path string, err error) error { + if err == nil { + return nil + } + var existing *Error + if errors.As(err, &existing) { + return err + } + if errors.Is(err, os.ErrPermission) { + return storeError(CategoryPermission, op, path, errors.Join(ErrPermission, err)) + } + return storeError(CategoryIO, op, path, errors.Join(ErrIO, err)) +} + +func classifyLockError(op, path string, err error) error { + if errors.Is(err, ErrLockTimeout) { + return storeError(CategoryLockTimeout, op, path, err) + } + return classifyError(op, path, err) +} diff --git a/internal/store/platform_unix.go b/internal/store/platform_unix.go new file mode 100644 index 0000000..7b8a38e --- /dev/null +++ b/internal/store/platform_unix.go @@ -0,0 +1,195 @@ +//go:build darwin || linux + +package store + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +func checkPrivatePath(path string, info os.FileInfo, want os.FileMode) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != uint32(os.Geteuid()) || info.Mode().Perm() != want { + return storeError(CategoryPermission, "inspect-private-metadata", path, ErrPermission) + } + if info.Mode().IsRegular() && stat.Nlink != 1 { + return storeError(CategoryUnsafePath, "inspect-hard-link", path, ErrUnsafePath) + } + return nil +} + +func checkStateRoot(path string, info os.FileInfo) error { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || stat.Uid != uint32(os.Geteuid()) || info.Mode().Perm()&0o022 != 0 { + return storeError(CategoryPermission, "inspect-state-root", path, ErrPermission) + } + return nil +} + +func checkOpenPrivateFile(file *os.File, want os.FileMode) error { + info, err := file.Stat() + if err != nil { + return err + } + return checkPrivatePath(file.Name(), info, want) +} + +func createPrivateDir(path string) error { return os.Mkdir(path, 0o700) } + +func normalizeSystemRootAlias(path string) (string, error) { + remainder := strings.TrimPrefix(path, string(filepath.Separator)) + if remainder == "" { + return path, nil + } + first := strings.SplitN(remainder, string(filepath.Separator), 2)[0] + candidate := filepath.Join(string(filepath.Separator), first) + info, err := os.Lstat(candidate) + if errors.Is(err, os.ErrNotExist) { + return path, nil + } + if err != nil { + return "", classifyError("normalize-system-root", candidate, err) + } + if info.Mode()&os.ModeSymlink == 0 || !trustedSystemSymlink(candidate, info) { + return path, nil + } + resolved, err := filepath.EvalSymlinks(candidate) + if err != nil { + return "", classifyError("normalize-system-root", candidate, err) + } + relative, err := filepath.Rel(candidate, path) + if err != nil { + return "", classifyError("normalize-system-root", candidate, err) + } + if relative == "." { + return resolved, nil + } + return filepath.Join(resolved, relative), nil +} + +func trustedSystemSymlink(path string, info os.FileInfo) bool { + if filepath.Dir(path) != string(filepath.Separator) { + return false + } + linkStat, ok := info.Sys().(*syscall.Stat_t) + if !ok || linkStat.Uid != 0 { + return false + } + resolved, err := filepath.EvalSymlinks(path) + if err != nil || !filepath.IsAbs(resolved) || resolved == path { + return false + } + resolvedInfo, err := os.Stat(resolved) + if err != nil || !resolvedInfo.IsDir() { + return false + } + resolvedStat, ok := resolvedInfo.Sys().(*syscall.Stat_t) + if !ok || resolvedStat.Uid != 0 { + return false + } + worldWritable := resolvedInfo.Mode().Perm()&0o022 != 0 + return !worldWritable || resolvedInfo.Mode()&os.ModeSticky != 0 +} + +func validateStateRootAncestors(path string) error { + remainder := strings.TrimPrefix(filepath.Clean(path), string(filepath.Separator)) + current := string(filepath.Separator) + for _, component := range strings.Split(remainder, string(filepath.Separator)) { + if component == "" { + continue + } + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return classifyError("inspect-state-ancestors", current, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "inspect-state-ancestors", current, ErrUnsafePath) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return storeError(CategoryPermission, "inspect-state-ancestors", current, ErrPermission) + } + if info.Mode().Perm()&0o022 != 0 { + trustedSticky := stat.Uid == 0 && info.Mode()&os.ModeSticky != 0 + if !trustedSticky { + return storeError(CategoryPermission, "inspect-state-ancestors", current, ErrPermission) + } + } + } + return nil +} + +func openPrivateFile(path string, readWrite, create bool) (*os.File, error) { + flags := unix.O_RDONLY | unix.O_CLOEXEC | unix.O_NOFOLLOW + if readWrite { + flags = unix.O_RDWR | unix.O_CLOEXEC | unix.O_NOFOLLOW + } + if create { + flags |= unix.O_CREAT + } + fd, err := unix.Open(path, flags, 0o600) + if err != nil { + if errors.Is(err, unix.ELOOP) { + return nil, storeError(CategoryUnsafePath, "open-no-follow", path, ErrUnsafePath) + } + return nil, err + } + return os.NewFile(uintptr(fd), path), nil +} + +func createPrivateTemp(path string) (*os.File, error) { + fd, err := unix.Open(path, unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOFOLLOW|unix.O_CREAT|unix.O_EXCL, 0o600) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(fd), path), nil +} + +func acquireFileLock(ctx context.Context, file *os.File, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if err == nil { + return nil + } + if !errors.Is(err, unix.EWOULDBLOCK) && !errors.Is(err, unix.EAGAIN) { + return err + } + remaining := time.Until(deadline) + if remaining <= 0 { + return ErrLockTimeout + } + wait := lockPollInterval + if remaining < wait { + wait = remaining + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("%w: %v", ErrLockTimeout, ctx.Err()) + case <-timer.C: + } + } +} + +func releaseFileLock(file *os.File) { _ = unix.Flock(int(file.Fd()), unix.LOCK_UN) } + +func syncDirectory(directory *os.File) (bool, error) { + if err := directory.Sync(); err != nil { + return false, err + } + return true, nil +} diff --git a/internal/store/platform_windows.go b/internal/store/platform_windows.go new file mode 100644 index 0000000..63463d6 --- /dev/null +++ b/internal/store/platform_windows.go @@ -0,0 +1,468 @@ +//go:build windows + +package store + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + "unsafe" + + "golang.org/x/sys/windows" +) + +const windowsFullControl = windows.STANDARD_RIGHTS_REQUIRED | windows.SYNCHRONIZE | 0x1ff + +func checkPrivatePath(path string, info os.FileInfo, want os.FileMode) error { + if info == nil || info.Mode()&os.ModeSymlink != 0 { + return storeError(CategoryUnsafePath, "inspect-private-path", path, ErrUnsafePath) + } + wantDirectory, err := privatePathKind(want) + if err != nil { + return err + } + if info.IsDir() != wantDirectory || (!wantDirectory && !info.Mode().IsRegular()) { + return storeError(CategoryUnsafePath, "inspect-private-path", path, ErrUnsafePath) + } + + file, err := openForInspection(path, wantDirectory) + if err != nil { + return classifyError("open-private-path", path, err) + } + defer file.Close() + + openInfo, err := file.Stat() + if err != nil { + return classifyError("stat-private-path", path, err) + } + if !os.SameFile(info, openInfo) { + return storeError(CategoryUnsafePath, "inspect-private-path", path, ErrUnsafePath) + } + return checkOpenPrivateFile(file, want) +} + +func checkStateRoot(path string, info os.FileInfo) error { + if info == nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "inspect-state-root", path, ErrUnsafePath) + } + file, err := openForInspection(path, true) + if err != nil { + return classifyError("open-state-root", path, err) + } + defer file.Close() + var details windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(windows.Handle(file.Fd()), &details); err != nil { + return classifyError("inspect-state-root", path, err) + } + if details.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return storeError(CategoryUnsafePath, "inspect-state-root", path, ErrUnsafePath) + } + descriptor, err := windows.GetSecurityInfo( + windows.Handle(file.Fd()), windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + return classifyError("read-state-root-security", path, err) + } + owner, _, err := descriptor.Owner() + if err != nil || owner == nil { + return storeError(CategoryPermission, "read-state-root-owner", path, errors.Join(ErrPermission, err)) + } + user, err := currentUserSID() + if err != nil { + return classifyError("current-user", path, err) + } + if !windows.EqualSid(owner, user) { + return storeError(CategoryPermission, "check-state-root-owner", path, ErrPermission) + } + dacl, present, err := descriptor.DACL() + if err != nil || !present || dacl == nil { + return storeError(CategoryPermission, "read-state-root-dacl", path, errors.Join(ErrPermission, err)) + } + administrator, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) + if err != nil { + return classifyError("administrator-sid", path, err) + } + system, err := windows.CreateWellKnownSid(windows.WinLocalSystemSid) + if err != nil { + return classifyError("system-sid", path, err) + } + writeMask := windows.ACCESS_MASK( + windows.GENERIC_ALL | windows.GENERIC_WRITE | windows.FILE_GENERIC_WRITE | + windows.DELETE | windows.WRITE_DAC | windows.WRITE_OWNER | + windows.FILE_WRITE_DATA | windows.FILE_APPEND_DATA | 0x40, + ) + for index := uint16(0); index < dacl.AceCount; index++ { + var ace *windows.ACCESS_ALLOWED_ACE + if err := windows.GetAce(dacl, uint32(index), &ace); err != nil { + return classifyError("read-state-root-dacl-entry", path, err) + } + if ace == nil { + return storeError(CategoryPermission, "check-state-root-dacl-entry", path, ErrPermission) + } + switch ace.Header.AceType { + case windows.ACCESS_ALLOWED_ACE_TYPE: + sid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + trusted := sid.IsValid() && (windows.EqualSid(sid, user) || windows.EqualSid(sid, administrator) || windows.EqualSid(sid, system)) + if !trusted && ace.Mask&writeMask != 0 { + return storeError(CategoryPermission, "check-state-root-dacl-access", path, ErrPermission) + } + case windows.ACCESS_DENIED_ACE_TYPE: + default: + return storeError(CategoryPermission, "check-state-root-dacl-entry", path, ErrPermission) + } + } + return nil +} + +func checkOpenPrivateFile(file *os.File, want os.FileMode) error { + if file == nil { + return storeError(CategoryUnsafePath, "inspect-open-file", "", ErrUnsafePath) + } + path := file.Name() + wantDirectory, err := privatePathKind(want) + if err != nil { + return err + } + handle := windows.Handle(file.Fd()) + fileType, err := windows.GetFileType(handle) + if err != nil { + return classifyError("file-type", path, err) + } + if fileType != windows.FILE_TYPE_DISK { + return storeError(CategoryUnsafePath, "file-type", path, ErrUnsafePath) + } + + var details windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &details); err != nil { + return classifyError("file-information", path, err) + } + isDirectory := details.FileAttributes&windows.FILE_ATTRIBUTE_DIRECTORY != 0 + if isDirectory != wantDirectory || details.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return storeError(CategoryUnsafePath, "file-information", path, ErrUnsafePath) + } + if !isDirectory && details.NumberOfLinks != 1 { + return storeError(CategoryUnsafePath, "file-information", path, ErrUnsafePath) + } + + descriptor, err := windows.GetSecurityInfo( + handle, + windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + return classifyError("read-security", path, err) + } + owner, _, err := descriptor.Owner() + if err != nil || owner == nil { + return storeError(CategoryPermission, "read-owner", path, errors.Join(ErrPermission, err)) + } + user, err := currentUserSID() + if err != nil { + return classifyError("current-user", path, err) + } + if !windows.EqualSid(owner, user) { + return storeError(CategoryPermission, "check-owner", path, ErrPermission) + } + + control, _, err := descriptor.Control() + if err != nil { + return classifyError("read-security-control", path, err) + } + if control&windows.SE_DACL_PRESENT == 0 || control&windows.SE_DACL_PROTECTED == 0 { + return storeError(CategoryPermission, "check-dacl", path, ErrPermission) + } + dacl, _, err := descriptor.DACL() + if err != nil || dacl == nil { + return storeError(CategoryPermission, "read-dacl", path, errors.Join(ErrPermission, err)) + } + + var userAllow windows.ACCESS_MASK + for index := uint16(0); index < dacl.AceCount; index++ { + var ace *windows.ACCESS_ALLOWED_ACE + if err := windows.GetAce(dacl, uint32(index), &ace); err != nil { + return classifyError("read-dacl-entry", path, err) + } + if ace == nil || ace.Header.AceFlags&windows.INHERITED_ACE != 0 { + return storeError(CategoryPermission, "check-dacl-entry", path, ErrPermission) + } + switch ace.Header.AceType { + case windows.ACCESS_ALLOWED_ACE_TYPE: + sid := (*windows.SID)(unsafe.Pointer(&ace.SidStart)) + if !sid.IsValid() || !windows.EqualSid(sid, user) { + return storeError(CategoryPermission, "check-dacl-trustee", path, ErrPermission) + } + userAllow |= ace.Mask + case windows.ACCESS_DENIED_ACE_TYPE: + // A deny ACE cannot disclose data. The successful handle open and the + // accumulated current-user allow mask below establish usability. + default: + return storeError(CategoryPermission, "check-dacl-entry", path, ErrPermission) + } + } + if userAllow&windows.GENERIC_ALL == 0 && userAllow&windowsFullControl != windowsFullControl { + return storeError(CategoryPermission, "check-dacl-access", path, ErrPermission) + } + return nil +} + +func createPrivateDir(path string) error { + pathUTF16, err := windows.UTF16PtrFromString(path) + if err != nil { + return classifyError("create-directory", path, err) + } + attributes, err := privateSecurityAttributes() + if err != nil { + return classifyError("create-directory-security", path, err) + } + created := false + if err := windows.CreateDirectory(pathUTF16, attributes); err != nil { + if !errors.Is(err, windows.ERROR_ALREADY_EXISTS) { + return classifyError("create-directory", path, err) + } + } else { + created = true + } + info, err := os.Lstat(path) + if err != nil { + return classifyError("inspect-created-directory", path, err) + } + if err := checkPrivatePath(path, info, 0o700); err != nil { + if created { + _ = windows.RemoveDirectory(pathUTF16) + } + return err + } + return nil +} + +func validateStateRootAncestors(path string) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return storeError(CategoryUnsafePath, "inspect-state-ancestors", path, ErrUnsafePath) + } + volume := filepath.VolumeName(path) + root := volume + string(filepath.Separator) + relative, err := filepath.Rel(root, path) + if err != nil || relative == "." { + return nil + } + current := root + for _, component := range splitWindowsPath(relative) { + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return classifyError("inspect-state-ancestors", current, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "inspect-state-ancestors", current, ErrUnsafePath) + } + file, err := openForInspection(current, true) + if err != nil { + return classifyError("open-state-ancestor", current, err) + } + var details windows.ByHandleFileInformation + inspectErr := windows.GetFileInformationByHandle(windows.Handle(file.Fd()), &details) + _ = file.Close() + if inspectErr != nil { + return classifyError("inspect-state-ancestor", current, inspectErr) + } + if details.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return storeError(CategoryUnsafePath, "inspect-state-ancestor", current, ErrUnsafePath) + } + } + return nil +} + +func normalizeSystemRootAlias(path string) (string, error) { + return filepath.Clean(path), nil +} + +func openPrivateFile(path string, readWrite, create bool) (*os.File, error) { + access := uint32(windows.GENERIC_READ) + if readWrite { + access |= windows.GENERIC_WRITE + } + if !create { + return openWindowsFile(path, access, windows.OPEN_EXISTING, nil, windows.FILE_ATTRIBUTE_NORMAL) + } + + attributes, err := privateSecurityAttributes() + if err != nil { + return nil, err + } + file, err := openWindowsFile(path, access, windows.CREATE_NEW, attributes, windows.FILE_ATTRIBUTE_NORMAL) + if err != nil && (errors.Is(err, windows.ERROR_FILE_EXISTS) || errors.Is(err, windows.ERROR_ALREADY_EXISTS)) { + file, err = openWindowsFile(path, access, windows.OPEN_EXISTING, nil, windows.FILE_ATTRIBUTE_NORMAL) + } + return file, err +} + +func createPrivateTemp(path string) (*os.File, error) { + attributes, err := privateSecurityAttributes() + if err != nil { + return nil, err + } + return openWindowsFile( + path, + windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.CREATE_NEW, + attributes, + windows.FILE_ATTRIBUTE_TEMPORARY, + ) +} + +func acquireFileLock(ctx context.Context, file *os.File, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + err := windows.LockFileEx( + windows.Handle(file.Fd()), + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + &windows.Overlapped{}, + ) + if err == nil { + return nil + } + if !errors.Is(err, windows.ERROR_LOCK_VIOLATION) && !errors.Is(err, windows.ERROR_IO_PENDING) { + return err + } + remaining := time.Until(deadline) + if remaining <= 0 { + return ErrLockTimeout + } + wait := lockPollInterval + if remaining < wait { + wait = remaining + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return fmt.Errorf("%w: %v", ErrLockTimeout, ctx.Err()) + case <-timer.C: + } + } +} + +func releaseFileLock(file *os.File) { + _ = windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &windows.Overlapped{}) +} + +func syncDirectory(directory *os.File) (bool, error) { + err := windows.FlushFileBuffers(windows.Handle(directory.Fd())) + if errors.Is(err, windows.ERROR_ACCESS_DENIED) || errors.Is(err, windows.ERROR_INVALID_HANDLE) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func privatePathKind(want os.FileMode) (bool, error) { + switch want.Perm() { + case 0o700: + return true, nil + case 0o600: + return false, nil + default: + return false, storeError(CategoryUnsafePath, "private-mode", "", ErrUnsafePath) + } +} + +func currentUserSID() (*windows.SID, error) { + user, err := windows.GetCurrentProcessToken().GetTokenUser() + if err != nil { + return nil, err + } + if user == nil || user.User.Sid == nil || !user.User.Sid.IsValid() { + return nil, ErrPermission + } + return user.User.Sid, nil +} + +func privateSecurityAttributes() (*windows.SecurityAttributes, error) { + user, err := currentUserSID() + if err != nil { + return nil, err + } + descriptor, err := windows.SecurityDescriptorFromString( + fmt.Sprintf("D:P(A;;FA;;;%s)", user.String()), + ) + if err != nil { + return nil, err + } + return &windows.SecurityAttributes{ + Length: uint32(unsafe.Sizeof(windows.SecurityAttributes{})), + SecurityDescriptor: descriptor, + }, nil +} + +func openForInspection(path string, directory bool) (*os.File, error) { + flags := uint32(windows.FILE_FLAG_OPEN_REPARSE_POINT) + if directory { + flags |= windows.FILE_FLAG_BACKUP_SEMANTICS + } + return openWindowsFile( + path, + uint32(windows.READ_CONTROL)|windows.FILE_READ_ATTRIBUTES, + windows.OPEN_EXISTING, + nil, + flags, + ) +} + +func openWindowsFile( + path string, + access uint32, + disposition uint32, + attributes *windows.SecurityAttributes, + fileAttributes uint32, +) (*os.File, error) { + pathUTF16, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + handle, err := windows.CreateFile( + pathUTF16, + access, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + attributes, + disposition, + fileAttributes|windows.FILE_FLAG_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(handle), path) + if file == nil { + _ = windows.CloseHandle(handle) + return nil, fmt.Errorf("wrap Windows file handle") + } + return file, nil +} + +func splitWindowsPath(path string) []string { + var components []string + for path != "." && path != "" { + directory, base := filepath.Split(path) + if base != "" { + components = append([]string{base}, components...) + } + path = filepath.Clean(directory) + path = filepath.Clean(path) + if path == "." || path == string(filepath.Separator) { + break + } + } + return components +} diff --git a/internal/store/security.go b/internal/store/security.go new file mode 100644 index 0000000..cdd48ff --- /dev/null +++ b/internal/store/security.go @@ -0,0 +1,158 @@ +package store + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +func rejectSymlinkComponents(path string) error { + if !filepath.IsAbs(path) { + return storeError(CategoryUnsafePath, "symlink-check", path, ErrUnsafePath) + } + clean := filepath.Clean(path) + volume := filepath.VolumeName(clean) + root := volume + string(filepath.Separator) + remainder := strings.TrimPrefix(clean, root) + current := root + for _, component := range strings.Split(remainder, string(filepath.Separator)) { + if component == "" { + continue + } + current = filepath.Join(current, component) + info, err := os.Lstat(current) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return classifyError("symlink-check", current, err) + } + if info.Mode()&os.ModeSymlink != 0 { + return storeError(CategoryUnsafePath, "symlink-check", current, ErrUnsafePath) + } + } + return nil +} + +func ensureStateRoot(path string) error { + info, err := os.Lstat(path) + if err == nil { + return validateStateRoot(path, info) + } + if !errors.Is(err, fs.ErrNotExist) { + return classifyError("inspect-state-root", path, err) + } + return createPrivatePath(path) +} + +func createPrivatePath(path string) error { + var missing []string + current := path + for { + info, err := os.Lstat(current) + if err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "create-directory", current, ErrUnsafePath) + } + break + } + if !errors.Is(err, fs.ErrNotExist) { + return classifyError("inspect-directory", current, err) + } + missing = append(missing, current) + parent := filepath.Dir(current) + if parent == current { + return storeError(CategoryUnsafePath, "create-directory", path, ErrUnsafePath) + } + current = parent + } + for index := len(missing) - 1; index >= 0; index-- { + component := missing[index] + if err := createPrivateDir(component); err != nil && !errors.Is(err, fs.ErrExist) { + return classifyError("create-directory", component, err) + } + info, err := os.Lstat(component) + if err != nil { + return classifyError("inspect-directory", component, err) + } + if err := validatePrivateDir(component, info); err != nil { + return err + } + } + return nil +} + +func validateStateRoot(path string, info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "inspect-state-root", path, ErrUnsafePath) + } + return checkStateRoot(path, info) +} + +func validatePrivateDir(path string, info os.FileInfo) error { + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return storeError(CategoryUnsafePath, "inspect-directory", path, ErrUnsafePath) + } + return checkPrivatePath(path, info, 0o700) +} + +func inspectPrivateFile(path string) error { + info, err := os.Lstat(path) + if err != nil { + return classifyError("inspect-file", path, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return storeError(CategoryUnsafePath, "inspect-file", path, ErrUnsafePath) + } + return checkPrivatePath(path, info, 0o600) +} + +func openOrCreatePrivateFile(path string) (*os.File, error) { + if err := inspectPrivateFile(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + file, err := openPrivateFile(path, true, true) + if err != nil { + return nil, classifyError("open-private-file", path, err) + } + if err := inspectOpenPrivateFile(file, path); err != nil { + _ = file.Close() + return nil, err + } + return file, nil +} + +func inspectOpenPrivateFile(file *os.File, path string) error { + openInfo, err := file.Stat() + if err != nil { + return classifyError("stat-open-file", path, err) + } + pathInfo, err := os.Lstat(path) + if err != nil { + return classifyError("inspect-open-file", path, err) + } + if pathInfo.Mode()&os.ModeSymlink != 0 || !os.SameFile(openInfo, pathInfo) { + return storeError(CategoryUnsafePath, "inspect-open-file", path, ErrUnsafePath) + } + if !openInfo.Mode().IsRegular() { + return storeError(CategoryPermission, "inspect-open-file", path, ErrPermission) + } + return checkOpenPrivateFile(file, 0o600) +} + +func rejectGitWorktree(path string) error { + for current := path; ; current = filepath.Dir(current) { + marker := filepath.Join(current, ".git") + if _, err := os.Lstat(marker); err == nil { + return storeError(CategoryUnsafePath, "git-worktree-check", path, fmt.Errorf("Git worktree state is forbidden: %w", ErrUnsafePath)) + } else if !errors.Is(err, fs.ErrNotExist) { + return classifyError("git-worktree-check", marker, err) + } + if filepath.Dir(current) == current { + return nil + } + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..bbec825 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,437 @@ +// Package store provides a private, single-writer, crash-atomic JSONL store. +package store + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + DefaultLockTimeout = 2 * time.Second + lockPollInterval = 10 * time.Millisecond + maxJournalBytes = 256 << 10 +) + +type Transaction func(existing []byte) ([]byte, error) + +type Hooks struct { + WriteTemp func(file *os.File, prospective []byte) error + AfterTempWrite func(tempPath string) error + AfterTempSync func(tempPath string) error + BeforeReplace func(tempPath, targetPath string) error + AfterReplace func(targetPath string) error + BeforeDirectorySync func(directory string) error +} + +type Options struct { + LockTimeout time.Duration + Hooks Hooks +} + +// Commit distinguishes pre-commit failures from failures after atomic replace. +type Commit struct { + Committed bool + DurabilityConfirmed bool +} + +type Store struct { + root string + relativePath string + relativeDir string + journalName string + lockName string + path string + lockPath string + privateDirs []string + lockTimeout time.Duration + hooks Hooks +} + +func New(stateRoot, relativePath string, options Options) (*Store, error) { + if stateRoot == "" || !filepath.IsAbs(stateRoot) || filepath.Clean(stateRoot) != stateRoot || filepath.Dir(stateRoot) == stateRoot { + return nil, storeError(CategoryUnsafePath, "configure", stateRoot, ErrUnsafePath) + } + root, err := normalizeSystemRootAlias(stateRoot) + if err != nil { + return nil, err + } + relative := filepath.FromSlash(relativePath) + if relative == "" || filepath.IsAbs(relative) || filepath.Clean(relative) != relative || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return nil, storeError(CategoryUnsafePath, "configure", relativePath, ErrUnsafePath) + } + target := filepath.Join(root, relative) + inside, err := filepath.Rel(root, target) + if err != nil || inside == ".." || strings.HasPrefix(inside, ".."+string(filepath.Separator)) { + return nil, storeError(CategoryUnsafePath, "configure", target, ErrUnsafePath) + } + timeout := options.LockTimeout + if timeout == 0 { + timeout = DefaultLockTimeout + } + if timeout < 0 { + return nil, storeError(CategoryUnsafePath, "configure", target, ErrUnsafePath) + } + directory := filepath.Dir(relative) + var privateDirs []string + if directory != "." { + current := "" + for _, component := range strings.Split(directory, string(filepath.Separator)) { + current = filepath.Join(current, component) + privateDirs = append(privateDirs, current) + } + } + return &Store{ + root: root, relativePath: relative, relativeDir: directory, + journalName: filepath.Base(relative), lockName: filepath.Base(relative) + ".lock", + path: target, lockPath: target + ".lock", + privateDirs: privateDirs, lockTimeout: timeout, hooks: options.Hooks, + }, nil +} + +func (s *Store) Path() string { return s.path } +func (s *Store) LockPath() string { return s.lockPath } + +// Update holds the stable exclusive lock while deriving and validating a +// complete append-only journal. A committed result is authoritative even when +// the returned error means that final directory durability is unconfirmed. +func (s *Store) Update(ctx context.Context, transaction Transaction) (Commit, error) { + if transaction == nil { + return Commit{}, storeError(CategoryValidation, "update", s.path, ErrValidation) + } + root, err := s.prepare() + if err != nil { + return Commit{}, err + } + defer root.Close() + + lock, err := openOrCreatePrivateFile(s.lockPath) + if err != nil { + return Commit{}, err + } + defer lock.Close() + if err := acquireFileLock(ctx, lock, s.lockTimeout); err != nil { + return Commit{}, classifyLockError("write-lock", s.lockPath, err) + } + defer releaseFileLock(lock) + + if err := s.verifyTree(root); err != nil { + return Commit{}, err + } + if err := inspectOpenPrivateFile(lock, s.lockPath); err != nil { + return Commit{}, err + } + rootLock, err := root.OpenFile(s.relativePath+".lock", os.O_RDWR, 0) + if err != nil { + return Commit{}, classifyError("open-rooted-lock", s.lockPath, err) + } + if err := sameOpenFile(lock, rootLock, s.lockPath); err != nil { + rootLock.Close() + return Commit{}, err + } + rootLock.Close() + + journalRoot, err := root.OpenRoot(s.relativeDir) + if err != nil { + return Commit{}, classifyError("open-journal-root", filepath.Dir(s.path), err) + } + defer journalRoot.Close() + existing, err := s.readExisting(journalRoot) + if err != nil { + return Commit{}, err + } + if len(existing) > 0 && existing[len(existing)-1] != '\n' { + return Commit{}, storeError(CategoryValidation, "read-journal", s.path, ErrValidation) + } + prospective, err := transaction(bytes.Clone(existing)) + if err != nil { + return Commit{}, err + } + if err := validateAppendOnly(existing, prospective); err != nil { + return Commit{}, err + } + return s.replaceWith(journalRoot, prospective) +} + +func (s *Store) prepare() (*os.Root, error) { + if err := rejectSymlinkComponents(s.root); err != nil { + return nil, err + } + if err := rejectSymlinkComponents(s.path); err != nil { + return nil, err + } + if err := rejectSymlinkComponents(s.lockPath); err != nil { + return nil, err + } + if err := validateStateRootAncestors(s.root); err != nil { + return nil, err + } + if err := rejectGitWorktree(s.root); err != nil { + return nil, err + } + if err := ensureStateRoot(s.root); err != nil { + return nil, err + } + root, err := os.OpenRoot(s.root) + if err != nil { + return nil, classifyError("open-state-root", s.root, err) + } + for _, relative := range s.privateDirs { + absolute := filepath.Join(s.root, relative) + info, statErr := root.Lstat(relative) + if errors.Is(statErr, fs.ErrNotExist) { + if err := createPrivateDir(absolute); err != nil && !errors.Is(err, fs.ErrExist) { + root.Close() + return nil, classifyError("create-private-directory", absolute, err) + } + info, statErr = root.Lstat(relative) + } + if statErr != nil { + root.Close() + return nil, classifyError("inspect-private-directory", absolute, statErr) + } + if err := validatePrivateDir(absolute, info); err != nil { + root.Close() + return nil, err + } + } + if err := s.verifyDirectories(root); err != nil { + root.Close() + return nil, err + } + return root, nil +} + +func (s *Store) verifyTree(root *os.Root) error { + if err := s.verifyDirectories(root); err != nil { + return err + } + for _, pair := range []struct{ relative, absolute string }{ + {s.relativePath, s.path}, {s.relativePath + ".lock", s.lockPath}, + } { + rootInfo, rootErr := root.Lstat(pair.relative) + absoluteInfo, absoluteErr := os.Lstat(pair.absolute) + if errors.Is(rootErr, fs.ErrNotExist) && errors.Is(absoluteErr, fs.ErrNotExist) { + continue + } + if rootErr != nil || absoluteErr != nil || !os.SameFile(rootInfo, absoluteInfo) { + return storeError(CategoryUnsafePath, "inspect-private-file-identity", pair.absolute, ErrUnsafePath) + } + if err := inspectPrivateFile(pair.absolute); err != nil { + return err + } + } + return nil +} + +func (s *Store) verifyDirectories(root *os.Root) error { + stateInfo, err := os.Lstat(s.root) + if err != nil { + return classifyError("inspect-state-root", s.root, err) + } + rootInfo, err := root.Lstat(".") + if err != nil || !os.SameFile(stateInfo, rootInfo) { + return storeError(CategoryUnsafePath, "inspect-state-root-identity", s.root, ErrUnsafePath) + } + if err := validateStateRoot(s.root, stateInfo); err != nil { + return err + } + for _, relative := range s.privateDirs { + info, err := root.Lstat(relative) + if err != nil { + return classifyError("inspect-private-directory", filepath.Join(s.root, relative), err) + } + if info.Mode()&os.ModeSymlink != 0 { + return storeError(CategoryUnsafePath, "inspect-private-directory", filepath.Join(s.root, relative), ErrUnsafePath) + } + absolute := filepath.Join(s.root, relative) + absoluteInfo, err := os.Lstat(absolute) + if err != nil || !os.SameFile(info, absoluteInfo) { + return storeError(CategoryUnsafePath, "inspect-private-directory-identity", absolute, ErrUnsafePath) + } + if err := validatePrivateDir(absolute, info); err != nil { + return err + } + } + return nil +} + +func (s *Store) readExisting(root *os.Root) ([]byte, error) { + if err := inspectPrivateFile(s.path); errors.Is(err, fs.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, err + } + file, err := root.Open(s.journalName) + if err != nil { + return nil, classifyError("open-journal", s.path, err) + } + defer file.Close() + if err := inspectOpenPrivateFile(file, s.path); err != nil { + return nil, err + } + data, err := io.ReadAll(io.LimitReader(file, maxJournalBytes+1)) + if err != nil { + return nil, classifyError("read-journal", s.path, err) + } + if len(data) > maxJournalBytes { + return nil, storeError(CategoryValidation, "read-journal", s.path, ErrValidation) + } + return data, nil +} + +func (s *Store) replaceWith(root *os.Root, prospective []byte) (result Commit, retErr error) { + directory := filepath.Dir(s.path) + temp, tempName, tempPath, err := createTemp(root, directory, filepath.Base(s.path)) + if err != nil { + return Commit{}, classifyError("create-temp", directory, err) + } + committed := false + defer func() { + if temp != nil { + _ = temp.Close() + } + if !committed { + _ = root.Remove(tempName) + } + }() + if err := inspectOpenPrivateFile(temp, tempPath); err != nil { + return Commit{}, err + } + if hook := s.hooks.WriteTemp; hook != nil { + err = hook(temp, bytes.Clone(prospective)) + } else { + err = writeAll(temp, prospective) + } + if err != nil { + return Commit{}, classifyError("write-temp", tempPath, err) + } + if hook := s.hooks.AfterTempWrite; hook != nil { + if err := hook(tempPath); err != nil { + return Commit{}, classifyError("after-temp-write", tempPath, err) + } + } + if err := temp.Sync(); err != nil { + return Commit{}, classifyError("sync-temp", tempPath, err) + } + if hook := s.hooks.AfterTempSync; hook != nil { + if err := hook(tempPath); err != nil { + return Commit{}, classifyError("after-temp-sync", tempPath, err) + } + } + if err := temp.Close(); err != nil { + return Commit{}, classifyError("close-temp", tempPath, err) + } + temp = nil + if hook := s.hooks.BeforeReplace; hook != nil { + if err := hook(tempPath, s.path); err != nil { + return Commit{}, classifyError("before-replace", s.path, err) + } + } + if err := root.Rename(tempName, s.journalName); err != nil { + return Commit{}, classifyError("replace", s.path, err) + } + committed = true + result = Commit{Committed: true} + if hook := s.hooks.AfterReplace; hook != nil { + if err := hook(s.path); err != nil { + return Commit{Committed: true}, classifyError("after-replace", s.path, err) + } + } + if hook := s.hooks.BeforeDirectorySync; hook != nil { + if err := hook(directory); err != nil { + return Commit{Committed: true}, classifyError("before-directory-sync", directory, err) + } + } + directoryFile, err := root.Open(".") + if err != nil { + return Commit{Committed: true}, classifyError("open-directory-sync", directory, err) + } + defer directoryFile.Close() + durable, err := syncDirectory(directoryFile) + if err != nil { + return Commit{Committed: true}, classifyError("sync-directory", directory, err) + } + return Commit{Committed: true, DurabilityConfirmed: durable}, nil +} + +func createTemp(root *os.Root, directory, base string) (*os.File, string, string, error) { + for attempts := 0; attempts < 100; attempts++ { + var random [12]byte + if _, err := rand.Read(random[:]); err != nil { + return nil, "", "", err + } + name := "." + base + ".tmp-" + hex.EncodeToString(random[:]) + path := filepath.Join(directory, name) + file, err := createPrivateTemp(path) + if errors.Is(err, fs.ErrExist) { + continue + } + if err != nil { + return nil, "", "", err + } + rooted, err := root.OpenFile(name, os.O_RDWR, 0) + if err != nil { + file.Close() + _ = os.Remove(path) + return nil, "", "", err + } + if err := sameOpenFile(file, rooted, path); err != nil { + rooted.Close() + file.Close() + _ = os.Remove(path) + return nil, "", "", err + } + rooted.Close() + return file, name, path, nil + } + return nil, "", "", fmt.Errorf("temporary name collision") +} + +func sameOpenFile(left, right *os.File, path string) error { + leftInfo, leftErr := left.Stat() + rightInfo, rightErr := right.Stat() + if leftErr != nil || rightErr != nil || !os.SameFile(leftInfo, rightInfo) { + return storeError(CategoryUnsafePath, "inspect-rooted-file-identity", path, ErrUnsafePath) + } + return nil +} + +func validateAppendOnly(existing, prospective []byte) error { + if len(prospective) <= len(existing) || !bytes.Equal(existing, prospective[:len(existing)]) { + return storeError(CategoryValidation, "validate-append", "", ErrValidation) + } + appended := prospective[len(existing):] + if len(appended) < 3 || appended[len(appended)-1] != '\n' || bytes.Count(appended, []byte{'\n'}) != 1 { + return storeError(CategoryValidation, "validate-append", "", ErrValidation) + } + row := appended[:len(appended)-1] + if !bytes.Equal(row, bytes.TrimSpace(row)) || !json.Valid(row) || len(row) == 0 || row[0] != '{' { + return storeError(CategoryValidation, "validate-append", "", ErrValidation) + } + return nil +} + +func writeAll(file *os.File, data []byte) error { + for len(data) > 0 { + written, err := file.Write(data) + if err != nil { + return err + } + if written == 0 { + return io.ErrShortWrite + } + data = data[written:] + } + return nil +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..7726e19 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,3 @@ +package version + +const Current = "0.2.0-experiment.1" diff --git a/schemas/experiment-v1.schema.json b/schemas/experiment-v1.schema.json new file mode 100644 index 0000000..ae38193 --- /dev/null +++ b/schemas/experiment-v1.schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:jgoneit:eval:experiment:v1", + "title": "Eval Bounded Host Experiment Row v1", + "description": "One stored row from the bounded Ward and Seal host experiment.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "slot", + "recorded_at", + "outcome", + "ward", + "seal" + ], + "properties": { + "schema_version": {"const": "eval-experiment/v1"}, + "slot": {"type": "integer", "minimum": 1, "maximum": 20}, + "recorded_at": { + "type": "string", + "format": "date-time", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\\.[0-9]{0,8}[1-9])?Z$" + }, + "outcome": {"enum": ["completed", "failed", "abandoned"]}, + "rework_required": {"type": "boolean"}, + "ward": {"$ref": "#/$defs/module"}, + "seal": {"$ref": "#/$defs/module"} + }, + "$defs": { + "publicVersion": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._+-]*$" + }, + "nonNegativeInt64": { + "type": "integer", + "minimum": 0, + "maximum": 9223372036854775807 + }, + "module": { + "type": "object", + "additionalProperties": false, + "required": ["used", "version"], + "properties": { + "used": {"type": "boolean"}, + "version": { + "oneOf": [ + {"type": "null"}, + {"$ref": "#/$defs/publicVersion"} + ] + }, + "defects_caught_before_terminal": {"$ref": "#/$defs/nonNegativeInt64"}, + "added_user_interventions": {"$ref": "#/$defs/nonNegativeInt64"}, + "interaction_seconds": {"$ref": "#/$defs/nonNegativeInt64"}, + "normal_work_blocked": {"type": "boolean"} + }, + "allOf": [ + { + "if": { + "required": ["used"], + "properties": {"used": {"const": false}} + }, + "then": { + "properties": {"version": {"type": "null"}}, + "not": { + "anyOf": [ + {"required": ["defects_caught_before_terminal"]}, + {"required": ["added_user_interventions"]}, + {"required": ["interaction_seconds"]}, + {"required": ["normal_work_blocked"]} + ] + } + } + } + ] + } + } +} From a66dfbe706f5c0ca6f03cdcb5fb95a9c20b6b118 Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:02:49 +0900 Subject: [PATCH 03/14] test(eval): cover recorder safety boundaries Why: - The bounded experiment is useful only if malformed input and storage failures cannot corrupt or expand the journal. What: - Cover strict decoding, field relationships, canonical rows, and the twenty-slot limit. - Exercise concurrent writers, invalid state, permissions, symlinks, hard links, lock timeout, and rooted path replacement. - Inject partial writes and pre/post-replace failures to distinguish skipped writes from recorded durability uncertainty. - Add Windows owner, protected DACL, reparse, hard-link, and LockFileEx coverage. Validation: - go test ./... - go test -race ./... - go vet ./... - Darwin, Linux, and Windows test binaries compiled --- internal/cli/cli_test.go | 265 ++++++++++++++++++++++++ internal/cli/cli_unix_test.go | 26 +++ internal/experiment/decode_test.go | 150 ++++++++++++++ internal/experiment/journal_test.go | 165 +++++++++++++++ internal/experiment/schema_test.go | 36 ++++ internal/experiment/validate_test.go | 154 ++++++++++++++ internal/store/platform_unix_test.go | 45 ++++ internal/store/platform_windows_test.go | 259 +++++++++++++++++++++++ internal/store/store_test.go | 178 ++++++++++++++++ 9 files changed, 1278 insertions(+) create mode 100644 internal/cli/cli_test.go create mode 100644 internal/cli/cli_unix_test.go create mode 100644 internal/experiment/decode_test.go create mode 100644 internal/experiment/journal_test.go create mode 100644 internal/experiment/schema_test.go create mode 100644 internal/experiment/validate_test.go create mode 100644 internal/store/platform_unix_test.go create mode 100644 internal/store/platform_windows_test.go create mode 100644 internal/store/store_test.go diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go new file mode 100644 index 0000000..99e7e8f --- /dev/null +++ b/internal/cli/cli_test.go @@ -0,0 +1,265 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/jgoneit/eval/internal/experiment" + "github.com/jgoneit/eval/internal/state" + "github.com/jgoneit/eval/internal/store" +) + +const validDraft = `{"outcome":"completed","rework_required":false,"ward":{"used":true,"version":null,"defects_caught_before_terminal":1,"added_user_interventions":0,"interaction_seconds":2,"normal_work_blocked":false},"seal":{"used":false,"version":null}}` + +type capturedResult struct { + Status string `json:"status"` + Slot int64 `json:"slot"` + Durability string `json:"durability"` + Reason string `json:"reason"` +} + +func TestVersionAndUsageSurface(t *testing.T) { + t.Parallel() + var stdout, stderr bytes.Buffer + if exit := Run(context.Background(), []string{"--version"}, Runtime{Stdout: &stdout, Stderr: &stderr}); exit != ExitSuccess { + t.Fatalf("--version exit = %d", exit) + } + if got, want := stdout.String(), "evalctl 0.2.0-experiment.1\n"; got != want { + t.Fatalf("--version = %q, want %q", got, want) + } + for _, args := range [][]string{{}, {"validate"}, {"summarize"}, {"compare"}, {"observe", "extra"}, {"--version", "extra"}} { + stdout.Reset() + stderr.Reset() + if exit := Run(context.Background(), args, Runtime{Stdout: &stdout, Stderr: &stderr}); exit != ExitUsage { + t.Fatalf("Run(%q) exit = %d, want %d", args, exit, ExitUsage) + } + } +} + +func TestObserveRecordsCanonicalRow(t *testing.T) { + t.Parallel() + root := t.TempDir() + result, exit := captureObserve(t, root, validDraft, store.Options{}) + if exit != ExitSuccess || result.Status != "recorded" || result.Slot != 1 { + t.Fatalf("observe = %+v exit %d", result, exit) + } + if result.Durability != "confirmed" && result.Durability != "unconfirmed" { + t.Fatalf("durability = %q", result.Durability) + } + data, err := os.ReadFile(state.JournalPath(root)) + if err != nil { + t.Fatal(err) + } + rows, err := experiment.ParseJournal(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].Slot != 1 || rows[0].Ward.Version != nil || !rows[0].Ward.Used { + t.Fatalf("stored rows = %#v", rows) + } +} + +func TestObserveInvalidInputNeverCreatesJournal(t *testing.T) { + t.Parallel() + for name, draft := range map[string]string{ + "malformed": `{`, + "duplicate": `{"outcome":"completed","outcome":"failed","ward":{"used":false,"version":null},"seal":{"used":false,"version":null}}`, + "fractional": `{"outcome":"completed","ward":{"used":true,"version":null,"interaction_seconds":1.5},"seal":{"used":false,"version":null}}`, + "unusedEffect": `{"outcome":"completed","ward":{"used":false,"version":null,"interaction_seconds":1},` + + `"seal":{"used":false,"version":null}}`, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + root := t.TempDir() + result, exit := captureObserve(t, root, draft, store.Options{}) + if exit != ExitSuccess || result.Status != "skipped" || result.Reason != "invalid-observation" { + t.Fatalf("observe = %+v exit %d", result, exit) + } + if _, err := os.Lstat(state.JournalPath(root)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("journal exists after invalid input: %v", err) + } + }) + } +} + +func TestObserveConcurrentSlotsAndBound(t *testing.T) { + root := t.TempDir() + results := make(chan capturedResult, experiment.MaxRows) + var wait sync.WaitGroup + for index := int64(0); index < experiment.MaxRows; index++ { + wait.Add(1) + go func() { + defer wait.Done() + result, exit := captureObserve(t, root, validDraft, store.Options{}) + if exit != ExitSuccess { + t.Errorf("observe exit = %d", exit) + } + results <- result + }() + } + wait.Wait() + close(results) + var slots []int + for result := range results { + if result.Status != "recorded" { + t.Fatalf("concurrent result = %+v", result) + } + slots = append(slots, int(result.Slot)) + } + sort.Ints(slots) + for index, slot := range slots { + if slot != index+1 { + t.Fatalf("slots = %v", slots) + } + } + + result, exit := captureObserve(t, root, validDraft, store.Options{}) + if exit != ExitSuccess || result.Status != "skipped" || result.Reason != "journal-full" { + t.Fatalf("twenty-first observe = %+v exit %d", result, exit) + } + data, err := os.ReadFile(state.JournalPath(root)) + if err != nil { + t.Fatal(err) + } + if rows, err := experiment.ParseJournal(bytes.NewReader(data)); err != nil || len(rows) != int(experiment.MaxRows) { + t.Fatalf("final journal rows = %d, err = %v", len(rows), err) + } +} + +func TestObserveInvalidJournalIsPreserved(t *testing.T) { + t.Parallel() + root := t.TempDir() + if result, _ := captureObserve(t, root, validDraft, store.Options{}); result.Status != "recorded" { + t.Fatalf("seed result = %+v", result) + } + path := state.JournalPath(root) + invalid := []byte("{\"schema_version\":\"wrong\"}\n") + if err := os.WriteFile(path, invalid, 0o600); err != nil { + t.Fatal(err) + } + result, exit := captureObserve(t, root, validDraft, store.Options{}) + if exit != ExitSuccess || result.Status != "skipped" || result.Reason != "invalid-state-data" { + t.Fatalf("invalid journal result = %+v exit %d", result, exit) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, invalid) { + t.Fatalf("invalid journal changed: %q", got) + } +} + +func TestObserveFaultBoundaries(t *testing.T) { + t.Parallel() + precommit := map[string]store.Hooks{ + "partialWrite": { + WriteTemp: func(file *os.File, _ []byte) error { + _, _ = file.WriteString("{") + return io.ErrUnexpectedEOF + }, + }, + "beforeReplace": {BeforeReplace: func(_, _ string) error { return errors.New("injected") }}, + } + for name, hooks := range precommit { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + result, exit := captureObserve(t, root, validDraft, store.Options{Hooks: hooks}) + if exit != ExitSuccess || result.Status != "skipped" || result.Reason != "state-io-error" { + t.Fatalf("result = %+v exit %d", result, exit) + } + if _, err := os.Lstat(state.JournalPath(root)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("journal exists after precommit failure: %v", err) + } + }) + } + + postcommit := map[string]store.Hooks{ + "afterReplace": {AfterReplace: func(string) error { return errors.New("injected") }}, + "directorySync": {BeforeDirectorySync: func(string) error { return errors.New("injected") }}, + } + for name, hooks := range postcommit { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + result, exit := captureObserve(t, root, validDraft, store.Options{Hooks: hooks}) + if exit != ExitSuccess || result.Status != "recorded" || result.Slot != 1 || result.Durability != "unconfirmed" { + t.Fatalf("result = %+v exit %d", result, exit) + } + if _, err := os.Stat(state.JournalPath(root)); err != nil { + t.Fatalf("committed journal missing: %v", err) + } + }) + } +} + +func TestObservePrecommitFaultPreservesExistingJournal(t *testing.T) { + t.Parallel() + for name, hooks := range map[string]store.Hooks{ + "partialWrite": { + WriteTemp: func(file *os.File, _ []byte) error { + _, _ = file.WriteString("partial") + return io.ErrUnexpectedEOF + }, + }, + "beforeReplace": {BeforeReplace: func(_, _ string) error { return errors.New("injected") }}, + } { + t.Run(name, func(t *testing.T) { + root := t.TempDir() + if result, _ := captureObserve(t, root, validDraft, store.Options{}); result.Status != "recorded" { + t.Fatalf("seed result = %+v", result) + } + path := state.JournalPath(root) + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + result, _ := captureObserve(t, root, validDraft, store.Options{Hooks: hooks}) + if result.Status != "skipped" { + t.Fatalf("fault result = %+v", result) + } + after, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after, before) { + t.Fatalf("journal changed: before %q after %q", before, after) + } + }) + } +} + +func captureObserve(t *testing.T, root, draft string, options store.Options) (capturedResult, int) { + t.Helper() + var stdout, stderr bytes.Buffer + exit := Run(context.Background(), []string{"observe", "--state-root", root}, Runtime{ + Stdin: strings.NewReader(draft), Stdout: &stdout, Stderr: &stderr, + Now: func() time.Time { return time.Date(2026, 8, 26, 12, 34, 56, 123, time.UTC) }, + StoreOptions: options, + }) + var result capturedResult + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatalf("decode %q: %v (stderr %q)", stdout.String(), err, stderr.String()) + } + return result, exit +} + +func TestStateRootMustBeAbsoluteButFailureIsBestEffort(t *testing.T) { + t.Parallel() + var stdout bytes.Buffer + exit := Run(context.Background(), []string{"observe", "--state-root", filepath.Join("relative", "state")}, Runtime{ + Stdin: strings.NewReader(validDraft), Stdout: &stdout, Stderr: io.Discard, + }) + if exit != ExitSuccess || !strings.Contains(stdout.String(), `"reason":"unsafe-state-path"`) { + t.Fatalf("exit = %d output = %q", exit, stdout.String()) + } +} diff --git a/internal/cli/cli_unix_test.go b/internal/cli/cli_unix_test.go new file mode 100644 index 0000000..71b64c5 --- /dev/null +++ b/internal/cli/cli_unix_test.go @@ -0,0 +1,26 @@ +//go:build darwin || linux + +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/jgoneit/eval/internal/store" +) + +func TestObservePermissionFailureIsBestEffort(t *testing.T) { + root := t.TempDir() + if result, _ := captureObserve(t, root, validDraft, store.Options{}); result.Status != "recorded" { + t.Fatalf("seed result = %+v", result) + } + privateRoot := filepath.Join(root, "jgoneit") + if err := os.Chmod(privateRoot, 0o755); err != nil { + t.Fatal(err) + } + result, exit := captureObserve(t, root, validDraft, store.Options{}) + if exit != ExitSuccess || result.Status != "skipped" || result.Reason != "state-permission-denied" { + t.Fatalf("result = %+v exit %d", result, exit) + } +} diff --git a/internal/experiment/decode_test.go b/internal/experiment/decode_test.go new file mode 100644 index 0000000..4039cc3 --- /dev/null +++ b/internal/experiment/decode_test.go @@ -0,0 +1,150 @@ +package experiment + +import ( + "errors" + "strings" + "testing" +) + +const validDraftJSON = `{"outcome":"completed","ward":{"used":true,"version":null},"seal":{"used":false,"version":null}}` + +func TestDecodeDraftRejectsDuplicateKeysAtEveryDepth(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + { + name: "top level", + input: `{"outcome":"completed","outcome":"failed","ward":{"used":true,"version":null},"seal":{"used":false,"version":null}}`, + }, + { + name: "module", + input: `{"outcome":"completed","ward":{"used":true,"used":false,"version":null},"seal":{"used":false,"version":null}}`, + }, + { + name: "unknown nested object", + input: `{"outcome":"completed","ward":{"used":true,"version":null,"unknown":{"value":1,"value":2}},"seal":{"used":false,"version":null}}`, + }, + { + name: "object nested in array", + input: `{"outcome":"completed","ward":{"used":true,"version":null,"unknown":[{"value":1,"value":2}]},"seal":{"used":false,"version":null}}`, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + _, err := DecodeDraft(strings.NewReader(test.input)) + if err == nil || !strings.Contains(err.Error(), "duplicate object key") { + t.Fatalf("DecodeDraft() error = %v, want duplicate-key rejection", err) + } + }) + } +} + +func TestDecodeDraftRejectsTrailingValues(t *testing.T) { + t.Parallel() + + for _, suffix := range []string{` {}`, ` null`, ` []`} { + _, err := DecodeDraft(strings.NewReader(validDraftJSON + suffix)) + if err == nil || !strings.Contains(err.Error(), "trailing JSON value") { + t.Errorf("DecodeDraft(%q) error = %v, want trailing-value rejection", suffix, err) + } + } +} + +func TestDecodeDraftRejectsUnknownAndMissingFields(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + { + name: "unknown top level", + input: `{"outcome":"completed","ward":{"used":true,"version":null},"seal":{"used":false,"version":null},"unknown":true}`, + }, + { + name: "unknown module field", + input: `{"outcome":"completed","ward":{"used":true,"version":null,"unknown":true},"seal":{"used":false,"version":null}}`, + }, + { + name: "missing outcome", + input: `{"ward":{"used":true,"version":null},"seal":{"used":false,"version":null}}`, + }, + { + name: "missing ward", + input: `{"outcome":"completed","seal":{"used":false,"version":null}}`, + }, + { + name: "missing seal", + input: `{"outcome":"completed","ward":{"used":true,"version":null}}`, + }, + { + name: "missing used", + input: `{"outcome":"completed","ward":{"version":null},"seal":{"used":false,"version":null}}`, + }, + { + name: "missing version", + input: `{"outcome":"completed","ward":{"used":true},"seal":{"used":false,"version":null}}`, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if _, err := DecodeDraft(strings.NewReader(test.input)); err == nil { + t.Fatal("DecodeDraft() succeeded, want strict shape rejection") + } + }) + } +} + +func TestDecodeDraftRejectsNonIntegerRepresentations(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + field string + value string + }{ + {name: "fractional defects", field: "defects_caught_before_terminal", value: "1.0"}, + {name: "exponent defects", field: "defects_caught_before_terminal", value: "1e0"}, + {name: "fractional interventions", field: "added_user_interventions", value: "2.5"}, + {name: "exponent interaction", field: "interaction_seconds", value: "6E1"}, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := `{"outcome":"completed","ward":{"used":true,"version":null,"` + + test.field + `":` + test.value + `},"seal":{"used":false,"version":null}}` + if _, err := DecodeDraft(strings.NewReader(input)); err == nil { + t.Fatal("DecodeDraft() succeeded, want int64 lexical rejection") + } + }) + } +} + +func TestDecodeDraftInputLimit(t *testing.T) { + t.Parallel() + + exact := validDraftJSON + strings.Repeat(" ", int(MaxInputBytes)-len(validDraftJSON)) + if got := int64(len(exact)); got != MaxInputBytes { + t.Fatalf("test input length = %d, want %d", got, MaxInputBytes) + } + if _, err := DecodeDraft(strings.NewReader(exact)); err != nil { + t.Fatalf("DecodeDraft() at limit error = %v", err) + } + + over := exact + " " + _, err := DecodeDraft(strings.NewReader(over)) + if !errors.Is(err, ErrInputTooLarge) { + t.Fatalf("DecodeDraft() over limit error = %v, want ErrInputTooLarge", err) + } +} diff --git a/internal/experiment/journal_test.go b/internal/experiment/journal_test.go new file mode 100644 index 0000000..d6f089b --- /dev/null +++ b/internal/experiment/journal_test.go @@ -0,0 +1,165 @@ +package experiment + +import ( + "errors" + "strings" + "testing" + "time" +) + +func TestMarshalCanonicalRowExactBytes(t *testing.T) { + t.Parallel() + + version := "0.1.0" + rework := false + zero := int64(0) + two := int64(2) + three := int64(3) + blocked := false + row := Row{ + SchemaVersion: SchemaVersion, + Slot: 1, + RecordedAt: "2026-08-26T03:34:56.12Z", + Outcome: OutcomeCompleted, + ReworkRequired: &rework, + Ward: Module{ + Used: true, + Version: &version, + DefectsCaughtBeforeTerminal: &zero, + AddedUserInterventions: &two, + InteractionSeconds: &three, + NormalWorkBlocked: &blocked, + }, + Seal: Module{Used: false, Version: nil}, + } + want := `{"schema_version":"eval-experiment/v1","slot":1,"recorded_at":"2026-08-26T03:34:56.12Z",` + + `"outcome":"completed","rework_required":false,"ward":{"used":true,"version":"0.1.0",` + + `"defects_caught_before_terminal":0,"added_user_interventions":2,"interaction_seconds":3,` + + `"normal_work_blocked":false},"seal":{"used":false,"version":null}}` + + got, err := MarshalCanonicalRow(row) + if err != nil { + t.Fatalf("MarshalCanonicalRow() error = %v", err) + } + if string(got) != want { + t.Fatalf("MarshalCanonicalRow() =\n%s\nwant:\n%s", got, want) + } +} + +func TestParseJournalRejectsFractionalAndExponentSlots(t *testing.T) { + t.Parallel() + + base := `{"schema_version":"eval-experiment/v1","slot":%s,"recorded_at":"2026-08-26T03:34:56Z",` + + `"outcome":"completed","ward":{"used":true,"version":null},"seal":{"used":false,"version":null}}` + for _, slot := range []string{"1.0", "1e0", "1E+0"} { + input := strings.Replace(base, "%s", slot, 1) + "\n" + if _, err := ParseJournal(strings.NewReader(input)); err == nil { + t.Errorf("ParseJournal(slot=%s) succeeded, want int64 lexical rejection", slot) + } + } +} + +func TestValidateJournalRejectsInvalidSlots(t *testing.T) { + t.Parallel() + + row := canonicalTestRow(t, 1) + tests := []struct { + name string + rows []Row + }{ + {name: "slot zero", rows: []Row{withSlot(row, 0)}}, + {name: "slot over maximum", rows: []Row{withSlot(row, MaxRows+1)}}, + {name: "starts at two", rows: []Row{withSlot(row, 2)}}, + {name: "gap", rows: []Row{row, withSlot(row, 3)}}, + {name: "duplicate", rows: []Row{row, row}}, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if err := ValidateJournal(test.rows); err == nil { + t.Fatal("ValidateJournal() succeeded, want slot rejection") + } + }) + } +} + +func TestParseJournalRejectsGappedAndOverTwentyRows(t *testing.T) { + t.Parallel() + + first := canonicalTestRow(t, 1) + third := canonicalTestRow(t, 3) + gapped := canonicalLine(t, first) + "\n" + canonicalLine(t, third) + "\n" + if _, err := ParseJournal(strings.NewReader(gapped)); err == nil { + t.Fatal("ParseJournal(gapped) succeeded") + } + + row := canonicalLine(t, first) + "\n" + over := strings.Repeat(row, int(MaxRows)+1) + if _, err := ParseJournal(strings.NewReader(over)); err == nil { + t.Fatal("ParseJournal(21 rows) succeeded") + } +} + +func TestValidateJournalRejectsMoreThanTwentyRows(t *testing.T) { + t.Parallel() + + rows := make([]Row, MaxRows+1) + row := canonicalTestRow(t, 1) + for index := range rows { + rows[index] = row + } + if err := ValidateJournal(rows); err == nil { + t.Fatal("ValidateJournal(21 rows) succeeded") + } +} + +func TestParseJournalAcceptsTwentyContiguousRows(t *testing.T) { + t.Parallel() + + var journal strings.Builder + for slot := int64(1); slot <= MaxRows; slot++ { + journal.WriteString(canonicalLine(t, canonicalTestRow(t, slot))) + journal.WriteByte('\n') + } + rows, err := ParseJournal(strings.NewReader(journal.String())) + if err != nil { + t.Fatalf("ParseJournal() error = %v", err) + } + if int64(len(rows)) != MaxRows || rows[MaxRows-1].Slot != MaxRows { + t.Fatalf("rows = %d last slot = %d, want twenty contiguous rows", len(rows), rows[len(rows)-1].Slot) + } +} + +func TestParseJournalInputLimit(t *testing.T) { + t.Parallel() + + _, err := ParseJournal(strings.NewReader(strings.Repeat(" ", int(MaxInputBytes)+1))) + if !errors.Is(err, ErrInputTooLarge) { + t.Fatalf("ParseJournal() over limit error = %v, want ErrInputTooLarge", err) + } +} + +func canonicalTestRow(t *testing.T, slot int64) Row { + t.Helper() + row, err := NewRow(validDraft(t), slot, time.Date(2026, 8, 26, 3, 34, 56, 0, time.UTC)) + if err != nil { + t.Fatalf("NewRow(slot=%d) error = %v", slot, err) + } + return row +} + +func canonicalLine(t *testing.T, row Row) string { + t.Helper() + encoded, err := MarshalCanonicalRow(row) + if err != nil { + t.Fatalf("MarshalCanonicalRow() error = %v", err) + } + return string(encoded) +} + +func withSlot(row Row, slot int64) Row { + row.Slot = slot + return row +} diff --git a/internal/experiment/schema_test.go b/internal/experiment/schema_test.go new file mode 100644 index 0000000..9baf58e --- /dev/null +++ b/internal/experiment/schema_test.go @@ -0,0 +1,36 @@ +package experiment + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestStoredRowSchemaIsTheOnlySchema(t *testing.T) { + t.Parallel() + _, source, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve test source") + } + directory := filepath.Clean(filepath.Join(filepath.Dir(source), "..", "..", "schemas")) + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Name() != "experiment-v1.schema.json" { + t.Fatalf("schema entries = %v", entries) + } + data, err := os.ReadFile(filepath.Join(directory, entries[0].Name())) + if err != nil { + t.Fatal(err) + } + var schema map[string]any + if err := json.Unmarshal(data, &schema); err != nil { + t.Fatalf("schema JSON: %v", err) + } + if schema["$id"] != "urn:jgoneit:eval:experiment:v1" || schema["additionalProperties"] != false { + t.Fatalf("unexpected stored-row schema identity: %#v", schema) + } +} diff --git a/internal/experiment/validate_test.go b/internal/experiment/validate_test.go new file mode 100644 index 0000000..5e9a3ea --- /dev/null +++ b/internal/experiment/validate_test.go @@ -0,0 +1,154 @@ +package experiment + +import ( + "strings" + "testing" + "time" +) + +func TestDecodeDraftModuleRelationships(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + { + name: "unused with version", + input: `{"outcome":"completed","ward":{"used":false,"version":"0.1.0"},"seal":{"used":false,"version":null}}`, + }, + { + name: "unused with defects", + input: `{"outcome":"completed","ward":{"used":false,"version":null,"defects_caught_before_terminal":0},"seal":{"used":false,"version":null}}`, + }, + { + name: "unused with interventions", + input: `{"outcome":"completed","ward":{"used":false,"version":null,"added_user_interventions":0},"seal":{"used":false,"version":null}}`, + }, + { + name: "unused with interaction", + input: `{"outcome":"completed","ward":{"used":false,"version":null,"interaction_seconds":0},"seal":{"used":false,"version":null}}`, + }, + { + name: "unused with false blocked effect", + input: `{"outcome":"completed","ward":{"used":false,"version":null,"normal_work_blocked":false},"seal":{"used":false,"version":null}}`, + }, + { + name: "negative defects", + input: `{"outcome":"completed","ward":{"used":true,"version":null,"defects_caught_before_terminal":-1},"seal":{"used":false,"version":null}}`, + }, + { + name: "negative interventions", + input: `{"outcome":"completed","ward":{"used":true,"version":null,"added_user_interventions":-1},"seal":{"used":false,"version":null}}`, + }, + { + name: "negative interaction", + input: `{"outcome":"completed","ward":{"used":true,"version":null,"interaction_seconds":-1},"seal":{"used":false,"version":null}}`, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if _, err := DecodeDraft(strings.NewReader(test.input)); err == nil { + t.Fatal("DecodeDraft() succeeded, want module relationship rejection") + } + }) + } +} + +func TestDecodeDraftAllowsUsedWithUnavailableVersion(t *testing.T) { + t.Parallel() + + draft, err := DecodeDraft(strings.NewReader(validDraftJSON)) + if err != nil { + t.Fatalf("DecodeDraft() error = %v", err) + } + if !draft.Ward.Used || draft.Ward.Version != nil { + t.Fatalf("ward = %+v, want used with null version", draft.Ward) + } +} + +func TestDecodeDraftOutcomeEnum(t *testing.T) { + t.Parallel() + + for _, outcome := range []string{"completed", "failed", "abandoned"} { + input := strings.Replace(validDraftJSON, `"completed"`, `"`+outcome+`"`, 1) + if _, err := DecodeDraft(strings.NewReader(input)); err != nil { + t.Errorf("DecodeDraft(outcome=%q) error = %v", outcome, err) + } + } + input := strings.Replace(validDraftJSON, `"completed"`, `"running"`, 1) + if _, err := DecodeDraft(strings.NewReader(input)); err == nil { + t.Fatal("DecodeDraft(outcome=running) succeeded") + } +} + +func TestDecodeDraftPreservesOptionalEffects(t *testing.T) { + t.Parallel() + + input := `{"outcome":"failed","rework_required":false,` + + `"ward":{"used":true,"version":"0.1.0","defects_caught_before_terminal":0,` + + `"added_user_interventions":2,"interaction_seconds":3,"normal_work_blocked":false},` + + `"seal":{"used":true,"version":null}}` + draft, err := DecodeDraft(strings.NewReader(input)) + if err != nil { + t.Fatalf("DecodeDraft() error = %v", err) + } + if draft.ReworkRequired == nil || *draft.ReworkRequired { + t.Fatalf("rework_required = %v, want present false", draft.ReworkRequired) + } + if draft.Ward.DefectsCaughtBeforeTerminal == nil || *draft.Ward.DefectsCaughtBeforeTerminal != 0 { + t.Fatalf("defects = %v, want present zero", draft.Ward.DefectsCaughtBeforeTerminal) + } + if draft.Ward.AddedUserInterventions == nil || *draft.Ward.AddedUserInterventions != 2 { + t.Fatalf("interventions = %v, want 2", draft.Ward.AddedUserInterventions) + } + if draft.Ward.InteractionSeconds == nil || *draft.Ward.InteractionSeconds != 3 { + t.Fatalf("interaction = %v, want 3", draft.Ward.InteractionSeconds) + } + if draft.Ward.NormalWorkBlocked == nil || *draft.Ward.NormalWorkBlocked { + t.Fatalf("normal_work_blocked = %v, want present false", draft.Ward.NormalWorkBlocked) + } + if !draft.Seal.Used || draft.Seal.Version != nil { + t.Fatalf("seal = %+v, want used with unavailable version", draft.Seal) + } +} + +func TestNewRowNormalizesAndValidatesTimestamp(t *testing.T) { + t.Parallel() + + draft := validDraft(t) + zone := time.FixedZone("KST", 9*60*60) + recorded := time.Date(2026, 8, 26, 12, 34, 56, 120_000_000, zone) + row, err := NewRow(draft, 1, recorded) + if err != nil { + t.Fatalf("NewRow() error = %v", err) + } + if row.RecordedAt != "2026-08-26T03:34:56.12Z" { + t.Fatalf("recorded_at = %q, want canonical UTC", row.RecordedAt) + } + + invalid := []string{ + "2026-08-26T03:34:56+00:00", + "2026-08-26T03:34:56.120Z", + "2026-02-30T03:34:56Z", + "2026-08-26 03:34:56Z", + } + for _, timestamp := range invalid { + row.RecordedAt = timestamp + if err := ValidateRow(row); err == nil { + t.Errorf("ValidateRow(recorded_at=%q) succeeded", timestamp) + } + } +} + +func validDraft(t *testing.T) Draft { + t.Helper() + draft, err := DecodeDraft(strings.NewReader(validDraftJSON)) + if err != nil { + t.Fatalf("DecodeDraft(valid) error = %v", err) + } + return draft +} diff --git a/internal/store/platform_unix_test.go b/internal/store/platform_unix_test.go new file mode 100644 index 0000000..e35fbb3 --- /dev/null +++ b/internal/store/platform_unix_test.go @@ -0,0 +1,45 @@ +//go:build darwin || linux + +package store + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestUpdateRejectsWritableAncestor(t *testing.T) { + parent := t.TempDir() + shared := filepath.Join(parent, "shared") + if err := os.Mkdir(shared, 0o777); err != nil { + t.Fatal(err) + } + if err := os.Chmod(shared, 0o777); err != nil { + t.Fatal(err) + } + root := filepath.Join(shared, "state") + if err := os.Mkdir(root, 0o700); err != nil { + t.Fatal(err) + } + journal := mustStore(t, root, Options{}) + commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)) + if err == nil || commit.Committed || CategoryOf(err) != CategoryPermission { + t.Fatalf("commit = %+v err = %v category = %s", commit, err, CategoryOf(err)) + } +} + +func TestUpdateRejectsBroadFileMode(t *testing.T) { + root := t.TempDir() + journal := mustStore(t, root, Options{}) + if commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)); err != nil || !commit.Committed { + t.Fatalf("seed commit = %+v err = %v", commit, err) + } + if err := os.Chmod(journal.Path(), 0o644); err != nil { + t.Fatal(err) + } + commit, err := journal.Update(context.Background(), appendObject(`{"value":2}`)) + if err == nil || commit.Committed || CategoryOf(err) != CategoryPermission { + t.Fatalf("commit = %+v err = %v category = %s", commit, err, CategoryOf(err)) + } +} diff --git a/internal/store/platform_windows_test.go b/internal/store/platform_windows_test.go new file mode 100644 index 0000000..996b767 --- /dev/null +++ b/internal/store/platform_windows_test.go @@ -0,0 +1,259 @@ +//go:build windows + +package store + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +func TestWindowsPrivateDirectoryAndFileAccepted(t *testing.T) { + directory := filepath.Join(t.TempDir(), "private") + if err := createPrivateDir(directory); err != nil { + t.Fatalf("create private directory: %v", err) + } + directoryInfo, err := os.Lstat(directory) + if err != nil { + t.Fatalf("stat private directory: %v", err) + } + if err := checkStateRoot(directory, directoryInfo); err != nil { + t.Fatalf("private directory rejected: %v", err) + } + assertWindowsCurrentUserProtectedDACL(t, directory) + + path := filepath.Join(directory, "journal.jsonl") + file, err := openPrivateFile(path, true, true) + if err != nil { + t.Fatalf("create private file: %v", err) + } + defer file.Close() + if err := checkOpenPrivateFile(file, 0o600); err != nil { + t.Fatalf("private file rejected: %v", err) + } + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("stat private file: %v", err) + } + if err := checkPrivatePath(path, info, 0o600); err != nil { + t.Fatalf("private path rejected: %v", err) + } + assertWindowsCurrentUserProtectedDACL(t, path) +} + +func TestWindowsBroadenedDACLRejected(t *testing.T) { + path := newWindowsPrivateFile(t) + user, err := currentUserSID() + if err != nil { + t.Fatalf("current user SID: %v", err) + } + setWindowsDACL(t, path, fmt.Sprintf("D:P(A;;FA;;;%s)(A;;FR;;;WD)", user.String()), true) + + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("stat broadened file: %v", err) + } + err = checkPrivatePath(path, info, 0o600) + if err == nil || CategoryOf(err) != CategoryPermission { + t.Fatalf("broadened DACL error = %v, want permission rejection", err) + } +} + +func TestWindowsInheritedDACLRejected(t *testing.T) { + path := newWindowsPrivateFile(t) + user, err := currentUserSID() + if err != nil { + t.Fatalf("current user SID: %v", err) + } + setWindowsDACL(t, path, fmt.Sprintf("D:AI(A;ID;FA;;;%s)", user.String()), false) + + descriptor, err := windows.GetNamedSecurityInfo( + path, + windows.SE_FILE_OBJECT, + windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + t.Fatalf("read inherited DACL: %v", err) + } + control, _, err := descriptor.Control() + if err != nil { + t.Fatalf("read inherited DACL control: %v", err) + } + if control&windows.SE_DACL_PROTECTED != 0 { + t.Fatal("test setup retained a protected DACL") + } + + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("stat inherited file: %v", err) + } + err = checkPrivatePath(path, info, 0o600) + if err == nil || CategoryOf(err) != CategoryPermission { + t.Fatalf("inherited DACL error = %v, want permission rejection", err) + } +} + +func TestWindowsHardLinkRejectedWhenSupported(t *testing.T) { + path := newWindowsPrivateFile(t) + alias := path + ".alias" + if err := os.Link(path, alias); err != nil { + if optionalWindowsFilesystemFeature(err) { + t.Skipf("hard links unavailable: %v", err) + } + t.Fatalf("create hard link: %v", err) + } + + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("stat hard-linked file: %v", err) + } + err = checkPrivatePath(path, info, 0o600) + if err == nil || CategoryOf(err) != CategoryUnsafePath { + t.Fatalf("hard-link error = %v, want unsafe-path rejection", err) + } +} + +func TestWindowsSymlinkRejectedWhenPermitted(t *testing.T) { + path := newWindowsPrivateFile(t) + link := path + ".link" + if err := os.Symlink(path, link); err != nil { + if optionalWindowsFilesystemFeature(err) { + t.Skipf("symlink creation unavailable without Developer Mode or privilege: %v", err) + } + t.Fatalf("create symlink: %v", err) + } + + info, err := os.Lstat(link) + if err != nil { + t.Fatalf("lstat symlink: %v", err) + } + err = checkPrivatePath(link, info, 0o600) + if err == nil || CategoryOf(err) != CategoryUnsafePath { + t.Fatalf("symlink error = %v, want unsafe-path rejection", err) + } +} + +func TestWindowsExclusiveLockTimeoutAndRelease(t *testing.T) { + directory := filepath.Join(t.TempDir(), "private") + if err := createPrivateDir(directory); err != nil { + t.Fatalf("create private directory: %v", err) + } + path := filepath.Join(directory, "journal.lock") + first, err := openPrivateFile(path, true, true) + if err != nil { + t.Fatalf("open first lock handle: %v", err) + } + defer first.Close() + second, err := openPrivateFile(path, true, true) + if err != nil { + t.Fatalf("open second lock handle: %v", err) + } + defer second.Close() + + if err := acquireFileLock(context.Background(), first, time.Second); err != nil { + t.Fatalf("acquire first lock: %v", err) + } + if err := acquireFileLock(context.Background(), second, 30*time.Millisecond); !errors.Is(err, ErrLockTimeout) { + t.Fatalf("second lock error = %v, want lock timeout", err) + } + releaseFileLock(first) + if err := acquireFileLock(context.Background(), second, time.Second); err != nil { + t.Fatalf("acquire released lock: %v", err) + } + releaseFileLock(second) +} + +func newWindowsPrivateFile(t *testing.T) string { + t.Helper() + directory := filepath.Join(t.TempDir(), "private") + if err := createPrivateDir(directory); err != nil { + t.Fatalf("create private directory: %v", err) + } + path := filepath.Join(directory, "journal.jsonl") + file, err := openPrivateFile(path, true, true) + if err != nil { + t.Fatalf("create private file: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("close private file: %v", err) + } + return path +} + +func setWindowsDACL(t *testing.T, path, sddl string, protected bool) { + t.Helper() + descriptor, err := windows.SecurityDescriptorFromString(sddl) + if err != nil { + t.Fatalf("parse test DACL: %v", err) + } + dacl, _, err := descriptor.DACL() + if err != nil { + t.Fatalf("extract test DACL: %v", err) + } + information := windows.SECURITY_INFORMATION( + windows.DACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION, + ) + if protected { + information = windows.SECURITY_INFORMATION( + windows.DACL_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION, + ) + } + if err := windows.SetNamedSecurityInfo( + path, + windows.SE_FILE_OBJECT, + information, + nil, + nil, + dacl, + nil, + ); err != nil { + t.Fatalf("set test DACL: %v", err) + } +} + +func assertWindowsCurrentUserProtectedDACL(t *testing.T, path string) { + t.Helper() + descriptor, err := windows.GetNamedSecurityInfo( + path, + windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + ) + if err != nil { + t.Fatalf("read private security descriptor: %v", err) + } + owner, _, err := descriptor.Owner() + if err != nil { + t.Fatalf("read private owner: %v", err) + } + user, err := currentUserSID() + if err != nil { + t.Fatalf("read current user SID: %v", err) + } + if owner == nil || !windows.EqualSid(owner, user) { + t.Fatal("private object owner is not the current user") + } + control, _, err := descriptor.Control() + if err != nil { + t.Fatalf("read private DACL control: %v", err) + } + if control&windows.SE_DACL_PRESENT == 0 || control&windows.SE_DACL_PROTECTED == 0 { + t.Fatalf("private DACL control = %#x, want present and protected", control) + } + dacl, _, err := descriptor.DACL() + if err != nil || dacl == nil || dacl.AceCount == 0 { + t.Fatalf("private DACL is absent or empty: dacl=%v err=%v", dacl, err) + } +} + +func optionalWindowsFilesystemFeature(err error) bool { + return errors.Is(err, windows.ERROR_ACCESS_DENIED) || + errors.Is(err, windows.ERROR_PRIVILEGE_NOT_HELD) || + errors.Is(err, windows.ERROR_NOT_SUPPORTED) || + errors.Is(err, windows.ERROR_INVALID_FUNCTION) +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go new file mode 100644 index 0000000..bd31095 --- /dev/null +++ b/internal/store/store_test.go @@ -0,0 +1,178 @@ +package store + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +const testRelativeJournal = "jgoneit/eval-experiment/v1/journal.jsonl" + +func TestUpdateRejectsGitWorktreeRoot(t *testing.T) { + t.Parallel() + root := privateTestRoot(t) + if err := os.Mkdir(filepath.Join(root, ".git"), 0o700); err != nil { + t.Fatal(err) + } + journal := mustStore(t, root, Options{}) + commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)) + if err == nil || commit.Committed || CategoryOf(err) != CategoryUnsafePath { + t.Fatalf("commit = %+v err = %v category = %s", commit, err, CategoryOf(err)) + } +} + +func TestUpdateRejectsSymlinkComponent(t *testing.T) { + t.Parallel() + root := privateTestRoot(t) + outside := t.TempDir() + link := filepath.Join(root, "jgoneit") + if err := os.Symlink(outside, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + journal := mustStore(t, root, Options{}) + commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)) + if err == nil || commit.Committed || CategoryOf(err) != CategoryUnsafePath { + t.Fatalf("commit = %+v err = %v category = %s", commit, err, CategoryOf(err)) + } +} + +func TestUpdateRejectsHardLinkedJournal(t *testing.T) { + t.Parallel() + root := privateTestRoot(t) + journal := mustStore(t, root, Options{}) + if commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)); err != nil || !commit.Committed { + t.Fatalf("seed commit = %+v err = %v", commit, err) + } + alias := journal.Path() + ".alias" + if err := os.Link(journal.Path(), alias); err != nil { + t.Skipf("hard links unavailable: %v", err) + } + commit, err := journal.Update(context.Background(), appendObject(`{"value":2}`)) + if err == nil || commit.Committed || CategoryOf(err) != CategoryUnsafePath { + t.Fatalf("commit = %+v err = %v category = %s", commit, err, CategoryOf(err)) + } +} + +func TestUpdateLockTimeoutPreservesJournal(t *testing.T) { + root := privateTestRoot(t) + journal := mustStore(t, root, Options{LockTimeout: 25 * time.Millisecond}) + if commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)); err != nil || !commit.Committed { + t.Fatalf("seed commit = %+v err = %v", commit, err) + } + before, err := os.ReadFile(journal.Path()) + if err != nil { + t.Fatal(err) + } + lock, err := openPrivateFile(journal.LockPath(), true, false) + if err != nil { + t.Fatal(err) + } + defer lock.Close() + if err := acquireFileLock(context.Background(), lock, time.Second); err != nil { + t.Fatal(err) + } + defer releaseFileLock(lock) + commit, err := journal.Update(context.Background(), appendObject(`{"value":2}`)) + if err == nil || commit.Committed || CategoryOf(err) != CategoryLockTimeout { + t.Fatalf("commit = %+v err = %v category = %s", commit, err, CategoryOf(err)) + } + after, err := os.ReadFile(journal.Path()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after, before) { + t.Fatalf("journal changed under lock: before %q after %q", before, after) + } +} + +func TestUpdateAnchorsCommitWhenStateRootPathMoves(t *testing.T) { + parent := t.TempDir() + root := filepath.Join(parent, "state") + if err := createPrivateDir(root); err != nil { + t.Fatal(err) + } + moved := filepath.Join(parent, "state-moved") + journal := mustStore(t, root, Options{Hooks: Hooks{ + BeforeReplace: func(_, _ string) error { + if err := os.Rename(root, moved); err != nil { + return err + } + return createPrivateDir(root) + }, + }}) + commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)) + if err != nil || !commit.Committed { + t.Fatalf("commit = %+v err = %v", commit, err) + } + newPath := filepath.Join(root, filepath.FromSlash(testRelativeJournal)) + if _, err := os.Lstat(newPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("replacement root was mutated: %v", err) + } + movedPath := filepath.Join(moved, filepath.FromSlash(testRelativeJournal)) + if got, err := os.ReadFile(movedPath); err != nil || string(got) != "{\"value\":1}\n" { + t.Fatalf("anchored journal = %q err = %v", got, err) + } +} + +func TestUpdateRequiresExactAppend(t *testing.T) { + t.Parallel() + for name, transaction := range map[string]Transaction{ + "empty": func([]byte) ([]byte, error) { return nil, nil }, + "twoRows": func(existing []byte) ([]byte, error) { + return append(existing, []byte("{\"a\":1}\n{\"b\":2}\n")...), nil + }, + } { + t.Run(name, func(t *testing.T) { + root := privateTestRoot(t) + journal := mustStore(t, root, Options{}) + commit, err := journal.Update(context.Background(), transaction) + if err == nil || commit.Committed || CategoryOf(err) != CategoryValidation { + t.Fatalf("commit = %+v err = %v category = %s", commit, err, CategoryOf(err)) + } + }) + } + t.Run("replaceExisting", func(t *testing.T) { + root := privateTestRoot(t) + journal := mustStore(t, root, Options{}) + if commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)); err != nil || !commit.Committed { + t.Fatalf("seed commit = %+v err = %v", commit, err) + } + commit, err := journal.Update(context.Background(), func([]byte) ([]byte, error) { + return []byte("{\"other\":1}\n"), nil + }) + if err == nil || commit.Committed || CategoryOf(err) != CategoryValidation { + t.Fatalf("commit = %+v err = %v category = %s", commit, err, CategoryOf(err)) + } + }) +} + +func privateTestRoot(t *testing.T) string { + t.Helper() + root := filepath.Join(t.TempDir(), "state") + if err := createPrivateDir(root); err != nil { + t.Fatal(err) + } + return root +} + +func mustStore(t *testing.T, root string, options Options) *Store { + t.Helper() + journal, err := New(root, testRelativeJournal, options) + if err != nil { + t.Fatal(err) + } + return journal +} + +func appendObject(object string) Transaction { + return func(existing []byte) ([]byte, error) { + prospective := append([]byte(nil), existing...) + prospective = append(prospective, object...) + prospective = append(prospective, '\n') + return prospective, nil + } +} From ab23436594ae0cd377de3625108c7536198b63fd Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:03:38 +0900 Subject: [PATCH 04/14] ci(eval): run recorder checks Why: - The experiment recorder needs remote proof on every supported operating system before the host policy is enabled. What: - Run formatting, vet, tests, and evalctl builds on Linux, macOS, and Windows. - Run the race detector on Linux. - Reject reintroduction of removed Plugin, compatibility, analyzer, validator-command, and local-path surfaces. Validation: - Workflow YAML parsed locally - Surface commands passed locally - go test, race, and vet passed locally --- .github/workflows/verify.yml | 69 ++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/verify.yml diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..2a10296 --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,69 @@ +name: verify + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.x" + cache: true + - name: Check formatting + shell: bash + run: test -z "$(gofmt -l .)" + - name: Vet + run: go vet ./... + - name: Test + run: go test ./... + - name: Build evalctl + run: go build ./cmd/evalctl + + race: + name: race + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.x" + cache: true + - run: go test -race ./... + + surface: + name: clean-slate surface + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - name: Reject removed product and compatibility surfaces + shell: bash + run: | + test ! -e .codex-plugin + test ! -e skills + test ! -e fixtures + test ! -e templates + test ! -e toolkit-module.json + test ! -e requirements-dev.txt + test ! -e scripts/verify.sh + test ! -e internal/analyze + test "$(find schemas -type f | wc -l | tr -d ' ')" = "1" + test -f schemas/experiment-v1.schema.json + ! rg -n 'eval-observation|observations\.jsonl|task_id|supersedes|observe-draft-v2' cmd internal schemas go.mod + ! rg -n 'runValidate|runSummarize|runCompare|case "(validate|summarize|compare)"' cmd internal/cli --glob '!**/*_test.go' + ! rg -n '/Users/[[:alnum:]_.-]+/' . --hidden --glob '!.git/**' From 6618f0391dada1b65546281968c1c8ba9b2c9946 Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:04:09 +0900 Subject: [PATCH 05/14] docs(eval): define twenty task experiment Why: - The recorder needs an explicit experimental population and human decision boundary without restoring product documentation layers. What: - Document the observe-only contract, private journal, and failure semantics. - Define eligible host tasks, silent terminal recording, denominator ownership, and the twentieth-task stop. - Record promotion, privacy review, and cohort-size boundaries. Validation: - Markdown and links reviewed locally - go test ./... passed - git diff --check passed --- README.md | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..aa8fde8 --- /dev/null +++ b/README.md @@ -0,0 +1,129 @@ +# Eval 20-Task Experiment + +Eval is a bounded recorder experiment, not a Toolkit product. It records a +small set of facts after eligible Codex tasks terminate so a person can decide +whether a larger evaluation tool is justified. + +Version: `evalctl 0.2.0-experiment.1` + +## Public surface + +```text +evalctl --version +evalctl observe [--state-root ABS] +``` + +`observe` reads exactly one JSON object from standard input. Only malformed CLI +arguments return exit 64. Input, permission, lock, or storage failures are +best-effort skips with exit 0. + +```json +{"status":"skipped","reason":"invalid-observation"} +``` + +A successful atomic replacement returns its immutable slot. If the replacement +completed but final directory durability could not be confirmed, the row is +still recorded and reported honestly. + +```json +{"status":"recorded","slot":1,"durability":"confirmed"} +{"status":"recorded","slot":1,"durability":"unconfirmed"} +``` + +## Observation contract + +```json +{ + "outcome": "completed", + "rework_required": false, + "ward": { + "used": true, + "version": null, + "defects_caught_before_terminal": 1, + "added_user_interventions": 0, + "interaction_seconds": 2, + "normal_work_blocked": false + }, + "seal": { + "used": false, + "version": null + } +} +``` + +`outcome`, `ward`, `seal`, and each module's `used` and `version` are required. +Effects and `rework_required` are optional and should be supplied only when the +host already knows them. An unused module requires `version: null` and forbids +effects. A used module may have `version: null`; that row belongs to the usage +cohort but cannot support an exact-version comparison. + +The recorder rejects unknown or duplicate keys, non-integral count or duration +representations, negative values, free text, and input-supplied identity or +dates. It generates only `schema_version`, `slot`, and `recorded_at`. + +## Private journal + +The default journal is: + +```text +$XDG_STATE_HOME/jgoneit/eval-experiment/v1/journal.jsonl +$HOME/.local/state/jgoneit/eval-experiment/v1/journal.jsonl +``` + +The journal contains successful observations only and stops after 20 contiguous +slots. Existing rows are immutable. Before each append, the writer validates +the complete journal while holding a kernel-backed exclusive lock. It uses a +private same-directory temporary file, file sync, atomic replacement, and a +directory durability attempt. State traversal is anchored with Go `os.Root`; +unsafe ownership, permissions, symlinks, reparse points, and regular-file hard +links fail closed on Darwin, Linux, and Windows. + +Legacy Eval v1/v2 state is neither read nor migrated. + +## Host experiment policy + +The experiment population is the next 20 eligible root Codex tasks after the +managed host policy becomes active. Codex loads global `~/.codex/AGENTS.md` +guidance at session start, so only fresh tasks are eligible after installation. +See the [OpenAI AGENTS.md documentation](https://learn.chatgpt.com/docs/agent-configuration/agents-md). + +Eligible terminal outcomes are `completed`, `failed`, and `abandoned`. Eval +development, pure Q&A, and subagent child tasks are excluded in advance. Ward +or Seal usage and the quality of the result never exclude a task afterward. + +For an eligible task, the primary Agent makes one silent `evalctl observe` +attempt after the task outcome is fixed. It does not: + +- run Ward or Seal to discover a metric or version; +- ask a question or request approval; +- retry, emit progress, or add an Eval message; +- mutate the task result or turn an observation failure into a task failure. + +The Host task history supplies the population denominator and missing-attempt +count. The journal intentionally does not store skipped rows or task identity. +The managed host block is removed after the twentieth eligible task. + +## Decision boundary + +After 20 eligible tasks, a person performs the privacy review and manual report. +Eval does not infer causality, recommend a release, or retain, promote, or remove +another module. + +- Fewer than 10 successful rows: do not promote; reduce input or stop Eval. +- Provisioning failures: report separately from observation burden. +- Ward or Seal used/unused cohort below 5: do not compare that module. +- Reintroduce `validate`, `summarize`, `compare`, Plugin packaging, or Harness + registration only after the corresponding repeated need is observed. + +## Development verification + +```sh +gofmt -w cmd internal +go vet ./... +go test ./... +go test -race ./... +go build ./cmd/evalctl +``` + +GitHub Actions runs formatting, vet, tests, and builds on macOS, Linux, and +Windows, plus the race detector on Linux. From 90d13e0e8ce70c50ca308cefd303eaf9337ff046 Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:16:17 +0900 Subject: [PATCH 06/14] ci(eval): enforce LF for Go sources Why: - Windows checkout converted Go sources before the shared gofmt gate, so the Windows job failed before tests ran. What: - Mark Go sources as LF in repository attributes across all runners. Validation: - gofmt -l . produced no output - git check-attr reports eol=lf for Unix and Windows Go files --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d207b18 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.go text eol=lf From 08e0f3e8de33f53ba09fdbb2b2465f707a70462d Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:18:38 +0900 Subject: [PATCH 07/14] fix(store): validate Windows security descriptors Why: - Elevated Windows runners can use a token default owner that differs from the current-user SID. - The DACL accessor boolean reports whether the ACL was defaulted, not whether it is present. What: - Set the current user explicitly as owner and sole full-control trustee on new private objects. - Check SE_DACL_PRESENT through descriptor control bits before inspecting state-root access. Validation: - Windows amd64 store and CLI test binaries compiled - Windows vet passed - Local go test and race test passed - GitHub Windows execution pending on the updated head --- internal/store/platform_windows.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/store/platform_windows.go b/internal/store/platform_windows.go index 63463d6..9c15161 100644 --- a/internal/store/platform_windows.go +++ b/internal/store/platform_windows.go @@ -78,8 +78,15 @@ func checkStateRoot(path string, info os.FileInfo) error { if !windows.EqualSid(owner, user) { return storeError(CategoryPermission, "check-state-root-owner", path, ErrPermission) } - dacl, present, err := descriptor.DACL() - if err != nil || !present || dacl == nil { + control, _, err := descriptor.Control() + if err != nil { + return classifyError("read-state-root-security-control", path, err) + } + if control&windows.SE_DACL_PRESENT == 0 { + return storeError(CategoryPermission, "read-state-root-dacl", path, ErrPermission) + } + dacl, _, err := descriptor.DACL() + if err != nil || dacl == nil { return storeError(CategoryPermission, "read-state-root-dacl", path, errors.Join(ErrPermission, err)) } administrator, err := windows.CreateWellKnownSid(windows.WinBuiltinAdministratorsSid) @@ -394,8 +401,9 @@ func privateSecurityAttributes() (*windows.SecurityAttributes, error) { if err != nil { return nil, err } + userString := user.String() descriptor, err := windows.SecurityDescriptorFromString( - fmt.Sprintf("D:P(A;;FA;;;%s)", user.String()), + fmt.Sprintf("O:%sD:P(A;;FA;;;%s)", userString, userString), ) if err != nil { return nil, err From 278d407ccfc8e7d43825cf72e7eae93552968643 Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:22:05 +0900 Subject: [PATCH 08/14] test(store): cover Windows private root semantics Why: - GitHub Windows temp directories inherit broader ACLs than an accepted private state root. - Windows may block a state-root rename while os.Root handles are open instead of allowing the Unix path-move scenario. What: - Run CLI tests against recorder-created private state roots. - Keep the Unix anchored-rename test and add a Windows assertion for either a blocked move or an anchored commit. Validation: - Local unit and race tests passed - Windows amd64 store and CLI test binaries compiled - Windows vet passed - GitHub Windows execution pending on the updated head --- internal/cli/cli_test.go | 19 ++++++---- internal/store/platform_unix_test.go | 30 +++++++++++++++ internal/store/platform_windows_test.go | 49 +++++++++++++++++++++++++ internal/store/store_test.go | 30 --------------- 4 files changed, 91 insertions(+), 37 deletions(-) diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index 99e7e8f..c777366 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -48,7 +48,7 @@ func TestVersionAndUsageSurface(t *testing.T) { func TestObserveRecordsCanonicalRow(t *testing.T) { t.Parallel() - root := t.TempDir() + root := testStateRoot(t) result, exit := captureObserve(t, root, validDraft, store.Options{}) if exit != ExitSuccess || result.Status != "recorded" || result.Slot != 1 { t.Fatalf("observe = %+v exit %d", result, exit) @@ -80,7 +80,7 @@ func TestObserveInvalidInputNeverCreatesJournal(t *testing.T) { } { t.Run(name, func(t *testing.T) { t.Parallel() - root := t.TempDir() + root := testStateRoot(t) result, exit := captureObserve(t, root, draft, store.Options{}) if exit != ExitSuccess || result.Status != "skipped" || result.Reason != "invalid-observation" { t.Fatalf("observe = %+v exit %d", result, exit) @@ -93,7 +93,7 @@ func TestObserveInvalidInputNeverCreatesJournal(t *testing.T) { } func TestObserveConcurrentSlotsAndBound(t *testing.T) { - root := t.TempDir() + root := testStateRoot(t) results := make(chan capturedResult, experiment.MaxRows) var wait sync.WaitGroup for index := int64(0); index < experiment.MaxRows; index++ { @@ -138,7 +138,7 @@ func TestObserveConcurrentSlotsAndBound(t *testing.T) { func TestObserveInvalidJournalIsPreserved(t *testing.T) { t.Parallel() - root := t.TempDir() + root := testStateRoot(t) if result, _ := captureObserve(t, root, validDraft, store.Options{}); result.Status != "recorded" { t.Fatalf("seed result = %+v", result) } @@ -173,7 +173,7 @@ func TestObserveFaultBoundaries(t *testing.T) { } for name, hooks := range precommit { t.Run(name, func(t *testing.T) { - root := t.TempDir() + root := testStateRoot(t) result, exit := captureObserve(t, root, validDraft, store.Options{Hooks: hooks}) if exit != ExitSuccess || result.Status != "skipped" || result.Reason != "state-io-error" { t.Fatalf("result = %+v exit %d", result, exit) @@ -190,7 +190,7 @@ func TestObserveFaultBoundaries(t *testing.T) { } for name, hooks := range postcommit { t.Run(name, func(t *testing.T) { - root := t.TempDir() + root := testStateRoot(t) result, exit := captureObserve(t, root, validDraft, store.Options{Hooks: hooks}) if exit != ExitSuccess || result.Status != "recorded" || result.Slot != 1 || result.Durability != "unconfirmed" { t.Fatalf("result = %+v exit %d", result, exit) @@ -214,7 +214,7 @@ func TestObservePrecommitFaultPreservesExistingJournal(t *testing.T) { "beforeReplace": {BeforeReplace: func(_, _ string) error { return errors.New("injected") }}, } { t.Run(name, func(t *testing.T) { - root := t.TempDir() + root := testStateRoot(t) if result, _ := captureObserve(t, root, validDraft, store.Options{}); result.Status != "recorded" { t.Fatalf("seed result = %+v", result) } @@ -253,6 +253,11 @@ func captureObserve(t *testing.T, root, draft string, options store.Options) (ca return result, exit } +func testStateRoot(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "state") +} + func TestStateRootMustBeAbsoluteButFailureIsBestEffort(t *testing.T) { t.Parallel() var stdout bytes.Buffer diff --git a/internal/store/platform_unix_test.go b/internal/store/platform_unix_test.go index e35fbb3..b3ac454 100644 --- a/internal/store/platform_unix_test.go +++ b/internal/store/platform_unix_test.go @@ -4,11 +4,41 @@ package store import ( "context" + "errors" "os" "path/filepath" "testing" ) +func TestUpdateAnchorsCommitWhenStateRootPathMoves(t *testing.T) { + parent := t.TempDir() + root := filepath.Join(parent, "state") + if err := createPrivateDir(root); err != nil { + t.Fatal(err) + } + moved := filepath.Join(parent, "state-moved") + journal := mustStore(t, root, Options{Hooks: Hooks{ + BeforeReplace: func(_, _ string) error { + if err := os.Rename(root, moved); err != nil { + return err + } + return createPrivateDir(root) + }, + }}) + commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)) + if err != nil || !commit.Committed { + t.Fatalf("commit = %+v err = %v", commit, err) + } + newPath := filepath.Join(root, filepath.FromSlash(testRelativeJournal)) + if _, err := os.Lstat(newPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("replacement root was mutated: %v", err) + } + movedPath := filepath.Join(moved, filepath.FromSlash(testRelativeJournal)) + if got, err := os.ReadFile(movedPath); err != nil || string(got) != "{\"value\":1}\n" { + t.Fatalf("anchored journal = %q err = %v", got, err) + } +} + func TestUpdateRejectsWritableAncestor(t *testing.T) { parent := t.TempDir() shared := filepath.Join(parent, "shared") diff --git a/internal/store/platform_windows_test.go b/internal/store/platform_windows_test.go index 996b767..9deb1f1 100644 --- a/internal/store/platform_windows_test.go +++ b/internal/store/platform_windows_test.go @@ -169,6 +169,55 @@ func TestWindowsExclusiveLockTimeoutAndRelease(t *testing.T) { releaseFileLock(second) } +func TestWindowsStateRootMoveCannotRedirectCommit(t *testing.T) { + parent := t.TempDir() + root := filepath.Join(parent, "state") + if err := createPrivateDir(root); err != nil { + t.Fatal(err) + } + moved := filepath.Join(parent, "state-moved") + moveSucceeded := false + journal := mustStore(t, root, Options{Hooks: Hooks{ + BeforeReplace: func(_, _ string) error { + if err := os.Rename(root, moved); err != nil { + return err + } + moveSucceeded = true + return createPrivateDir(root) + }, + }}) + commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)) + if moveSucceeded { + if err != nil || !commit.Committed { + t.Fatalf("anchored commit = %+v err = %v", commit, err) + } + newPath := filepath.Join(root, filepath.FromSlash(testRelativeJournal)) + if _, err := os.Lstat(newPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("replacement root was mutated: %v", err) + } + movedPath := filepath.Join(moved, filepath.FromSlash(testRelativeJournal)) + if got, err := os.ReadFile(movedPath); err != nil || string(got) != "{\"value\":1}\n" { + t.Fatalf("anchored journal = %q err = %v", got, err) + } + return + } + if err == nil || commit.Committed { + t.Fatalf("commit = %+v err = %v, want blocked precommit move", commit, err) + } + if !errors.Is(err, windows.ERROR_SHARING_VIOLATION) && !errors.Is(err, windows.ERROR_ACCESS_DENIED) { + t.Fatalf("move error = %v, want Windows sharing or access denial", err) + } + if _, err := os.Stat(root); err != nil { + t.Fatalf("original root unavailable after blocked move: %v", err) + } + if _, err := os.Lstat(moved); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("moved root exists after blocked move: %v", err) + } + if _, err := os.Lstat(journal.Path()); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("journal committed after blocked move: %v", err) + } +} + func newWindowsPrivateFile(t *testing.T) string { t.Helper() directory := filepath.Join(t.TempDir(), "private") diff --git a/internal/store/store_test.go b/internal/store/store_test.go index bd31095..362219f 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -3,7 +3,6 @@ package store import ( "bytes" "context" - "errors" "os" "path/filepath" "testing" @@ -89,35 +88,6 @@ func TestUpdateLockTimeoutPreservesJournal(t *testing.T) { } } -func TestUpdateAnchorsCommitWhenStateRootPathMoves(t *testing.T) { - parent := t.TempDir() - root := filepath.Join(parent, "state") - if err := createPrivateDir(root); err != nil { - t.Fatal(err) - } - moved := filepath.Join(parent, "state-moved") - journal := mustStore(t, root, Options{Hooks: Hooks{ - BeforeReplace: func(_, _ string) error { - if err := os.Rename(root, moved); err != nil { - return err - } - return createPrivateDir(root) - }, - }}) - commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)) - if err != nil || !commit.Committed { - t.Fatalf("commit = %+v err = %v", commit, err) - } - newPath := filepath.Join(root, filepath.FromSlash(testRelativeJournal)) - if _, err := os.Lstat(newPath); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("replacement root was mutated: %v", err) - } - movedPath := filepath.Join(moved, filepath.FromSlash(testRelativeJournal)) - if got, err := os.ReadFile(movedPath); err != nil || string(got) != "{\"value\":1}\n" { - t.Fatalf("anchored journal = %q err = %v", got, err) - } -} - func TestUpdateRequiresExactAppend(t *testing.T) { t.Parallel() for name, transaction := range map[string]Transaction{ From d18fe81af09bc8f67bc2755f34733afe38b095d5 Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:46:07 +0900 Subject: [PATCH 09/14] fix(store): anchor journal existence checks Why: - An absolute-path miss during a state-root move could classify an anchored Journal as empty and risk replacing prior rows. What: - Treat the Journal as absent only when rooted and absolute entries are both absent. - Reject one-sided absence and file-identity mismatch as unsafe. Validation: - go test -count=1 ./internal/store - go test -count=1 ./... - go test -race -count=1 ./... --- internal/store/store.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/store/store.go b/internal/store/store.go index bbec825..4207c88 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -267,9 +267,15 @@ func (s *Store) verifyDirectories(root *os.Root) error { } func (s *Store) readExisting(root *os.Root) ([]byte, error) { - if err := inspectPrivateFile(s.path); errors.Is(err, fs.ErrNotExist) { + rootInfo, rootErr := root.Lstat(s.journalName) + absoluteInfo, absoluteErr := os.Lstat(s.path) + if errors.Is(rootErr, fs.ErrNotExist) && errors.Is(absoluteErr, fs.ErrNotExist) { return nil, nil - } else if err != nil { + } + if rootErr != nil || absoluteErr != nil || !os.SameFile(rootInfo, absoluteInfo) { + return nil, storeError(CategoryUnsafePath, "inspect-private-file-identity", s.path, ErrUnsafePath) + } + if err := inspectPrivateFile(s.path); err != nil { return nil, err } file, err := root.Open(s.journalName) From ed75e17bc0a9d92a1b603428d4590bc9324f190c Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:46:14 +0900 Subject: [PATCH 10/14] test(store): cover moved root journal reads Why: - The rooted-read regression requires a deterministic state-root move that Unix permits while os.Root handles remain open. What: - Seed a Journal, move its state root, and assert the anchored read fails closed without changing prior bytes. Validation: - go test -count=1 ./internal/store - go test -race -count=1 ./... - git diff --check --- internal/store/platform_unix_test.go | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/internal/store/platform_unix_test.go b/internal/store/platform_unix_test.go index b3ac454..eb5f386 100644 --- a/internal/store/platform_unix_test.go +++ b/internal/store/platform_unix_test.go @@ -39,6 +39,39 @@ func TestUpdateAnchorsCommitWhenStateRootPathMoves(t *testing.T) { } } +func TestReadExistingRejectsMovedAbsolutePath(t *testing.T) { + root := privateTestRoot(t) + journal := mustStore(t, root, Options{}) + if commit, err := journal.Update(context.Background(), appendObject(`{"value":1}`)); err != nil || !commit.Committed { + t.Fatalf("seed commit = %+v err = %v", commit, err) + } + + stateRoot, err := os.OpenRoot(journal.root) + if err != nil { + t.Fatal(err) + } + defer stateRoot.Close() + journalRoot, err := stateRoot.OpenRoot(journal.relativeDir) + if err != nil { + t.Fatal(err) + } + defer journalRoot.Close() + + moved := journal.root + "-moved" + if err := os.Rename(journal.root, moved); err != nil { + t.Fatal(err) + } + existing, err := journal.readExisting(journalRoot) + if err == nil || !errors.Is(err, ErrUnsafePath) || CategoryOf(err) != CategoryUnsafePath || len(existing) != 0 { + t.Fatalf("existing = %q err = %v category = %s", existing, err, CategoryOf(err)) + } + + movedPath := filepath.Join(moved, journal.relativePath) + if got, err := os.ReadFile(movedPath); err != nil || string(got) != "{\"value\":1}\n" { + t.Fatalf("moved journal = %q err = %v", got, err) + } +} + func TestUpdateRejectsWritableAncestor(t *testing.T) { parent := t.TempDir() shared := filepath.Join(parent, "shared") From 60698f9ff66057778b6ec519acbf1692d72b519d Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:46:21 +0900 Subject: [PATCH 11/14] fix(state): resolve platform user home Why: - Native Windows normally provides USERPROFILE rather than HOME, so the default recorder path could be skipped as unsafe. What: - Resolve explicit and XDG roots before a platform-aware user-home fallback. - Inject os.UserHomeDir through the internal CLI runtime for deterministic tests. Validation: - go test -count=1 ./internal/state ./internal/cli - go vet ./... - GOOS=windows GOARCH=amd64 go vet ./... - Darwin, Linux, and Windows evalctl cross-builds --- internal/cli/cli.go | 6 +++++- internal/state/state.go | 16 +++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index fd753d7..ca60cdb 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -32,6 +32,7 @@ type Runtime struct { Stdout io.Writer Stderr io.Writer Getenv func(string) string + UserHomeDir func() (string, error) Now func() time.Time StoreOptions store.Options } @@ -73,7 +74,7 @@ func runObserve(ctx context.Context, args []string, runtime Runtime) int { writeResult(runtime.Stdout, observeResult{Status: "skipped", Reason: "invalid-observation"}) return ExitSuccess } - root, err := state.Root(*stateRoot, runtime.Getenv) + root, err := state.Root(*stateRoot, runtime.Getenv, runtime.UserHomeDir) if err != nil { writeResult(runtime.Stdout, observeResult{Status: "skipped", Reason: "unsafe-state-path"}) return ExitSuccess @@ -171,6 +172,9 @@ func defaults(runtime Runtime) Runtime { if runtime.Getenv == nil { runtime.Getenv = os.Getenv } + if runtime.UserHomeDir == nil { + runtime.UserHomeDir = os.UserHomeDir + } if runtime.Now == nil { runtime.Now = time.Now } diff --git a/internal/state/state.go b/internal/state/state.go index 5448012..cff598f 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -8,19 +8,25 @@ import ( const JournalRelativePath = "jgoneit/eval-experiment/v1/journal.jsonl" -// Root resolves the XDG state root without creating it. Store owns creation -// and security checks; this package only defines the public location contract. -func Root(explicit string, getenv func(string) string) (string, error) { +// Root resolves the recorder state root without creating it. Store owns +// creation and security checks; this package only defines the location contract. +func Root(explicit string, getenv func(string) string, userHomeDir func() (string, error)) (string, error) { if getenv == nil { getenv = os.Getenv } + if userHomeDir == nil { + userHomeDir = os.UserHomeDir + } root := explicit if root == "" { root = getenv("XDG_STATE_HOME") if root == "" { - home := getenv("HOME") + home, err := userHomeDir() + if err != nil { + return "", fmt.Errorf("resolve user home: %w", err) + } if home == "" || !filepath.IsAbs(home) || filepath.Clean(home) != home { - return "", fmt.Errorf("HOME must be an absolute clean path") + return "", fmt.Errorf("user home must be an absolute clean path") } root = filepath.Join(home, ".local", "state") } From f126027ec2ceb99a662e95f25080d09555c4aaff Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:46:33 +0900 Subject: [PATCH 12/14] test(state): cover platform home fallback Why: - Root precedence and the Windows USERPROFILE fallback need regression coverage independent of the host environment. What: - Cover explicit, XDG, resolver-error, and invalid-home cases with injected dependencies. - Add a Windows integration test that records without --state-root when only USERPROFILE is set. Validation: - go test -count=1 ./internal/state ./internal/cli - go test -race -count=1 ./internal/state ./internal/cli - Windows state, CLI, and Store test binaries cross-compiled - Windows execution pending GitHub Actions --- internal/cli/cli_windows_test.go | 55 ++++++++++++ internal/state/state_test.go | 146 +++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 internal/cli/cli_windows_test.go create mode 100644 internal/state/state_test.go diff --git a/internal/cli/cli_windows_test.go b/internal/cli/cli_windows_test.go new file mode 100644 index 0000000..aa32033 --- /dev/null +++ b/internal/cli/cli_windows_test.go @@ -0,0 +1,55 @@ +//go:build windows + +package cli + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jgoneit/eval/internal/experiment" + "github.com/jgoneit/eval/internal/state" +) + +func TestObserveUsesWindowsUserProfileByDefault(t *testing.T) { + profile := filepath.Join(t.TempDir(), "profile") + t.Setenv("XDG_STATE_HOME", "") + t.Setenv("HOME", "") + t.Setenv("USERPROFILE", profile) + + var stdout, stderr bytes.Buffer + exit := Run(context.Background(), []string{"observe"}, Runtime{ + Stdin: strings.NewReader(validDraft), + Stdout: &stdout, + Stderr: &stderr, + Now: func() time.Time { return time.Date(2026, 8, 26, 12, 34, 56, 123, time.UTC) }, + }) + var result capturedResult + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatalf("decode %q: %v (stderr %q)", stdout.String(), err, stderr.String()) + } + if exit != ExitSuccess || result.Status != "recorded" || result.Slot != 1 { + t.Fatalf("observe = %+v exit %d (stderr %q)", result, exit, stderr.String()) + } + if result.Durability != "confirmed" && result.Durability != "unconfirmed" { + t.Fatalf("durability = %q", result.Durability) + } + + root := filepath.Join(profile, ".local", "state") + data, err := os.ReadFile(state.JournalPath(root)) + if err != nil { + t.Fatal(err) + } + rows, err := experiment.ParseJournal(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].Slot != 1 { + t.Fatalf("stored rows = %#v", rows) + } +} diff --git a/internal/state/state_test.go b/internal/state/state_test.go new file mode 100644 index 0000000..84d3fc0 --- /dev/null +++ b/internal/state/state_test.go @@ -0,0 +1,146 @@ +package state + +import ( + "errors" + "path/filepath" + "testing" +) + +func TestRootPrefersExplicitPath(t *testing.T) { + t.Parallel() + explicit := filepath.Join(t.TempDir(), "explicit") + getenvCalled := false + homeCalled := false + + got, err := Root( + explicit, + func(string) string { + getenvCalled = true + return filepath.Join(t.TempDir(), "xdg") + }, + func() (string, error) { + homeCalled = true + return t.TempDir(), nil + }, + ) + if err != nil { + t.Fatal(err) + } + if got != explicit { + t.Fatalf("root = %q, want %q", got, explicit) + } + if getenvCalled || homeCalled { + t.Fatalf("explicit path consulted environment: getenv=%t home=%t", getenvCalled, homeCalled) + } +} + +func TestRootPrefersXDGStateHome(t *testing.T) { + t.Parallel() + xdg := filepath.Join(t.TempDir(), "xdg") + homeCalled := false + + got, err := Root( + "", + func(name string) string { + if name == "XDG_STATE_HOME" { + return xdg + } + return "" + }, + func() (string, error) { + homeCalled = true + return t.TempDir(), nil + }, + ) + if err != nil { + t.Fatal(err) + } + if got != xdg { + t.Fatalf("root = %q, want %q", got, xdg) + } + if homeCalled { + t.Fatal("XDG resolution consulted user home") + } +} + +func TestRootFallsBackToInjectedUserHome(t *testing.T) { + t.Parallel() + home := t.TempDir() + homeCalls := 0 + + got, err := Root( + "", + func(string) string { return "" }, + func() (string, error) { + homeCalls++ + return home, nil + }, + ) + if err != nil { + t.Fatal(err) + } + want := filepath.Join(home, ".local", "state") + if got != want { + t.Fatalf("root = %q, want %q", got, want) + } + if homeCalls != 1 { + t.Fatalf("user home calls = %d, want 1", homeCalls) + } +} + +func TestRootRejectsInvalidXDGWithoutHomeFallback(t *testing.T) { + t.Parallel() + homeCalled := false + + _, err := Root( + "", + func(name string) string { + if name == "XDG_STATE_HOME" { + return "relative-state" + } + return "" + }, + func() (string, error) { + homeCalled = true + return t.TempDir(), nil + }, + ) + if err == nil { + t.Fatal("relative XDG state root was accepted") + } + if homeCalled { + t.Fatal("invalid nonempty XDG state root fell back to user home") + } +} + +func TestRootReportsUserHomeFailure(t *testing.T) { + t.Parallel() + wantErr := errors.New("home unavailable") + + _, err := Root( + "", + func(string) string { return "" }, + func() (string, error) { return "", wantErr }, + ) + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want wrapped %v", err, wantErr) + } +} + +func TestRootRejectsInvalidUserHome(t *testing.T) { + t.Parallel() + unclean := t.TempDir() + string(filepath.Separator) + "." + for _, home := range []string{"", "relative-home", unclean} { + home := home + t.Run(home, func(t *testing.T) { + t.Parallel() + if _, err := Root( + "", + func(string) string { return "" }, + func() (string, error) { return home, nil }, + ); err == nil { + t.Fatalf("invalid user home %q was accepted", home) + } + }) + } +} From e6d97261573d6638bf65b4c55f7ef50e9e449922 Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:46:40 +0900 Subject: [PATCH 13/14] docs(eval): document Windows journal fallback Why: - The default Journal documentation described only the Unix HOME fallback. What: - Distinguish the shared XDG path from Darwin/Linux HOME and Windows USERPROFILE fallbacks. Validation: - Markdown source reviewed - git diff --check --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aa8fde8..b89c475 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,9 @@ dates. It generates only `schema_version`, `slot`, and `recorded_at`. The default journal is: ```text -$XDG_STATE_HOME/jgoneit/eval-experiment/v1/journal.jsonl -$HOME/.local/state/jgoneit/eval-experiment/v1/journal.jsonl +$XDG_STATE_HOME/jgoneit/eval-experiment/v1/journal.jsonl (all platforms when set) +$HOME/.local/state/jgoneit/eval-experiment/v1/journal.jsonl (Darwin/Linux fallback) +%USERPROFILE%\.local\state\jgoneit\eval-experiment\v1\journal.jsonl (Windows fallback) ``` The journal contains successful observations only and stops after 20 contiguous From 6c05f97cd54be92c7cac56713115ebc4b567655e Mon Sep 17 00:00:00 2001 From: jgoneit Date: Wed, 26 Aug 2026 23:46:57 +0900 Subject: [PATCH 14/14] chore(config): quarantine legacy private data Why: - Upgraded checkouts may still contain previously ignored private observations that git add -A must never stage. What: - Keep only the two retired checkout-local private paths ignored. - Add a CI canary that verifies they remain untracked, ignored, and absent from dry-run staging. Validation: - git check-ignore passed for both root-only paths - git ls-files reported no tracked legacy private data - workflow static check passed when actionlint was available - git diff --check --- .github/workflows/verify.yml | 10 ++++++++++ .gitignore | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2a10296..b5a4a80 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -67,3 +67,13 @@ jobs: ! rg -n 'eval-observation|observations\.jsonl|task_id|supersedes|observe-draft-v2' cmd internal schemas go.mod ! rg -n 'runValidate|runSummarize|runCompare|case "(validate|summarize|compare)"' cmd internal/cli --glob '!**/*_test.go' ! rg -n '/Users/[[:alnum:]_.-]+/' . --hidden --glob '!.git/**' + - name: Verify legacy privacy quarantine + shell: bash + run: | + mkdir -p reports/private + printf '%s\n' canary > observations.jsonl + printf '%s\n' canary > reports/private/canary.jsonl + test -z "$(git ls-files -- observations.jsonl reports/private)" + git check-ignore --quiet observations.jsonl + git check-ignore --quiet reports/private/canary.jsonl + test -z "$(git add --dry-run -A)" diff --git a/.gitignore b/.gitignore index 3d79cfd..7849393 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ .DS_Store .venv/ + +# Privacy quarantine for checkout-local data from legacy Eval versions. +/observations.jsonl +/reports/private/ + *.tmp *.swp coverage.out